loading objects from doab!

main
eric 2023-02-20 16:03:07 -05:00
parent ec779b2290
commit 1c64bc33a8
18 changed files with 4942 additions and 0 deletions

15
Pipfile Normal file
View File

@ -0,0 +1,15 @@
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[packages]
django = "*"
psycopg2 = "*"
pyoai = "*"
requests = "*"
[dev-packages]
[requires]
python_version = "3.9"

287
Pipfile.lock generated Normal file
View File

@ -0,0 +1,287 @@
{
"_meta": {
"hash": {
"sha256": "3f2b9a189c4fc85dfefbc27be2773392d202020860a5314e3bafbe6e139e4339"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.9"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"asgiref": {
"hashes": [
"sha256:71e68008da809b957b7ee4b43dbccff33d1b23519fb8344e33f049897077afac",
"sha256:9567dfe7bd8d3c8c892227827c41cce860b368104c3431da67a0c5a65a949506"
],
"markers": "python_version >= '3.7'",
"version": "==3.6.0"
},
"certifi": {
"hashes": [
"sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3",
"sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"
],
"markers": "python_version >= '3.6'",
"version": "==2022.12.7"
},
"charset-normalizer": {
"hashes": [
"sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b",
"sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42",
"sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d",
"sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b",
"sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a",
"sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59",
"sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154",
"sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1",
"sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c",
"sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a",
"sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d",
"sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6",
"sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b",
"sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b",
"sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783",
"sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5",
"sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918",
"sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555",
"sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639",
"sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786",
"sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e",
"sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed",
"sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820",
"sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8",
"sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3",
"sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541",
"sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14",
"sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be",
"sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e",
"sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76",
"sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b",
"sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c",
"sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b",
"sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3",
"sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc",
"sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6",
"sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59",
"sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4",
"sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d",
"sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d",
"sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3",
"sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a",
"sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea",
"sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6",
"sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e",
"sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603",
"sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24",
"sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a",
"sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58",
"sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678",
"sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a",
"sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c",
"sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6",
"sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18",
"sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174",
"sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317",
"sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f",
"sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc",
"sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837",
"sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41",
"sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c",
"sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579",
"sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753",
"sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8",
"sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291",
"sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087",
"sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866",
"sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3",
"sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d",
"sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1",
"sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca",
"sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e",
"sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db",
"sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72",
"sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d",
"sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc",
"sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539",
"sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d",
"sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af",
"sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b",
"sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602",
"sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f",
"sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478",
"sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c",
"sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e",
"sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479",
"sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7",
"sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8"
],
"version": "==3.0.1"
},
"django": {
"hashes": [
"sha256:44f714b81c5f190d9d2ddad01a532fe502fa01c4cb8faf1d081f4264ed15dcd8",
"sha256:f2f431e75adc40039ace496ad3b9f17227022e8b11566f4b363da44c7e44761e"
],
"index": "pypi",
"version": "==4.1.7"
},
"idna": {
"hashes": [
"sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
"sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
],
"markers": "python_version >= '3.5'",
"version": "==3.4"
},
"lxml": {
"hashes": [
"sha256:01d36c05f4afb8f7c20fd9ed5badca32a2029b93b1750f571ccc0b142531caf7",
"sha256:04876580c050a8c5341d706dd464ff04fd597095cc8c023252566a8826505726",
"sha256:05ca3f6abf5cf78fe053da9b1166e062ade3fa5d4f92b4ed688127ea7d7b1d03",
"sha256:090c6543d3696cbe15b4ac6e175e576bcc3f1ccfbba970061b7300b0c15a2140",
"sha256:0dc313ef231edf866912e9d8f5a042ddab56c752619e92dfd3a2c277e6a7299a",
"sha256:0f2b1e0d79180f344ff9f321327b005ca043a50ece8713de61d1cb383fb8ac05",
"sha256:13598ecfbd2e86ea7ae45ec28a2a54fb87ee9b9fdb0f6d343297d8e548392c03",
"sha256:16efd54337136e8cd72fb9485c368d91d77a47ee2d42b057564aae201257d419",
"sha256:1ab8f1f932e8f82355e75dda5413a57612c6ea448069d4fb2e217e9a4bed13d4",
"sha256:223f4232855ade399bd409331e6ca70fb5578efef22cf4069a6090acc0f53c0e",
"sha256:2455cfaeb7ac70338b3257f41e21f0724f4b5b0c0e7702da67ee6c3640835b67",
"sha256:2899456259589aa38bfb018c364d6ae7b53c5c22d8e27d0ec7609c2a1ff78b50",
"sha256:2a29ba94d065945944016b6b74e538bdb1751a1db6ffb80c9d3c2e40d6fa9894",
"sha256:2a87fa548561d2f4643c99cd13131acb607ddabb70682dcf1dff5f71f781a4bf",
"sha256:2e430cd2824f05f2d4f687701144556646bae8f249fd60aa1e4c768ba7018947",
"sha256:36c3c175d34652a35475a73762b545f4527aec044910a651d2bf50de9c3352b1",
"sha256:3818b8e2c4b5148567e1b09ce739006acfaa44ce3156f8cbbc11062994b8e8dd",
"sha256:3ab9fa9d6dc2a7f29d7affdf3edebf6ece6fb28a6d80b14c3b2fb9d39b9322c3",
"sha256:3efea981d956a6f7173b4659849f55081867cf897e719f57383698af6f618a92",
"sha256:4c8f293f14abc8fd3e8e01c5bd86e6ed0b6ef71936ded5bf10fe7a5efefbaca3",
"sha256:5344a43228767f53a9df6e5b253f8cdca7dfc7b7aeae52551958192f56d98457",
"sha256:58bfa3aa19ca4c0f28c5dde0ff56c520fbac6f0daf4fac66ed4c8d2fb7f22e74",
"sha256:5b4545b8a40478183ac06c073e81a5ce4cf01bf1734962577cf2bb569a5b3bbf",
"sha256:5f50a1c177e2fa3ee0667a5ab79fdc6b23086bc8b589d90b93b4bd17eb0e64d1",
"sha256:63da2ccc0857c311d764e7d3d90f429c252e83b52d1f8f1d1fe55be26827d1f4",
"sha256:6749649eecd6a9871cae297bffa4ee76f90b4504a2a2ab528d9ebe912b101975",
"sha256:6804daeb7ef69e7b36f76caddb85cccd63d0c56dedb47555d2fc969e2af6a1a5",
"sha256:689bb688a1db722485e4610a503e3e9210dcc20c520b45ac8f7533c837be76fe",
"sha256:699a9af7dffaf67deeae27b2112aa06b41c370d5e7633e0ee0aea2e0b6c211f7",
"sha256:6b418afe5df18233fc6b6093deb82a32895b6bb0b1155c2cdb05203f583053f1",
"sha256:76cf573e5a365e790396a5cc2b909812633409306c6531a6877c59061e42c4f2",
"sha256:7b515674acfdcadb0eb5d00d8a709868173acece5cb0be3dd165950cbfdf5409",
"sha256:7b770ed79542ed52c519119473898198761d78beb24b107acf3ad65deae61f1f",
"sha256:7d2278d59425777cfcb19735018d897ca8303abe67cc735f9f97177ceff8027f",
"sha256:7e91ee82f4199af8c43d8158024cbdff3d931df350252288f0d4ce656df7f3b5",
"sha256:821b7f59b99551c69c85a6039c65b75f5683bdc63270fec660f75da67469ca24",
"sha256:822068f85e12a6e292803e112ab876bc03ed1f03dddb80154c395f891ca6b31e",
"sha256:8340225bd5e7a701c0fa98284c849c9b9fc9238abf53a0ebd90900f25d39a4e4",
"sha256:85cabf64adec449132e55616e7ca3e1000ab449d1d0f9d7f83146ed5bdcb6d8a",
"sha256:880bbbcbe2fca64e2f4d8e04db47bcdf504936fa2b33933efd945e1b429bea8c",
"sha256:8d0b4612b66ff5d62d03bcaa043bb018f74dfea51184e53f067e6fdcba4bd8de",
"sha256:8e20cb5a47247e383cf4ff523205060991021233ebd6f924bca927fcf25cf86f",
"sha256:925073b2fe14ab9b87e73f9a5fde6ce6392da430f3004d8b72cc86f746f5163b",
"sha256:998c7c41910666d2976928c38ea96a70d1aa43be6fe502f21a651e17483a43c5",
"sha256:9b22c5c66f67ae00c0199f6055705bc3eb3fcb08d03d2ec4059a2b1b25ed48d7",
"sha256:9f102706d0ca011de571de32c3247c6476b55bb6bc65a20f682f000b07a4852a",
"sha256:a08cff61517ee26cb56f1e949cca38caabe9ea9fbb4b1e10a805dc39844b7d5c",
"sha256:a0a336d6d3e8b234a3aae3c674873d8f0e720b76bc1d9416866c41cd9500ffb9",
"sha256:a35f8b7fa99f90dd2f5dc5a9fa12332642f087a7641289ca6c40d6e1a2637d8e",
"sha256:a38486985ca49cfa574a507e7a2215c0c780fd1778bb6290c21193b7211702ab",
"sha256:a5da296eb617d18e497bcf0a5c528f5d3b18dadb3619fbdadf4ed2356ef8d941",
"sha256:a6e441a86553c310258aca15d1c05903aaf4965b23f3bc2d55f200804e005ee5",
"sha256:a82d05da00a58b8e4c0008edbc8a4b6ec5a4bc1e2ee0fb6ed157cf634ed7fa45",
"sha256:ab323679b8b3030000f2be63e22cdeea5b47ee0abd2d6a1dc0c8103ddaa56cd7",
"sha256:b1f42b6921d0e81b1bcb5e395bc091a70f41c4d4e55ba99c6da2b31626c44892",
"sha256:b23e19989c355ca854276178a0463951a653309fb8e57ce674497f2d9f208746",
"sha256:b264171e3143d842ded311b7dccd46ff9ef34247129ff5bf5066123c55c2431c",
"sha256:b26a29f0b7fc6f0897f043ca366142d2b609dc60756ee6e4e90b5f762c6adc53",
"sha256:b64d891da92e232c36976c80ed7ebb383e3f148489796d8d31a5b6a677825efe",
"sha256:b9cc34af337a97d470040f99ba4282f6e6bac88407d021688a5d585e44a23184",
"sha256:bc718cd47b765e790eecb74d044cc8d37d58562f6c314ee9484df26276d36a38",
"sha256:be7292c55101e22f2a3d4d8913944cbea71eea90792bf914add27454a13905df",
"sha256:c83203addf554215463b59f6399835201999b5e48019dc17f182ed5ad87205c9",
"sha256:c9ec3eaf616d67db0764b3bb983962b4f385a1f08304fd30c7283954e6a7869b",
"sha256:ca34efc80a29351897e18888c71c6aca4a359247c87e0b1c7ada14f0ab0c0fb2",
"sha256:ca989b91cf3a3ba28930a9fc1e9aeafc2a395448641df1f387a2d394638943b0",
"sha256:d02a5399126a53492415d4906ab0ad0375a5456cc05c3fc0fc4ca11771745cda",
"sha256:d17bc7c2ccf49c478c5bdd447594e82692c74222698cfc9b5daae7ae7e90743b",
"sha256:d5bf6545cd27aaa8a13033ce56354ed9e25ab0e4ac3b5392b763d8d04b08e0c5",
"sha256:d6b430a9938a5a5d85fc107d852262ddcd48602c120e3dbb02137c83d212b380",
"sha256:da248f93f0418a9e9d94b0080d7ebc407a9a5e6d0b57bb30db9b5cc28de1ad33",
"sha256:da4dd7c9c50c059aba52b3524f84d7de956f7fef88f0bafcf4ad7dde94a064e8",
"sha256:df0623dcf9668ad0445e0558a21211d4e9a149ea8f5666917c8eeec515f0a6d1",
"sha256:e5168986b90a8d1f2f9dc1b841467c74221bd752537b99761a93d2d981e04889",
"sha256:efa29c2fe6b4fdd32e8ef81c1528506895eca86e1d8c4657fda04c9b3786ddf9",
"sha256:f1496ea22ca2c830cbcbd473de8f114a320da308438ae65abad6bab7867fe38f",
"sha256:f49e52d174375a7def9915c9f06ec4e569d235ad428f70751765f48d5926678c"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==4.9.2"
},
"psycopg2": {
"hashes": [
"sha256:093e3894d2d3c592ab0945d9eba9d139c139664dcf83a1c440b8a7aa9bb21955",
"sha256:190d51e8c1b25a47484e52a79638a8182451d6f6dff99f26ad9bd81e5359a0fa",
"sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e",
"sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a",
"sha256:322fd5fca0b1113677089d4ebd5222c964b1760e361f151cbb2706c4912112c5",
"sha256:4cb9936316d88bfab614666eb9e32995e794ed0f8f6b3b718666c22819c1d7ee",
"sha256:920bf418000dd17669d2904472efeab2b20546efd0548139618f8fa305d1d7ad",
"sha256:922cc5f0b98a5f2b1ff481f5551b95cd04580fd6f0c72d9b22e6c0145a4840e0",
"sha256:a5246d2e683a972e2187a8714b5c2cf8156c064629f9a9b1a873c1730d9e245a",
"sha256:b9ac1b0d8ecc49e05e4e182694f418d27f3aedcfca854ebd6c05bb1cffa10d6d",
"sha256:d3ef67e630b0de0779c42912fe2cbae3805ebaba30cda27fea2a3de650a9414f",
"sha256:f5b6320dbc3cf6cfb9f25308286f9f7ab464e65cfb105b64cc9c52831748ced2",
"sha256:fc04dd5189b90d825509caa510f20d1d504761e78b8dfb95a0ede180f71d50e5"
],
"index": "pypi",
"version": "==2.9.5"
},
"pyoai": {
"hashes": [
"sha256:029521e1f6a819511feb4299a6181b5c312e8a71f7cddc4547e27001e7552be0"
],
"index": "pypi",
"version": "==2.5.0"
},
"requests": {
"hashes": [
"sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa",
"sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"
],
"index": "pypi",
"version": "==2.28.2"
},
"six": {
"hashes": [
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
"sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==1.16.0"
},
"sqlparse": {
"hashes": [
"sha256:0323c0ec29cd52bceabc1b4d9d579e311f3e4961b98d174201d5622a23b85e34",
"sha256:69ca804846bb114d2ec380e4360a8a340db83f0ccf3afceeb1404df028f57268"
],
"markers": "python_version >= '3.5'",
"version": "==0.4.3"
},
"urllib3": {
"hashes": [
"sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72",
"sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==1.26.14"
}
},
"develop": {}
}

0
doab_check/__init__.py Normal file
View File

12
doab_check/admin.py Normal file
View File

@ -0,0 +1,12 @@
from django.contrib import admin
# Register your models here.
from . import models
admin.site.register(models.Item)
admin.site.register(models.Link)
admin.site.register(models.Timestamp)
admin.site.register(models.Record)
admin.site.register(models.LinkRel)
admin.site.register(models.Check)

16
doab_check/asgi.py Normal file
View File

@ -0,0 +1,16 @@
"""
ASGI config for doab_check project.
It exposes the ASGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/4.1/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'doab_check.settings')
application = get_asgi_application()

128
doab_check/doab_oai.py Normal file
View File

@ -0,0 +1,128 @@
#!/usr/bin/env python
# encoding: utf-8
import datetime
import logging
import re
from oaipmh.client import Client
from oaipmh.error import IdDoesNotExistError, NoRecordsMatchError
from oaipmh.metadata import MetadataRegistry
import requests
from .doab_utils import doab_reader
from .models import Item, Link, Record, Timestamp
DOAB_OAIURL = 'https://directory.doabooks.org/oai/request'
DOAB_PATT = re.compile(r'oai:directory\.doabooks\.org:(.*)')
logger = logging.getLogger(__name__)
mdregistry = MetadataRegistry()
mdregistry.registerReader('oai_dc', doab_reader)
doab_client = Client(DOAB_OAIURL, mdregistry)
def unlist(alist):
if not alist:
return None
return alist[0]
def getdoab(url):
id_match = DOAB_PATT.search(url)
if id_match:
return f'oai:doab-books:{id_match.group(1)}'
return False
def add_by_doab(doab_id, record=None):
try:
record = record if record else doab_client.getRecord(
metadataPrefix='oai_dc',
identifier=doab_id
)
if not record[1]:
logger.error('No content in record %s', record)
return None
metadata = record[1].getMap()
urls = []
for ident in metadata.pop('identifier', []):
if ident.find('doabooks.org') >= 0:
# should already know the doab_id
continue
if ident.startswith('http'):
urls.append(ident)
title = unlist(metadata.pop('title', ['']))
item_type = unlist(metadata.pop('type', []))
timestamps = metadata.pop('timestamp', [])
added_record = load_doab_record(
doab_id,
title,
item_type,
urls,
timestamps,
**metadata
)
return added_record
except IdDoesNotExistError as e:
logger.error(e)
return None
def load_doab_record(doab_id, title, item_type, urls, timestamps, **kwargs):
"""
create a record from doabooks.org represented by input parameters
"""
logger.info('load doab %s', doab_id)
(new_item, created) = Item.objects.get_or_create(doab=doab_id)
new_record = Record.objects.create(item=new_item)
for timestamp in timestamps:
(new_timestamp, created) = Timestamp.objects.get_or_create(
datetime=timestamp,
record=new_record)
for url in urls:
url = url.strip()
(link, created) = Link.objects.get_or_create(url=url)
link.items.add(new_item)
return new_record
def load_doab_oai(from_date, until_date, limit=100):
'''
use oai feed to get oai updates
'''
start = datetime.datetime.now()
if from_date:
from_ = from_date
else:
# last 15 days
from_ = datetime.datetime.now() - datetime.timedelta(days=15)
num_doabs = 0
new_doabs = 0
lasttime = datetime.datetime(2000, 1, 1)
try:
for record in doab_client.listRecords(metadataPrefix='oai_dc', from_=from_,
until=until_date):
if not record[1]:
continue
item_type = unlist(record[1].getMap().get('type', None))
ident = record[0].identifier()
responsestamp = record[0].datestamp()
lasttime = responsestamp if responsestamp > lasttime else lasttime
doab = getdoab(ident)
if doab:
num_doabs += 1
rec = add_by_doab(doab, record=record)
if not rec:
logger.error('error for doab #%s', doab)
continue
if lasttime > start:
new_doabs += 1
title = rec.item.title
logger.info(u'updated:\t%s\t%s', doab, title)
if num_doabs >= limit:
break
except NoRecordsMatchError:
pass
return num_doabs, new_doabs, lasttime

86
doab_check/doab_utils.py Normal file
View File

@ -0,0 +1,86 @@
"""
doab_utils.py
"""
import logging
import re
from ssl import SSLError
from urllib.parse import urljoin
import requests
from oaipmh.metadata import MetadataReader
from django.conf import settings
logger = logging.getLogger(__name__)
doab_reader = MetadataReader(
fields={
'title': ('textList', 'oai_dc:dc/datacite:title/text()'),
'creator': ('textList', 'oai_dc:dc/datacite:creator/text()'),
'subject': ('textList', 'oai_dc:dc/datacite:subject/text()'),
'description': ('textList', 'oai_dc:dc/dc:description/text()'),
'publisher': ('textList', 'oai_dc:dc/dc:publisher/text()'),
'editor': ('textList', 'oai_dc:dc/datacite:contributor[@type="Editor"]/text()'),
'date': ('textList', 'oai_dc:dc/datacite:date[@type="Issued"]/text()'),
'timestamp': ('textList', 'oai_dc:dc/dc:date/text()'),
'type': ('textList', 'oai_dc:dc/oaire:resourceType/text()'),
'format': ('textList', 'oai_dc:dc/dc:format/text()'),
'identifier': ('textList', 'oai_dc:dc/dc:identifier/text()'),
'source': ('textList', 'oai_dc:dc/dc:source/text()'),
'language': ('textList', 'oai_dc:dc/dc:language/text()'),
'relation': ('textList', 'oai_dc:dc/dc:relation/text()'),
'coverage': ('textList', 'oai_dc:dc/dc:coverage/text()'),
'rights': ('textList', 'oai_dc:dc/oaire:licenseCondition/@uri'),
'isbn': ('textList', 'oai_dc:dc/datacite:alternateIdentifier[@type="ISBN"]/text()'),
'doi': ('textList', 'oai_dc:dc/datacite:alternateIdentifier[@type="DOI"]/text()'),
},
namespaces={
'oai_dc': 'http://www.openarchives.org/OAI/2.0/oai_dc/',
'dc' : 'http://purl.org/dc/elements/1.1/',
'grantor': 'http://purl.org/dc/elements/1.1/',
'publisher': 'http://purl.org/dc/elements/1.1/',
'oapen': 'http://purl.org/dc/elements/1.1/',
'oaire': 'https://raw.githubusercontent.com/rcic/openaire4/master/schemas/4.0/oaire.xsd',
'datacite': 'https://schema.datacite.org/meta/kernel-4.1/metadata.xsd',
'doc': 'http://www.lyncode.com/xoai'
}
)
STREAM_QUERY = 'https://directory.doabooks.org/rest/search?query=handle:{}&expand=bitstreams'
def get_streamdata(handle):
url = STREAM_QUERY.format(handle)
try:
response = requests.get(url, headers={"User-Agent": settings.USER_AGENT})
items = response.json()
if items:
for stream in items[0]['bitstreams']:
if stream['bundleName'] == "THUMBNAIL":
stream['handle'] = handle
return stream
else:
logger.error("No items in streamdata for %s", handle)
except requests.exceptions.RequestException as e:
logger.error(e)
except SSLError as e:
logger.error(e)
except ValueError as e:
# decoder error
logger.error(e)
COVER_FSTRING = "https://directory.doabooks.org/bitstream/handle/{handle}/{name}?sequence={sequenceId}&isAllowed=y"
def doab_cover(doab_id):
stream_data = get_streamdata(doab_id)
if not stream_data:
logger.error('get_streamdata failed for %s', doab_id)
return None
return COVER_FSTRING.format(**stream_data)

View File

View File

@ -0,0 +1,29 @@
import datetime
from django.core.management.base import BaseCommand
from doab_check.doab_oai import load_doab_oai
def timefromiso(datestring):
try:
return datetime.datetime.strptime(datestring, "%Y-%m-%d")
except:
return datetime.datetime.strptime(datestring, "%Y-%m-%dT%H:%M:%S")
class Command(BaseCommand):
help = "load doab books via oai"
def add_arguments(self, parser):
parser.add_argument('from_date', nargs='?', type=timefromiso,
default=None, help="YYYY-MM-DD to start")
parser.add_argument('--until', nargs='?', type=timefromiso,
default=None, help="YYYY-MM-DD to end")
parser.add_argument('--max', nargs='?', type=int, default=None, help="max desired records")
def handle(self, from_date, **options):
until_date = options['until']
max = options['max']
self.stdout.write('starting at date:{} until:{}, max: {}'.format(
from_date, until_date, max))
records, new_doabs, last_time = load_doab_oai(from_date, until_date, limit=max)
self.stdout.write('loaded {} records ({} new), ending at {}'.format(
records, new_doabs, last_time))

View File

@ -0,0 +1,76 @@
# Generated by Django 4.1.1 on 2023-02-20 18:43
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Item',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('doab', models.CharField(max_length=40, unique=True)),
('created', models.DateTimeField(auto_now_add=True, db_index=True)),
('resource_type', models.CharField(max_length=20, null=True)),
('title', models.CharField(max_length=1000)),
],
),
migrations.CreateModel(
name='Link',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('url', models.URLField(max_length=1024, unique=True)),
('created', models.DateTimeField(auto_now_add=True)),
('live', models.BooleanField(default=True)),
],
),
migrations.CreateModel(
name='Record',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created', models.DateTimeField(auto_now_add=True)),
('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='records', to='doab_check.item')),
],
),
migrations.CreateModel(
name='Timestamp',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created', models.DateTimeField()),
('datetime', models.DateTimeField()),
('record', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='timestamps', to='doab_check.record')),
],
),
migrations.CreateModel(
name='LinkRel',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('role', models.CharField(max_length=10, null=True)),
('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='related', to='doab_check.item')),
('link', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='related', to='doab_check.link')),
],
),
migrations.AddField(
model_name='link',
name='items',
field=models.ManyToManyField(db_index=True, related_name='links', through='doab_check.LinkRel', to='doab_check.item'),
),
migrations.CreateModel(
name='Check',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created', models.DateTimeField(auto_now_add=True)),
('return_code', models.IntegerField()),
('content_type', models.CharField(max_length=30, null=True)),
('link', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='checks', to='doab_check.link')),
('location', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='redirects_from', to='doab_check.link')),
],
),
]

View File

58
doab_check/models.py Normal file
View File

@ -0,0 +1,58 @@
from django.db import models
class Item(models.Model):
''' an object in DOAB'''
# for example, oai:doab-books:20.500.12854/25932
doab = models.CharField(max_length=40, null=False, unique=True)
created = models.DateTimeField(auto_now_add=True, db_index=True)
# book, chapter, etc.
resource_type = models.CharField(max_length=20, null=True)
# titles can change
title = models.CharField(max_length=1000)
class Link(models.Model):
''' these are the links we're going to check '''
url = models.URLField(max_length=1024, unique=True)
created = models.DateTimeField(auto_now_add=True)
# the items reporting this link
items = models.ManyToManyField("Item", related_name="links", db_index=True, through="LinkRel")
# so we can set it to dead instead of deleting
live = models.BooleanField(default=True)
class Timestamp(models.Model):
''' timestamp of the record returned by doab. records can have multiple timestamps '''
created = models.DateTimeField(auto_now_add=True)
datetime = models.DateTimeField()
record = models.ForeignKey("Record", related_name="timestamps", null=False,
on_delete=models.CASCADE)
class Record(models.Model):
''' a harvested record '''
created = models.DateTimeField(auto_now_add=True)
item = models.ForeignKey("Item", related_name="records", on_delete=models.CASCADE)
class LinkRel(models.Model):
''' association between an item and a link '''
# might be 'cover'
role = models.CharField(max_length=10, null=True)
link = models.ForeignKey("Link", related_name='related', on_delete=models.CASCADE)
item = models.ForeignKey("Item", related_name='related', on_delete=models.CASCADE)
class Check(models.Model):
''' The results of a link check '''
created = models.DateTimeField(auto_now_add=True)
link = models.ForeignKey("Link", related_name='checks', on_delete=models.CASCADE)
return_code = models.IntegerField()
content_type = models.CharField(max_length=30, null=True)
location = models.ForeignKey("Link", related_name='redirects_from', null=True,
on_delete=models.SET_NULL)

129
doab_check/settings.py Normal file
View File

@ -0,0 +1,129 @@
"""
Django settings for doab_check project.
Generated by 'django-admin startproject' using Django 4.1.1.
For more information on this file, see
https://docs.djangoproject.com/en/4.1/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/4.1/ref/settings/
"""
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'django-insecure-$@5jch%4&#^6%r0t!p#n+24mfb)+f#*=s8c7hqbn&htu&pd4ej'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'doab_check',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'doab_check.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'doab_check.wsgi.application'
# Database
# https://docs.djangoproject.com/en/4.1/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql',
'NAME': 'doabcheck',
'USER': 'postgres',
'HOST': '127.0.0.1',
'PORT': '5433', }
}
# Password validation
# https://docs.djangoproject.com/en/4.1/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/4.1/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/4.1/howto/static-files/
STATIC_URL = 'static/'
# Default primary key field type
# https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
USER_AGENT = "doab_check_bot v0.0.1 <https://github.com/EbookFoundation/doab_check>"

4047
doab_check/test.xml Normal file

File diff suppressed because it is too large Load Diff

21
doab_check/urls.py Normal file
View File

@ -0,0 +1,21 @@
"""doab_check URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/4.1/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
urlpatterns = [
path('admin/', admin.site.urls),
]

16
doab_check/wsgi.py Normal file
View File

@ -0,0 +1,16 @@
"""
WSGI config for doab_check project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/4.1/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'doab_check.settings')
application = get_wsgi_application()

22
manage.py Executable file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys
def main():
"""Run administrative tasks."""
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'doab_check.settings')
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
if __name__ == '__main__':
main()