EbookFoundation
/
regluit
mirror of https://github.com/EbookFoundation/regluit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

The notebook is a bit out of date but I want to put it under git control before updating it.

pull/1/head
Raymond Yee 2014-07-03 09:57:30 -07:00
parent 2fac485f08
commit fbc6c61587
1 changed files with 278 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

278
notebooks/doab_loading.ipynb Normal file
View File

@ -0,0 +1,278 @@
{
"metadata": {
"name": "",
"signature": "sha256:2b951eb591ac0924fe8694712aba3bc6e6af53bb70dbd53908789042963d28ec"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"http://doabooks.org/doab?func=about&uiLanguage=en#metadata\n"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"import regluit\n",
"from regluit.core.bookloader import (add_by_isbn, \n",
" get_google_isbn_results)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"It's going to be a bit complicated, it seems. Trying to learn from how I loaded Gutenberg books: https://github.com/Gluejar/regluit/blob/master/core/bookloader.py#L629\n",
"\n",
"* add doab id?\n",
"\n",
"I need to remind myself: how does clustering happen on unglue.it?"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# sister_edition = add_by_isbn(seed_isbn)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"get_google_isbn_results('9781847881298')"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"import regluit\n",
"from regluit.core import (models,tasks)\n",
"from regluit.core.bookloader import add_by_isbn\n",
"\n",
"def load_doab_edition(title, doab_id, seed_isbn, url, format, rights, \n",
" provider='Directory of Open Access Books'):\n",
" \n",
" # can we find doab_id as an identifier? \n",
" # doab work or edition id\n",
" \n",
" try:\n",
" work = models.Identifier.objects.get(type='doab',value=doab_id).work\n",
" except models.Identifier.DoesNotExist: # try to find an Edition with the seed_isbn and use that work to hang off of\n",
" sister_edition = add_by_isbn(seed_isbn)\n",
" if sister_edition.new:\n",
" # add related editions asynchronously\n",
" tasks.populate_edition.delay(sister_edition.isbn_13)\n",
" work = sister_edition.work\n",
" # attach the olwk identifier to this work if it's not none.\n",
" if doab_id is not None:\n",
" work_id = models.Identifier.get_or_add(type='doab',value=doab_id, \n",
" work=work,\n",
" edition=sister_edition)\n",
"\n",
" # Now pull out any existing DOAB editions tied to the work with the proper DOAB ID\n",
" try:\n",
" edition = models.Identifier.objects.get( type='doab', value=doab_id).edition \n",
" except models.Identifier.DoesNotExist:\n",
" edition = models.Edition()\n",
" edition.title = title\n",
" edition.work = work\n",
" \n",
" edition.save()\n",
" edition_id = models.Identifier.get_or_add(type='doab',value=doab_id, \n",
" edition=edition, work=work)\n",
" \n",
" # check to see whether the Edition hasn't already been loaded first\n",
" # search by url\n",
" ebooks = models.Ebook.objects.filter(url=url)\n",
" \n",
" # format: what's the controlled vocab? -- from Google -- alternative would be mimetype\n",
" \n",
" if len(ebooks): \n",
" ebook = ebooks[0]\n",
" elif len(ebooks) == 0: # need to create new ebook\n",
" ebook = models.Ebook()\n",
"\n",
" if len(ebooks) > 1:\n",
" warnings.warn(\"There is more than one Ebook matching url {0}\".format(url))\n",
" \n",
" \n",
" ebook.format = format\n",
" ebook.provider = provider\n",
" ebook.url = url\n",
" ebook.rights = rights\n",
" \n",
" # is an Ebook instantiable without a corresponding Edition? (No, I think)\n",
" \n",
" ebook.edition = edition\n",
" ebook.save()\n",
" \n",
" return ebook"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# let's do one by hand\n",
"\n",
"book1 = {'doab_id': u'12555',\n",
" 'format': 'pdf',\n",
" 'rights': u'CC BY-NC-ND',\n",
" 'seed_isbn': u'9781847881298',\n",
" 'title': u'Jihad Beyond Islam',\n",
" 'url': 'http://www.oapen.org/download?type=document&docid=390768'}\n",
"\n",
"edition = load_doab_edition(**book1)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"df4.urls"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "heading",
"level": 1,
"metadata": {},
"source": [
"Loading the list of books"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"import json\n",
"s = open(\"/Users/raymondyee/D/Document/Working_with_Open_Data/working-open-data-2014/notebooks/doab.json\").read()\n",
"\n",
"records = json.loads(s)\n",
"records[:5]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"from itertools import islice\n",
"\n",
"for (i, book) in enumerate(islice(records,30)):\n",
" print i, \n",
" d = dict(book)\n",
" if d['format'] == 'pdf':\n",
" try:\n",
" edition = load_doab_edition(**dict(book))\n",
" print \"success\"\n",
" except Exception, e:\n",
" print e\n",
" else:\n",
" print \"non-pdf\"\n",
" "
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "heading",
"level": 1,
"metadata": {},
"source": [
"Using regluit.core.doab functions"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"from regluit.core import doab\n"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"DOAB_DATA_FILE = \"/Users/raymondyee/D/Document/Working_with_Open_Data/working-open-data-2014/notebooks/doab.json\"\n",
"doab.load_doab_records(DOAB_DATA_FILE, limit=50, async=True)\n"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"!tail logs/unglue.it.log"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"import os"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"os.path.exists(os.path.join(\"/Users/raymondyee/C/src/Gluejar/regluit/core/management/commands\", \n",
" \"../../../bookdata/doab.json\"))"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": []
}
],
"metadata": {}
}
]
}