regluit/notebooks/doab_loading.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:ed8b1c5d25bddcabb988082214fa205de769890440e681157f4177e2836b0053"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from __future__ import print_function"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# optionally copy doab.json from DOAB repo\n",
      "\n",
      "import shutil\n",
      "\n",
      "# toggle the boolean to set whether to copy\n",
      "if (False):\n",
      "    shutil.copyfile(\"/Users/raymondyee/D/Document/Gluejar/Gluejar.github/DOAB/doab.json\",\n",
      "                    \"../bookdata/doab.json\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Loading the list of books"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import json\n",
      "import codecs\n",
      "s = codecs.open(\"../bookdata/doab.json\", encoding='UTF-8').read()\n",
      "records = json.loads(s)\n",
      "records[:1]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# how many PDFs are there to load?\n",
      "\n",
      "pdf_records = [record for record in records if dict(record).get('format') == 'pdf']\n",
      "len(pdf_records)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from collections import Counter\n",
      "\n",
      "# doab_ids unique for the PDFs?\n",
      "[c for c in Counter([dict(r).get('doab_id') for r in pdf_records]).items() if c[1] > 1]\n",
      "\n",
      "# 2 of the doab records have more than 1 pdf\n",
      "# doab_id of 15968 and 15969 "
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# do the loading from the local version of load_doab_editon or core.doab\n",
      "\n",
      "from regluit.core import models\n",
      "\n",
      "def attach_more_doab_metadata(ebook, description, subjects,\n",
      "                              publication_date, publisher_name):\n",
      "    \n",
      "    # if edition doesn't have a publication date, update it\n",
      "    edition = ebook.edition\n",
      "    edition_to_save = False\n",
      "    \n",
      "    if not edition.publication_date:\n",
      "        edition.publication_date = publication_date\n",
      "        edition_to_save = True\n",
      "    \n",
      "    # if edition.publisher_name is empty, set it\n",
      "    if not edition.publisher_name:\n",
      "        edition.publisher_name = models.PublisherName.objects.get_or_create(name=publisher_name)[0]\n",
      "        edition_to_save = True\n",
      "        \n",
      "    if edition_to_save:\n",
      "        edition.save()\n",
      "        \n",
      "    # attach description to work if it's not empty\n",
      "    work = edition.work\n",
      "    if not work.description:\n",
      "        work.description = description\n",
      "        work.save()\n",
      "        \n",
      "    # update subjects\n",
      "    work.subjects.add(*[models.Subject.objects.get_or_create(name=s)[0] for s in subjects])\n",
      "            \n",
      "    return ebook\n",
      "\n",
      "def load_doab_edition(title, doab_id, seed_isbn, url, format, rights,\n",
      "                      language, isbns,\n",
      "                      provider='Directory of Open Access Books', **kwargs):\n",
      "    \n",
      "\n",
      "    from django.db.models import (Q, F)\n",
      "    \n",
      "    from regluit.core import tasks\n",
      "    from regluit.core import (models, bookloader)\n",
      "    \n",
      "    from regluit.core.doab import update_cover_doab \n",
      "   \n",
      "    # check to see whether the Edition hasn't already been loaded first\n",
      "    # search by url\n",
      "    ebooks = models.Ebook.objects.filter(url=url)\n",
      "       \n",
      "    # 1 match\n",
      "    # > 1 matches\n",
      "    # 0 match\n",
      "\n",
      "    # simplest case -- if match (1 or more), we could check whether any\n",
      "    # ebook.edition.work has a doab id matching given doab_id\n",
      "    \n",
      "    # put a migration to force Ebook.url to be unique id\n",
      "    \n",
      "    # if yes, then return one of the Edition(s) whose work is doab_id\n",
      "    # if no, then \n",
      "    \n",
      "    if len(ebooks) > 1:\n",
      "        raise Exception(\"There is more than one Ebook matching url {0}\".format(url))    \n",
      "    elif len(ebooks) == 1:  \n",
      "        ebook = ebooks[0]\n",
      "        doab_identifer = models.Identifier.get_or_add(type='doab',value=doab_id, \n",
      "                                               work=ebook.edition.work)\n",
      "        # update the cover id \n",
      "        cover_url = update_cover_doab(doab_id)\n",
      "        \n",
      "        # attach more metadata\n",
      "        attach_more_doab_metadata(ebook, \n",
      "                                  description=kwargs.get('description'),\n",
      "                                  subjects=kwargs.get('subject'),\n",
      "                                  publication_date=kwargs.get('date'),\n",
      "                                  publisher_name=kwargs.get('publisher'))\n",
      "        \n",
      "        return ebook\n",
      "    \n",
      "    # remaining case --> need to create a new Ebook \n",
      "    assert len(ebooks) == 0\n",
      "            \n",
      "    # make sure we have isbns to work with before creating ebook\n",
      "    if len(isbns) == 0:\n",
      "        return None\n",
      "    \n",
      "    ebook = models.Ebook()\n",
      "    ebook.format = format\n",
      "    ebook.provider = provider\n",
      "    ebook.url =  url\n",
      "    ebook.rights = rights\n",
      "\n",
      "    # we still need to find the right Edition/Work to tie Ebook to...\n",
      "    \n",
      "    # look for the Edition with which to associate ebook.\n",
      "    # loop through the isbns to see whether we get one that is not None\n",
      "        \n",
      "    for isbn in isbns:\n",
      "        edition = bookloader.add_by_isbn(isbn)\n",
      "        if edition is not None: break        \n",
      "    \n",
      "    if edition is not None:\n",
      "        # if this is a new edition, then add related editions asynchronously\n",
      "        if getattr(edition,'new', False):\n",
      "            tasks.populate_edition.delay(edition.isbn_13)\n",
      "            \n",
      "        # QUESTION:  Is this good enough?\n",
      "        # what's going to happen to edition.work if there's merging   \n",
      "        doab_identifer = models.Identifier.get_or_add(type='doab',value=doab_id, \n",
      "                                work=edition.work)\n",
      "\n",
      "    # we need to create Edition(s) de novo    \n",
      "    else: \n",
      "        # if there is a Work with doab_id already, attach any new Edition(s)\n",
      "        try:\n",
      "            work = models.Identifier.objects.get(type='doab',value=doab_id).work\n",
      "        except models.Identifier.DoesNotExist:\n",
      "            work = models.Work(language=language,title=title)\n",
      "            work.save()\n",
      "            doab_identifer = models.Identifier.get_or_add(type='doab',value=doab_id, \n",
      "                                               work=work)\n",
      "            \n",
      "        \n",
      "        # create Edition(s) for each of the isbn from the input info\n",
      "        editions = []\n",
      "        for isbn in isbns:\n",
      "            edition = models.Edition(title=title, work=work)\n",
      "            edition.save()\n",
      "            \n",
      "            isbn_id = models.Identifier.get_or_add(type='isbn',value=isbn,work=work)\n",
      "            \n",
      "            editions.append(edition)\n",
      "  \n",
      "        # if work has any ebooks already, attach the ebook to the corresponding edition\n",
      "        # otherwise pick the first one\n",
      "        # pick the first edition as the one to tie ebook to \n",
      "        editions_with_ebooks = models.Edition.objects.filter(Q(work__id=work.id) & \\\n",
      "                                                      Q(ebooks__isnull=False)).distinct()\n",
      "        if editions_with_ebooks:\n",
      "            edition = editions_with_ebooks[0]\n",
      "        else:\n",
      "            edition = editions[0]\n",
      "        \n",
      "    # make the edition the selected_edition of the work\n",
      "    edition.work.selected_edition = edition\n",
      "    \n",
      "    # tie the edition to ebook\n",
      "    \n",
      "    ebook.edition = edition\n",
      "    ebook.save()\n",
      "    \n",
      "    # update the cover id (could be done separately)\n",
      "    cover_url = update_cover_doab(doab_id)\n",
      "    \n",
      "    # attach more metadata\n",
      "    attach_more_doab_metadata(ebook, \n",
      "                              description=kwargs.get('description'),\n",
      "                              subjects=kwargs.get('subject'),\n",
      "                              publication_date=kwargs.get('date'),\n",
      "                              publisher_name=kwargs.get('publisher'))    \n",
      "    return ebook\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from itertools import islice\n",
      "\n",
      "import json\n",
      "import codecs\n",
      "\n",
      "from regluit.core import doab\n",
      "\n",
      "USE_LOCAL_DOAB_FUNCTION = False\n",
      "\n",
      "loading_problems = []\n",
      "\n",
      "s = codecs.open(\"../bookdata/doab.json\", encoding='UTF-8').read()\n",
      "records = json.loads(s)\n",
      "\n",
      "# filter out the pdf records\n",
      "\n",
      "pdf_records = [record for record in records if dict(record).get('format') == 'pdf']\n",
      "records_to_load = list(islice(pdf_records,100))\n",
      "\n",
      "for (i, book) in enumerate(records_to_load):\n",
      "    print (i,) \n",
      "    d = dict(book)\n",
      "    try:\n",
      "        if USE_LOCAL_DOAB_FUNCTION:\n",
      "            edition = load_doab_edition(**dict(book))\n",
      "        else:\n",
      "            edition = doab.load_doab_edition(**dict(book))\n",
      "        print (\"success\")\n",
      "    except Exception, e:\n",
      "        import sys\n",
      "        import traceback\n",
      "        loading_problems.append((d, e))\n",
      "        exc_type, exc_value, exc_traceback = sys.exc_info()\n",
      "        print (\"*** print_tb:\")\n",
      "        traceback.print_tb(exc_traceback, limit=1, file=sys.stdout)\n",
      "        print (\"*** print_exception:\")\n",
      "        traceback.print_exception(exc_type, exc_value, exc_traceback,\n",
      "                                  limit=2, file=sys.stdout)\n",
      "        print (\"*** print_exc:\")\n",
      "#         traceback.print_exc()\n",
      "#         print \"*** format_exc, first and last line:\"\n",
      "#         formatted_lines = traceback.format_exc().splitlines()\n",
      "#         print formatted_lines[0]\n",
      "#         print formatted_lines[-1]\n",
      "#         print \"*** format_exception:\"\n",
      "#         print repr(traceback.format_exception(exc_type, exc_value,\n",
      "#                                               exc_traceback))\n",
      "#         print \"*** extract_tb:\"\n",
      "#         print repr(traceback.extract_tb(exc_traceback))\n",
      "#         print \"*** format_tb:\"\n",
      "#         print repr(traceback.format_tb(exc_traceback))\n",
      "#         print \"*** tb_lineno:\", exc_traceback.tb_lineno\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "I need to remind myself of how to check that there are no outstanding celery jobs after I do this loading. \n",
      "\n",
      "I have a technique for using `django-celery` monitoring that works on redis (what we use on just and production) -- but not laptop (http://stackoverflow.com/a/5451479/7782).  I think a workable way is to look at the celery_taskmeta table."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import djcelery\n",
      "[t.status for t in djcelery.models.TaskMeta.objects.all()]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# Tests for the loading\n",
      "\n",
      "   * can we find all the the URLs?\n",
      "   * is it associated with the the right doab_id?\n",
      "   * all the ISBNs loaded?\n",
      "   * which books are not matched with Google Books IDs -- and therefore might require URLs for covers?\n",
      "   * did I make sure the edition I'm attaching the ebooks to is the \"selected edition\"?\n",
      "   * for editions that I create [and maybe all editions?], attach a cover_image from DOAB.\n",
      "   * all clustered around the same work?  (or do I need to do further merging?)\n",
      "   * are we creating extraneous works?\n",
      "   * subject metadata\n",
      "   * are we loading all the useful metadata? \n",
      "   * is the loading script idempotent?\n"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## important limit to testing\n",
      "\n",
      "I have written code to handle the loading of all associated ISBNs with DOAB records -- but we upload only records with non-null licenses, we will have only one ISBN per DOAB record for records with known licenses. So the loading of works for which we know the license won't exercise the code in question:\n",
      "https://github.com/Gluejar/regluit/blob/5b3a8d7b1302bc1b1985c675add06c345567a7a1/core/doab.py#L91\n",
      "I also checked that there is no intersection of DOAB ids betwen records with known licenses and those that don't."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from regluit.core.models import Work, Edition, Ebook, Identifier\n",
      "from regluit.core.isbn import ISBN\n",
      "\n",
      "tests_exceptions = []\n",
      "no_google_book_id  = []\n",
      "all_problems = []\n",
      "cover_problems = []\n",
      "\n",
      "for record in islice(records_to_load, None ):\n",
      "    d = dict(record)\n",
      "    ebooks = Ebook.objects.filter(url=d.get('url'))\n",
      "    \n",
      "    problems = []\n",
      "    \n",
      "    try:\n",
      "        # check only one ebook with this URL.\n",
      "        if len(ebooks) != 1:\n",
      "            problems.append(\"len(ebooks): \" + len(ebooks))\n",
      "            \n",
      "        # ebook.edition.work is the work with the doab_id\n",
      "        if not(ebooks[0].edition.work == Identifier.objects.get(type='doab', \n",
      "                                                                value=d.get('doab_id')).work):\n",
      "            problems.append(\"ebook.edition.work is the work with the doab_id\")\n",
      "        # all the ISBNs loaded?\n",
      "        # this code might be a bit inefficient given there might only be one isbn per record\n",
      "        \n",
      "        isbns = [ISBN(i).to_string() for i in d.get('isbns')]\n",
      "        if not(set(isbns) == set([id_.value for id_ in Identifier.objects.filter(type=\"isbn\", \n",
      "                                                                   value__in=isbns)])):\n",
      "            problems.append(\"isbns not matching\")\n",
      "            \n",
      "        if problems:\n",
      "            all_problems.append((d, problems))\n",
      "            \n",
      "        # check on presence of Google books id\n",
      "        if len(ebooks[0].edition.identifiers.filter(type=\"goog\")) < 1:\n",
      "            no_google_book_id.append(d)\n",
      "\n",
      "        # check on the cover URLs\n",
      "        #print (ebooks[0].edition.work.cover_image_small())\n",
      "        if ebooks[0].edition.work.cover_image_small().find(\"amazonaws\") < 0:\n",
      "            cover_problems.append((d))\n",
      "    \n",
      "    except Exception, e:\n",
      "        tests_exceptions.append((d, e))\n",
      "  \n",
      "print (\"number of records loaded\",  len(records_to_load))\n",
      "print ()\n",
      "print (\"all_problems\", all_problems)\n",
      "print ()\n",
      "print (\"tests_exceptions\", tests_exceptions)\n",
      "print ()\n",
      "print (\"no_google_book_id\", no_google_book_id)\n",
      "print ()\n",
      "print (\"cover problems\", cover_problems)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Stop"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "raise Exception(\"Stop here\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# invalid ISBNs?\n",
      "\n",
      "for (d, p) in all_problems:\n",
      "    print (d['isbns'][0], ISBN(d['isbns'][0]).valid)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[(d['doab_id'], d['isbns'][0]) for d in no_google_book_id]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# it is possible to do a query for a whole set of values, a technique I might make use of.\n",
      "# http://stackoverflow.com/a/9304968\n",
      "# e.g., Blog.objects.filter(pk__in=[1,4,7])\n",
      "\n",
      "urls = [dict(record).get('url') for record in records_to_load]\n",
      "set([ebook.url for ebook in Ebook.objects.filter(url__in=urls)]) == set(urls)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Code I was working out to use Django querysets to pull out relationships among ebooks, editions, works"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from regluit.core.models import (Ebook, Edition, Work)\n",
      "from django.db.models import (Q, F)\n",
      "\n",
      "# models.Identifier.objects.filter(edition__isnull=False).filter(~Q(edition__work__id = F('work__id'))).count()\n",
      "\n",
      "editions_with_ebooks = Edition.objects.filter(ebooks__isnull=False)\n",
      "editions_with_ebooks\n",
      "\n",
      "edition = editions_with_ebooks[0]\n",
      "print (edition.work_id)\n",
      "work = edition.work\n",
      "print (work.editions.all())\n",
      "# didn't know you should use distinct()\n",
      "Edition.objects.filter(Q(work__id=edition.work_id) & Q(ebooks__isnull=False)).distinct()\n",
      "#Edition.objects.filter(Q(work__id=edition.work_id))\n",
      "#work.objects.filter(editions__ebooks__isnull=False)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# let me grab ebooks and look at their parent works\n",
      "\n",
      "from regluit.core.models import Ebook\n",
      "\n",
      "[eb.edition for eb in Ebook.objects.all()]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Extra"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "raise Exception(\"Stop here\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Checking Celery Results"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Checking the results of a local celery task \n",
      "from regluit.core import tasks\n",
      "\n",
      "task_id = \"28982485-efc3-44d7-9cf6-439645180d5d\"\n",
      "result = tasks.fac.AsyncResult(task_id)\n",
      "result.get()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}