2013-06-03 16:31:39 +00:00
|
|
|
"""
|
|
|
|
external library imports
|
|
|
|
"""
|
2011-09-07 09:34:03 +00:00
|
|
|
import json
|
|
|
|
import logging
|
2016-05-16 19:43:11 +00:00
|
|
|
import re
|
2011-09-07 09:34:03 +00:00
|
|
|
import requests
|
2011-10-19 03:00:07 +00:00
|
|
|
|
2012-03-07 19:42:16 +00:00
|
|
|
from datetime import timedelta
|
2016-03-04 20:09:30 +00:00
|
|
|
from itertools import (izip, islice)
|
2013-06-03 16:31:39 +00:00
|
|
|
from xml.etree import ElementTree
|
2016-03-04 20:09:30 +00:00
|
|
|
from urlparse import (urljoin, urlparse)
|
|
|
|
|
2012-03-07 19:42:16 +00:00
|
|
|
|
2013-06-03 16:31:39 +00:00
|
|
|
"""
|
|
|
|
django imports
|
|
|
|
"""
|
2011-11-06 22:42:09 +00:00
|
|
|
from django.conf import settings
|
2016-07-21 20:05:57 +00:00
|
|
|
from django_comments.models import Comment
|
2013-06-03 16:31:39 +00:00
|
|
|
from django.db import IntegrityError
|
|
|
|
from django.db.models import Q
|
2011-09-09 05:38:28 +00:00
|
|
|
|
2016-03-04 20:09:30 +00:00
|
|
|
from github3 import (login, GitHub)
|
|
|
|
from github3.repos.release import Release
|
|
|
|
|
2015-08-03 14:04:07 +00:00
|
|
|
from gitenberg.metadata.pandata import Pandata
|
2015-07-30 03:01:43 +00:00
|
|
|
from ..marc.models import inverse_marc_rels
|
|
|
|
|
2013-06-03 16:31:39 +00:00
|
|
|
"""
|
|
|
|
regluit imports
|
|
|
|
"""
|
2012-02-15 02:01:13 +00:00
|
|
|
import regluit
|
2011-12-20 04:26:55 +00:00
|
|
|
import regluit.core.isbn
|
2011-09-09 05:38:28 +00:00
|
|
|
|
2013-06-03 16:31:39 +00:00
|
|
|
from regluit.core import models
|
|
|
|
from regluit.utils.localdatetime import now
|
2015-07-30 03:01:43 +00:00
|
|
|
import regluit.core.cc as cc
|
2012-03-07 19:42:16 +00:00
|
|
|
|
2011-09-10 11:36:38 +00:00
|
|
|
logger = logging.getLogger(__name__)
|
2013-10-30 16:42:59 +00:00
|
|
|
request_log = logging.getLogger("requests")
|
|
|
|
request_log.setLevel(logging.WARNING)
|
2011-09-09 05:38:28 +00:00
|
|
|
|
2012-01-09 18:55:22 +00:00
|
|
|
def add_by_oclc(isbn, work=None):
|
|
|
|
# this is indirection in case we have a data source other than google
|
|
|
|
return add_by_oclc_from_google(isbn)
|
2011-12-19 06:33:13 +00:00
|
|
|
|
2011-11-06 21:33:04 +00:00
|
|
|
|
2012-01-09 18:55:22 +00:00
|
|
|
def add_by_oclc_from_google(oclc):
|
|
|
|
if oclc:
|
2012-01-10 20:20:02 +00:00
|
|
|
logger.info("adding book by oclc %s" , oclc)
|
2012-01-09 18:55:22 +00:00
|
|
|
else:
|
2011-11-06 21:33:04 +00:00
|
|
|
return None
|
|
|
|
try:
|
2012-01-09 18:55:22 +00:00
|
|
|
return models.Identifier.objects.get(type='oclc', value=oclc).edition
|
|
|
|
except:
|
|
|
|
url = "https://www.googleapis.com/books/v1/volumes"
|
2012-01-31 04:57:23 +00:00
|
|
|
try:
|
|
|
|
results = _get_json(url, {"q": '"OCLC%s"' % oclc})
|
|
|
|
except LookupFailure, e:
|
|
|
|
logger.exception("lookup failure for %s", oclc)
|
|
|
|
return None
|
2012-01-09 18:55:22 +00:00
|
|
|
if not results.has_key('items') or len(results['items']) == 0:
|
2012-01-10 20:20:02 +00:00
|
|
|
logger.warn("no google hits for %s" , oclc)
|
2012-01-09 18:55:22 +00:00
|
|
|
return None
|
|
|
|
|
|
|
|
try:
|
2012-01-18 04:22:07 +00:00
|
|
|
e = add_by_googlebooks_id(results['items'][0]['id'], results=results['items'][0])
|
2012-01-09 18:55:22 +00:00
|
|
|
models.Identifier(type='oclc', value=oclc, edition=e, work=e.work).save()
|
|
|
|
return e
|
|
|
|
except LookupFailure, e:
|
|
|
|
logger.exception("failed to add edition for %s", oclc)
|
|
|
|
except IntegrityError, e:
|
|
|
|
logger.exception("google books data for %s didn't fit our db", oclc)
|
|
|
|
return None
|
2011-11-06 22:42:09 +00:00
|
|
|
|
2016-12-30 15:24:01 +00:00
|
|
|
def valid_isbn(isbn):
|
|
|
|
try:
|
|
|
|
isbn = regluit.core.isbn.ISBN(isbn)
|
|
|
|
except:
|
|
|
|
logger.exception("invalid isbn: %s", isbn)
|
|
|
|
return None
|
|
|
|
if not isbn.valid:
|
|
|
|
return None
|
|
|
|
return isbn.to_string()
|
|
|
|
|
2016-10-28 00:05:43 +00:00
|
|
|
def add_by_isbn(isbn, work=None, language='xx', title=''):
|
2012-01-09 18:55:22 +00:00
|
|
|
if not isbn:
|
|
|
|
return None
|
2012-01-31 04:57:23 +00:00
|
|
|
try:
|
|
|
|
e = add_by_isbn_from_google(isbn, work=work)
|
|
|
|
except LookupFailure:
|
|
|
|
logger.exception("failed google lookup for %s", isbn)
|
|
|
|
# try again some other time
|
|
|
|
return None
|
2012-01-09 18:55:22 +00:00
|
|
|
if e:
|
|
|
|
return e
|
2012-01-10 20:20:02 +00:00
|
|
|
|
|
|
|
logger.info("null came back from add_by_isbn_from_google: %s", isbn)
|
|
|
|
|
2016-10-28 00:05:43 +00:00
|
|
|
# if there's a a title, we want to create stub editions and
|
2012-01-18 04:15:24 +00:00
|
|
|
# works, even if google doesn't know about it # but if it's not valid,
|
|
|
|
# forget it!
|
|
|
|
|
2016-10-28 00:05:43 +00:00
|
|
|
if work:
|
|
|
|
title = work.title if work.title else title
|
|
|
|
if not title:
|
|
|
|
return None
|
|
|
|
if not title:
|
|
|
|
return None
|
|
|
|
|
2016-12-30 15:24:01 +00:00
|
|
|
isbn = valid_isbn(isbn)
|
|
|
|
if not isbn:
|
2012-01-09 18:55:22 +00:00
|
|
|
return None
|
2012-01-10 20:20:02 +00:00
|
|
|
|
2016-10-28 00:05:43 +00:00
|
|
|
if not language or language == 'xx': # don't add unknown language
|
|
|
|
# we don't know the language ->'xx'
|
|
|
|
work = models.Work(title=title, language='xx')
|
|
|
|
work.save()
|
|
|
|
elif not work:
|
|
|
|
work = models.Work(title=title, language=language)
|
|
|
|
work.save()
|
|
|
|
e = models.Edition(title=title, work=work)
|
2012-01-09 18:55:22 +00:00
|
|
|
e.save()
|
2012-01-18 04:22:07 +00:00
|
|
|
e.new = True
|
2016-10-28 00:05:43 +00:00
|
|
|
models.Identifier(type='isbn', value=isbn, work=work, edition=e).save()
|
2012-01-09 18:55:22 +00:00
|
|
|
return e
|
2012-02-02 14:05:08 +00:00
|
|
|
|
|
|
|
def get_google_isbn_results(isbn):
|
|
|
|
url = "https://www.googleapis.com/books/v1/volumes"
|
|
|
|
try:
|
|
|
|
results = _get_json(url, {"q": "isbn:%s" % isbn})
|
|
|
|
except LookupFailure:
|
|
|
|
logger.exception("lookup failure for %s", isbn)
|
|
|
|
return None
|
|
|
|
if not results.has_key('items') or len(results['items']) == 0:
|
|
|
|
logger.warn("no google hits for %s" , isbn)
|
|
|
|
return None
|
|
|
|
else:
|
|
|
|
return results
|
|
|
|
|
|
|
|
def add_ebooks(item, edition):
|
|
|
|
access_info = item.get('accessInfo')
|
|
|
|
if access_info:
|
|
|
|
epub = access_info.get('epub')
|
|
|
|
if epub and epub.get('downloadLink'):
|
|
|
|
ebook = models.Ebook(edition=edition, format='epub',
|
|
|
|
url=epub.get('downloadLink'),
|
2012-02-28 22:27:48 +00:00
|
|
|
provider='Google Books')
|
2012-02-02 14:05:08 +00:00
|
|
|
try:
|
|
|
|
ebook.save()
|
|
|
|
except IntegrityError:
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
pdf = access_info.get('pdf')
|
|
|
|
if pdf and pdf.get('downloadLink'):
|
|
|
|
ebook = models.Ebook(edition=edition, format='pdf',
|
|
|
|
url=pdf.get('downloadLink', None),
|
2012-02-28 22:27:48 +00:00
|
|
|
provider='Google Books')
|
2012-02-02 14:05:08 +00:00
|
|
|
try:
|
|
|
|
ebook.save()
|
|
|
|
except IntegrityError:
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
def update_edition(edition):
|
2012-02-15 21:23:10 +00:00
|
|
|
"""
|
|
|
|
attempt to update data associated with input edition and return that updated edition
|
|
|
|
"""
|
|
|
|
|
|
|
|
# if there is no ISBN associated with edition, just return the input edition
|
2012-02-02 14:05:08 +00:00
|
|
|
try:
|
|
|
|
isbn=edition.identifiers.filter(type='isbn')[0].value
|
2012-08-03 02:36:11 +00:00
|
|
|
except (models.Identifier.DoesNotExist, IndexError):
|
2012-02-02 14:05:08 +00:00
|
|
|
return edition
|
|
|
|
|
2012-02-15 21:23:10 +00:00
|
|
|
# do a Google Books lookup on the isbn associated with the edition (there should be either 0 or 1 isbns associated
|
|
|
|
# with an edition because of integrity constraint in Identifier)
|
|
|
|
|
|
|
|
# if we get some data about this isbn back from Google, update the edition data accordingly
|
2012-02-02 14:05:08 +00:00
|
|
|
results=get_google_isbn_results(isbn)
|
|
|
|
if not results:
|
|
|
|
return edition
|
|
|
|
item=results['items'][0]
|
|
|
|
googlebooks_id=item['id']
|
|
|
|
d = item['volumeInfo']
|
2012-02-02 16:33:51 +00:00
|
|
|
if d.has_key('title'):
|
|
|
|
title = d['title']
|
|
|
|
else:
|
|
|
|
title=''
|
2012-02-02 14:05:08 +00:00
|
|
|
if len(title)==0:
|
|
|
|
# need a title to make an edition record; some crap records in GB. use title from parent if available
|
|
|
|
title=edition.work.title
|
|
|
|
|
|
|
|
# check for language change
|
|
|
|
language = d['language']
|
2014-06-06 18:47:36 +00:00
|
|
|
# allow variants in main language (e.g., 'zh-tw')
|
|
|
|
if len(language)>5:
|
|
|
|
language= language[0:5]
|
2012-02-15 21:23:10 +00:00
|
|
|
|
|
|
|
# if the language of the edition no longer matches that of the parent work, attach edition to the
|
2012-02-02 14:05:08 +00:00
|
|
|
if edition.work.language != language:
|
|
|
|
logger.info("reconnecting %s since it is %s instead of %s" %(googlebooks_id, language, edition.work.language))
|
|
|
|
old_work=edition.work
|
2012-02-15 21:23:10 +00:00
|
|
|
|
2012-02-02 14:05:08 +00:00
|
|
|
new_work = models.Work(title=title, language=language)
|
|
|
|
new_work.save()
|
|
|
|
edition.work = new_work
|
|
|
|
edition.save()
|
|
|
|
for identifier in edition.identifiers.all():
|
|
|
|
logger.info("moving identifier %s" % identifier.value)
|
|
|
|
identifier.work=new_work
|
|
|
|
identifier.save()
|
2012-08-05 17:03:37 +00:00
|
|
|
if old_work and old_work.editions.count()==0:
|
2012-02-02 14:05:08 +00:00
|
|
|
#a dangling work; make sure nothing else is attached!
|
|
|
|
merge_works(new_work,old_work)
|
|
|
|
|
|
|
|
# update the edition
|
|
|
|
edition.title = title
|
|
|
|
edition.publication_date = d.get('publishedDate', '')
|
2013-03-26 03:41:19 +00:00
|
|
|
edition.set_publisher(d.get('publisher'))
|
2012-02-02 14:05:08 +00:00
|
|
|
edition.save()
|
2012-01-09 18:55:22 +00:00
|
|
|
|
2012-02-02 14:05:08 +00:00
|
|
|
# create identifier if needed
|
|
|
|
models.Identifier.get_or_add(type='goog',value=googlebooks_id,edition=edition,work=edition.work)
|
|
|
|
|
|
|
|
for a in d.get('authors', []):
|
2014-10-20 20:54:19 +00:00
|
|
|
edition.add_author(a)
|
2012-02-02 14:05:08 +00:00
|
|
|
|
|
|
|
add_ebooks(item, edition)
|
|
|
|
|
|
|
|
return edition
|
|
|
|
|
2012-01-09 18:55:22 +00:00
|
|
|
|
|
|
|
def add_by_isbn_from_google(isbn, work=None):
|
|
|
|
"""add a book to the UnglueIt database from google based on ISBN. The work parameter
|
2011-10-14 04:12:20 +00:00
|
|
|
is optional, and if not supplied the edition will be associated with
|
|
|
|
a stub work.
|
|
|
|
"""
|
2011-12-22 19:29:46 +00:00
|
|
|
if not isbn:
|
|
|
|
return None
|
2011-12-20 04:26:55 +00:00
|
|
|
if len(isbn)==10:
|
2012-01-18 04:22:07 +00:00
|
|
|
isbn = regluit.core.isbn.convert_10_to_13(isbn)
|
2011-12-20 04:26:55 +00:00
|
|
|
|
2012-02-02 14:05:08 +00:00
|
|
|
|
2012-01-09 18:55:22 +00:00
|
|
|
# check if we already have this isbn
|
|
|
|
edition = get_edition_by_id(type='isbn',value=isbn)
|
|
|
|
if edition:
|
2011-12-22 19:29:46 +00:00
|
|
|
edition.new = False
|
2011-10-19 03:00:07 +00:00
|
|
|
return edition
|
|
|
|
|
2012-05-25 17:13:15 +00:00
|
|
|
logger.info("adding new book by isbn %s", isbn)
|
2012-02-02 14:05:08 +00:00
|
|
|
results=get_google_isbn_results(isbn)
|
|
|
|
if results:
|
|
|
|
try:
|
|
|
|
return add_by_googlebooks_id(results['items'][0]['id'], work=work, results=results['items'][0], isbn=isbn)
|
|
|
|
except LookupFailure, e:
|
|
|
|
logger.exception("failed to add edition for %s", isbn)
|
|
|
|
except IntegrityError, e:
|
|
|
|
logger.exception("google books data for %s didn't fit our db", isbn)
|
2011-10-10 16:57:10 +00:00
|
|
|
return None
|
2012-02-02 14:05:08 +00:00
|
|
|
else:
|
|
|
|
return None
|
|
|
|
|
2012-01-09 18:55:22 +00:00
|
|
|
def get_work_by_id(type,value):
|
|
|
|
if value:
|
|
|
|
try:
|
|
|
|
return models.Identifier.objects.get(type=type,value=value).work
|
|
|
|
except models.Identifier.DoesNotExist:
|
|
|
|
return None
|
2011-09-07 09:34:03 +00:00
|
|
|
|
2012-01-09 18:55:22 +00:00
|
|
|
def get_edition_by_id(type,value):
|
|
|
|
if value:
|
|
|
|
try:
|
|
|
|
return models.Identifier.objects.get(type=type,value=value).edition
|
|
|
|
except models.Identifier.DoesNotExist:
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
2012-01-28 02:44:02 +00:00
|
|
|
def add_by_googlebooks_id(googlebooks_id, work=None, results=None, isbn=None):
|
2011-10-14 04:12:20 +00:00
|
|
|
"""add a book to the UnglueIt database based on the GoogleBooks ID. The
|
|
|
|
work parameter is optional, and if not supplied the edition will be
|
2012-01-28 02:44:02 +00:00
|
|
|
associated with a stub work. isbn can be passed because sometimes passed data won't include it
|
2012-01-09 18:55:22 +00:00
|
|
|
|
2011-10-14 04:12:20 +00:00
|
|
|
"""
|
2016-12-30 15:24:01 +00:00
|
|
|
isbn = valid_isbn(isbn)
|
|
|
|
|
2011-10-14 04:02:19 +00:00
|
|
|
# don't ping google again if we already know about the edition
|
2011-12-13 14:55:26 +00:00
|
|
|
try:
|
2012-01-31 20:07:14 +00:00
|
|
|
edition = models.Identifier.objects.get(type='goog', value=googlebooks_id).edition
|
|
|
|
edition.new = False
|
2016-12-30 15:24:01 +00:00
|
|
|
if isbn:
|
|
|
|
# check that the isbn is in db; if not, then there are two isbns for the edition
|
|
|
|
try:
|
|
|
|
isbn_edition = models.Identifier.objects.get(type='isbn', value=isbn).edition
|
|
|
|
# not going to worry about isbn_edition != edition
|
|
|
|
except models.Identifier.DoesNotExist:
|
|
|
|
models.Identifier.objects.create(type='isbn', value=isbn, edition=edition, work=edition.work)
|
2012-01-31 20:07:14 +00:00
|
|
|
return edition
|
2012-01-09 18:55:22 +00:00
|
|
|
except models.Identifier.DoesNotExist:
|
2011-12-13 14:55:26 +00:00
|
|
|
pass
|
2012-01-09 18:55:22 +00:00
|
|
|
|
|
|
|
# if google has been queried by caller, don't call again
|
|
|
|
if results:
|
|
|
|
item =results
|
|
|
|
else:
|
|
|
|
logger.info("loading metadata from google for %s", googlebooks_id)
|
|
|
|
url = "https://www.googleapis.com/books/v1/volumes/%s" % googlebooks_id
|
|
|
|
item = _get_json(url)
|
2011-11-06 21:33:04 +00:00
|
|
|
d = item['volumeInfo']
|
2012-01-31 04:56:20 +00:00
|
|
|
|
2012-02-02 16:33:51 +00:00
|
|
|
if d.has_key('title'):
|
|
|
|
title = d['title']
|
|
|
|
else:
|
|
|
|
title=''
|
2012-01-31 04:56:20 +00:00
|
|
|
if len(title)==0:
|
|
|
|
# need a title to make an edition record; some crap records in GB. use title from parent if available
|
|
|
|
if work:
|
|
|
|
title=work.title
|
|
|
|
else:
|
|
|
|
return None
|
2011-10-14 04:02:19 +00:00
|
|
|
|
2011-12-13 14:55:26 +00:00
|
|
|
# don't add the edition to a work with a different language
|
|
|
|
# https://www.pivotaltracker.com/story/show/17234433
|
2012-01-09 18:55:22 +00:00
|
|
|
language = d['language']
|
2014-06-06 18:47:36 +00:00
|
|
|
if len(language)>5:
|
|
|
|
language= language[0:5]
|
2011-12-13 14:55:26 +00:00
|
|
|
if work and work.language != language:
|
2012-01-10 20:20:02 +00:00
|
|
|
logger.info("not connecting %s since it is %s instead of %s" %
|
2011-12-13 14:55:26 +00:00
|
|
|
(googlebooks_id, language, work.language))
|
2012-01-18 04:22:07 +00:00
|
|
|
work = None
|
2012-01-28 02:44:02 +00:00
|
|
|
# isbn = None
|
|
|
|
if not isbn:
|
|
|
|
for i in d.get('industryIdentifiers', []):
|
|
|
|
if i['type'] == 'ISBN_10' and not isbn:
|
|
|
|
isbn = regluit.core.isbn.convert_10_to_13(i['identifier'])
|
|
|
|
elif i['type'] == 'ISBN_13':
|
|
|
|
isbn = i['identifier']
|
2012-01-09 18:55:22 +00:00
|
|
|
|
|
|
|
# now check to see if there's an existing Work
|
2012-01-31 20:07:14 +00:00
|
|
|
if work:
|
|
|
|
work.new = False
|
2012-01-27 14:35:00 +00:00
|
|
|
if isbn and not work:
|
2012-01-09 18:55:22 +00:00
|
|
|
work = get_work_by_id(type='isbn',value=isbn)
|
2012-01-31 20:07:14 +00:00
|
|
|
if work:
|
|
|
|
work.new = False
|
2012-01-09 18:55:22 +00:00
|
|
|
if not work:
|
2012-01-31 04:56:20 +00:00
|
|
|
work = models.Work.objects.create(title=title, language=language)
|
2012-01-18 04:22:07 +00:00
|
|
|
work.new = True
|
2012-01-09 18:55:22 +00:00
|
|
|
work.save()
|
|
|
|
|
2012-01-27 14:35:00 +00:00
|
|
|
# going off to google can take some time, so we want to make sure this edition has not
|
|
|
|
# been created in another thread while we were waiting
|
|
|
|
try:
|
2012-01-31 15:01:10 +00:00
|
|
|
e = models.Identifier.objects.get(type='goog', value=googlebooks_id).edition
|
2012-01-31 20:07:14 +00:00
|
|
|
e.new = False
|
2016-10-28 18:40:16 +00:00
|
|
|
logger.warning( " whoa nellie, somebody else created an edition while we were working.")
|
2012-01-31 15:01:10 +00:00
|
|
|
if work.new:
|
|
|
|
work.delete()
|
|
|
|
return e
|
2012-01-27 14:35:00 +00:00
|
|
|
except models.Identifier.DoesNotExist:
|
|
|
|
pass
|
|
|
|
|
2012-01-09 18:55:22 +00:00
|
|
|
# because this is a new google id, we have to create a new edition
|
|
|
|
e = models.Edition(work=work)
|
2012-01-31 04:56:20 +00:00
|
|
|
e.title = title
|
2011-11-21 19:13:29 +00:00
|
|
|
e.publication_date = d.get('publishedDate', '')
|
2013-03-26 03:41:19 +00:00
|
|
|
e.set_publisher(d.get('publisher'))
|
2011-12-13 14:55:26 +00:00
|
|
|
e.save()
|
2012-01-18 04:22:07 +00:00
|
|
|
e.new = True
|
2012-01-09 18:55:22 +00:00
|
|
|
|
|
|
|
# create identifier where needed
|
|
|
|
models.Identifier(type='goog',value=googlebooks_id,edition=e,work=work).save()
|
|
|
|
if isbn:
|
|
|
|
models.Identifier.get_or_add(type='isbn',value=isbn,edition=e,work=work)
|
2011-12-13 14:55:26 +00:00
|
|
|
|
2011-10-10 16:57:10 +00:00
|
|
|
for a in d.get('authors', []):
|
|
|
|
a, created = models.Author.objects.get_or_create(name=a)
|
2015-07-28 13:41:55 +00:00
|
|
|
e.add_author(a)
|
2011-09-09 05:38:28 +00:00
|
|
|
|
2012-02-02 14:05:08 +00:00
|
|
|
add_ebooks(item, e)
|
2012-01-09 18:55:22 +00:00
|
|
|
|
2011-10-10 16:57:10 +00:00
|
|
|
return e
|
2011-09-09 05:38:28 +00:00
|
|
|
|
|
|
|
|
2012-08-05 17:05:35 +00:00
|
|
|
def relate_isbn(isbn, cluster_size=1):
|
|
|
|
"""add a book by isbn and then see if there's an existing work to add it to so as to make a cluster
|
|
|
|
bigger than cluster_size.
|
|
|
|
"""
|
|
|
|
logger.info("finding a related work for %s", isbn)
|
|
|
|
|
|
|
|
edition = add_by_isbn(isbn)
|
|
|
|
if edition is None:
|
|
|
|
return None
|
|
|
|
if edition.work is None:
|
2012-08-05 22:30:13 +00:00
|
|
|
logger.info("didn't add related to null work")
|
2012-08-05 17:05:35 +00:00
|
|
|
return None
|
|
|
|
if edition.work.editions.count()>cluster_size:
|
2012-08-05 22:30:13 +00:00
|
|
|
return edition.work
|
2012-08-05 17:05:35 +00:00
|
|
|
for other_isbn in thingisbn(isbn):
|
|
|
|
# 979's come back as 13
|
|
|
|
logger.debug("other_isbn: %s", other_isbn)
|
|
|
|
if len(other_isbn)==10:
|
|
|
|
other_isbn = regluit.core.isbn.convert_10_to_13(other_isbn)
|
|
|
|
related_edition = add_by_isbn(other_isbn, work=edition.work)
|
|
|
|
if related_edition:
|
|
|
|
related_language = related_edition.work.language
|
|
|
|
if edition.work.language == related_language:
|
|
|
|
if related_edition.work is None:
|
|
|
|
related_edition.work = edition.work
|
2012-08-05 22:30:13 +00:00
|
|
|
related_edition.save()
|
|
|
|
elif related_edition.work.id != edition.work.id:
|
2012-08-05 17:05:35 +00:00
|
|
|
logger.debug("merge_works path 1 %s %s", edition.work.id, related_edition.work.id )
|
|
|
|
merge_works(related_edition.work, edition.work)
|
2012-08-05 22:30:13 +00:00
|
|
|
|
|
|
|
if related_edition.work.editions.count()>cluster_size:
|
|
|
|
return related_edition.work
|
2012-08-05 17:05:35 +00:00
|
|
|
return edition.work
|
|
|
|
|
2011-10-14 04:12:20 +00:00
|
|
|
def add_related(isbn):
|
|
|
|
"""add all books related to a particular ISBN to the UnglueIt database.
|
|
|
|
The initial seed ISBN will be added if it's not already there.
|
|
|
|
"""
|
2011-10-14 04:02:19 +00:00
|
|
|
# make sure the seed edition is there
|
2011-10-20 03:31:16 +00:00
|
|
|
logger.info("adding related editions for %s", isbn)
|
2012-01-28 00:16:46 +00:00
|
|
|
|
|
|
|
new_editions = []
|
|
|
|
|
2011-10-14 04:02:19 +00:00
|
|
|
edition = add_by_isbn(isbn)
|
2012-01-28 00:16:46 +00:00
|
|
|
if edition is None:
|
|
|
|
return new_editions
|
2012-08-05 17:03:37 +00:00
|
|
|
if edition.work is None:
|
|
|
|
logger.warning("didn't add related to null work")
|
|
|
|
return new_editions
|
2011-10-14 04:02:19 +00:00
|
|
|
# this is the work everything will hang off
|
|
|
|
work = edition.work
|
2012-01-10 20:20:02 +00:00
|
|
|
other_editions = {}
|
2011-10-14 04:02:19 +00:00
|
|
|
for other_isbn in thingisbn(isbn):
|
2011-12-20 04:26:55 +00:00
|
|
|
# 979's come back as 13
|
2012-02-13 22:35:08 +00:00
|
|
|
logger.debug("other_isbn: %s", other_isbn)
|
2011-12-20 04:26:55 +00:00
|
|
|
if len(other_isbn)==10:
|
2012-01-18 04:22:07 +00:00
|
|
|
other_isbn = regluit.core.isbn.convert_10_to_13(other_isbn)
|
2012-01-09 18:55:22 +00:00
|
|
|
related_edition = add_by_isbn(other_isbn, work=work)
|
2011-10-20 05:23:30 +00:00
|
|
|
|
2012-01-10 20:20:02 +00:00
|
|
|
if related_edition:
|
2012-01-18 04:22:07 +00:00
|
|
|
related_language = related_edition.work.language
|
2012-01-10 20:20:02 +00:00
|
|
|
if edition.work.language == related_language:
|
|
|
|
new_editions.append(related_edition)
|
2012-08-05 17:03:37 +00:00
|
|
|
if related_edition.work is None:
|
|
|
|
related_edition.work = work
|
|
|
|
related_edition.save()
|
2012-08-05 22:30:13 +00:00
|
|
|
elif related_edition.work.id != work.id:
|
2012-08-05 17:03:37 +00:00
|
|
|
logger.debug("merge_works path 1 %s %s", work.id, related_edition.work.id )
|
|
|
|
merge_works(work, related_edition.work)
|
2012-01-10 20:20:02 +00:00
|
|
|
else:
|
|
|
|
if other_editions.has_key(related_language):
|
|
|
|
other_editions[related_language].append(related_edition)
|
|
|
|
else:
|
|
|
|
other_editions[related_language]=[related_edition]
|
|
|
|
|
|
|
|
# group the other language editions together
|
|
|
|
for lang_group in other_editions.itervalues():
|
2012-02-13 22:35:08 +00:00
|
|
|
logger.debug("lang_group (ed, work): %s", [(ed.id, ed.work.id) for ed in lang_group])
|
2012-01-10 20:20:02 +00:00
|
|
|
if len(lang_group)>1:
|
|
|
|
lang_edition = lang_group[0]
|
2012-02-13 22:35:08 +00:00
|
|
|
logger.debug("lang_edition.id: %s", lang_edition.id)
|
|
|
|
# compute the distinct set of works to merge into lang_edition.work
|
|
|
|
works_to_merge = set([ed.work for ed in lang_group[1:]]) - set([lang_edition.work])
|
|
|
|
for w in works_to_merge:
|
|
|
|
logger.debug("merge_works path 2 %s %s", lang_edition.work.id, w.id )
|
|
|
|
merge_works(lang_edition.work, w)
|
2016-08-16 15:42:58 +00:00
|
|
|
models.WorkRelation.objects.get_or_create(to_work=lang_edition.work, from_work=work, relation='translation')
|
2012-01-10 20:20:02 +00:00
|
|
|
|
2011-10-20 05:23:30 +00:00
|
|
|
return new_editions
|
2012-01-10 20:20:02 +00:00
|
|
|
|
2011-10-14 04:12:20 +00:00
|
|
|
|
2011-10-13 01:59:46 +00:00
|
|
|
def thingisbn(isbn):
|
2011-10-14 04:12:20 +00:00
|
|
|
"""given an ISBN return a list of related edition ISBNs, according to
|
2011-12-20 04:26:55 +00:00
|
|
|
Library Thing. (takes isbn_10 or isbn_13, returns isbn_10, except for 979 isbns, which come back as isbn_13')
|
2011-10-14 04:12:20 +00:00
|
|
|
"""
|
2012-01-10 20:20:02 +00:00
|
|
|
logger.info("looking up %s at ThingISBN" , isbn)
|
2011-10-13 01:59:46 +00:00
|
|
|
url = "http://www.librarything.com/api/thingISBN/%s" % isbn
|
|
|
|
xml = requests.get(url, headers={"User-Agent": settings.USER_AGENT}).content
|
|
|
|
doc = ElementTree.fromstring(xml)
|
|
|
|
return [e.text for e in doc.findall('isbn')]
|
|
|
|
|
2011-10-19 03:45:02 +00:00
|
|
|
|
2012-02-28 22:27:48 +00:00
|
|
|
def merge_works(w1, w2, user=None):
|
2011-10-19 03:00:07 +00:00
|
|
|
"""will merge the second work (w2) into the first (w1)
|
|
|
|
"""
|
2013-10-30 16:42:59 +00:00
|
|
|
logger.info("merging work %s into %s", w2.id, w1.id)
|
2012-02-13 22:35:08 +00:00
|
|
|
# don't merge if the works are the same or at least one of the works has no id (for example, when w2 has already been deleted)
|
2012-08-05 22:30:13 +00:00
|
|
|
if w1 is None or w2 is None or w1.id == w2.id or w1.id is None or w2.id is None:
|
2012-02-13 22:35:08 +00:00
|
|
|
return
|
2014-04-14 21:43:06 +00:00
|
|
|
if w2.selected_edition != None and w1.selected_edition == None:
|
|
|
|
#the merge should be reversed
|
|
|
|
temp = w1
|
|
|
|
w1 = w2
|
|
|
|
w2 = temp
|
2013-11-08 17:13:34 +00:00
|
|
|
models.WasWork(was=w2.pk, work=w1, user=user).save()
|
|
|
|
for ww in models.WasWork.objects.filter(work = w2):
|
|
|
|
ww.work = w1
|
|
|
|
ww.save()
|
2014-07-01 17:07:07 +00:00
|
|
|
if w2.description and not w1.description:
|
|
|
|
w1.description = w2.description
|
|
|
|
if w2.featured and not w1.featured:
|
|
|
|
w1.featured = w2.featured
|
2014-12-13 17:37:35 +00:00
|
|
|
if w2.is_free and not w1.is_free:
|
|
|
|
w1.is_free = True
|
2016-10-12 17:55:27 +00:00
|
|
|
if w2.age_level and not w1.age_level:
|
|
|
|
w1.age_level = w2.age_level
|
2014-07-01 17:07:07 +00:00
|
|
|
w1.save()
|
2013-11-08 17:13:34 +00:00
|
|
|
for wishlist in models.Wishlist.objects.filter(works__in=[w2]):
|
|
|
|
w2source = wishlist.work_source(w2)
|
|
|
|
wishlist.remove_work(w2)
|
|
|
|
wishlist.add_work(w1, w2source)
|
2012-01-09 18:55:22 +00:00
|
|
|
for identifier in w2.identifiers.all():
|
|
|
|
identifier.work = w1
|
|
|
|
identifier.save()
|
2012-02-10 01:51:10 +00:00
|
|
|
for comment in Comment.objects.for_model(w2):
|
|
|
|
comment.object_pk = w1.pk
|
|
|
|
comment.save()
|
2011-10-19 03:00:07 +00:00
|
|
|
for edition in w2.editions.all():
|
|
|
|
edition.work = w1
|
|
|
|
edition.save()
|
|
|
|
for campaign in w2.campaigns.all():
|
|
|
|
campaign.work = w1
|
|
|
|
campaign.save()
|
2012-10-05 21:55:14 +00:00
|
|
|
for claim in w2.claim.all():
|
|
|
|
claim.work = w1
|
2013-11-08 17:13:34 +00:00
|
|
|
claim.dont_notify = True
|
2012-10-05 21:55:14 +00:00
|
|
|
claim.save()
|
2013-06-12 20:43:11 +00:00
|
|
|
for offer in w2.offers.all():
|
|
|
|
offer.work = w1
|
|
|
|
offer.save()
|
2013-11-08 17:13:34 +00:00
|
|
|
for acq in w2.acqs.all():
|
|
|
|
acq.work = w1
|
|
|
|
acq.save()
|
|
|
|
for hold in w2.holds.all():
|
|
|
|
hold.work = w1
|
|
|
|
hold.save()
|
2017-03-16 15:50:10 +00:00
|
|
|
for landing in w2.landings.all():
|
|
|
|
landing.object_id = w1.id
|
|
|
|
landing.save()
|
2013-03-01 18:13:14 +00:00
|
|
|
for subject in w2.subjects.all():
|
|
|
|
if subject not in w1.subjects.all():
|
|
|
|
w1.subjects.add(subject)
|
2012-02-10 03:30:33 +00:00
|
|
|
|
2012-02-13 22:35:08 +00:00
|
|
|
|
2011-10-19 03:00:07 +00:00
|
|
|
w2.delete()
|
2013-04-16 20:46:25 +00:00
|
|
|
|
|
|
|
def detach_edition(e):
|
|
|
|
"""will detach edition from its work, creating a new stub work. if remerge=true, will see if there's another work to attach to
|
|
|
|
"""
|
|
|
|
logger.info("splitting edition %s from %s", e, e.work)
|
|
|
|
w = models.Work(title=e.title, language = e.work.language)
|
|
|
|
w.save()
|
|
|
|
|
|
|
|
for identifier in e.identifiers.all():
|
|
|
|
identifier.work = w
|
|
|
|
identifier.save()
|
|
|
|
|
|
|
|
e.work = w
|
|
|
|
e.save()
|
2011-10-14 04:12:20 +00:00
|
|
|
|
2013-03-07 16:37:40 +00:00
|
|
|
def despam_description(description):
|
|
|
|
""" a lot of descriptions from openlibrary have free-book promotion text; this removes some of it."""
|
|
|
|
if description.find("GeneralBooksClub.com")>-1 or description.find("AkashaPublishing.Com")>-1:
|
|
|
|
return ""
|
|
|
|
pieces=description.split("1stWorldLibrary.ORG -")
|
|
|
|
if len(pieces)>1:
|
|
|
|
return pieces[1]
|
|
|
|
pieces=description.split("a million books for free.")
|
|
|
|
if len(pieces)>1:
|
|
|
|
return pieces[1]
|
|
|
|
return description
|
2011-10-19 03:45:02 +00:00
|
|
|
|
2012-07-27 19:58:16 +00:00
|
|
|
def add_openlibrary(work, hard_refresh = False):
|
|
|
|
if (not hard_refresh) and work.openlibrary_lookup is not None:
|
2012-01-09 18:55:22 +00:00
|
|
|
# don't hit OL if we've visited in the past month or so
|
2012-03-07 19:42:16 +00:00
|
|
|
if now()- work.openlibrary_lookup < timedelta(days=30):
|
2012-01-09 18:55:22 +00:00
|
|
|
return
|
2012-03-07 19:42:16 +00:00
|
|
|
work.openlibrary_lookup = now()
|
2011-12-19 06:33:13 +00:00
|
|
|
work.save()
|
|
|
|
|
|
|
|
# find the first ISBN match in OpenLibrary
|
|
|
|
logger.info("looking up openlibrary data for work %s", work.id)
|
|
|
|
found = False
|
|
|
|
e = None # openlibrary edition json
|
|
|
|
w = None # openlibrary work json
|
|
|
|
|
|
|
|
# get the 1st openlibrary match by isbn that has an associated work
|
|
|
|
url = "http://openlibrary.org/api/books"
|
|
|
|
params = {"format": "json", "jscmd": "details"}
|
2012-05-07 02:31:38 +00:00
|
|
|
subjects = []
|
2011-12-19 06:33:13 +00:00
|
|
|
for edition in work.editions.all():
|
2011-12-20 04:26:55 +00:00
|
|
|
isbn_key = "ISBN:%s" % edition.isbn_13
|
2011-12-19 06:33:13 +00:00
|
|
|
params['bibkeys'] = isbn_key
|
2012-01-31 04:57:23 +00:00
|
|
|
try:
|
|
|
|
e = _get_json(url, params, type='ol')
|
|
|
|
except LookupFailure:
|
|
|
|
logger.exception("OL lookup failed for %s", isbn_key)
|
|
|
|
e = {}
|
2012-05-07 02:31:38 +00:00
|
|
|
if e.has_key(isbn_key):
|
|
|
|
if e[isbn_key].has_key('details'):
|
|
|
|
if e[isbn_key]['details'].has_key('oclc_numbers'):
|
|
|
|
for oclcnum in e[isbn_key]['details']['oclc_numbers']:
|
2012-05-07 05:18:11 +00:00
|
|
|
models.Identifier.get_or_add(type='oclc',value=oclcnum,work=work, edition=edition)
|
2012-05-07 02:31:38 +00:00
|
|
|
if e[isbn_key]['details'].has_key('identifiers'):
|
|
|
|
ids = e[isbn_key]['details']['identifiers']
|
|
|
|
if ids.has_key('goodreads'):
|
|
|
|
models.Identifier.get_or_add(type='gdrd',value=ids['goodreads'][0],work=work,edition=edition)
|
|
|
|
if ids.has_key('librarything'):
|
|
|
|
models.Identifier.get_or_add(type='ltwk',value=ids['librarything'][0],work=work)
|
|
|
|
if ids.has_key('google'):
|
|
|
|
models.Identifier.get_or_add(type='goog',value=ids['google'][0],work=work)
|
|
|
|
if ids.has_key('project_gutenberg'):
|
|
|
|
models.Identifier.get_or_add(type='gute',value=ids['project_gutenberg'][0],work=work)
|
|
|
|
if e[isbn_key]['details'].has_key('works'):
|
|
|
|
work_key = e[isbn_key]['details']['works'].pop(0)['key']
|
|
|
|
logger.info("got openlibrary work %s for isbn %s", work_key, isbn_key)
|
2012-05-07 05:18:11 +00:00
|
|
|
models.Identifier.get_or_add(type='olwk',value=work_key,work=work)
|
2012-05-07 02:31:38 +00:00
|
|
|
try:
|
|
|
|
w = _get_json("http://openlibrary.org" + work_key,type='ol')
|
|
|
|
if w.has_key('description'):
|
2012-07-26 16:06:39 +00:00
|
|
|
description=w['description']
|
|
|
|
if isinstance(description,dict):
|
|
|
|
if description.has_key('value'):
|
|
|
|
description=description['value']
|
2013-03-07 16:37:40 +00:00
|
|
|
description=despam_description(description)
|
2012-07-26 16:06:39 +00:00
|
|
|
if not work.description or work.description.startswith('{') or len(description) > len(work.description):
|
|
|
|
work.description = description
|
2012-05-07 05:18:11 +00:00
|
|
|
work.save()
|
2012-05-07 02:31:38 +00:00
|
|
|
if w.has_key('subjects') and len(w['subjects']) > len(subjects):
|
|
|
|
subjects = w['subjects']
|
|
|
|
except LookupFailure:
|
|
|
|
logger.exception("OL lookup failed for %s", work_key)
|
|
|
|
if not subjects:
|
2011-12-19 06:33:13 +00:00
|
|
|
logger.warn("unable to find work %s at openlibrary", work.id)
|
|
|
|
return
|
|
|
|
|
|
|
|
# add the subjects to the Work
|
2012-05-07 02:31:38 +00:00
|
|
|
for s in subjects:
|
2015-02-03 23:41:38 +00:00
|
|
|
if valid_subject(s):
|
2015-01-30 16:33:22 +00:00
|
|
|
logger.info("adding subject %s to work %s", s, work.id)
|
|
|
|
subject, created = models.Subject.objects.get_or_create(name=s)
|
|
|
|
work.subjects.add(subject)
|
2012-05-07 02:31:38 +00:00
|
|
|
|
2011-12-19 06:33:13 +00:00
|
|
|
work.save()
|
2012-01-09 18:55:22 +00:00
|
|
|
|
2015-01-30 16:33:22 +00:00
|
|
|
def valid_xml_char_ordinal(c):
|
|
|
|
codepoint = ord(c)
|
|
|
|
# conditions ordered by presumed frequency
|
|
|
|
return (
|
|
|
|
0x20 <= codepoint <= 0xD7FF or
|
|
|
|
codepoint in (0x9, 0xA, 0xD) or
|
|
|
|
0xE000 <= codepoint <= 0xFFFD or
|
|
|
|
0x10000 <= codepoint <= 0x10FFFF
|
|
|
|
)
|
|
|
|
|
|
|
|
def valid_subject( subject_name ):
|
|
|
|
num_commas = 0
|
|
|
|
for c in subject_name:
|
|
|
|
if not valid_xml_char_ordinal(c):
|
|
|
|
return False
|
|
|
|
if c == ',':
|
2015-02-02 18:31:10 +00:00
|
|
|
num_commas += 1
|
2015-01-30 16:33:22 +00:00
|
|
|
if num_commas > 2:
|
|
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
2011-12-19 06:33:13 +00:00
|
|
|
|
2012-01-09 18:55:22 +00:00
|
|
|
def _get_json(url, params={}, type='gb'):
|
2011-10-10 19:57:12 +00:00
|
|
|
# TODO: should X-Forwarded-For change based on the request from client?
|
|
|
|
headers = {'User-Agent': settings.USER_AGENT,
|
|
|
|
'Accept': 'application/json',
|
|
|
|
'X-Forwarded-For': '69.174.114.214'}
|
2012-01-18 04:22:07 +00:00
|
|
|
if type == 'gb':
|
2012-01-09 18:55:22 +00:00
|
|
|
params['key'] = settings.GOOGLE_BOOKS_API_KEY
|
2012-07-31 20:06:11 +00:00
|
|
|
params['country'] = 'us'
|
2011-09-09 05:38:28 +00:00
|
|
|
response = requests.get(url, params=params, headers=headers)
|
2011-09-07 09:34:03 +00:00
|
|
|
if response.status_code == 200:
|
|
|
|
return json.loads(response.content)
|
|
|
|
else:
|
2011-09-10 11:36:38 +00:00
|
|
|
logger.error("unexpected HTTP response: %s" % response)
|
2012-01-18 04:15:24 +00:00
|
|
|
if response.content:
|
|
|
|
logger.error("response content: %s" % response.content)
|
2011-09-07 09:34:03 +00:00
|
|
|
raise LookupFailure("GET failed: url=%s and params=%s" % (url, params))
|
|
|
|
|
2011-09-09 05:38:28 +00:00
|
|
|
|
2012-02-15 02:01:13 +00:00
|
|
|
def load_gutenberg_edition(title, gutenberg_etext_id, ol_work_id, seed_isbn, url, format, license, lang, publication_date):
|
|
|
|
|
|
|
|
# let's start with instantiating the relevant Work and Edition if they don't already exist
|
|
|
|
|
|
|
|
try:
|
|
|
|
work = models.Identifier.objects.get(type='olwk',value=ol_work_id).work
|
|
|
|
except models.Identifier.DoesNotExist: # try to find an Edition with the seed_isbn and use that work to hang off of
|
|
|
|
sister_edition = add_by_isbn(seed_isbn)
|
|
|
|
if sister_edition.new:
|
|
|
|
# add related editions asynchronously
|
2012-02-16 18:19:36 +00:00
|
|
|
regluit.core.tasks.populate_edition.delay(sister_edition.isbn_13)
|
2012-02-15 02:01:13 +00:00
|
|
|
work = sister_edition.work
|
|
|
|
# attach the olwk identifier to this work if it's not none.
|
|
|
|
if ol_work_id is not None:
|
|
|
|
work_id = models.Identifier.get_or_add(type='olwk',value=ol_work_id, work=work)
|
|
|
|
|
|
|
|
# Now pull out any existing Gutenberg editions tied to the work with the proper Gutenberg ID
|
|
|
|
try:
|
|
|
|
edition = models.Identifier.objects.get( type='gtbg', value=gutenberg_etext_id ).edition
|
|
|
|
except models.Identifier.DoesNotExist:
|
|
|
|
edition = models.Edition()
|
|
|
|
edition.title = title
|
|
|
|
edition.work = work
|
|
|
|
|
|
|
|
edition.save()
|
|
|
|
edition_id = models.Identifier.get_or_add(type='gtbg',value=gutenberg_etext_id, edition=edition, work=work)
|
|
|
|
|
|
|
|
# check to see whether the Edition hasn't already been loaded first
|
|
|
|
# search by url
|
|
|
|
ebooks = models.Ebook.objects.filter(url=url)
|
|
|
|
|
|
|
|
# format: what's the controlled vocab? -- from Google -- alternative would be mimetype
|
|
|
|
|
|
|
|
if len(ebooks):
|
|
|
|
ebook = ebooks[0]
|
|
|
|
elif len(ebooks) == 0: # need to create new ebook
|
|
|
|
ebook = models.Ebook()
|
|
|
|
|
|
|
|
if len(ebooks) > 1:
|
|
|
|
warnings.warn("There is more than one Ebook matching url {0}".format(url))
|
|
|
|
|
|
|
|
|
|
|
|
ebook.format = format
|
2012-02-28 21:39:08 +00:00
|
|
|
ebook.provider = 'Project Gutenberg'
|
2012-02-15 02:01:13 +00:00
|
|
|
ebook.url = url
|
|
|
|
ebook.rights = license
|
|
|
|
|
|
|
|
# is an Ebook instantiable without a corresponding Edition? (No, I think)
|
|
|
|
|
|
|
|
ebook.edition = edition
|
|
|
|
ebook.save()
|
|
|
|
|
|
|
|
return ebook
|
|
|
|
|
2012-02-16 03:36:18 +00:00
|
|
|
def add_missing_isbn_to_editions(max_num=None, confirm=False):
|
|
|
|
"""For each of the editions with Google Books ids, do a lookup and attach ISBNs. Set confirm to True to check db changes made correctly"""
|
2012-02-16 18:44:13 +00:00
|
|
|
logger.info("Number of editions with Google Books IDs but not ISBNs (before): %d",
|
|
|
|
models.Edition.objects.filter(identifiers__type='goog').exclude(identifiers__type='isbn').count())
|
2012-02-16 03:36:18 +00:00
|
|
|
|
|
|
|
from regluit.experimental import bookdata
|
|
|
|
|
|
|
|
gb = bookdata.GoogleBooks(key=settings.GOOGLE_BOOKS_API_KEY)
|
|
|
|
|
|
|
|
new_isbns = []
|
2012-02-16 18:44:13 +00:00
|
|
|
google_id_not_found = []
|
2012-02-16 03:36:18 +00:00
|
|
|
no_isbn_found = []
|
|
|
|
editions_to_merge = []
|
|
|
|
exceptions = []
|
|
|
|
|
2012-02-16 18:44:13 +00:00
|
|
|
|
2012-02-16 03:36:18 +00:00
|
|
|
for (i, ed) in enumerate(islice(models.Edition.objects.filter(identifiers__type='goog').exclude(identifiers__type='isbn'), max_num)):
|
|
|
|
try:
|
|
|
|
g_id = ed.identifiers.get(type='goog').value
|
|
|
|
except Exception, e:
|
|
|
|
# we might get an exception if there is, for example, more than one Google id attached to this Edition
|
|
|
|
logger.exception("add_missing_isbn_to_editions for edition.id %s: %s", ed.id, e)
|
|
|
|
exceptions.append((ed.id, e))
|
|
|
|
continue
|
|
|
|
|
|
|
|
# try to get ISBN from Google Books
|
|
|
|
try:
|
2012-02-16 18:44:13 +00:00
|
|
|
vol_id = gb.volumeid(g_id)
|
|
|
|
if vol_id is None:
|
|
|
|
google_id_not_found.append((ed.id, g_id))
|
|
|
|
logger.debug("g_id not found: %s", g_id)
|
2012-02-16 03:36:18 +00:00
|
|
|
else:
|
2012-02-16 18:44:13 +00:00
|
|
|
isbn = vol_id.get('isbn')
|
|
|
|
logger.info("g_id, isbn: %s %s", g_id, isbn)
|
|
|
|
if isbn is not None:
|
|
|
|
# check to see whether the isbn is actually already in the db but attached to another Edition
|
|
|
|
existing_isbn_ids = models.Identifier.objects.filter(type='isbn', value=isbn)
|
|
|
|
if len(existing_isbn_ids):
|
|
|
|
# don't try to merge editions right now, just note the need to merge
|
|
|
|
ed2 = existing_isbn_ids[0].edition
|
|
|
|
editions_to_merge.append((ed.id, g_id, isbn, ed2.id))
|
|
|
|
else:
|
|
|
|
new_id = models.Identifier(type='isbn', value=isbn, edition=ed, work=ed.work)
|
|
|
|
new_id.save()
|
|
|
|
new_isbns.append((ed.id, g_id, isbn))
|
|
|
|
else:
|
|
|
|
no_isbn_found.append((ed.id, g_id, None))
|
2012-02-16 03:36:18 +00:00
|
|
|
except Exception, e:
|
|
|
|
logger.exception("add_missing_isbn_to_editions for edition.id %s: %s", ed.id, e)
|
|
|
|
exceptions.append((ed.id, g_id, None, e))
|
|
|
|
|
2012-02-16 18:44:13 +00:00
|
|
|
logger.info("Number of editions with Google Books IDs but not ISBNs (after): %d",
|
|
|
|
models.Edition.objects.filter(identifiers__type='goog').exclude(identifiers__type='isbn').count())
|
2012-02-16 03:36:18 +00:00
|
|
|
|
|
|
|
ok = None
|
|
|
|
|
|
|
|
if confirm:
|
|
|
|
ok = True
|
|
|
|
for (ed_id, g_id, isbn) in new_isbns:
|
|
|
|
if models.Edition.objects.get(id=ed_id).identifiers.get(type='isbn').value != isbn:
|
|
|
|
ok = False
|
|
|
|
break
|
|
|
|
|
|
|
|
return {
|
|
|
|
'new_isbns': new_isbns,
|
|
|
|
'no_isbn_found': no_isbn_found,
|
|
|
|
'editions_to_merge': editions_to_merge,
|
|
|
|
'exceptions': exceptions,
|
2012-02-16 21:18:22 +00:00
|
|
|
'google_id_not_found': google_id_not_found,
|
2012-02-16 03:36:18 +00:00
|
|
|
'confirm': ok
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2011-09-07 09:34:03 +00:00
|
|
|
class LookupFailure(Exception):
|
|
|
|
pass
|
2013-03-07 16:37:40 +00:00
|
|
|
|
2015-09-24 21:58:34 +00:00
|
|
|
IDTABLE = [('librarything','ltwk'),('goodreads','gdrd'),('openlibrary','olwk'),('gutenberg','gtbg'),('isbn','isbn'),('oclc','oclc'),('edition_id','edid') ]
|
2015-07-30 03:01:43 +00:00
|
|
|
|
2015-09-12 23:20:08 +00:00
|
|
|
def unreverse(name):
|
|
|
|
if not ',' in name:
|
|
|
|
return name
|
|
|
|
(last, rest) = name.split(',', 1)
|
|
|
|
if not ',' in rest:
|
|
|
|
return '%s %s' % (rest.strip(),last.strip())
|
|
|
|
(first, rest) = rest.split(',', 1)
|
|
|
|
return '%s %s, %s' % (first.strip(),last.strip(),rest.strip())
|
|
|
|
|
|
|
|
|
2016-03-07 21:17:33 +00:00
|
|
|
def load_from_yaml(yaml_url, test_mode=False):
|
2016-03-04 20:09:30 +00:00
|
|
|
"""
|
|
|
|
if mock_ebook is True, don't construct list of ebooks from a release -- rather use an epub
|
|
|
|
"""
|
2015-09-24 21:58:34 +00:00
|
|
|
all_metadata = Pandata(yaml_url)
|
|
|
|
for metadata in all_metadata.get_edition_list():
|
|
|
|
#find an work to associate
|
|
|
|
work = edition = None
|
|
|
|
if metadata.url:
|
|
|
|
new_ids = [('http','http', metadata.url)]
|
2015-07-30 03:01:43 +00:00
|
|
|
else:
|
2015-09-24 21:58:34 +00:00
|
|
|
new_ids = []
|
|
|
|
for (identifier, id_code) in IDTABLE:
|
|
|
|
# note that the work chosen is the last associated
|
|
|
|
value = metadata.edition_identifiers.get(identifier,None)
|
|
|
|
if not value:
|
|
|
|
value = metadata.identifiers.get(identifier,None)
|
|
|
|
if value:
|
|
|
|
value = value[0] if isinstance(value, list) else value
|
|
|
|
try:
|
|
|
|
id = models.Identifier.objects.get(type=id_code, value=value)
|
|
|
|
work = id.work
|
|
|
|
if id.edition and not edition:
|
|
|
|
edition = id.edition
|
|
|
|
except models.Identifier.DoesNotExist:
|
|
|
|
new_ids.append((identifier, id_code, value))
|
|
|
|
|
|
|
|
|
|
|
|
if not work:
|
|
|
|
work = models.Work.objects.create(title=metadata.title, language=metadata.language)
|
|
|
|
if not edition:
|
|
|
|
edition = models.Edition.objects.create(title=metadata.title, work=work)
|
|
|
|
for (identifier, id_code, value) in new_ids:
|
|
|
|
models.Identifier.set(type=id_code, value=value, edition=edition if id_code in ('isbn', 'oclc','edid') else None, work=work)
|
|
|
|
if metadata.publisher: #always believe yaml
|
|
|
|
edition.set_publisher(metadata.publisher)
|
|
|
|
if metadata.publication_date: #always believe yaml
|
|
|
|
edition.publication_date = metadata.publication_date
|
|
|
|
if metadata.description and len(metadata.description)>len(work.description): #be careful about overwriting the work description
|
|
|
|
work.description = metadata.description
|
|
|
|
if metadata.creator: #always believe yaml
|
|
|
|
edition.authors.clear()
|
|
|
|
for key in metadata.creator.keys():
|
|
|
|
creators=metadata.creator[key]
|
|
|
|
rel_code=inverse_marc_rels.get(key,'aut')
|
|
|
|
creators = creators if isinstance(creators,list) else [creators]
|
|
|
|
for creator in creators:
|
|
|
|
edition.add_author(unreverse(creator.get('agent_name','')),relation=rel_code)
|
|
|
|
for yaml_subject in metadata.subjects: #always add yaml subjects (don't clear)
|
|
|
|
if isinstance(yaml_subject, tuple):
|
|
|
|
(authority, heading) = yaml_subject
|
|
|
|
elif isinstance(yaml_subject, str):
|
|
|
|
(authority, heading) = ( '', yaml_subject)
|
|
|
|
else:
|
|
|
|
continue
|
|
|
|
(subject, created) = models.Subject.objects.get_or_create(name=heading)
|
|
|
|
if not subject.authority and authority:
|
|
|
|
subject.authority = authority
|
|
|
|
subject.save()
|
|
|
|
subject.works.add(work)
|
|
|
|
# the default edition uses the first cover in covers.
|
|
|
|
for cover in metadata.covers:
|
|
|
|
if cover.get('image_path', False):
|
|
|
|
edition.cover_image=urljoin(yaml_url,cover['image_path'])
|
|
|
|
break
|
|
|
|
edition.save()
|
2016-03-04 20:09:30 +00:00
|
|
|
# create Ebook for any ebook in the corresponding GitHub release
|
|
|
|
# assuming yaml_url of form (from GitHub, though not necessarily GITenberg)
|
|
|
|
# https://github.com/GITenberg/Adventures-of-Huckleberry-Finn_76/raw/master/metadata.yaml
|
|
|
|
|
|
|
|
url_path = urlparse(yaml_url).path.split("/")
|
|
|
|
(repo_owner, repo_name) = (url_path[1], url_path[2])
|
|
|
|
repo_tag = metadata._version
|
|
|
|
# allow for there not to be a token in the settings
|
|
|
|
try:
|
|
|
|
token = settings.GITHUB_PUBLIC_TOKEN
|
|
|
|
except:
|
|
|
|
token = None
|
|
|
|
|
2015-09-24 21:58:34 +00:00
|
|
|
if metadata._version and not metadata._version.startswith('0.0.'):
|
2016-03-04 20:09:30 +00:00
|
|
|
# use GitHub API to compute the ebooks in release until we're in test mode
|
2016-03-07 21:17:33 +00:00
|
|
|
if test_mode:
|
2016-03-04 20:09:30 +00:00
|
|
|
# not using ebook_name in this code
|
2016-05-16 22:11:03 +00:00
|
|
|
ebooks_in_release = [('epub', 'book.epub')]
|
2016-03-04 20:09:30 +00:00
|
|
|
else:
|
|
|
|
ebooks_in_release = ebooks_in_github_release(repo_owner, repo_name, repo_tag, token=token)
|
|
|
|
|
|
|
|
for (ebook_format, ebook_name) in ebooks_in_release:
|
2016-05-16 19:43:11 +00:00
|
|
|
(book_name_prefix, _ ) = re.search(r'(.*)\.([^\.]*)$', ebook_name).groups()
|
2016-03-04 20:09:30 +00:00
|
|
|
(ebook, created)= models.Ebook.objects.get_or_create(
|
2016-05-16 19:43:11 +00:00
|
|
|
url=git_download_from_yaml_url(yaml_url,metadata._version,edition_name=book_name_prefix,
|
2016-03-04 20:09:30 +00:00
|
|
|
format_= ebook_format),
|
|
|
|
provider='Github',
|
2016-07-14 23:02:22 +00:00
|
|
|
rights = cc.match_license(metadata.rights),
|
2016-03-04 20:09:30 +00:00
|
|
|
format = ebook_format,
|
|
|
|
edition = edition,
|
|
|
|
)
|
2016-09-23 18:53:54 +00:00
|
|
|
ebook.set_version(metadata._version)
|
2016-03-04 20:09:30 +00:00
|
|
|
|
2015-07-30 04:34:05 +00:00
|
|
|
return work.id
|
2015-07-30 03:01:43 +00:00
|
|
|
|
2016-03-04 20:09:30 +00:00
|
|
|
def git_download_from_yaml_url(yaml_url, version, edition_name='book', format_='epub'):
|
2015-07-30 03:01:43 +00:00
|
|
|
# go from https://github.com/GITenberg/Adventures-of-Huckleberry-Finn_76/raw/master/metadata.yaml
|
|
|
|
# to https://github.com/GITenberg/Adventures-of-Huckleberry-Finn_76/releases/download/v0.0.3/Adventures-of-Huckleberry-Finn.epub
|
|
|
|
if yaml_url.endswith('raw/master/metadata.yaml'):
|
|
|
|
repo_url = yaml_url[0:-24]
|
2015-08-03 14:04:07 +00:00
|
|
|
#print (repo_url,version,edition_name)
|
2016-03-04 20:09:30 +00:00
|
|
|
ebook_url = repo_url + 'releases/download/' + version + '/' + edition_name + '.' + format_
|
|
|
|
return ebook_url
|
|
|
|
|
|
|
|
|
|
|
|
def release_from_tag(repo, tag_name):
|
|
|
|
"""Get a release by tag name.
|
|
|
|
release_from_tag() returns a release with specified tag
|
|
|
|
while release() returns a release with specified release id
|
|
|
|
:param str tag_name: (required) name of tag
|
|
|
|
:returns: :class:`Release <github3.repos.release.Release>`
|
|
|
|
"""
|
|
|
|
# release_from_tag adapted from
|
|
|
|
# https://github.com/sigmavirus24/github3.py/blob/38de787e465bffc63da73d23dc51f50d86dc903d/github3/repos/repo.py#L1781-L1793
|
|
|
|
|
|
|
|
url = repo._build_url('releases', 'tags', tag_name,
|
|
|
|
base_url=repo._api)
|
|
|
|
json = repo._json(repo._get(url), 200)
|
|
|
|
return Release(json, repo) if json else None
|
|
|
|
|
|
|
|
|
|
|
|
def ebooks_in_github_release(repo_owner, repo_name, tag, token=None):
|
|
|
|
"""
|
|
|
|
returns a list of (book_type, book_name) for a given GitHub release (specified by
|
|
|
|
owner, name, tag). token is a GitHub authorization token -- useful for accessing
|
|
|
|
higher rate limit in the GitHub API
|
|
|
|
"""
|
|
|
|
|
|
|
|
# map mimetype to file extension
|
2016-03-07 21:30:40 +00:00
|
|
|
EBOOK_FORMATS = dict([(v,k) for (k,v) in settings.CONTENT_TYPES.items()])
|
2016-03-04 20:09:30 +00:00
|
|
|
|
|
|
|
if token is not None:
|
|
|
|
gh = login(token=token)
|
|
|
|
else:
|
|
|
|
# anonymous access
|
|
|
|
gh = GitHub()
|
|
|
|
|
|
|
|
repo = gh.repository(repo_owner, repo_name)
|
|
|
|
release = release_from_tag(repo, tag)
|
|
|
|
|
|
|
|
return [(EBOOK_FORMATS.get(asset.content_type), asset.name)
|
|
|
|
for asset in release.iter_assets()
|
|
|
|
if EBOOK_FORMATS.get(asset.content_type) is not None]
|