moved to using googlebooks api for data. required moving authors and subjects to be associated to editions, since we google books does not have a notion of a work

pull/1/head
Ed Summers 2011-10-10 12:57:10 -04:00
parent 4e65134ab1
commit 70a620b27f
10 changed files with 274 additions and 150 deletions

View File

@ -88,13 +88,6 @@ class SubjectResource(ModelResource):
queryset = models.Subject.objects.all()
resource_name = 'subject'
class EditionCoverResource(ModelResource):
edition = fields.ToManyField(EditionResource, 'editions')
class Meta:
authentication = ApiKeyAuthentication()
queryset = models.EditionCover.objects.all()
resource_name = 'editioncover'
class WishlistResource(ModelResource):
user = fields.ToOneField(UserResource, 'user')
works = fields.ToManyField(WorkResource, 'works')

View File

@ -7,7 +7,6 @@ v1_api = Api(api_name='v1')
v1_api.register(resources.UserResource())
v1_api.register(resources.WorkResource())
v1_api.register(resources.EditionResource())
v1_api.register(resources.EditionCoverResource())
v1_api.register(resources.CampaignResource())
v1_api.register(resources.AuthorResource())
v1_api.register(resources.SubjectResource())

View File

@ -1,8 +1,3 @@
"""
The module handles fetching books from OpenLibrary and adding them
to the local database.
"""
import json
import logging
@ -10,94 +5,59 @@ import requests
from django.conf import settings
from regluit.core import models
from regluit.core.isbn import convert_10_to_13
logger = logging.getLogger(__name__)
def add_book(isbn):
url = "http://openlibrary.org/api/books"
bibkeys = "ISBN:%s" % isbn
params = {"bibkeys": bibkeys, "jscmd": "details", "format": "json"}
results = _get_json(url, params)
edition = None
if results.has_key(bibkeys):
logger.info("saving book info for %s", isbn)
edition = _save_edition(results[bibkeys]['details'])
elif len(isbn) == 10:
isbn_13 = convert_10_to_13(isbn)
logger.info("lookup failed for %s trying isbn13 %s", isbn, isbn_13)
edition = add_book(isbn_13)
else:
logger.info("lookup failed for %s", isbn)
def add_by_isbn(isbn):
url = "https://www.googleapis.com/books/v1/volumes"
results = _get_json(url, {"q": "isbn:%s" % isbn})
return edition
if len(results['items']) == 0:
logger.warn("no google hits for %s" % isbn)
return None
return add_by_googlebooks_id(results['items'][0]['id'])
def _save_edition(edition_data):
edition_key = edition_data['key']
edition, created = models.Edition.objects.get_or_create(openlibrary_id=edition_key)
edition.title = edition_data.get('title')
edition.description = edition_data.get('description')
edition.publisher = _first(edition_data, 'publishers')
edition.publication_date = edition_data.get('publish_date')
def add_by_googlebooks_id(googlebooks_id):
url = "https://www.googleapis.com/books/v1/volumes/%s" % googlebooks_id
d = _get_json(url)['volumeInfo']
# assumption: OL has only one isbn_10 or isbn_13 for an edition
edition.isbn_10 = _first(edition_data, 'isbn_10')
edition.isbn_13 = _first(edition_data, 'isbn_13')
e, created = models.Edition.objects.get_or_create(googlebooks_id=googlebooks_id)
if not created:
return e
edition.save()
e.title = d.get('title')
e.description = d.get('description')
e.publisher = d.get('publisher')
e.publication_date = d.get('publishedDate')
for work_data in edition_data.get('works', []):
_save_work(work_data['key'], edition)
for i in d.get('industryIdentifiers', []):
if i['type'] == 'ISBN_10':
e.isbn_10 = i['identifier']
elif i['type'] == 'ISBN_13':
e.isbn_13 = i['identifier']
for cover_id in edition_data.get('covers', []):
models.EditionCover.objects.get_or_create(openlibrary_id=cover_id, edition=edition)
for a in d.get('authors', []):
a, created = models.Author.objects.get_or_create(name=a)
a.editions.add(e)
return edition
for s in d.get('categories', []):
s, created = models.Subject.objects.get_or_create(name=s)
s.editions.add(e)
# add a stub Work for the edition
if e.work == None:
w = models.Work.objects.create(title=e.title)
w.editions.add(e)
def _save_work(work_key, edition):
url = "http://openlibrary.org" + work_key
work_data = _get_json(url)
work, created = models.Work.objects.get_or_create(openlibrary_id=work_key)
work.title = work_data.get('title')
work.save()
for author_data in work_data.get('authors', []):
_save_author(author_data['author']['key'], work)
for subject_name in work_data.get('subjects', []):
subject, created = models.Subject.objects.get_or_create(name=subject_name)
work.subjects.add(subject)
work.editions.add(edition)
return work
def _save_author(author_key, work):
url = "http://openlibrary.org" + author_key
author_data = _get_json(url)
author, created = models.Author.objects.get_or_create(openlibrary_id=author_key)
author.name = author_data['name']
author.save()
author.works.add(work)
return author
def _first(dictionary, key):
l = dictionary.get(key, [])
if len(l) == 0: return None
return l[0]
return e
def _get_json(url, params={}):
headers = {'User-Agent': settings.USER_AGENT, 'Accept': 'application/json'}
params['key'] = settings.GOOGLE_BOOKS_API_KEY
response = requests.get(url, params=params, headers=headers)
if response.status_code == 200:
return json.loads(response.content)

View File

@ -0,0 +1,186 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Deleting model 'EditionCover'
db.delete_table('core_editioncover')
# Deleting field 'Edition.openlibrary_id'
db.delete_column('core_edition', 'openlibrary_id')
# Adding field 'Edition.googlebooks_id'
db.add_column('core_edition', 'googlebooks_id', self.gf('django.db.models.fields.CharField')(default='initial', max_length=50), keep_default=False)
# Removing M2M table for field works on 'Subject'
db.delete_table('core_subject_works')
# Adding M2M table for field editions on 'Subject'
db.create_table('core_subject_editions', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('subject', models.ForeignKey(orm['core.subject'], null=False)),
('edition', models.ForeignKey(orm['core.edition'], null=False))
))
db.create_unique('core_subject_editions', ['subject_id', 'edition_id'])
# Deleting field 'Author.openlibrary_id'
db.delete_column('core_author', 'openlibrary_id')
# Removing M2M table for field works on 'Author'
db.delete_table('core_author_works')
# Adding M2M table for field editions on 'Author'
db.create_table('core_author_editions', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('author', models.ForeignKey(orm['core.author'], null=False)),
('edition', models.ForeignKey(orm['core.edition'], null=False))
))
db.create_unique('core_author_editions', ['author_id', 'edition_id'])
def backwards(self, orm):
# Adding model 'EditionCover'
db.create_table('core_editioncover', (
('edition', self.gf('django.db.models.fields.related.ForeignKey')(related_name='covers', to=orm['core.Edition'])),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('openlibrary_id', self.gf('django.db.models.fields.IntegerField')()),
))
db.send_create_signal('core', ['EditionCover'])
# Adding field 'Edition.openlibrary_id'
db.add_column('core_edition', 'openlibrary_id', self.gf('django.db.models.fields.CharField')(max_length=50, null=True), keep_default=False)
# Deleting field 'Edition.googlebooks_id'
db.delete_column('core_edition', 'googlebooks_id')
# Adding M2M table for field works on 'Subject'
db.create_table('core_subject_works', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('subject', models.ForeignKey(orm['core.subject'], null=False)),
('work', models.ForeignKey(orm['core.work'], null=False))
))
db.create_unique('core_subject_works', ['subject_id', 'work_id'])
# Removing M2M table for field editions on 'Subject'
db.delete_table('core_subject_editions')
# Adding field 'Author.openlibrary_id'
db.add_column('core_author', 'openlibrary_id', self.gf('django.db.models.fields.CharField')(max_length=50, null=True), keep_default=False)
# Adding M2M table for field works on 'Author'
db.create_table('core_author_works', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('author', models.ForeignKey(orm['core.author'], null=False)),
('work', models.ForeignKey(orm['core.work'], null=False))
))
db.create_unique('core_author_works', ['author_id', 'work_id'])
# Removing M2M table for field editions on 'Author'
db.delete_table('core_author_editions')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'core.author': {
'Meta': {'object_name': 'Author'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'editions': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'authors'", 'symmetrical': 'False', 'to': "orm['core.Edition']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '500'})
},
'core.campaign': {
'Meta': {'object_name': 'Campaign'},
'amazon_receiver': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'}),
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'deadline': ('django.db.models.fields.DateTimeField', [], {}),
'description': ('django.db.models.fields.CharField', [], {'max_length': '10000'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
'paypal_receiver': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'}),
'target': ('django.db.models.fields.DecimalField', [], {'max_digits': '14', 'decimal_places': '2'}),
'work': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'campaigns'", 'to': "orm['core.Work']"})
},
'core.edition': {
'Meta': {'object_name': 'Edition'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True'}),
'googlebooks_id': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'isbn_10': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True'}),
'isbn_13': ('django.db.models.fields.CharField', [], {'max_length': '13', 'null': 'True'}),
'publication_date': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
'publisher': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '1000'}),
'work': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editions'", 'null': 'True', 'to': "orm['core.Work']"})
},
'core.subject': {
'Meta': {'object_name': 'Subject'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'editions': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'subjects'", 'symmetrical': 'False', 'to': "orm['core.Edition']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '500'})
},
'core.userprofile': {
'Meta': {'object_name': 'UserProfile'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'tagline': ('django.db.models.fields.CharField', [], {'max_length': '140', 'blank': 'True'}),
'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'})
},
'core.wishlist': {
'Meta': {'object_name': 'Wishlist'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'wishlist'", 'unique': 'True', 'to': "orm['auth.User']"}),
'works': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'wishlists'", 'symmetrical': 'False', 'to': "orm['core.Work']"})
},
'core.work': {
'Meta': {'object_name': 'Work'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'openlibrary_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True'}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '1000'})
}
}
complete_apps = ['core']

View File

@ -1,7 +1,10 @@
import random
from decimal import Decimal
from django.db import models
from django.db.models import Q
from django.contrib.auth.models import User
from decimal import Decimal
class Campaign(models.Model):
created = models.DateTimeField(auto_now_add=True)
@ -29,8 +32,9 @@ class Work(models.Model):
return None
def cover_image_small(self):
first_isbn = self.editions.all()[0].isbn_10
return "http://covers.openlibrary.org/b/isbn/%s-S.jpg" % first_isbn
server_id = random.randint(0, 9)
gb_id = self.editions.all()[0].googlebooks_id
return "http://bks%i.books.google.com/books?id=%s&printsec=frontcover&img=1&zoom=5" % (server_id, gb_id)
def __unicode__(self):
return self.title
@ -39,8 +43,7 @@ class Work(models.Model):
class Author(models.Model):
created = models.DateTimeField(auto_now_add=True)
name = models.CharField(max_length=500)
openlibrary_id = models.CharField(max_length=50, null=True)
works = models.ManyToManyField("Work", related_name="authors")
editions = models.ManyToManyField("Edition", related_name="authors")
def __unicode__(self):
return self.name
@ -49,13 +52,14 @@ class Author(models.Model):
class Subject(models.Model):
created = models.DateTimeField(auto_now_add=True)
name = models.CharField(max_length=500)
works = models.ManyToManyField("Work", related_name="subjects")
editions = models.ManyToManyField("Edition", related_name="subjects")
def __unicode__(self):
return self.name
class Edition(models.Model):
googlebooks_id = models.CharField(max_length=50, null=False)
created = models.DateTimeField(auto_now_add=True)
title = models.CharField(max_length=1000)
description = models.TextField(default='', null=True)
@ -63,7 +67,6 @@ class Edition(models.Model):
publication_date = models.CharField(max_length=50)
isbn_10 = models.CharField(max_length=10, null=True)
isbn_13 = models.CharField(max_length=13, null=True)
openlibrary_id = models.CharField(max_length=50, null=True)
work = models.ForeignKey("Work", related_name="editions", null=True)
def __unicode__(self):
@ -75,11 +78,6 @@ class Edition(models.Model):
return e
return None
class EditionCover(models.Model):
openlibrary_id = models.IntegerField()
edition = models.ForeignKey("Edition", related_name="covers")
class Wishlist(models.Model):
created = models.DateTimeField(auto_now_add=True)
user = models.OneToOneField(User, related_name='wishlist')

View File

@ -13,7 +13,7 @@ def gluejar_search(q):
r = {'title': v.get('title', ""),
'description': v.get('description', ""),
'publisher': v.get('publisher', ""),
'google_id': item.get('selfLink')}
'googlebooks_id': item.get('id')}
# TODO: allow multiple authors
if v.has_key('authors') and len(v['authors']) > 0:

View File

@ -6,40 +6,33 @@ class TestBooks(TestCase):
def test_add_book(self):
# edition
edition = bookloader.add_book(isbn='0441012035')
edition = bookloader.add_by_isbn('0441012035')
self.assertEqual(edition.title, 'Neuromancer')
self.assertEqual(edition.publication_date, '2004')
self.assertEqual(edition.publisher, 'Ace Books')
self.assertEqual(edition.isbn_10, '0441012035')
self.assertEqual(edition.isbn_13, None)
self.assertEqual(edition.openlibrary_id, "/books/OL3305354M")
# edition covers
covers = edition.covers.all()
self.assertEqual(len(covers), 1)
self.assertEqual(covers[0].openlibrary_id, 284192)
# work
work = edition.work
self.assertTrue(work)
self.assertEqual(work.authors.all()[0].name, 'William F. Gibson')
self.assertEqual(edition.isbn_13, '9780441012039')
self.assertEqual(edition.googlebooks_id, "2NyiPwAACAAJ")
# subjects
subject_names = [subject.name for subject in work.subjects.all()]
self.assertEqual(len(subject_names), 18)
self.assertTrue('Fiction' in subject_names)
subject_names = [subject.name for subject in edition.subjects.all()]
self.assertEqual(len(subject_names), 11)
self.assertTrue('Japan' in subject_names)
# authors
author_names = [author.name for author in work.authors.all()]
self.assertEqual(len(author_names), 1)
self.assertEqual(author_names[0], "William F. Gibson")
self.assertEqual(edition.authors.all().count(), 1)
self.assertEqual(edition.authors.all()[0].name, 'William Gibson')
# work
self.assertTrue(edition.work)
def test_double_add(self):
bookloader.add_book(isbn='0441012035')
bookloader.add_book(isbn='0441012035')
bookloader.add_by_isbn('0441012035')
bookloader.add_by_isbn('0441012035')
self.assertEqual(models.Edition.objects.all().count(), 1)
self.assertEqual(models.Author.objects.all().count(), 1)
self.assertEqual(models.Work.objects.all().count(), 1)
self.assertEqual(models.Subject.objects.all().count(), 18)
self.assertEqual(models.Subject.objects.all().count(), 11)
class SearchTests(TestCase):
@ -55,6 +48,7 @@ class SearchTests(TestCase):
self.assertTrue(r.has_key('image'))
self.assertTrue(r.has_key('publisher'))
self.assertTrue(r.has_key('isbn_10'))
self.assertTrue(r.has_key('googlebooks_id'))
def test_googlebooks_search(self):
response = search.googlebooks_search('melville')

View File

@ -6,9 +6,9 @@ $(document).ready(function() {
$(".add-wishlist").each(function (index, element) {
$(element).click(function() {
var span = $(element).find("span");
var isbn = span.attr('id')
if (!isbn) return;
$.post('/wishlist/', {'isbn': isbn}, function(data) {
var gb_id = span.attr('id')
if (!gb_id) return;
$.post('/wishlist/', {'googlebooks_id': gb_id}, function(data) {
span.fadeOut();
var newSpan = $("<span>On Your Wishlist!</span>").hide();
span.replaceWith(newSpan);
@ -68,7 +68,7 @@ $(document).ready(function() {
{% if result.on_wishlist %}
<span>On Your Wishlist!</span>
{% else %}
<span id="{{ result.isbn_10 }}">Add to Wishlist</span>
<span id="{{ result.googlebooks_id }}">Add to Wishlist</span>
{% endif %}
</div>
<div class="booklist-status">

View File

@ -1,23 +1,22 @@
"""
This file demonstrates two different styles of tests (one doctest and one
unittest). These will both pass when you run "manage.py test".
Replace these with more appropriate tests for your application.
"""
from django.test import TestCase
from django.test.client import Client
from django.contrib.auth.models import User
class SimpleTest(TestCase):
def test_basic_addition(self):
"""
Tests that 1 + 1 always equals 2.
"""
self.failUnlessEqual(1 + 1, 2)
class WishlistTests(TestCase):
__test__ = {"doctest": """
Another way to test that 1 + 1 is equal to 2.
def setUp(self):
self.user = User.objects.create_user('test', 'test@example.org', 'test')
self.client = Client()
self.client.login(username='test', password='test')
>>> 1 + 1 == 2
True
"""}
def test_add_remove(self):
# add a book to the wishlist
r = self.client.post("/wishlist/", {"googlebooks_id": "2NyiPwAACAAJ"},
HTTP_X_REQUESTED_WITH="XMLHttpRequest")
self.assertEqual(r.status_code, 302)
self.assertEqual(self.user.wishlist.works.all().count(), 1)
# remove the book
r = self.client.post("/wishlist/", {"remove_work_id": "1"},
HTTP_X_REQUESTED_WITH="XMLHttpRequest")
self.assertEqual(self.user.wishlist.works.all().count(), 0)

View File

@ -69,19 +69,14 @@ def search(request):
return render(request, 'search.html', context)
# TODO: perhaps this functionality belongs in the API?
@csrf_exempt
@require_POST
@login_required
@csrf_exempt
def wishlist(request):
isbn = request.POST.get('isbn', None)
googlebooks_id = request.POST.get('googlebooks_id', None)
remove_work_id = request.POST.get('remove_work_id', None)
if isbn:
edition = models.Edition.get_by_isbn(isbn)
if not edition:
print "loading book"
edition = bookloader.add_book(isbn)
if edition:
print "adding edition"
if googlebooks_id:
edition = bookloader.add_by_googlebooks_id(googlebooks_id)
request.user.wishlist.works.add(edition.work)
# TODO: redirect to work page, when it exists
return HttpResponseRedirect('/')