diff --git a/api/resources.py b/api/resources.py index c652a9df..5d3fb56c 100755 --- a/api/resources.py +++ b/api/resources.py @@ -88,13 +88,6 @@ class SubjectResource(ModelResource): queryset = models.Subject.objects.all() resource_name = 'subject' -class EditionCoverResource(ModelResource): - edition = fields.ToManyField(EditionResource, 'editions') - class Meta: - authentication = ApiKeyAuthentication() - queryset = models.EditionCover.objects.all() - resource_name = 'editioncover' - class WishlistResource(ModelResource): user = fields.ToOneField(UserResource, 'user') works = fields.ToManyField(WorkResource, 'works') diff --git a/api/urls.py b/api/urls.py index 17734f5b..7c17213b 100644 --- a/api/urls.py +++ b/api/urls.py @@ -7,7 +7,6 @@ v1_api = Api(api_name='v1') v1_api.register(resources.UserResource()) v1_api.register(resources.WorkResource()) v1_api.register(resources.EditionResource()) -v1_api.register(resources.EditionCoverResource()) v1_api.register(resources.CampaignResource()) v1_api.register(resources.AuthorResource()) v1_api.register(resources.SubjectResource()) diff --git a/core/bookloader.py b/core/bookloader.py index b699cd2a..007aa3bd 100755 --- a/core/bookloader.py +++ b/core/bookloader.py @@ -1,8 +1,3 @@ -""" -The module handles fetching books from OpenLibrary and adding them -to the local database. -""" - import json import logging @@ -10,94 +5,59 @@ import requests from django.conf import settings from regluit.core import models -from regluit.core.isbn import convert_10_to_13 logger = logging.getLogger(__name__) -def add_book(isbn): - url = "http://openlibrary.org/api/books" - bibkeys = "ISBN:%s" % isbn - params = {"bibkeys": bibkeys, "jscmd": "details", "format": "json"} - results = _get_json(url, params) - edition = None - if results.has_key(bibkeys): - logger.info("saving book info for %s", isbn) - edition = _save_edition(results[bibkeys]['details']) - elif len(isbn) == 10: - isbn_13 = convert_10_to_13(isbn) - logger.info("lookup failed for %s trying isbn13 %s", isbn, isbn_13) - edition = add_book(isbn_13) - else: - logger.info("lookup failed for %s", isbn) +def add_by_isbn(isbn): + url = "https://www.googleapis.com/books/v1/volumes" + results = _get_json(url, {"q": "isbn:%s" % isbn}) - return edition + if len(results['items']) == 0: + logger.warn("no google hits for %s" % isbn) + return None + + return add_by_googlebooks_id(results['items'][0]['id']) -def _save_edition(edition_data): - edition_key = edition_data['key'] - edition, created = models.Edition.objects.get_or_create(openlibrary_id=edition_key) - edition.title = edition_data.get('title') - edition.description = edition_data.get('description') - edition.publisher = _first(edition_data, 'publishers') - edition.publication_date = edition_data.get('publish_date') +def add_by_googlebooks_id(googlebooks_id): + url = "https://www.googleapis.com/books/v1/volumes/%s" % googlebooks_id + d = _get_json(url)['volumeInfo'] - # assumption: OL has only one isbn_10 or isbn_13 for an edition - edition.isbn_10 = _first(edition_data, 'isbn_10') - edition.isbn_13 = _first(edition_data, 'isbn_13') + e, created = models.Edition.objects.get_or_create(googlebooks_id=googlebooks_id) + if not created: + return e - edition.save() + e.title = d.get('title') + e.description = d.get('description') + e.publisher = d.get('publisher') + e.publication_date = d.get('publishedDate') - for work_data in edition_data.get('works', []): - _save_work(work_data['key'], edition) + for i in d.get('industryIdentifiers', []): + if i['type'] == 'ISBN_10': + e.isbn_10 = i['identifier'] + elif i['type'] == 'ISBN_13': + e.isbn_13 = i['identifier'] - for cover_id in edition_data.get('covers', []): - models.EditionCover.objects.get_or_create(openlibrary_id=cover_id, edition=edition) + for a in d.get('authors', []): + a, created = models.Author.objects.get_or_create(name=a) + a.editions.add(e) - return edition + for s in d.get('categories', []): + s, created = models.Subject.objects.get_or_create(name=s) + s.editions.add(e) + # add a stub Work for the edition + if e.work == None: + w = models.Work.objects.create(title=e.title) + w.editions.add(e) -def _save_work(work_key, edition): - url = "http://openlibrary.org" + work_key - work_data = _get_json(url) - - work, created = models.Work.objects.get_or_create(openlibrary_id=work_key) - work.title = work_data.get('title') - work.save() - - for author_data in work_data.get('authors', []): - _save_author(author_data['author']['key'], work) - - for subject_name in work_data.get('subjects', []): - subject, created = models.Subject.objects.get_or_create(name=subject_name) - work.subjects.add(subject) - - work.editions.add(edition) - - return work - - -def _save_author(author_key, work): - url = "http://openlibrary.org" + author_key - author_data = _get_json(url) - - author, created = models.Author.objects.get_or_create(openlibrary_id=author_key) - author.name = author_data['name'] - author.save() - - author.works.add(work) - - return author - - -def _first(dictionary, key): - l = dictionary.get(key, []) - if len(l) == 0: return None - return l[0] + return e def _get_json(url, params={}): headers = {'User-Agent': settings.USER_AGENT, 'Accept': 'application/json'} + params['key'] = settings.GOOGLE_BOOKS_API_KEY response = requests.get(url, params=params, headers=headers) if response.status_code == 200: return json.loads(response.content) diff --git a/core/migrations/0014_use_more_googlebooks_api.py b/core/migrations/0014_use_more_googlebooks_api.py new file mode 100644 index 00000000..e6d33809 --- /dev/null +++ b/core/migrations/0014_use_more_googlebooks_api.py @@ -0,0 +1,186 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Deleting model 'EditionCover' + db.delete_table('core_editioncover') + + # Deleting field 'Edition.openlibrary_id' + db.delete_column('core_edition', 'openlibrary_id') + + # Adding field 'Edition.googlebooks_id' + db.add_column('core_edition', 'googlebooks_id', self.gf('django.db.models.fields.CharField')(default='initial', max_length=50), keep_default=False) + + # Removing M2M table for field works on 'Subject' + db.delete_table('core_subject_works') + + # Adding M2M table for field editions on 'Subject' + db.create_table('core_subject_editions', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('subject', models.ForeignKey(orm['core.subject'], null=False)), + ('edition', models.ForeignKey(orm['core.edition'], null=False)) + )) + db.create_unique('core_subject_editions', ['subject_id', 'edition_id']) + + # Deleting field 'Author.openlibrary_id' + db.delete_column('core_author', 'openlibrary_id') + + # Removing M2M table for field works on 'Author' + db.delete_table('core_author_works') + + # Adding M2M table for field editions on 'Author' + db.create_table('core_author_editions', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('author', models.ForeignKey(orm['core.author'], null=False)), + ('edition', models.ForeignKey(orm['core.edition'], null=False)) + )) + db.create_unique('core_author_editions', ['author_id', 'edition_id']) + + + def backwards(self, orm): + + # Adding model 'EditionCover' + db.create_table('core_editioncover', ( + ('edition', self.gf('django.db.models.fields.related.ForeignKey')(related_name='covers', to=orm['core.Edition'])), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('openlibrary_id', self.gf('django.db.models.fields.IntegerField')()), + )) + db.send_create_signal('core', ['EditionCover']) + + # Adding field 'Edition.openlibrary_id' + db.add_column('core_edition', 'openlibrary_id', self.gf('django.db.models.fields.CharField')(max_length=50, null=True), keep_default=False) + + # Deleting field 'Edition.googlebooks_id' + db.delete_column('core_edition', 'googlebooks_id') + + # Adding M2M table for field works on 'Subject' + db.create_table('core_subject_works', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('subject', models.ForeignKey(orm['core.subject'], null=False)), + ('work', models.ForeignKey(orm['core.work'], null=False)) + )) + db.create_unique('core_subject_works', ['subject_id', 'work_id']) + + # Removing M2M table for field editions on 'Subject' + db.delete_table('core_subject_editions') + + # Adding field 'Author.openlibrary_id' + db.add_column('core_author', 'openlibrary_id', self.gf('django.db.models.fields.CharField')(max_length=50, null=True), keep_default=False) + + # Adding M2M table for field works on 'Author' + db.create_table('core_author_works', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('author', models.ForeignKey(orm['core.author'], null=False)), + ('work', models.ForeignKey(orm['core.work'], null=False)) + )) + db.create_unique('core_author_works', ['author_id', 'work_id']) + + # Removing M2M table for field editions on 'Author' + db.delete_table('core_author_editions') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'core.author': { + 'Meta': {'object_name': 'Author'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'editions': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'authors'", 'symmetrical': 'False', 'to': "orm['core.Edition']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'core.campaign': { + 'Meta': {'object_name': 'Campaign'}, + 'amazon_receiver': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'deadline': ('django.db.models.fields.DateTimeField', [], {}), + 'description': ('django.db.models.fields.CharField', [], {'max_length': '10000'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '500'}), + 'paypal_receiver': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'}), + 'target': ('django.db.models.fields.DecimalField', [], {'max_digits': '14', 'decimal_places': '2'}), + 'work': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'campaigns'", 'to': "orm['core.Work']"}) + }, + 'core.edition': { + 'Meta': {'object_name': 'Edition'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True'}), + 'googlebooks_id': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isbn_10': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True'}), + 'isbn_13': ('django.db.models.fields.CharField', [], {'max_length': '13', 'null': 'True'}), + 'publication_date': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'publisher': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '1000'}), + 'work': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editions'", 'null': 'True', 'to': "orm['core.Work']"}) + }, + 'core.subject': { + 'Meta': {'object_name': 'Subject'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'editions': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'subjects'", 'symmetrical': 'False', 'to': "orm['core.Edition']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'core.userprofile': { + 'Meta': {'object_name': 'UserProfile'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'tagline': ('django.db.models.fields.CharField', [], {'max_length': '140', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'}) + }, + 'core.wishlist': { + 'Meta': {'object_name': 'Wishlist'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'wishlist'", 'unique': 'True', 'to': "orm['auth.User']"}), + 'works': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'wishlists'", 'symmetrical': 'False', 'to': "orm['core.Work']"}) + }, + 'core.work': { + 'Meta': {'object_name': 'Work'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'openlibrary_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '1000'}) + } + } + + complete_apps = ['core'] diff --git a/core/models.py b/core/models.py index 42c305a3..f473c0b5 100755 --- a/core/models.py +++ b/core/models.py @@ -1,7 +1,10 @@ +import random +from decimal import Decimal + from django.db import models from django.db.models import Q from django.contrib.auth.models import User -from decimal import Decimal + class Campaign(models.Model): created = models.DateTimeField(auto_now_add=True) @@ -29,8 +32,9 @@ class Work(models.Model): return None def cover_image_small(self): - first_isbn = self.editions.all()[0].isbn_10 - return "http://covers.openlibrary.org/b/isbn/%s-S.jpg" % first_isbn + server_id = random.randint(0, 9) + gb_id = self.editions.all()[0].googlebooks_id + return "http://bks%i.books.google.com/books?id=%s&printsec=frontcover&img=1&zoom=5" % (server_id, gb_id) def __unicode__(self): return self.title @@ -39,8 +43,7 @@ class Work(models.Model): class Author(models.Model): created = models.DateTimeField(auto_now_add=True) name = models.CharField(max_length=500) - openlibrary_id = models.CharField(max_length=50, null=True) - works = models.ManyToManyField("Work", related_name="authors") + editions = models.ManyToManyField("Edition", related_name="authors") def __unicode__(self): return self.name @@ -49,13 +52,14 @@ class Author(models.Model): class Subject(models.Model): created = models.DateTimeField(auto_now_add=True) name = models.CharField(max_length=500) - works = models.ManyToManyField("Work", related_name="subjects") + editions = models.ManyToManyField("Edition", related_name="subjects") def __unicode__(self): return self.name class Edition(models.Model): + googlebooks_id = models.CharField(max_length=50, null=False) created = models.DateTimeField(auto_now_add=True) title = models.CharField(max_length=1000) description = models.TextField(default='', null=True) @@ -63,7 +67,6 @@ class Edition(models.Model): publication_date = models.CharField(max_length=50) isbn_10 = models.CharField(max_length=10, null=True) isbn_13 = models.CharField(max_length=13, null=True) - openlibrary_id = models.CharField(max_length=50, null=True) work = models.ForeignKey("Work", related_name="editions", null=True) def __unicode__(self): @@ -75,11 +78,6 @@ class Edition(models.Model): return e return None -class EditionCover(models.Model): - openlibrary_id = models.IntegerField() - edition = models.ForeignKey("Edition", related_name="covers") - - class Wishlist(models.Model): created = models.DateTimeField(auto_now_add=True) user = models.OneToOneField(User, related_name='wishlist') diff --git a/core/search.py b/core/search.py index cdd92304..d69842d3 100644 --- a/core/search.py +++ b/core/search.py @@ -13,7 +13,7 @@ def gluejar_search(q): r = {'title': v.get('title', ""), 'description': v.get('description', ""), 'publisher': v.get('publisher', ""), - 'google_id': item.get('selfLink')} + 'googlebooks_id': item.get('id')} # TODO: allow multiple authors if v.has_key('authors') and len(v['authors']) > 0: diff --git a/core/tests.py b/core/tests.py index 54e17c56..ea18d014 100755 --- a/core/tests.py +++ b/core/tests.py @@ -6,40 +6,33 @@ class TestBooks(TestCase): def test_add_book(self): # edition - edition = bookloader.add_book(isbn='0441012035') + edition = bookloader.add_by_isbn('0441012035') self.assertEqual(edition.title, 'Neuromancer') self.assertEqual(edition.publication_date, '2004') self.assertEqual(edition.publisher, 'Ace Books') self.assertEqual(edition.isbn_10, '0441012035') - self.assertEqual(edition.isbn_13, None) - self.assertEqual(edition.openlibrary_id, "/books/OL3305354M") - - # edition covers - covers = edition.covers.all() - self.assertEqual(len(covers), 1) - self.assertEqual(covers[0].openlibrary_id, 284192) - - # work - work = edition.work - self.assertTrue(work) - self.assertEqual(work.authors.all()[0].name, 'William F. Gibson') + self.assertEqual(edition.isbn_13, '9780441012039') + self.assertEqual(edition.googlebooks_id, "2NyiPwAACAAJ") # subjects - subject_names = [subject.name for subject in work.subjects.all()] - self.assertEqual(len(subject_names), 18) - self.assertTrue('Fiction' in subject_names) + subject_names = [subject.name for subject in edition.subjects.all()] + self.assertEqual(len(subject_names), 11) + self.assertTrue('Japan' in subject_names) # authors - author_names = [author.name for author in work.authors.all()] - self.assertEqual(len(author_names), 1) - self.assertEqual(author_names[0], "William F. Gibson") + self.assertEqual(edition.authors.all().count(), 1) + self.assertEqual(edition.authors.all()[0].name, 'William Gibson') + + # work + self.assertTrue(edition.work) def test_double_add(self): - bookloader.add_book(isbn='0441012035') - bookloader.add_book(isbn='0441012035') + bookloader.add_by_isbn('0441012035') + bookloader.add_by_isbn('0441012035') + self.assertEqual(models.Edition.objects.all().count(), 1) self.assertEqual(models.Author.objects.all().count(), 1) self.assertEqual(models.Work.objects.all().count(), 1) - self.assertEqual(models.Subject.objects.all().count(), 18) + self.assertEqual(models.Subject.objects.all().count(), 11) class SearchTests(TestCase): @@ -55,6 +48,7 @@ class SearchTests(TestCase): self.assertTrue(r.has_key('image')) self.assertTrue(r.has_key('publisher')) self.assertTrue(r.has_key('isbn_10')) + self.assertTrue(r.has_key('googlebooks_id')) def test_googlebooks_search(self): response = search.googlebooks_search('melville') diff --git a/frontend/templates/search.html b/frontend/templates/search.html index dd3c5093..d5171abd 100644 --- a/frontend/templates/search.html +++ b/frontend/templates/search.html @@ -6,9 +6,9 @@ $(document).ready(function() { $(".add-wishlist").each(function (index, element) { $(element).click(function() { var span = $(element).find("span"); - var isbn = span.attr('id') - if (!isbn) return; - $.post('/wishlist/', {'isbn': isbn}, function(data) { + var gb_id = span.attr('id') + if (!gb_id) return; + $.post('/wishlist/', {'googlebooks_id': gb_id}, function(data) { span.fadeOut(); var newSpan = $("On Your Wishlist!").hide(); span.replaceWith(newSpan); @@ -68,7 +68,7 @@ $(document).ready(function() { {% if result.on_wishlist %} On Your Wishlist! {% else %} - Add to Wishlist + Add to Wishlist {% endif %}