diff --git a/.travis.yml b/.travis.yml index 5720ab98..96d4df1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,4 +23,3 @@ install: - pip install -r requirements_versioned.pip script: django-admin test - diff --git a/core/bookloader.py b/core/bookloader.py index 64208755..a9fed0c2 100755 --- a/core/bookloader.py +++ b/core/bookloader.py @@ -242,6 +242,17 @@ def update_edition(edition): return edition +def get_isbn_item(items, isbn): + # handle case where google sends back several items + for item in items: + volumeInfo = item.get('volumeInfo', {}) + industryIdentifiers = volumeInfo.get('industryIdentifiers', []) + for ident in industryIdentifiers: + if ident['identifier'] == isbn: + return item + else: + return None # no items + return item def add_by_isbn_from_google(isbn, work=None): """add a book to the UnglueIt database from google based on ISBN. The work parameter @@ -262,12 +273,16 @@ def add_by_isbn_from_google(isbn, work=None): logger.info(u"adding new book by isbn %s", isbn) results = get_google_isbn_results(isbn) - if results: + if results and 'items' in results: + item = get_isbn_item(results['items'], isbn) + if not item: + logger.exception(u"no items for %s", isbn) + return None try: return add_by_googlebooks_id( - results['items'][0]['id'], + item['id'], work=work, - results=results['items'][0], + results=item, isbn=isbn ) except LookupFailure, e: @@ -521,6 +536,20 @@ def merge_works(w1, w2, user=None): #(for example, when w2 has already been deleted) if w1 is None or w2 is None or w1.id == w2.id or w1.id is None or w2.id is None: return w1 + + #don't merge if the works are related. + if w2 in w1.works_related_to.all() or w1 in w2.works_related_to.all(): + return w1 + + # check if one of the works is a series with parts (that have their own isbn) + if w1.works_related_from.filter(relation='part'): + models.WorkRelation.objects.get_or_create(to_work=w2, from_work=w1, relation='part') + return w1 + if w2.works_related_from.filter(relation='part'): + models.WorkRelation.objects.get_or_create(to_work=w1, from_work=w2, relation='part') + return w1 + + if w2.selected_edition is not None and w1.selected_edition is None: #the merge should be reversed temp = w1 @@ -583,7 +612,7 @@ def merge_works(w1, w2, user=None): for work_relation in w2.works_related_from.all(): work_relation.from_work = w1 work_relation.save() - w2.delete() + w2.delete(cascade=False) return w1 def detach_edition(e): diff --git a/core/loaders/doab.py b/core/loaders/doab.py index 14f90c43..6d364328 100644 --- a/core/loaders/doab.py +++ b/core/loaders/doab.py @@ -142,10 +142,11 @@ def add_all_isbns(isbns, work, language=None, title=None): if edition: first_edition = first_edition if first_edition else edition if work and (edition.work_id != work.id): - if work.created < edition.work.created: - work = merge_works(work, edition.work) - else: - work = merge_works(edition.work, work) + if work.doab and edition.work.doab and work.doab != edition.work.doab: + if work.created < edition.work.created: + work = merge_works(work, edition.work) + else: + work = merge_works(edition.work, work) else: work = edition.work return work, first_edition @@ -393,6 +394,20 @@ def add_by_doab(doab_id, record=None): url_to_provider(dl_url) if dl_url else None, **metadata ) + else: + if 'format' in metadata: + del metadata['format'] + edition = load_doab_edition( + title, + doab_id, + '', + '', + license, + language, + isbns, + None, + **metadata + ) return edition except IdDoesNotExistError: return None @@ -411,8 +426,8 @@ def load_doab_oai(from_year=None, limit=100000): if from_year: from_ = datetime.datetime(year=from_year, month=1, day=1) else: - # last 45 days - from_ = datetime.datetime.now() - datetime.timedelta(days=45) + # last 15 days + from_ = datetime.datetime.now() - datetime.timedelta(days=15) doab_ids = [] for record in doab_client.listRecords(metadataPrefix='oai_dc', from_=from_): if not record[1]: diff --git a/core/loaders/doab_utils.py b/core/loaders/doab_utils.py index e03c2348..ceef8bb7 100644 --- a/core/loaders/doab_utils.py +++ b/core/loaders/doab_utils.py @@ -103,6 +103,8 @@ FRONTIERSIN = re.compile(r'frontiersin.org/books/[^/]+/(\d+)') def online_to_download(url): urls = [] + if not url: + return urls if url.find(u'mdpi.com/books/pdfview/book/') >= 0: doc = get_soup(url) if doc: diff --git a/core/loaders/tests.py b/core/loaders/tests.py index 6a8f485e..f94e1ad3 100644 --- a/core/loaders/tests.py +++ b/core/loaders/tests.py @@ -19,10 +19,10 @@ class LoaderTests(TestCase): dropbox_url = 'https://www.dropbox.com/s/h5jzpb4vknk8n7w/Jakobsson_The_Troll_Inside_You_EBook.pdf?dl=0' dropbox_ebook = Ebook.objects.create(format='online', url=dropbox_url, edition=edition) - dropbox_ebf = dl_online(dropbox_ebook) + dropbox_ebf, new_ebf = dl_online(dropbox_ebook) self.assertTrue(dropbox_ebf.ebook.filesize) jbe_url = 'http://www.jbe-platform.com/content/books/9789027295958' jbe_ebook = Ebook.objects.create(format='online', url=jbe_url, edition=edition) - jbe_ebf = dl_online(jbe_ebook) + jbe_ebf, new_ebf = dl_online(jbe_ebook) self.assertTrue(jbe_ebf.ebook.filesize) diff --git a/core/management/commands/load_by_doab.py b/core/management/commands/load_by_doab.py index 611f18ed..beb32483 100644 --- a/core/management/commands/load_by_doab.py +++ b/core/management/commands/load_by_doab.py @@ -4,7 +4,10 @@ from regluit.core.loaders import doab class Command(BaseCommand): help = "load doab books by doab_id via oai" - args = "" + + def add_arguments(self, parser): + parser.add_argument('doab_ids', nargs='+', type=int, default=1, help="doab ids to add") - def handle(self, doab_id, **options): - doab.add_by_doab(doab_id) + def handle(self, doab_ids, **options): + for doab_id in doab_ids: + doab.add_by_doab(doab_id) diff --git a/core/management/commands/make_missing_mobis.py b/core/management/commands/make_missing_mobis.py index af2d5461..562aa9cc 100644 --- a/core/management/commands/make_missing_mobis.py +++ b/core/management/commands/make_missing_mobis.py @@ -1,22 +1,27 @@ from django.core.management.base import BaseCommand -from regluit.core.models import Work +from regluit.core.models import Work, EbookFile class Command(BaseCommand): help = "generate mobi ebooks where needed and possible." - args = "" - + + def add_arguments(self, parser): + parser.add_argument('max', nargs='?', type=int, default=1, help="maximum mobis to make") + parser.add_argument('--reset', '-r', action='store_true', help="reset failed mobi conversions") + + def handle(self, max=None, **options): - if max: - try: - max = int(max) - except ValueError: - max = 1 - else: - max = 1 + maxbad = 10 + if options['reset']: + bads = EbookFile.objects.filter(mobied__lt=0) + for bad in bads: + bad.mobied = 0 + bad.save() + epubs = Work.objects.filter(editions__ebooks__format='epub').distinct().order_by('-id') i = 0 + n_bad = 0 for work in epubs: if not work.ebooks().filter(format="mobi"): for ebook in work.ebooks().filter(format="epub"): @@ -26,11 +31,14 @@ class Command(BaseCommand): print u'making mobi for {}'.format(work.title) if ebf.make_mobi(): print 'made mobi' - i = i + 1 + i += 1 break else: - print 'failed to make mobi' + self.stdout.write('failed to make mobi') + n_bad += 1 + except: - print 'failed to make mobi' - if i >= max: + self.stdout.write('failed to make mobi') + n_bad += 1 + if i >= max or n_bad >= maxbad: break diff --git a/core/migrations/0014_auto_20180618_1646.py b/core/migrations/0014_auto_20180618_1646.py new file mode 100644 index 00000000..756c895f --- /dev/null +++ b/core/migrations/0014_auto_20180618_1646.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0013_ebookfile_mobied'), + ] + + operations = [ + migrations.AlterField( + model_name='workrelation', + name='relation', + field=models.CharField(max_length=15, choices=[(b'translation', b'translation'), (b'revision', b'revision'), (b'sequel', b'sequel'), (b'part', b'part')]), + ), + ] diff --git a/core/models/__init__.py b/core/models/__init__.py index 420405d3..866f0666 100755 --- a/core/models/__init__.py +++ b/core/models/__init__.py @@ -13,7 +13,8 @@ from tempfile import SpooledTemporaryFile import requests from ckeditor.fields import RichTextField from notification import models as notification -from postmonkey import PostMonkey, MailChimpException +from mailchimp3 import MailChimp +from mailchimp3.mailchimpclient import MailChimpError #django imports from django.apps import apps @@ -96,7 +97,7 @@ from .bibmodels import ( ) from .rh_models import Claim, RightsHolder -pm = PostMonkey(settings.MAILCHIMP_API_KEY) +mc_client = MailChimp(mc_api=settings.MAILCHIMP_API_KEY) logger = logging.getLogger(__name__) @@ -319,14 +320,17 @@ class Acq(models.Model): self.expire_in(timedelta(days=14)) self.user.wishlist.add_work(self.work, "borrow") notification.send([self.user], "library_borrow", {'acq':self}) - return self + result = self elif self.borrowable and user: user.wishlist.add_work(self.work, "borrow") borrowed = Acq.objects.create(user=user, work=self.work, license=BORROWED, lib_acq=self) from regluit.core.tasks import watermark_acq notification.send([user], "library_borrow", {'acq':borrowed}) watermark_acq.delay(borrowed) - return borrowed + result = borrowed + from regluit.core.tasks import emit_notifications + emit_notifications.delay() + return result @property def borrowable(self): @@ -1257,10 +1261,17 @@ class UserProfile(models.Model): # use @example.org email addresses for testing! return False try: - return settings.MAILCHIMP_NEWS_ID in pm.listsForEmail(email_address=self.user.email) - except MailChimpException, e: - if e.code != 215: # don't log case where user is not on a list + member = mc_client.lists.members.get( + list_id=settings.MAILCHIMP_NEWS_ID, + subscriber_hash=self.user.email + ) + if member['status'] == 'subscribed': + return 'True' + except MailChimpError, e: + if e[0]['status'] != 404: # don't log case where user is not on a list logger.error("error getting mailchimp status %s" % (e)) + except ValueError, e: + logger.error("bad email address %s" % (self.user.email)) except Exception, e: logger.error("error getting mailchimp status %s" % (e)) return False @@ -1268,7 +1279,7 @@ class UserProfile(models.Model): def ml_subscribe(self, **kwargs): if "@example.org" in self.user.email: # use @example.org email addresses for testing! - return True + return from regluit.core.tasks import ml_subscribe_task ml_subscribe_task.delay(self, **kwargs) @@ -1277,7 +1288,14 @@ class UserProfile(models.Model): # use @example.org email addresses for testing! return True try: - return pm.listUnsubscribe(id=settings.MAILCHIMP_NEWS_ID, email_address=self.user.email) + mc_client.lists.members.delete( + list_id=settings.MAILCHIMP_NEWS_ID, + subscriber_hash=self.user.email, + ) + return True + except MailChimpError, e: + if e[0]['status'] != 404: # don't log case where user is not on a list + logger.error("error getting mailchimp status %s" % (e)) except Exception, e: logger.error("error unsubscribing from mailchimp list %s" % (e)) return False @@ -1358,6 +1376,9 @@ class Gift(models.Model): self.used = now() self.save() notification.send([self.giver], "purchase_got_gift", {'gift': self}, True) + from regluit.core.tasks import emit_notifications + emit_notifications.delay() + # this was causing a circular import problem and we do not seem to be using diff --git a/core/models/bibmodels.py b/core/models/bibmodels.py index d8e16d64..f8b0af23 100644 --- a/core/models/bibmodels.py +++ b/core/models/bibmodels.py @@ -22,6 +22,8 @@ from django.db.models import F from django.db.models.signals import post_save, pre_delete from django.utils.timezone import now +from django_comments.models import Comment + import regluit from regluit.marc.models import MARCRecord as NewMARC from questionnaire.models import Landing @@ -131,6 +133,7 @@ class Work(models.Model): class Meta: ordering = ['title'] + def __unicode__(self): return self.title @@ -138,6 +141,31 @@ class Work(models.Model): self._last_campaign = None super(Work, self).__init__(*args, **kwargs) + def delete(self, cascade=True, *args, **kwargs): + if cascade: + if self.offers.all() or self.claim.all() or self.campaigns.all() or self.acqs.all() \ + or self.holds.all() or self.landings.all(): + return + for wishlist in self.wishlists.all(): + wishlist.remove_work(self) + for userprofile in self.contributors.all(): + userprofile.works.remove(self) + for identifier in self.identifiers.all(): + identifier.delete() + for comment in Comment.objects.for_model(self): + comment.delete() + for edition in self.editions.all(): + for ebook in edition.ebooks.all(): + ebook.delete() + for ebookfile in edition.ebook_files.all(): + ebookfile.delete() + edition.delete() + for work_relation in self.works_related_to.all(): + work_relation.delete() + for work_relation in self.works_related_from.all(): + work_relation.delete() + super(Work, self).delete(*args, **kwargs) # Call the "real" save() method. + def id_for(self, type): return id_for(self, type) diff --git a/core/models/rh_models.py b/core/models/rh_models.py index 792e729b..72ca471b 100644 --- a/core/models/rh_models.py +++ b/core/models/rh_models.py @@ -109,5 +109,7 @@ def notify_rh(sender, created, instance, **kwargs): for claim in instance.claim.filter(status='pending'): claim.status = 'active' claim.save() + from regluit.core.tasks import emit_notifications + emit_notifications.delay() post_save.connect(notify_rh, sender=RightsHolder) diff --git a/core/parameters.py b/core/parameters.py index 9db6a10c..a7fffae8 100644 --- a/core/parameters.py +++ b/core/parameters.py @@ -20,7 +20,7 @@ TEXT_RELATION_CHOICES = ( ('translation', 'translation'), ('revision', 'revision'), ('sequel', 'sequel'), - ('compilation', 'compilation') + ('part', 'part') ) ID_CHOICES = ( diff --git a/core/signals.py b/core/signals.py index ea17ac4f..96f94cf1 100644 --- a/core/signals.py +++ b/core/signals.py @@ -212,9 +212,8 @@ def handle_transaction_charged(sender,transaction=None, **kwargs): from regluit.core.tasks import send_mail_task message = render_to_string("notification/purchase_complete/full.txt", context ) send_mail_task.delay('unglue.it transaction confirmation', message, 'notices@gluejar.com', [transaction.receipt]) - if transaction.user: - from regluit.core.tasks import emit_notifications - emit_notifications.delay() + from regluit.core.tasks import emit_notifications + emit_notifications.delay() transaction_charged.connect(handle_transaction_charged) diff --git a/core/tasks.py b/core/tasks.py index 8a10b2f1..36dc8f2b 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -17,6 +17,10 @@ from django.utils.timezone import now from notification.engine import send_all from notification import models as notification +from mailchimp3 import MailChimp +from mailchimp3.mailchimpclient import MailChimpError + + """ regluit imports """ @@ -33,6 +37,7 @@ from regluit.core.parameters import RESERVE, REWARDS, THANKS from regluit.utils.localdatetime import date_today logger = logging.getLogger(__name__) +mc_client = MailChimp(mc_api=settings.MAILCHIMP_API_KEY) @task def populate_edition(isbn): @@ -168,7 +173,7 @@ def refresh_acqs(): # notify the user with the hold if 'example.org' not in reserve_acq.user.email: - notification.send([reserve_acq.user], "library_reserve", {'acq':reserve_acq}) + notification.send_now([reserve_acq.user], "library_reserve", {'acq':reserve_acq}) # delete the hold hold.delete() break @@ -183,14 +188,17 @@ def convert_to_mobi(input_url, input_format="application/epub+zip"): def generate_mobi_ebook_for_edition(edition): return mobigen.generate_mobi_ebook_for_edition(edition) -from postmonkey import PostMonkey, MailChimpException -pm = PostMonkey(settings.MAILCHIMP_API_KEY) - @task def ml_subscribe_task(profile, **kwargs): try: if not profile.on_ml: - return pm.listSubscribe(id=settings.MAILCHIMP_NEWS_ID, email_address=profile.user.email, **kwargs) + data = {"email_address": profile.user.email, "status_if_new": "pending"} + mc_client.lists.members.create_or_update( + list_id=settings.MAILCHIMP_NEWS_ID, + subscriber_hash=profile.user.email, + data=data, + ) + return True except Exception, e: logger.error("error subscribing to mailchimp list %s" % (e)) return False diff --git a/core/tests.py b/core/tests.py index 35b6c387..5c5050bd 100755 --- a/core/tests.py +++ b/core/tests.py @@ -298,7 +298,13 @@ class BookLoaderTests(TestCase): # first try to merge work 1 into itself -- should not do anything bookloader.merge_works(w1, w1) self.assertEqual(models.Work.objects.count(), before + 2) - + + # first try to merge related works -- should not do anything + rel, created = models.WorkRelation.objects.get_or_create(to_work=w1, from_work=w2, relation='part') + bookloader.merge_works(w1, w2) + self.assertEqual(models.Work.objects.count(), before + 2) + rel.delete() + # merge the second work into the first bookloader.merge_works(e1.work, e2.work) self.assertEqual(models.Work.objects.count(), before + 1) @@ -1004,10 +1010,11 @@ class MailingListTests(TestCase): #mostly to check that MailChimp account is setp correctly def test_mailchimp(self): - from postmonkey import PostMonkey - pm = PostMonkey(settings.MAILCHIMP_API_KEY) + from mailchimp3 import MailChimp + mc_client = MailChimp(settings.MAILCHIMP_API_KEY) if settings.TEST_INTEGRATION: - self.assertEqual(pm.ping(), "Everything's Chimpy!") + root = mc_client.root.get() + self.assertEqual(root[u'account_id'], u'15472878790f9faa11317e085') self.user = User.objects.create_user('chimp_test', 'eric@gluejar.com', 'chimp_test') self.assertTrue(self.user.profile.on_ml) diff --git a/frontend/templates/claim.html b/frontend/templates/claim.html index 2bdc4ce2..9ff84225 100644 --- a/frontend/templates/claim.html +++ b/frontend/templates/claim.html @@ -44,6 +44,6 @@ {% endif %} {% else %} -Please find a work to claim. +It appears you have reached this page in the wrong context. Please see information for rights holders. {% endif %} {% endblock %} \ No newline at end of file diff --git a/frontend/views/__init__.py b/frontend/views/__init__.py index 4c887cd6..fef8080d 100755 --- a/frontend/views/__init__.py +++ b/frontend/views/__init__.py @@ -2889,6 +2889,8 @@ def receive_gift(request, nonce): gift.acq.expire_in(0) gift.use() notification.send([giftee], "purchase_gift", context, True) + from regluit.core.tasks import emit_notifications + emit_notifications.delay() return render(request, 'gift_duplicate.html', context) context['form'] = RegiftForm() return render(request, 'gift_duplicate.html', context) diff --git a/frontend/views/rh_views.py b/frontend/views/rh_views.py index 02e5990f..03410506 100644 --- a/frontend/views/rh_views.py +++ b/frontend/views/rh_views.py @@ -1,5 +1,6 @@ from datetime import timedelta from decimal import Decimal as D +import logging from django.conf import settings from django.core.urlresolvers import reverse, reverse_lazy @@ -23,6 +24,8 @@ from regluit.frontend.forms import ( ) from regluit.utils.localdatetime import date_today +logger = logging.getLogger(__name__) + class RHAgree(CreateView): template_name = "rh_agree.html" form_class = RightsHolderForm @@ -74,7 +77,7 @@ class ClaimView(CreateView): return UserClaimForm(self.request.user, data=self.request.POST, prefix='claim') def form_valid(self, form): - print form.cleaned_data + logger.info(form.cleaned_data) work = form.cleaned_data['work'] rights_holder = form.cleaned_data['rights_holder'] if not rights_holder.approved: @@ -88,9 +91,10 @@ class ClaimView(CreateView): return HttpResponseRedirect(reverse('rightsholders')) def get_context_data(self, form): - if not form.is_valid(): - return {'form': form} - work = form.cleaned_data['work'] + try: + work = form.cleaned_data['work'] + except AttributeError: + return {} rights_holder = form.cleaned_data['rights_holder'] active_claims = work.claim.exclude(status = 'release') return { diff --git a/requirements_versioned.pip b/requirements_versioned.pip index b668a74d..f1e0f02d 100644 --- a/requirements_versioned.pip +++ b/requirements_versioned.pip @@ -58,7 +58,7 @@ oauth2==1.5.211 oauthlib==1.1.2 pandas==0.19.1 paramiko==1.14.1 -postmonkey==1.0b +mailchimp3==3.0.4 pycrypto==2.6 pymarc==3.0.2 pyoai==2.5.0