Merge pull request #93 from EbookFoundation/upstream-staging

merge from upstream

.travis.yml

@@ -23,4 +23,3 @@ install:
   - pip install -r requirements_versioned.pip
 
 script: django-admin test
-

core/bookloader.py

@@ -242,6 +242,17 @@ def update_edition(edition):
 
     return edition
 
+def get_isbn_item(items, isbn):
+    # handle case where google sends back several items
+    for item in items:
+        volumeInfo = item.get('volumeInfo', {})
+        industryIdentifiers = volumeInfo.get('industryIdentifiers', [])
+        for ident in industryIdentifiers:
+            if ident['identifier'] == isbn:
+                return item
+    else:
+        return None # no items
+    return item
 
 def add_by_isbn_from_google(isbn, work=None):
     """add a book to the UnglueIt database from google based on ISBN. The work parameter
@@ -262,12 +273,16 @@ def add_by_isbn_from_google(isbn, work=None):
 
     logger.info(u"adding new book by isbn %s", isbn)
     results = get_google_isbn_results(isbn)
-    if results:
+    if results and 'items' in results:
+        item = get_isbn_item(results['items'], isbn)
+        if not item:
+            logger.exception(u"no items for %s", isbn)
+            return None
         try:
             return add_by_googlebooks_id(
-                results['items'][0]['id'],
+                item['id'],
                 work=work,
-                results=results['items'][0],
+                results=item,
                 isbn=isbn
             )
         except LookupFailure, e:
@@ -521,6 +536,20 @@ def merge_works(w1, w2, user=None):
     #(for example, when w2 has already been deleted)
     if w1 is None or w2 is None or w1.id == w2.id or w1.id is None or w2.id is None:
         return w1
+
+    #don't merge if the works are related.
+    if w2 in w1.works_related_to.all() or w1 in w2.works_related_to.all():
+        return w1
+    
+    # check if one of the works is a series with parts (that have their own isbn)
+    if w1.works_related_from.filter(relation='part'):
+        models.WorkRelation.objects.get_or_create(to_work=w2, from_work=w1, relation='part')
+        return w1
+    if w2.works_related_from.filter(relation='part'):
+        models.WorkRelation.objects.get_or_create(to_work=w1, from_work=w2, relation='part')
+        return w1
+        
+        
     if w2.selected_edition is not None and w1.selected_edition is None:
         #the merge should be reversed
         temp = w1
@@ -583,7 +612,7 @@ def merge_works(w1, w2, user=None):
     for work_relation in w2.works_related_from.all():
         work_relation.from_work = w1
         work_relation.save()
-    w2.delete()
+    w2.delete(cascade=False)
     return w1
 
 def detach_edition(e):

core/loaders/doab.py

@@ -142,10 +142,11 @@ def add_all_isbns(isbns, work, language=None, title=None):
         if edition:
             first_edition = first_edition if first_edition else edition
             if work and (edition.work_id != work.id):
-                if work.created < edition.work.created:
-                    work = merge_works(work, edition.work)
-                else:
-                    work = merge_works(edition.work, work)
+                if work.doab and edition.work.doab and work.doab != edition.work.doab:
+                    if work.created < edition.work.created:
+                        work = merge_works(work, edition.work)
+                    else:
+                        work = merge_works(edition.work, work)
             else:
                 work = edition.work
     return work, first_edition
@@ -393,6 +394,20 @@ def add_by_doab(doab_id, record=None):
                 url_to_provider(dl_url) if dl_url else None,
                 **metadata
             )
+        else:
+            if 'format' in metadata:
+                del metadata['format']
+            edition = load_doab_edition(
+                title,
+                doab_id,
+                '',
+                '',
+                license,
+                language,
+                isbns,
+                None,
+                **metadata
+            )
         return edition
     except IdDoesNotExistError:
         return None
@@ -411,8 +426,8 @@ def load_doab_oai(from_year=None, limit=100000):
     if from_year:
         from_ = datetime.datetime(year=from_year, month=1, day=1)
     else: 
-        # last 45 days
-        from_ = datetime.datetime.now() - datetime.timedelta(days=45)
+        # last 15 days
+        from_ = datetime.datetime.now() - datetime.timedelta(days=15)
     doab_ids = []
     for record in doab_client.listRecords(metadataPrefix='oai_dc', from_=from_):
         if not record[1]:

core/loaders/doab_utils.py

@@ -103,6 +103,8 @@ FRONTIERSIN = re.compile(r'frontiersin.org/books/[^/]+/(\d+)')
 
 def online_to_download(url):
     urls = []
+    if not url:
+        return urls
     if url.find(u'mdpi.com/books/pdfview/book/') >= 0:
         doc = get_soup(url)
         if doc:

core/loaders/tests.py

@@ -19,10 +19,10 @@ class LoaderTests(TestCase):
 
         dropbox_url = 'https://www.dropbox.com/s/h5jzpb4vknk8n7w/Jakobsson_The_Troll_Inside_You_EBook.pdf?dl=0'
         dropbox_ebook = Ebook.objects.create(format='online', url=dropbox_url, edition=edition)
-        dropbox_ebf = dl_online(dropbox_ebook)
+        dropbox_ebf, new_ebf = dl_online(dropbox_ebook)
         self.assertTrue(dropbox_ebf.ebook.filesize)
 
         jbe_url = 'http://www.jbe-platform.com/content/books/9789027295958'
         jbe_ebook = Ebook.objects.create(format='online', url=jbe_url, edition=edition)
-        jbe_ebf = dl_online(jbe_ebook)
+        jbe_ebf, new_ebf = dl_online(jbe_ebook)
         self.assertTrue(jbe_ebf.ebook.filesize)

core/management/commands/load_by_doab.py

@@ -4,7 +4,10 @@ from regluit.core.loaders import doab
 
 class Command(BaseCommand):
     help = "load doab books by doab_id via oai"
-    args = "<doab_id>"
 
-    def handle(self, doab_id, **options):
-        doab.add_by_doab(doab_id)
+    def add_arguments(self, parser):
+        parser.add_argument('doab_ids', nargs='+', type=int, default=1, help="doab ids to add")    
+    
+    def handle(self, doab_ids, **options):
+        for doab_id in doab_ids:
+            doab.add_by_doab(doab_id)

core/management/commands/make_missing_mobis.py

@@ -1,22 +1,27 @@
 from django.core.management.base import BaseCommand
-from regluit.core.models import Work
+from regluit.core.models import Work, EbookFile
 
 
 class Command(BaseCommand):
     help = "generate mobi ebooks where needed and possible."
-    args = "<max>"
+
+    def add_arguments(self, parser):
+        parser.add_argument('max', nargs='?', type=int, default=1, help="maximum mobis to make")    
+        parser.add_argument('--reset', '-r', action='store_true', help="reset failed mobi conversions")    
+        
         
     def handle(self, max=None, **options):
-        if max:
-            try:
-                max = int(max)
-            except ValueError:
-                max = 1
-        else:
-            max = 1
+        maxbad = 10
+        if options['reset']:
+            bads = EbookFile.objects.filter(mobied__lt=0)
+            for bad in bads:
+                bad.mobied = 0
+                bad.save()
+        
         epubs = Work.objects.filter(editions__ebooks__format='epub').distinct().order_by('-id')
 
         i = 0
+        n_bad = 0
         for work in epubs:
             if not work.ebooks().filter(format="mobi"):
                 for ebook in work.ebooks().filter(format="epub"):
@@ -26,11 +31,14 @@ class Command(BaseCommand):
                             print u'making mobi for {}'.format(work.title)
                             if ebf.make_mobi():
                                 print 'made mobi'
-                                i = i + 1
+                                i += 1
                                 break
                             else:
-                                print 'failed to make mobi'
+                                self.stdout.write('failed to make mobi')
+                                n_bad += 1
+                                
                         except:
-                            print 'failed to make mobi'
-            if i >= max:
+                            self.stdout.write('failed to make mobi')
+                            n_bad += 1
+            if i >= max or n_bad >= maxbad:
                 break

core/migrations/0014_auto_20180618_1646.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0013_ebookfile_mobied'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='workrelation',
+            name='relation',
+            field=models.CharField(max_length=15, choices=[(b'translation', b'translation'), (b'revision', b'revision'), (b'sequel', b'sequel'), (b'part', b'part')]),
+        ),
+    ]

core/models/__init__.py

@@ -13,7 +13,8 @@ from tempfile import SpooledTemporaryFile
 import requests
 from ckeditor.fields import RichTextField
 from notification import models as notification
-from postmonkey import PostMonkey, MailChimpException
+from mailchimp3 import MailChimp
+from mailchimp3.mailchimpclient import MailChimpError
 
 #django imports
 from django.apps import apps
@@ -96,7 +97,7 @@ from .bibmodels import (
 )
 
 from .rh_models import Claim, RightsHolder
-pm = PostMonkey(settings.MAILCHIMP_API_KEY)
+mc_client = MailChimp(mc_api=settings.MAILCHIMP_API_KEY)
 
 logger = logging.getLogger(__name__)
 
@@ -319,14 +320,17 @@ class Acq(models.Model):
             self.expire_in(timedelta(days=14))
             self.user.wishlist.add_work(self.work, "borrow")
             notification.send([self.user], "library_borrow", {'acq':self})
-            return self
+            result = self
         elif self.borrowable and user:
             user.wishlist.add_work(self.work, "borrow")
             borrowed = Acq.objects.create(user=user, work=self.work, license=BORROWED, lib_acq=self)
             from regluit.core.tasks import watermark_acq
             notification.send([user], "library_borrow", {'acq':borrowed})
             watermark_acq.delay(borrowed)
-            return borrowed
+            result = borrowed
+        from regluit.core.tasks import emit_notifications
+        emit_notifications.delay()
+        return result
 
     @property
     def borrowable(self):
@@ -1257,10 +1261,17 @@ class UserProfile(models.Model):
             # use @example.org email addresses for testing!
             return False
         try:
-            return settings.MAILCHIMP_NEWS_ID in pm.listsForEmail(email_address=self.user.email)
-        except MailChimpException, e:
-            if e.code != 215: # don't log case where user is not on a list
+            member = mc_client.lists.members.get(
+                list_id=settings.MAILCHIMP_NEWS_ID,
+                subscriber_hash=self.user.email
+            )
+            if member['status'] == 'subscribed':
+                return 'True'
+        except MailChimpError, e:
+            if e[0]['status'] != 404: # don't log case where user is not on a list
                 logger.error("error getting mailchimp status  %s" % (e))
+        except ValueError, e:
+            logger.error("bad email address  %s" % (self.user.email))
         except Exception, e:
             logger.error("error getting mailchimp status  %s" % (e))
         return False
@@ -1268,7 +1279,7 @@ class UserProfile(models.Model):
     def ml_subscribe(self, **kwargs):
         if "@example.org" in self.user.email:
             # use @example.org email addresses for testing!
-            return True
+            return
         from regluit.core.tasks import ml_subscribe_task
         ml_subscribe_task.delay(self, **kwargs)
 
@@ -1277,7 +1288,14 @@ class UserProfile(models.Model):
             # use @example.org email addresses for testing!
             return True
         try:
-            return pm.listUnsubscribe(id=settings.MAILCHIMP_NEWS_ID, email_address=self.user.email)
+            mc_client.lists.members.delete(
+                list_id=settings.MAILCHIMP_NEWS_ID,
+                subscriber_hash=self.user.email,
+            )
+            return True
+        except MailChimpError, e:
+            if e[0]['status'] != 404: # don't log case where user is not on a list
+                logger.error("error getting mailchimp status  %s" % (e))
         except Exception, e:
             logger.error("error unsubscribing from mailchimp list  %s" % (e))
         return False
@@ -1358,6 +1376,9 @@ class Gift(models.Model):
         self.used = now()
         self.save()
         notification.send([self.giver], "purchase_got_gift", {'gift': self}, True)
+        from regluit.core.tasks import emit_notifications
+        emit_notifications.delay()
+
 
 
 # this was causing a circular import problem and we do not seem to be using

core/models/bibmodels.py

@@ -22,6 +22,8 @@ from django.db.models import F
 from django.db.models.signals import post_save, pre_delete
 from django.utils.timezone import now
 
+from django_comments.models import Comment
+
 import regluit
 from regluit.marc.models import MARCRecord as NewMARC
 from questionnaire.models import Landing
@@ -131,6 +133,7 @@ class Work(models.Model):
 
     class Meta:
         ordering = ['title']
+    
     def __unicode__(self):
         return self.title
 
@@ -138,6 +141,31 @@ class Work(models.Model):
         self._last_campaign = None
         super(Work, self).__init__(*args, **kwargs)
 
+    def delete(self, cascade=True, *args, **kwargs):
+        if cascade:
+            if self.offers.all() or self.claim.all() or self.campaigns.all() or self.acqs.all() \
+                 or self.holds.all() or self.landings.all():
+                 return
+            for wishlist in self.wishlists.all():
+                wishlist.remove_work(self)
+            for userprofile in self.contributors.all():
+                userprofile.works.remove(self)
+            for identifier in self.identifiers.all():
+                identifier.delete()
+            for comment in Comment.objects.for_model(self):
+                comment.delete()
+            for edition in self.editions.all():
+                for ebook in edition.ebooks.all():
+                    ebook.delete()
+                for ebookfile in edition.ebook_files.all():
+                    ebookfile.delete()
+                edition.delete()
+            for work_relation in self.works_related_to.all():
+                work_relation.delete()
+            for work_relation in self.works_related_from.all():
+                work_relation.delete()
+        super(Work, self).delete(*args, **kwargs)  # Call the "real" save() method.
+
     def id_for(self, type):
         return id_for(self, type)
 

core/models/rh_models.py

@@ -109,5 +109,7 @@ def notify_rh(sender, created, instance, **kwargs):
         for claim in instance.claim.filter(status='pending'):
             claim.status = 'active'
             claim.save()
+    from regluit.core.tasks import emit_notifications
+    emit_notifications.delay()
 
 post_save.connect(notify_rh, sender=RightsHolder)

core/parameters.py

@@ -20,7 +20,7 @@ TEXT_RELATION_CHOICES = (
     ('translation', 'translation'),
     ('revision', 'revision'),
     ('sequel', 'sequel'),
-    ('compilation', 'compilation')
+    ('part', 'part')
 )
 
 ID_CHOICES = (

core/signals.py

@@ -212,9 +212,8 @@ def handle_transaction_charged(sender,transaction=None, **kwargs):
             from regluit.core.tasks import send_mail_task
             message = render_to_string("notification/purchase_complete/full.txt", context )
             send_mail_task.delay('unglue.it transaction confirmation', message, 'notices@gluejar.com', [transaction.receipt])
-    if transaction.user:
-        from regluit.core.tasks import emit_notifications
-        emit_notifications.delay()
+    from regluit.core.tasks import emit_notifications
+    emit_notifications.delay()
 
 transaction_charged.connect(handle_transaction_charged)
 

core/tasks.py

@@ -17,6 +17,10 @@ from django.utils.timezone import now
 from notification.engine import send_all
 from notification import models as notification
 
+from mailchimp3 import MailChimp
+from mailchimp3.mailchimpclient import MailChimpError
+
+
 """
 regluit imports
 """
@@ -33,6 +37,7 @@ from regluit.core.parameters import RESERVE, REWARDS, THANKS
 from regluit.utils.localdatetime import date_today
 
 logger = logging.getLogger(__name__)
+mc_client = MailChimp(mc_api=settings.MAILCHIMP_API_KEY)
 
 @task 
 def populate_edition(isbn):
@@ -168,7 +173,7 @@ def refresh_acqs():
             
             # notify the user with the hold
             if 'example.org' not in reserve_acq.user.email:
-                notification.send([reserve_acq.user], "library_reserve", {'acq':reserve_acq})
+                notification.send_now([reserve_acq.user], "library_reserve", {'acq':reserve_acq})
             # delete the hold
             hold.delete()
             break
@@ -183,14 +188,17 @@ def convert_to_mobi(input_url, input_format="application/epub+zip"):
 def generate_mobi_ebook_for_edition(edition):
     return mobigen.generate_mobi_ebook_for_edition(edition)
 
-from postmonkey import PostMonkey, MailChimpException
-pm = PostMonkey(settings.MAILCHIMP_API_KEY)
-
 @task
 def ml_subscribe_task(profile, **kwargs):
     try:
         if not profile.on_ml:
-            return pm.listSubscribe(id=settings.MAILCHIMP_NEWS_ID, email_address=profile.user.email, **kwargs)
+            data = {"email_address": profile.user.email, "status_if_new": "pending"}
+            mc_client.lists.members.create_or_update(
+                list_id=settings.MAILCHIMP_NEWS_ID,
+                subscriber_hash=profile.user.email,
+                data=data,
+            )
+            return True
     except Exception, e:
         logger.error("error subscribing to mailchimp list %s" % (e))
         return False

core/tests.py

@@ -299,6 +299,12 @@ class BookLoaderTests(TestCase):
         bookloader.merge_works(w1, w1)
         self.assertEqual(models.Work.objects.count(), before + 2)
         
+        # first try to merge related works -- should not do anything
+        rel, created = models.WorkRelation.objects.get_or_create(to_work=w1, from_work=w2, relation='part')
+        bookloader.merge_works(w1, w2)
+        self.assertEqual(models.Work.objects.count(), before + 2)
+        rel.delete()
+        
         # merge the second work into the first
         bookloader.merge_works(e1.work, e2.work)
         self.assertEqual(models.Work.objects.count(), before + 1)
@@ -1004,10 +1010,11 @@ class MailingListTests(TestCase):
     #mostly to check that MailChimp account is setp correctly
 
     def test_mailchimp(self):
-        from postmonkey import PostMonkey
-        pm = PostMonkey(settings.MAILCHIMP_API_KEY)
+        from mailchimp3 import MailChimp
+        mc_client = MailChimp(settings.MAILCHIMP_API_KEY)
         if settings.TEST_INTEGRATION:
-            self.assertEqual(pm.ping(), "Everything's Chimpy!")
+            root = mc_client.root.get()
+            self.assertEqual(root[u'account_id'], u'15472878790f9faa11317e085')
             self.user = User.objects.create_user('chimp_test', 'eric@gluejar.com', 'chimp_test')
             self.assertTrue(self.user.profile.on_ml)
 

frontend/templates/claim.html

@@ -44,6 +44,6 @@
     </form>
 {% endif %}
 {% else %}
-Please find a work to claim.
+It appears you have reached this page in the wrong context. Please see <a href="{% url 'rightsholders' %}">information for rights holders</a>.
 {% endif %}
 {% endblock %}

frontend/views/__init__.py

@@ -2889,6 +2889,8 @@ def receive_gift(request, nonce):
                         gift.acq.expire_in(0)
                         gift.use()
                         notification.send([giftee], "purchase_gift", context, True)
+                        from regluit.core.tasks import emit_notifications
+                        emit_notifications.delay()
                         return render(request, 'gift_duplicate.html', context)
                 context['form'] = RegiftForm()
                 return render(request, 'gift_duplicate.html', context)

frontend/views/rh_views.py

@@ -1,5 +1,6 @@
 from datetime import timedelta
 from decimal import Decimal as D
+import logging
 
 from django.conf import settings
 from django.core.urlresolvers import reverse, reverse_lazy
@@ -23,6 +24,8 @@ from regluit.frontend.forms import (
 )
 from regluit.utils.localdatetime import date_today
 
+logger = logging.getLogger(__name__)
+
 class RHAgree(CreateView):
     template_name = "rh_agree.html"
     form_class = RightsHolderForm
@@ -74,7 +77,7 @@ class ClaimView(CreateView):
         return UserClaimForm(self.request.user, data=self.request.POST, prefix='claim')
 
     def form_valid(self, form):
-        print form.cleaned_data
+        logger.info(form.cleaned_data)
         work = form.cleaned_data['work']
         rights_holder = form.cleaned_data['rights_holder']
         if not rights_holder.approved:
@@ -88,9 +91,10 @@ class ClaimView(CreateView):
         return HttpResponseRedirect(reverse('rightsholders'))
 
     def get_context_data(self, form):
-        if not form.is_valid():
-            return {'form': form}
-        work = form.cleaned_data['work']
+        try:
+            work = form.cleaned_data['work']
+        except AttributeError:
+            return {}
         rights_holder = form.cleaned_data['rights_holder']
         active_claims = work.claim.exclude(status = 'release')
         return {

requirements_versioned.pip

@@ -58,7 +58,7 @@ oauth2==1.5.211
 oauthlib==1.1.2
 pandas==0.19.1
 paramiko==1.14.1
-postmonkey==1.0b
+mailchimp3==3.0.4
 pycrypto==2.6
 pymarc==3.0.2
 pyoai==2.5.0