EbookFoundation
/
regluit
mirror of https://github.com/EbookFoundation/regluit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #93 from EbookFoundation/upstream-staging 

merge from upstream
update-staging
eshellman 2018-06-27 12:59:42 -04:00 committed by GitHub
parent dad5b4b3cc 3e7fefb0a5
commit 0a59165099
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 204 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.travis.yml
View File

@ -23,4 +23,3 @@ install:
- pip install -r requirements_versioned.pip
script: django-admin test

37
core/bookloader.py
View File

@ -242,6 +242,17 @@ def update_edition(edition):
return edition
def get_isbn_item(items, isbn):
# handle case where google sends back several items
for item in items:
volumeInfo = item.get('volumeInfo', {})
industryIdentifiers = volumeInfo.get('industryIdentifiers', [])
for ident in industryIdentifiers:
if ident['identifier'] == isbn:
return item
else:
return None # no items
return item
def add_by_isbn_from_google(isbn, work=None):
"""add a book to the UnglueIt database from google based on ISBN. The work parameter
@ -262,12 +273,16 @@ def add_by_isbn_from_google(isbn, work=None):
logger.info(u"adding new book by isbn %s", isbn)
results = get_google_isbn_results(isbn)
if results:
if results and 'items' in results:
item = get_isbn_item(results['items'], isbn)
if not item:
logger.exception(u"no items for %s", isbn)
return None
try:
return add_by_googlebooks_id(
results['items'][0]['id'],
item['id'],
work=work,
results=results['items'][0],
results=item,
isbn=isbn
)
except LookupFailure, e:
@ -521,6 +536,20 @@ def merge_works(w1, w2, user=None):
#(for example, when w2 has already been deleted)
if w1 is None or w2 is None or w1.id == w2.id or w1.id is None or w2.id is None:
return w1
#don't merge if the works are related.
if w2 in w1.works_related_to.all() or w1 in w2.works_related_to.all():
return w1
# check if one of the works is a series with parts (that have their own isbn)
if w1.works_related_from.filter(relation='part'):
models.WorkRelation.objects.get_or_create(to_work=w2, from_work=w1, relation='part')
return w1
if w2.works_related_from.filter(relation='part'):
models.WorkRelation.objects.get_or_create(to_work=w1, from_work=w2, relation='part')
return w1
if w2.selected_edition is not None and w1.selected_edition is None:
#the merge should be reversed
temp = w1
@ -583,7 +612,7 @@ def merge_works(w1, w2, user=None):
for work_relation in w2.works_related_from.all():
work_relation.from_work = w1
work_relation.save()
w2.delete()
w2.delete(cascade=False)
return w1
def detach_edition(e):

27
core/loaders/doab.py
View File

@ -142,10 +142,11 @@ def add_all_isbns(isbns, work, language=None, title=None):
if edition:
first_edition = first_edition if first_edition else edition
if work and (edition.work_id != work.id):
if work.created < edition.work.created:
work = merge_works(work, edition.work)
else:
work = merge_works(edition.work, work)
if work.doab and edition.work.doab and work.doab != edition.work.doab:
if work.created < edition.work.created:
work = merge_works(work, edition.work)
else:
work = merge_works(edition.work, work)
else:
work = edition.work
return work, first_edition
@ -393,6 +394,20 @@ def add_by_doab(doab_id, record=None):
url_to_provider(dl_url) if dl_url else None,
**metadata
)
else:
if 'format' in metadata:
del metadata['format']
edition = load_doab_edition(
title,
doab_id,
'',
'',
license,
language,
isbns,
None,
**metadata
)
return edition
except IdDoesNotExistError:
return None
@ -411,8 +426,8 @@ def load_doab_oai(from_year=None, limit=100000):
if from_year:
from_ = datetime.datetime(year=from_year, month=1, day=1)
else:
# last 45 days
from_ = datetime.datetime.now() - datetime.timedelta(days=45)
# last 15 days
from_ = datetime.datetime.now() - datetime.timedelta(days=15)
doab_ids = []
for record in doab_client.listRecords(metadataPrefix='oai_dc', from_=from_):
if not record[1]:

2
core/loaders/doab_utils.py
View File

@ -103,6 +103,8 @@ FRONTIERSIN = re.compile(r'frontiersin.org/books/[^/]+/(\d+)')
def online_to_download(url):
urls = []
if not url:
return urls
if url.find(u'mdpi.com/books/pdfview/book/') >= 0:
doc = get_soup(url)
if doc:

4
core/loaders/tests.py
View File

@ -19,10 +19,10 @@ class LoaderTests(TestCase):
dropbox_url = 'https://www.dropbox.com/s/h5jzpb4vknk8n7w/Jakobsson_The_Troll_Inside_You_EBook.pdf?dl=0'
dropbox_ebook = Ebook.objects.create(format='online', url=dropbox_url, edition=edition)
dropbox_ebf = dl_online(dropbox_ebook)
dropbox_ebf, new_ebf = dl_online(dropbox_ebook)
self.assertTrue(dropbox_ebf.ebook.filesize)
jbe_url = 'http://www.jbe-platform.com/content/books/9789027295958'
jbe_ebook = Ebook.objects.create(format='online', url=jbe_url, edition=edition)
jbe_ebf = dl_online(jbe_ebook)
jbe_ebf, new_ebf = dl_online(jbe_ebook)
self.assertTrue(jbe_ebf.ebook.filesize)

9
core/management/commands/load_by_doab.py
View File

@ -4,7 +4,10 @@ from regluit.core.loaders import doab
class Command(BaseCommand):
help = "load doab books by doab_id via oai"
args = "<doab_id>"
def add_arguments(self, parser):
parser.add_argument('doab_ids', nargs='+', type=int, default=1, help="doab ids to add")
def handle(self, doab_id, **options):
doab.add_by_doab(doab_id)
def handle(self, doab_ids, **options):
for doab_id in doab_ids:
doab.add_by_doab(doab_id)

36
core/management/commands/make_missing_mobis.py
View File

@ -1,22 +1,27 @@
from django.core.management.base import BaseCommand
from regluit.core.models import Work
from regluit.core.models import Work, EbookFile
class Command(BaseCommand):
help = "generate mobi ebooks where needed and possible."
args = "<max>"
def add_arguments(self, parser):
parser.add_argument('max', nargs='?', type=int, default=1, help="maximum mobis to make")
parser.add_argument('--reset', '-r', action='store_true', help="reset failed mobi conversions")
def handle(self, max=None, **options):
if max:
try:
max = int(max)
except ValueError:
max = 1
else:
max = 1
maxbad = 10
if options['reset']:
bads = EbookFile.objects.filter(mobied__lt=0)
for bad in bads:
bad.mobied = 0
bad.save()
epubs = Work.objects.filter(editions__ebooks__format='epub').distinct().order_by('-id')
i = 0
n_bad = 0
for work in epubs:
if not work.ebooks().filter(format="mobi"):
for ebook in work.ebooks().filter(format="epub"):
@ -26,11 +31,14 @@ class Command(BaseCommand):
print u'making mobi for {}'.format(work.title)
if ebf.make_mobi():
print 'made mobi'
i = i + 1
i += 1
break
else:
print 'failed to make mobi'
self.stdout.write('failed to make mobi')
n_bad += 1
except:
print 'failed to make mobi'
if i >= max:
self.stdout.write('failed to make mobi')
n_bad += 1
if i >= max or n_bad >= maxbad:
break

19
core/migrations/0014_auto_20180618_1646.py Normal file
View File

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0013_ebookfile_mobied'),
]
operations = [
migrations.AlterField(
model_name='workrelation',
name='relation',
field=models.CharField(max_length=15, choices=[(b'translation', b'translation'), (b'revision', b'revision'), (b'sequel', b'sequel'), (b'part', b'part')]),
),
]

39
core/models/__init__.py
View File

@ -13,7 +13,8 @@ from tempfile import SpooledTemporaryFile
import requests
from ckeditor.fields import RichTextField
from notification import models as notification
from postmonkey import PostMonkey, MailChimpException
from mailchimp3 import MailChimp
from mailchimp3.mailchimpclient import MailChimpError
#django imports
from django.apps import apps
@ -96,7 +97,7 @@ from .bibmodels import (
)
from .rh_models import Claim, RightsHolder
pm = PostMonkey(settings.MAILCHIMP_API_KEY)
mc_client = MailChimp(mc_api=settings.MAILCHIMP_API_KEY)
logger = logging.getLogger(__name__)
@ -319,14 +320,17 @@ class Acq(models.Model):
self.expire_in(timedelta(days=14))
self.user.wishlist.add_work(self.work, "borrow")
notification.send([self.user], "library_borrow", {'acq':self})
return self
result = self
elif self.borrowable and user:
user.wishlist.add_work(self.work, "borrow")
borrowed = Acq.objects.create(user=user, work=self.work, license=BORROWED, lib_acq=self)
from regluit.core.tasks import watermark_acq
notification.send([user], "library_borrow", {'acq':borrowed})
watermark_acq.delay(borrowed)
return borrowed
result = borrowed
from regluit.core.tasks import emit_notifications
emit_notifications.delay()
return result
@property
def borrowable(self):
@ -1257,10 +1261,17 @@ class UserProfile(models.Model):
# use @example.org email addresses for testing!
return False
try:
return settings.MAILCHIMP_NEWS_ID in pm.listsForEmail(email_address=self.user.email)
except MailChimpException, e:
if e.code != 215: # don't log case where user is not on a list
member = mc_client.lists.members.get(
list_id=settings.MAILCHIMP_NEWS_ID,
subscriber_hash=self.user.email
)
if member['status'] == 'subscribed':
return 'True'
except MailChimpError, e:
if e[0]['status'] != 404: # don't log case where user is not on a list
logger.error("error getting mailchimp status %s" % (e))
except ValueError, e:
logger.error("bad email address %s" % (self.user.email))
except Exception, e:
logger.error("error getting mailchimp status %s" % (e))
return False
@ -1268,7 +1279,7 @@ class UserProfile(models.Model):
def ml_subscribe(self, **kwargs):
if "@example.org" in self.user.email:
# use @example.org email addresses for testing!
return True
return
from regluit.core.tasks import ml_subscribe_task
ml_subscribe_task.delay(self, **kwargs)
@ -1277,7 +1288,14 @@ class UserProfile(models.Model):
# use @example.org email addresses for testing!
return True
try:
return pm.listUnsubscribe(id=settings.MAILCHIMP_NEWS_ID, email_address=self.user.email)
mc_client.lists.members.delete(
list_id=settings.MAILCHIMP_NEWS_ID,
subscriber_hash=self.user.email,
)
return True
except MailChimpError, e:
if e[0]['status'] != 404: # don't log case where user is not on a list
logger.error("error getting mailchimp status %s" % (e))
except Exception, e:
logger.error("error unsubscribing from mailchimp list %s" % (e))
return False
@ -1358,6 +1376,9 @@ class Gift(models.Model):
self.used = now()
self.save()
notification.send([self.giver], "purchase_got_gift", {'gift': self}, True)
from regluit.core.tasks import emit_notifications
emit_notifications.delay()
# this was causing a circular import problem and we do not seem to be using

28
core/models/bibmodels.py
View File

@ -22,6 +22,8 @@ from django.db.models import F
from django.db.models.signals import post_save, pre_delete
from django.utils.timezone import now
from django_comments.models import Comment
import regluit
from regluit.marc.models import MARCRecord as NewMARC
from questionnaire.models import Landing
@ -131,6 +133,7 @@ class Work(models.Model):
class Meta:
ordering = ['title']
def __unicode__(self):
return self.title
@ -138,6 +141,31 @@ class Work(models.Model):
self._last_campaign = None
super(Work, self).__init__(*args, **kwargs)
def delete(self, cascade=True, *args, **kwargs):
if cascade:
if self.offers.all() or self.claim.all() or self.campaigns.all() or self.acqs.all() \
or self.holds.all() or self.landings.all():
return
for wishlist in self.wishlists.all():
wishlist.remove_work(self)
for userprofile in self.contributors.all():
userprofile.works.remove(self)
for identifier in self.identifiers.all():
identifier.delete()
for comment in Comment.objects.for_model(self):
comment.delete()
for edition in self.editions.all():
for ebook in edition.ebooks.all():
ebook.delete()
for ebookfile in edition.ebook_files.all():
ebookfile.delete()
edition.delete()
for work_relation in self.works_related_to.all():
work_relation.delete()
for work_relation in self.works_related_from.all():
work_relation.delete()
super(Work, self).delete(*args, **kwargs) # Call the "real" save() method.
def id_for(self, type):
return id_for(self, type)

2
core/models/rh_models.py
View File

@ -109,5 +109,7 @@ def notify_rh(sender, created, instance, **kwargs):
for claim in instance.claim.filter(status='pending'):
claim.status = 'active'
claim.save()
from regluit.core.tasks import emit_notifications
emit_notifications.delay()
post_save.connect(notify_rh, sender=RightsHolder)

2
core/parameters.py
View File

@ -20,7 +20,7 @@ TEXT_RELATION_CHOICES = (
('translation', 'translation'),
('revision', 'revision'),
('sequel', 'sequel'),
('compilation', 'compilation')
('part', 'part')
)
ID_CHOICES = (

5
core/signals.py
View File

@ -212,9 +212,8 @@ def handle_transaction_charged(sender,transaction=None, **kwargs):
from regluit.core.tasks import send_mail_task
message = render_to_string("notification/purchase_complete/full.txt", context )
send_mail_task.delay('unglue.it transaction confirmation', message, 'notices@gluejar.com', [transaction.receipt])
if transaction.user:
from regluit.core.tasks import emit_notifications
emit_notifications.delay()
from regluit.core.tasks import emit_notifications
emit_notifications.delay()
transaction_charged.connect(handle_transaction_charged)

18
core/tasks.py
View File

@ -17,6 +17,10 @@ from django.utils.timezone import now
from notification.engine import send_all
from notification import models as notification
from mailchimp3 import MailChimp
from mailchimp3.mailchimpclient import MailChimpError
"""
regluit imports
"""
@ -33,6 +37,7 @@ from regluit.core.parameters import RESERVE, REWARDS, THANKS
from regluit.utils.localdatetime import date_today
logger = logging.getLogger(__name__)
mc_client = MailChimp(mc_api=settings.MAILCHIMP_API_KEY)
@task
def populate_edition(isbn):
@ -168,7 +173,7 @@ def refresh_acqs():
# notify the user with the hold
if 'example.org' not in reserve_acq.user.email:
notification.send([reserve_acq.user], "library_reserve", {'acq':reserve_acq})
notification.send_now([reserve_acq.user], "library_reserve", {'acq':reserve_acq})
# delete the hold
hold.delete()
break
@ -183,14 +188,17 @@ def convert_to_mobi(input_url, input_format="application/epub+zip"):
def generate_mobi_ebook_for_edition(edition):
return mobigen.generate_mobi_ebook_for_edition(edition)
from postmonkey import PostMonkey, MailChimpException
pm = PostMonkey(settings.MAILCHIMP_API_KEY)
@task
def ml_subscribe_task(profile, **kwargs):
try:
if not profile.on_ml:
return pm.listSubscribe(id=settings.MAILCHIMP_NEWS_ID, email_address=profile.user.email, **kwargs)
data = {"email_address": profile.user.email, "status_if_new": "pending"}
mc_client.lists.members.create_or_update(
list_id=settings.MAILCHIMP_NEWS_ID,
subscriber_hash=profile.user.email,
data=data,
)
return True
except Exception, e:
logger.error("error subscribing to mailchimp list %s" % (e))
return False

15
core/tests.py
View File

@ -298,7 +298,13 @@ class BookLoaderTests(TestCase):
# first try to merge work 1 into itself -- should not do anything
bookloader.merge_works(w1, w1)
self.assertEqual(models.Work.objects.count(), before + 2)
# first try to merge related works -- should not do anything
rel, created = models.WorkRelation.objects.get_or_create(to_work=w1, from_work=w2, relation='part')
bookloader.merge_works(w1, w2)
self.assertEqual(models.Work.objects.count(), before + 2)
rel.delete()
# merge the second work into the first
bookloader.merge_works(e1.work, e2.work)
self.assertEqual(models.Work.objects.count(), before + 1)
@ -1004,10 +1010,11 @@ class MailingListTests(TestCase):
#mostly to check that MailChimp account is setp correctly
def test_mailchimp(self):
from postmonkey import PostMonkey
pm = PostMonkey(settings.MAILCHIMP_API_KEY)
from mailchimp3 import MailChimp
mc_client = MailChimp(settings.MAILCHIMP_API_KEY)
if settings.TEST_INTEGRATION:
self.assertEqual(pm.ping(), "Everything's Chimpy!")
root = mc_client.root.get()
self.assertEqual(root[u'account_id'], u'15472878790f9faa11317e085')
self.user = User.objects.create_user('chimp_test', 'eric@gluejar.com', 'chimp_test')
self.assertTrue(self.user.profile.on_ml)

2
frontend/templates/claim.html
View File

@ -44,6 +44,6 @@
</form>
{% endif %}
{% else %}
Please find a work to claim.
It appears you have reached this page in the wrong context. Please see <a href="{% url 'rightsholders' %}">information for rights holders</a>.
{% endif %}
{% endblock %}

2
frontend/views/__init__.py
View File

@ -2889,6 +2889,8 @@ def receive_gift(request, nonce):
gift.acq.expire_in(0)
gift.use()
notification.send([giftee], "purchase_gift", context, True)
from regluit.core.tasks import emit_notifications
emit_notifications.delay()
return render(request, 'gift_duplicate.html', context)
context['form'] = RegiftForm()
return render(request, 'gift_duplicate.html', context)

12
frontend/views/rh_views.py
View File

@ -1,5 +1,6 @@
from datetime import timedelta
from decimal import Decimal as D
import logging
from django.conf import settings
from django.core.urlresolvers import reverse, reverse_lazy
@ -23,6 +24,8 @@ from regluit.frontend.forms import (
)
from regluit.utils.localdatetime import date_today
logger = logging.getLogger(__name__)
class RHAgree(CreateView):
template_name = "rh_agree.html"
form_class = RightsHolderForm
@ -74,7 +77,7 @@ class ClaimView(CreateView):
return UserClaimForm(self.request.user, data=self.request.POST, prefix='claim')
def form_valid(self, form):
print form.cleaned_data
logger.info(form.cleaned_data)
work = form.cleaned_data['work']
rights_holder = form.cleaned_data['rights_holder']
if not rights_holder.approved:
@ -88,9 +91,10 @@ class ClaimView(CreateView):
return HttpResponseRedirect(reverse('rightsholders'))
def get_context_data(self, form):
if not form.is_valid():
return {'form': form}
work = form.cleaned_data['work']
try:
work = form.cleaned_data['work']
except AttributeError:
return {}
rights_holder = form.cleaned_data['rights_holder']
active_claims = work.claim.exclude(status = 'release')
return {

2
requirements_versioned.pip
View File

@ -58,7 +58,7 @@ oauth2==1.5.211
oauthlib==1.1.2
pandas==0.19.1
paramiko==1.14.1
postmonkey==1.0b
mailchimp3==3.0.4
pycrypto==2.6
pymarc==3.0.2
pyoai==2.5.0