EbookFoundation
/
regluit
mirror of https://github.com/EbookFoundation/regluit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #555 from Gluejar/fix-mobi-making 

Fix mobi making
pull/1/head
Raymond Yee 2016-03-30 17:26:06 -07:00
parent bf1ba2890c 182e784144
commit 26520d92e6
12 changed files with 177 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
core/management/commands/set_campaign_editions.py Normal file
View File

@ -0,0 +1,14 @@
from django.core.management.base import BaseCommand
from regluit.core.models import Campaign
class Command(BaseCommand):
help = "set campaign edition for every campaign"
def handle(self, **options):
fixed = 0
for campaign in Campaign.objects.all():
if not campaign.edition:
campaign.edition = campaign.work.editions.all()[0]
campaign.save()
fixed +=1
print "{} campaign editions set".format(fixed)

30
core/mobi.py Normal file
View File

@ -0,0 +1,30 @@
import requests
from django.conf import settings
mobigen_url = settings.MOBIGEN_URL
mobigen_user_id = settings.MOBIGEN_USER_ID
mobigen_password = settings.MOBIGEN_PASSWORD
def convert_to_mobi(input_url, input_format="application/epub+zip"):
"""
return a string with the output of mobigen computation
"""
# using verify=False since at the moment, using a self-signed SSL cert.
payload = requests.get(input_url, verify=False).content
headers = {'Content-Type': input_format}
r = requests.post(mobigen_url, auth=(mobigen_user_id, mobigen_password),
data=payload, verify=False, headers=headers)
# if HTTP reponse code is ok, the output is the mobi file; else error message
if r.status_code == 200:
return r.content
else:
raise Exception("{0}: {1}".format(r.status_code, r.content))

43
core/mobigen.py
View File

@ -1,11 +1,10 @@
"""
Utility for calling mobigen
Utilities for calling mobigen for management. do not use in application
"""
from itertools import islice
from StringIO import StringIO
import requests
import uuid
from django.core.files.storage import default_storage
@ -13,45 +12,7 @@ from django.core.files.base import ContentFile, File
from regluit.core.models import (Campaign, Ebook)
from regluit.core import parameters
def convert_to_mobi(input_url, input_format="application/epub+zip"):
"""
return a string with the output of mobigen computation
"""
# substitute file_path with a local epub or html file
#file_path = "/Users/raymondyee/D/Document/Gluejar/Gluejar.github/regluit/test-data/pg2701.epub"
#file_type = "application/epub+zip"
# where to write the output
#output_path = "/Users/raymondyee/Downloads/pg2701.mobi"
# url of the mobigen service
mobigen_url = "https://docker.gluejar.com:5001/mobigen"
mobigen_user_id = "admin"
mobigen_password = "CXq5FSEQFgXtP_s"
# read the file and do a http post
# equivalent curl
# curl -k --user "admin:CXq5FSEQFgXtP_s" -X POST -H "Content-Type: application/epub+zip" --data-binary "@/Users/raymondyee/D/Document/Gluejar/Gluejar.github/regluit/test-data/pg2701.epub" https://docker.gluejar.com/mobigen:5001 > pg2701.mobi
# using verify=False since at the moment, using a self-signed SSL cert.
payload = requests.get(input_url, verify=False).content
headers = {'Content-Type': input_format}
r = requests.post(mobigen_url, auth=(mobigen_user_id, mobigen_password),
data=payload, verify=False, headers=headers)
# if HTTP reponse code is ok, the output is the mobi file; else error message
if r.status_code == 200:
return r.content
else:
raise Exception("{0}: {1}".format(r.status_code, r.content))
from regluit.core.mobi import convert_to_mobi
# compute whether we can apply mobigen to a given edition to produce a mobi file

132
core/models.py
View File

@ -4,6 +4,7 @@ external library imports
import binascii
import logging
import hashlib
import uuid
import re
import random
import urllib
@ -11,6 +12,7 @@ import urllib2
from urlparse import urlparse
import unicodedata
import math
import requests
from ckeditor.fields import RichTextField
from datetime import timedelta, datetime
@ -41,6 +43,7 @@ import regluit.core.isbn
import regluit.core.cc as cc
from regluit.core.epub import personalize, ungluify, test_epub, ask_epub
from regluit.core.pdf import ask_pdf, pdf_append
from regluit.core import mobi
from regluit.marc.models import MARCRecord as NewMARC
from regluit.core.signals import (
successful_campaign,
@ -519,7 +522,8 @@ class Campaign(models.Model):
self.problems.append(_('A buy-to-unglue campaign must have a target'))
may_launch = False
if self.type==THANKS:
if EbookFile.objects.filter(edition__work=self.work).count()==0:
# the case in which there is no EbookFile and no Ebook associated with work (We have ebooks without ebook files.)
if EbookFile.objects.filter(edition__work=self.work).count()==0 and self.work.ebooks().count()==0:
self.problems.append(_('You can\'t launch a thanks-for-ungluing campaign if you don\'t have any ebook files uploaded' ))
may_launch = False
except Exception as e :
@ -917,60 +921,82 @@ class Campaign(models.Model):
def latest_ending(cls):
return (timedelta(days=int(settings.UNGLUEIT_LONGEST_DEADLINE)) + now())
def make_mobi(self):
for ebf in self.work.ebookfiles().filter(format='epub').order_by('-created'):
if ebf.active:
new_mobi_ebf = EbookFile.objects.create(edition=ebf.edition, format='mobi', asking=ebf.asking)
new_mobi_ebf.file.save(path_for_file('ebf',None),ContentFile(mobi.convert_to_mobi(ebf.file.url)))
new_mobi_ebf.save()
self.work.make_ebooks_from_ebfs()
return True
return False
def add_ask_to_ebfs(self, position=0):
if not self.use_add_ask or self.type != THANKS :
return
done_formats= []
pdf_to_do = pdf_edition = None
epub_to_do = epub_edition = None
new_ebfs = {}
for ebf in self.work.ebookfiles().filter(asking = False).order_by('-created'):
if ebf.format=='pdf' and 'pdf' not in done_formats:
try:
added = ask_pdf({'campaign':self, 'work':self.work, 'site':Site.objects.get_current()})
new_file = SpooledTemporaryFile()
old_file = SpooledTemporaryFile()
ebf.file.open()
old_file.write(ebf.file.read())
if position==0:
pdf_append(added, old_file, new_file)
else:
pdf_append(old_file, added, new_file)
new_file.seek(0)
new_ebf = EbookFile.objects.create(edition=ebf.edition, format='pdf', asking=True)
new_ebf.file.save(path_for_file(ebf,None),ContentFile(new_file.read()))
new_ebf.save()
for old_ebf in self.work.ebookfiles().filter(asking = True, format='pdf').exclude(pk=new_ebf.pk):
obsolete = Ebook.objects.filter(url=old_ebf.file.url)
for eb in obsolete:
eb.deactivate()
old_ebf.delete()
done_formats.append('pdf')
except Exception as e:
logger.error("error appending pdf ask %s" % (e))
elif ebf.format=='epub' and 'epub' not in done_formats:
try:
new_file = SpooledTemporaryFile()
old_file = SpooledTemporaryFile()
ebf.file.open()
old_file.write(ebf.file.read())
new_file= ask_epub(old_file, {'campaign':self, 'work':self.work, 'site':Site.objects.get_current()})
new_file.seek(0)
new_ebf = EbookFile.objects.create(edition=ebf.edition, format='epub', asking=True)
new_ebf.file.save(path_for_file(ebf,None),ContentFile(new_file.read()))
new_ebf.save()
for old_ebf in self.work.ebookfiles().filter(asking = True, format='epub').exclude(pk=new_ebf.pk):
obsolete = Ebook.objects.filter(url=old_ebf.file.url)
for eb in obsolete:
eb.deactivate()
old_ebf.delete()
done_formats.append('epub')
except Exception as e:
logger.error("error making epub ask %s" % (e))
if ebf.format=='pdf' and not pdf_to_do:
ebf.file.open()
pdf_to_do = ebf.file.read()
pdf_edition = ebf.edition
elif ebf.format=='epub' and not epub_to_do:
ebf.file.open()
epub_to_do = ebf.file.read()
epub_edition = ebf.edition
for ebook in self.work.ebooks_all().exclude(provider='Unglue.it'):
if ebook.format=='pdf' and not pdf_to_do:
r= requests.get(ebook.url)
pdf_to_do = r.content
pdf_edition = ebook.edition
elif ebook.format=='epub' and not epub_to_do:
r= requests.get(ebook.url)
epub_to_do = r.content
epub_edition = ebook.edition
if pdf_to_do:
try:
added = ask_pdf({'campaign':self, 'work':self.work, 'site':Site.objects.get_current()})
new_file = SpooledTemporaryFile()
old_file = SpooledTemporaryFile()
old_file.write(pdf_to_do)
if position==0:
pdf_append(added, old_file, new_file)
else:
pdf_append(old_file, added, new_file)
new_file.seek(0)
new_pdf_ebf = EbookFile.objects.create(edition=pdf_edition, format='pdf', asking=True)
new_pdf_ebf.file.save(path_for_file('ebf',None),ContentFile(new_file.read()))
new_pdf_ebf.save()
new_ebfs['pdf']=new_pdf_ebf
except Exception as e:
logger.error("error appending pdf ask %s" % (e))
if epub_to_do:
try:
old_file = SpooledTemporaryFile()
old_file.write(epub_to_do)
new_file= ask_epub(old_file, {'campaign':self, 'work':self.work, 'site':Site.objects.get_current()})
new_file.seek(0)
new_epub_ebf = EbookFile.objects.create(edition=epub_edition, format='epub', asking=True)
new_epub_ebf.file.save(path_for_file(new_epub_ebf,None),ContentFile(new_file.read()))
new_epub_ebf.save()
new_ebfs['epub']=new_epub_ebf
# now make the mobi file
new_mobi_ebf = EbookFile.objects.create(edition=epub_edition, format='mobi', asking=True)
new_mobi_ebf.file.save(path_for_file('ebf',None),ContentFile(mobi.convert_to_mobi(new_epub_ebf.file.url)))
new_mobi_ebf.save()
new_ebfs['mobi']=new_mobi_ebf
except Exception as e:
logger.error("error making epub ask or mobi %s" % (e))
for key in new_ebfs.keys():
for old_ebf in self.work.ebookfiles().filter(asking = True, format=key).exclude(pk=new_ebfs[key].pk):
obsolete = Ebook.objects.filter(url=old_ebf.file.url)
for eb in obsolete:
eb.deactivate()
old_ebf.delete()
self.work.make_ebooks_from_ebfs(add_ask=True)
def make_unglued_ebf(self, format, watermarked):
ebf=EbookFile.objects.create(edition=self.work.preferred_edition, format=format)
r=urllib2.urlopen(watermarked.download_link(format))
@ -1331,6 +1357,7 @@ class Work(models.Model):
return EbookFile.objects.filter(edition__work=self, format='pdf').exclude(file='').order_by('-created')
def make_ebooks_from_ebfs(self, add_ask=True):
# either the ebf has been uploaded or a created (perhaps an ask was added or mobi generated)
if self.last_campaign().type != THANKS: # just to make sure that ebf's can be unglued by mistake
return
ebfs=EbookFile.objects.filter(edition__work=self).exclude(file='').order_by('-created')
@ -1844,7 +1871,7 @@ class Edition(models.Model):
#### following methods for compatibility with marc outputter
def downloads(self):
return self.ebooks.all()
return self.ebooks.filter(active=True)
def download_via_url(self):
return settings.BASE_URL_SECURE + reverse('download', args=[self.work.id])
@ -1928,10 +1955,7 @@ def safe_get_work(work_id):
FORMAT_CHOICES = (('pdf','PDF'),( 'epub','EPUB'), ('html','HTML'), ('text','TEXT'), ('mobi','MOBI'))
def path_for_file(instance, filename):
version = EbookFile.objects.filter(edition = instance.edition, format = instance.format).count()
hash = hashlib.md5('%s.%s.%d'%(settings.SOCIAL_AUTH_TWITTER_SECRET, instance.edition.pk, version)).hexdigest()
fn = "ebf/%s.%s"%(hash,instance.format)
return fn
return "ebf/{}.{}".format(uuid.uuid4().get_hex(), instance.format)
class EbookFile(models.Model):
file = models.FileField(upload_to=path_for_file)

4
core/tasks.py
View File

@ -137,6 +137,10 @@ def process_ebfs(campaign):
campaign.work.make_ebooks_from_ebfs(add_ask=False)
campaign.work.remove_old_ebooks()
@task
def make_mobi(campaign):
return campaign.make_mobi()
@task
def refresh_acqs():
in_10_min = now() + timedelta(minutes=10)

29
core/tests.py
View File

@ -965,7 +965,7 @@ class EbookFileTests(TestCase):
c.save()
url= acq.get_watermarked().download_link_epub
def test_ebookfile_pdf(self):
def test_ebookfile_thanks(self):
w = Work.objects.create(title="Work 2")
e = Edition.objects.create(title=w.title,work=w)
u = User.objects.create_user('test2', 'test@example.org', 'testpass')
@ -999,6 +999,29 @@ class EbookFileTests(TestCase):
asking_pdf = c.work.ebookfiles().filter(asking = True)[0].file.url
assert test_pdf(asking_pdf)
#Now do the same with epub
temp = NamedTemporaryFile(delete=False)
test_file_content = requests.get(settings.BOOXTREAM_TEST_EPUB_URL).content
temp.write(test_file_content)
temp.close()
try:
# now we can try putting the test pdf file into Django storage
temp_file = open(temp.name)
dj_file = DjangoFile(temp_file)
ebf = EbookFile( format='epub', edition=e, file=dj_file)
ebf.save()
temp_file.close()
finally:
# make sure we get rid of temp file
os.remove(temp.name)
#test the ask-appender
c.add_ask_to_ebfs()
self.assertTrue( c.work.ebookfiles().filter(asking = True, format='epub').count >0)
self.assertTrue( c.work.ebookfiles().filter(asking = True, format='mobi').count >0)
class MobigenTests(TestCase):
def test_convert_to_mobi(self):
@ -1007,8 +1030,8 @@ class MobigenTests(TestCase):
"""
from regluit.core.mobigen import convert_to_mobi
output = convert_to_mobi("https://archive.org/download/mobydickorthewha02701gut/pg2701.epub")
self.assertTrue(len(output)==2207877)
output = convert_to_mobi("https://github.com/GITenberg/Moby-Dick--Or-The-Whale_2701/releases/download/0.2.0/Moby-Dick-Or-The-Whale.epub")
self.assertTrue(len(output)>2207877)
from .signals import handle_transaction_charged
@override_settings(LOCAL_TEST=True)

2
frontend/forms.py
View File

@ -593,6 +593,8 @@ def getManageCampaignForm ( instance, data=None, initial=None, *args, **kwargs )
elif self.instance.license in ['GDFL' , 'LAL']:
raise forms.ValidationError(_('Once you start a campaign with GDFL or LAL, you can\'t use any other license.'))
return new_license
if initial and not initial.get('edition', None) and not instance.edition:
initial['edition']= instance.work.editions.all()[0]
return ManageCampaignForm(instance = instance, data=data, initial=initial)
class CampaignPurchaseForm(forms.Form):

7
frontend/templates/manage_campaign.html
View File

@ -134,8 +134,9 @@ Please fix the following before launching your campaign:
{{ form.edition.errors }}
{% for edition in campaign.work.editions.all %}
<div class="edition_form" id="edition_{{edition.id}}">
<p> Edition {{ edition.id }}: <input type="radio" {% ifequal edition.id campaign.edition.id %}checked="checked" {% endifequal %}id="id_edition_{{forloop.counter}}" value="{{edition.id}}" name="edition" /><label for="id_edition_{{forloop.counter}}"> Prefer this edition </label>
<p> Edition {{ edition.id }}: <input type="radio" {% ifequal edition.id form.edition.value %}checked="checked" {% endifequal %}id="id_edition_{{forloop.counter}}" value="{{edition.id}}" name="edition" /><label for="id_edition_{{forloop.counter}}"> Prefer this edition </label>
<ul style="text-indent:1em">
<li style="text-indent:2.5em">There are {{ edition.downloads.all.count }} downloadable ebook for this edition</li>
<li style="text-indent:2.5em"><a href="{% url 'new_edition' edition.work.id edition.id %}"> Edit </a> this edition</li>
{% ifnotequal campaign.type 1 %}
{% if campaign.rh.can_sell %}
@ -159,7 +160,7 @@ Please fix the following before launching your campaign:
{% endif %}
{% if campaign.work.epubfiles.0 %}
{% for ebf in campaign.work.epubfiles %}
<p>{% if ebf.active %}<span class="yikes">ACTIVE</span> {% endif %}EPUB file: <a href="{{ebf.file.url}}">{{ebf.file}}</a> <br />created {{ebf.created}} for edition <a href="#edition_{{ebf.edition.id}}">{{ebf.edition.id}}</a> {% if ebf.asking %}(This file has had the campaign 'ask' added.){% endif %}</p>
<p>{% if ebf.active %}<span class="yikes">ACTIVE</span> {% endif %}EPUB file: <a href="{{ebf.file.url}}">{{ebf.file}}</a> <br />created {{ebf.created}} for edition <a href="#edition_{{ebf.edition.id}}">{{ebf.edition.id}}</a> {% if ebf.asking %}(This file has had the campaign 'ask' added.){% endif %}<br />{% if ebf.active %}{% ifequal action 'mademobi' %}<span class="yikes">A MOBI file is being generated. </span> (Takes a minute or two.) {% else %}You can <a href="{% url 'makemobi' campaign.id %}">generate a MOBI file.</a> {% endifequal %}{% endif %}</p>
{% endfor %}
{% if campaign.work.test_acqs.0 %}
<ul>
@ -458,7 +459,7 @@ Please fix the following before launching your campaign:
{% endifequal %}
{% ifequal campaign.type 3 %}
<h3> Suggested Contributions </h3>
{% if not campaign.work.ebookfiles.0 %}
{% if not campaign.work.ebooks.0 %}
<p> <b>ebook files for this work <a class="tabs1">need to be loaded</a>!</b></p>
{% endif %}
<p> Enter a suggested per/copy contribution for each user type. You may change these numbers after the campaign has begun. Don't suggest contributions between $0.00 and $1.00, because these won't be processed. If you enter $0 for BOTH, then no contribution form will be displayed.</p>

2
frontend/templates/work.html
View File

@ -6,7 +6,7 @@
{% load lib_acqs %}
{% block title %}&#151;
{% if work.is_free %}
{{ work.title }} is a Free eBook
{{ work.title }} is a Free eBook. {% if work.first_pdf %}[PDF]{% endif %}{% if work.first_epub %}[EPUB, MOBI]{% endif %}
{% else %}
Help us make {{ work.title }} a Free eBook!
{% endif %}{% ifequal action 'editions' %} – All Editions{% endifequal %}

2
frontend/urls.py
View File

@ -58,6 +58,8 @@ urlpatterns = patterns(
url(r"^rightsholders/$", "rh_tools", name="rightsholders"),
url(r"^rightsholders/campaign/(?P<id>\d+)/$", "manage_campaign", name="manage_campaign"),
url(r"^rightsholders/campaign/(?P<id>\d+)/results/$", "manage_campaign", {'action': 'results'}, name="campaign_results"),
url(r"^rightsholders/campaign/(?P<id>\d+)/makemobi/$", "manage_campaign", {'action': 'makemobi'}, name="makemobi"),
url(r"^rightsholders/campaign/(?P<id>\d+)/mademobi/$", "manage_campaign", {'action': 'mademobi'}, name="mademobi"),
url(r"^rightsholders/edition/(?P<work_id>\d*)/(?P<edition_id>\d*)$", "new_edition",{'by': 'rh'}, name="rh_edition"),
url(r"^rightsholders/edition/(?P<edition_id>\d*)/upload/$", "edition_uploads", name="edition_uploads"),
url(r"^rightsholders/claim/$", "claim", name="claim"),

8
frontend/views.py
View File

@ -772,6 +772,11 @@ def manage_campaign(request, id, action='manage'):
new_premium_form = CustomPremiumForm(data={'campaign': campaign})
activetab = '#2'
else:
if action == 'makemobi':
tasks.make_mobi.delay(campaign)
return HttpResponseRedirect(reverse('mademobi', args=[campaign.id]))
elif action =='mademobi':
alerts.append('A MOBI file is being generated')
form = getManageCampaignForm(instance=campaign, initial={'work_description':campaign.work.description})
new_premium_form = CustomPremiumForm(data={'campaign': campaign})
@ -784,7 +789,8 @@ def manage_campaign(request, id, action='manage'):
'premium_form' : new_premium_form,
'work': campaign.work,
'activetab': activetab,
'offers':offers
'offers':offers,
'action':action,
})
def googlebooks(request, googlebooks_id):

4
settings/common.py
View File

@ -464,3 +464,7 @@ GITHUB_PUBLIC_TOKEN = 'f702409f913d7f9046f93c677710f829e2b599c9'
SOUTH_MIGRATION_MODULES = {
'default': 'social.apps.django_app.default.south_migrations'
}
MOBIGEN_URL = "https://docker.gluejar.com:5001/mobigen"
MOBIGEN_USER_ID = "admin"
MOBIGEN_PASSWORD = "CXq5FSEQFgXtP_s"