EbookFoundation
/
regluit
mirror of https://github.com/EbookFoundation/regluit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

generate mobi on ask

pull/1/head
eric 2016-03-28 14:30:05 -04:00
parent dfb7422857
commit 7180661fe3
5 changed files with 78 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
core/mobi.py Normal file
View File

@ -0,0 +1,30 @@
import requests
from django.conf import settings
mobigen_url = settings.MOBIGEN_URL
mobigen_user_id = settings.MOBIGEN_USER_ID
mobigen_password = settings.MOBIGEN_PASSWORD
def convert_to_mobi(input_url, input_format="application/epub+zip"):
"""
return a string with the output of mobigen computation
"""
# using verify=False since at the moment, using a self-signed SSL cert.
payload = requests.get(input_url, verify=False).content
headers = {'Content-Type': input_format}
r = requests.post(mobigen_url, auth=(mobigen_user_id, mobigen_password),
data=payload, verify=False, headers=headers)
# if HTTP reponse code is ok, the output is the mobi file; else error message
if r.status_code == 200:
return r.content
else:
raise Exception("{0}: {1}".format(r.status_code, r.content))

43
core/mobigen.py
View File

@ -1,11 +1,10 @@
"""
Utility for calling mobigen
Utilities for calling mobigen for management. do not use in application
"""
from itertools import islice
from StringIO import StringIO
import requests
import uuid
from django.core.files.storage import default_storage
@ -13,45 +12,7 @@ from django.core.files.base import ContentFile, File
from regluit.core.models import (Campaign, Ebook)
from regluit.core import parameters
def convert_to_mobi(input_url, input_format="application/epub+zip"):
"""
return a string with the output of mobigen computation
"""
# substitute file_path with a local epub or html file
#file_path = "/Users/raymondyee/D/Document/Gluejar/Gluejar.github/regluit/test-data/pg2701.epub"
#file_type = "application/epub+zip"
# where to write the output
#output_path = "/Users/raymondyee/Downloads/pg2701.mobi"
# url of the mobigen service
mobigen_url = "https://docker.gluejar.com:5001/mobigen"
mobigen_user_id = "admin"
mobigen_password = "CXq5FSEQFgXtP_s"
# read the file and do a http post
# equivalent curl
# curl -k --user "admin:CXq5FSEQFgXtP_s" -X POST -H "Content-Type: application/epub+zip" --data-binary "@/Users/raymondyee/D/Document/Gluejar/Gluejar.github/regluit/test-data/pg2701.epub" https://docker.gluejar.com/mobigen:5001 > pg2701.mobi
# using verify=False since at the moment, using a self-signed SSL cert.
payload = requests.get(input_url, verify=False).content
headers = {'Content-Type': input_format}
r = requests.post(mobigen_url, auth=(mobigen_user_id, mobigen_password),
data=payload, verify=False, headers=headers)
# if HTTP reponse code is ok, the output is the mobi file; else error message
if r.status_code == 200:
return r.content
else:
raise Exception("{0}: {1}".format(r.status_code, r.content))
from regluit.core.mobi import convert_to_mobi
# compute whether we can apply mobigen to a given edition to produce a mobi file

23
core/models.py
View File

@ -4,6 +4,7 @@ external library imports
import binascii
import logging
import hashlib
import uuid
import re
import random
import urllib
@ -41,6 +42,7 @@ import regluit.core.isbn
import regluit.core.cc as cc
from regluit.core.epub import personalize, ungluify, test_epub, ask_epub
from regluit.core.pdf import ask_pdf, pdf_append
from regluit.core import mobi
from regluit.marc.models import MARCRecord as NewMARC
from regluit.core.signals import (
successful_campaign,
@ -947,7 +949,6 @@ class Campaign(models.Model):
logger.error("error appending pdf ask %s" % (e))
elif ebf.format=='epub' and 'epub' not in done_formats:
try:
new_file = SpooledTemporaryFile()
old_file = SpooledTemporaryFile()
ebf.file.open()
old_file.write(ebf.file.read())
@ -962,8 +963,18 @@ class Campaign(models.Model):
eb.deactivate()
old_ebf.delete()
done_formats.append('epub')
# now make the mobi file
new_mobi_ebf = EbookFile.objects.create(edition=ebf.edition, format='mobi', asking=True)
new_mobi_ebf.file.save(path_for_file(ebf,None),ContentFile(mobi.convert_to_mobi(new_ebf.file.url)))
new_mobi_ebf.save()
for old_ebf in self.work.ebookfiles().filter(asking = True, format='mobi').exclude(pk=new_mobi_ebf.pk):
obsolete = Ebook.objects.filter(url=old_ebf.file.url)
for eb in obsolete:
eb.deactivate()
old_ebf.delete()
except Exception as e:
logger.error("error making epub ask %s" % (e))
logger.error("error making epub ask or mobi %s" % (e))
self.work.make_ebooks_from_ebfs(add_ask=True)
@ -1331,12 +1342,13 @@ class Work(models.Model):
return EbookFile.objects.filter(edition__work=self, format='pdf').exclude(file='').order_by('-created')
def make_ebooks_from_ebfs(self, add_ask=True):
# either the ebf has been uploaded or a created (perhaps an ask was added or mobi generated)
if self.last_campaign().type != THANKS: # just to make sure that ebf's can be unglued by mistake
return
ebfs=EbookFile.objects.filter(edition__work=self).exclude(file='').order_by('-created')
done_formats= []
for ebf in ebfs:
previous_ebooks=Ebook.objects.filter(url= ebf.file.url,)
previous_ebooks=Ebook.objects.filter(url= ebf.file.url,)
try:
previous_ebook = previous_ebooks[0]
for eb in previous_ebooks[1:]: #housekeeping
@ -1928,10 +1940,7 @@ def safe_get_work(work_id):
FORMAT_CHOICES = (('pdf','PDF'),( 'epub','EPUB'), ('html','HTML'), ('text','TEXT'), ('mobi','MOBI'))
def path_for_file(instance, filename):
version = EbookFile.objects.filter(edition = instance.edition, format = instance.format).count()
hash = hashlib.md5('%s.%s.%d'%(settings.SOCIAL_AUTH_TWITTER_SECRET, instance.edition.pk, version)).hexdigest()
fn = "ebf/%s.%s"%(hash,instance.format)
return fn
return "ebf/{}.{}".format(uuid.uuid4().get_hex(), instance.format)
class EbookFile(models.Model):
file = models.FileField(upload_to=path_for_file)

29
core/tests.py
View File

@ -965,7 +965,7 @@ class EbookFileTests(TestCase):
c.save()
url= acq.get_watermarked().download_link_epub
def test_ebookfile_pdf(self):
def test_ebookfile_thanks(self):
w = Work.objects.create(title="Work 2")
e = Edition.objects.create(title=w.title,work=w)
u = User.objects.create_user('test2', 'test@example.org', 'testpass')
@ -999,6 +999,29 @@ class EbookFileTests(TestCase):
asking_pdf = c.work.ebookfiles().filter(asking = True)[0].file.url
assert test_pdf(asking_pdf)
#Now do the same with epub
temp = NamedTemporaryFile(delete=False)
test_file_content = requests.get(settings.BOOXTREAM_TEST_EPUB_URL).content
temp.write(test_file_content)
temp.close()
try:
# now we can try putting the test pdf file into Django storage
temp_file = open(temp.name)
dj_file = DjangoFile(temp_file)
ebf = EbookFile( format='epub', edition=e, file=dj_file)
ebf.save()
temp_file.close()
finally:
# make sure we get rid of temp file
os.remove(temp.name)
#test the ask-appender
c.add_ask_to_ebfs()
self.assertTrue( c.work.ebookfiles().filter(asking = True, format='epub').count >0)
self.assertTrue( c.work.ebookfiles().filter(asking = True, format='mobi').count >0)
class MobigenTests(TestCase):
def test_convert_to_mobi(self):
@ -1007,8 +1030,8 @@ class MobigenTests(TestCase):
"""
from regluit.core.mobigen import convert_to_mobi
output = convert_to_mobi("https://archive.org/download/mobydickorthewha02701gut/pg2701.epub")
self.assertTrue(len(output)==2207877)
output = convert_to_mobi("https://github.com/GITenberg/Moby-Dick--Or-The-Whale_2701/releases/download/0.2.0/Moby-Dick-Or-The-Whale.epub")
self.assertTrue(len(output)>2207877)
from .signals import handle_transaction_charged
@override_settings(LOCAL_TEST=True)

4
settings/common.py
View File

@ -458,3 +458,7 @@ GITHUB_PUBLIC_TOKEN = 'f702409f913d7f9046f93c677710f829e2b599c9'
SOUTH_MIGRATION_MODULES = {
'default': 'social.apps.django_app.default.south_migrations'
}
MOBIGEN_URL = "https://docker.gluejar.com:5001/mobigen"
MOBIGEN_USER_ID = "admin"
MOBIGEN_PASSWORD = "CXq5FSEQFgXtP_s"