Merge pull request #86 from EbookFoundation/add-travis

WIP: see if we can easily get travis to do tests
pull/87/head
eshellman 2018-06-09 15:46:47 -04:00 committed by GitHub
commit d7751af8f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 215 additions and 118 deletions

26
.travis.yml Normal file
View File

@ -0,0 +1,26 @@
language: python
python:
- '2.7'
services:
- redis-server
- mysql
env:
global:
- DJANGO_SETTINGS_MODULE=regluit.settings.travis
- PYTHONPATH=/home/travis/build/EbookFoundation/
before_install:
- sudo mkdir /var/log/django
- sudo chmod 777 /var/log/django
- mkdir ~/build/EbookFoundation/regluit/settings/keys/
- cp ~/build/EbookFoundation/regluit/settings/dummy/__init__.py ~/build/EbookFoundation/regluit/settings/keys/__init__.py
- openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/EbookFoundation/regluit/test/travis-host.py.enc -out ~/build/EbookFoundation/regluit/settings/keys/host.py -d
install:
- pip install -r requirements_versioned.pip
script: django-admin test

View File

@ -1,10 +1,11 @@
import requests
import random
from django.conf import settings
from urllib import quote
from functools import partial
from urllib import quote
from xml.etree import ElementTree
import requests
from django.conf import settings
from django.apps import apps
from . exceptions import BooXtreamError
@ -34,10 +35,10 @@ class BooXtream(object):
if not apiuser:
apiuser = settings.BOOXTREAM_API_USER
self.endpoint = 'https://service.booxtream.com/'
self.postrequest = partial(requests.post, timeout=timeout, auth=(apiuser,apikey))
self.postrequest = partial(requests.post, timeout=timeout, auth=(apiuser, apikey))
def platform(self, epubfile=None, epub=True, kf8mobi=False, **kwargs):
def platform(self, epubfile=None, epub=True, kf8mobi=False, **kwargs):
""" Make an API request to BooXtream
``self.apikey``, ``epubfile`` and the supplied ``kwargs``.
Attempts to deserialize the XML response and return the download link.
@ -48,25 +49,25 @@ class BooXtream(object):
Boox = apps.get_model('booxtream', 'Boox')
url = self.endpoint + 'booxtream.xml'
kwargs['epub'] = '1' if epub else '0'
kwargs['epub'] = '1' if epub else '0'
kwargs['kf8mobi'] = '1' if kf8mobi else '0'
if epubfile:
if hasattr(epubfile,'name') and str(epubfile.name).endswith('.epub'):
files= {'epubfile': (str(epubfile.name),epubfile)}
if hasattr(epubfile, 'name') and str(epubfile.name).endswith('.epub'):
files = {'epubfile': (str(epubfile.name), epubfile)}
else:
# give it a random file name so that kindlegen doesn't choke
# needed for in-memory (StringIO) epubs
files= {'epubfile': ('%012x.epub' % random.randrange(16**12),epubfile)}
files= {'epubfile': ('%012x.epub' % random.randrange(16**12), epubfile)}
else:
files={}
files = {}
if settings.LOCAL_TEST:
# fake it, so you can test other functions without hitting booxtream
boox = Boox.objects.create(
download_link_epub='https://github.com/eshellman/42_ebook/blob/master/download/42.epub?raw=true&extra=download.booxtream.com/',
download_link_mobi='https://github.com/eshellman/42_ebook/blob/master/download/42.mobi?raw=true',
referenceid= kwargs.get('referenceid'),
downloads_remaining= kwargs.get('downloadlimit'),
expirydays=kwargs.get('expirydays'),
referenceid= kwargs.get('referenceid', '42'),
downloads_remaining=kwargs.get('downloadlimit', 10),
expirydays=kwargs.get('expirydays', 30),
)
return boox
@ -86,9 +87,8 @@ class BooXtream(object):
boox = Boox.objects.create(
download_link_epub=download_link_epub,
download_link_mobi=download_link_mobi,
referenceid= kwargs.get('referenceid'),
downloads_remaining= kwargs.get('downloadlimit'),
referenceid=kwargs.get('referenceid'),
downloads_remaining=kwargs.get('downloadlimit'),
expirydays=kwargs.get('expirydays'),
)
return boox
return boox

View File

@ -7,7 +7,7 @@ class BooXtreamError(Exception):
self.errors = errors
def __str__(self):
errormsg='BooXtream errors:'
errormsg = 'BooXtream errors:'
for error in self.errors:
errormsg += 'Error %s: %s\n'% (error.find('Code').text,error.find('Msg').text)
return errormsg
errormsg += 'Error %s: %s\n'% (error.find('Code').text, error.find('Msg').text)
return errormsg

View File

@ -6,22 +6,21 @@ class Boox(models.Model):
"""
keeps a record of a file that's been watermarked
"""
download_link_epub = models.URLField(null=True)
download_link_mobi = models.URLField(null=True)
download_link_epub = models.URLField(null=True)
download_link_mobi = models.URLField(null=True)
referenceid = models.CharField(max_length=32)
downloads_remaining = models.PositiveSmallIntegerField(default = 0)
downloads_remaining = models.PositiveSmallIntegerField(default=0)
expirydays = models.PositiveSmallIntegerField()
created = models.DateTimeField(auto_now_add=True)
@property
def expired(self):
return self.created+timedelta(days=self.expirydays) < datetime.now()
def download_link(self, format):
if format == 'epub':
return self.download_link_epub
elif format == 'mobi':
return self.download_link_mobi
else:
return None
return None

View File

@ -1,6 +1,6 @@
import unittest
import time
import urllib2
import urllib2
from tempfile import NamedTemporaryFile
from StringIO import StringIO
from django.conf import settings
@ -23,14 +23,16 @@ class TestBooXtream(unittest.TestCase):
return manager
def test_booxtream_errors(self):
if settings.LOCAL_TEST:
return
from .exceptions import BooXtreamError
inst = self._makeOne()
if not settings.BOOXTREAM_API_KEY:
return
with self.assertRaises(BooXtreamError) as cm:
inst.platform()
self.assertIn( 'expirydays not set',str(cm.exception))
params={
self.assertIn('expirydays not set', str(cm.exception))
params = {
'customername': 'Jane Test',
'languagecode':'1043',
'expirydays': 1,
@ -47,7 +49,7 @@ class TestBooXtream(unittest.TestCase):
def test_booxtream_good(self):
inst = self._makeOne()
params={
params = {
'customeremailaddress':'jane@example.com',
'customername': 'Jane Test',
'languagecode':'1043',
@ -57,16 +59,16 @@ class TestBooXtream(unittest.TestCase):
'chapterfooter':1,
'disclaimer':1,
}
params['referenceid']= 'order'+str(time.time())
boox=inst.platform(epubfile=self.epub2file, **params)
self.assertRegexpMatches(boox.download_link_epub,'download.booxtream.com/')
params['referenceid'] = 'order' + str(time.time())
boox = inst.platform(epubfile=self.epub2file, **params)
self.assertRegexpMatches(boox.download_link_epub, 'download.booxtream.com/')
self.assertFalse(boox.expired)
self.assertEqual(boox.downloads_remaining,3)
self.assertEqual(boox.downloads_remaining, 3)
# make sure it works with an in-memory file
self.epub2file.seek(0)
in_mem_epub = StringIO()
in_mem_epub.write(self.epub2file.read())
in_mem_epub.seek(0)
boox2=inst.platform(epubfile=in_mem_epub, **params)
self.assertRegexpMatches(boox2.download_link_epub,'download.booxtream.com/')
boox2 = inst.platform(epubfile=in_mem_epub, **params)
self.assertRegexpMatches(boox2.download_link_epub, 'download.booxtream.com/')

View File

@ -13,7 +13,8 @@ def convert_to_mobi(input_url, input_format="application/epub+zip"):
return a string with the output of mobigen computation
"""
if mobigen_url and mobigen_user_id and mobigen_password:
print 'settings ok'
# using verify=False since at the moment, using a self-signed SSL cert.
payload = requests.get(input_url, verify=False).content
@ -26,5 +27,6 @@ def convert_to_mobi(input_url, input_format="application/epub+zip"):
if r.status_code == 200:
return r.content
else:
print "{0}: {1}".format(r.status_code, r.content)
raise Exception("{0}: {1}".format(r.status_code, r.content))

View File

@ -1,6 +1,7 @@
"""
Utilities that manipulate pdf files
"""
import logging
import requests
from xhtml2pdf import pisa # import python module
from PyPDF2 import PdfFileMerger,PdfFileReader
@ -9,6 +10,7 @@ from tempfile import NamedTemporaryFile
from django.template.loader import render_to_string
from regluit import settings
logger = logging.getLogger(__name__)
# Utility function
def ask_pdf(context={}):
@ -35,7 +37,7 @@ def pdf_append( file1, file2, file_out ):
def test_pdf(pdf_file):
temp = None
try:
if isinstance(pdf_file , str):
if isinstance(pdf_file , (str, unicode)):
if pdf_file.startswith('http:') or pdf_file.startswith('https:'):
temp = NamedTemporaryFile(delete=False)
test_file_content = requests.get(pdf_file).content
@ -53,7 +55,9 @@ def test_pdf(pdf_file):
except:
success = False
return success
except:
except Exception:
pdf_file = unicode(pdf_file)
logger.exception('error testing a pdf: %s' % pdf_file[:100])
return False
def test_test_pdf(self):

View File

@ -992,8 +992,8 @@ class DownloadPageTest(TestCase):
anon_client = Client()
response = anon_client.get("/work/%s/download/" % w.id, follow=True)
self.assertContains(response, "/download_ebook/%s/"% eb1.id, count=11)
self.assertContains(response, "/download_ebook/%s/"% eb2.id, count=5)
self.assertContains(response, "/download_ebook/%s/"% eb1.id, count=11)
self.assertContains(response, "/download_ebook/%s/"% eb2.id, count=4)
self.assertTrue(eb1.edition.work.is_free)
eb1.delete()
self.assertTrue(eb2.edition.work.is_free)
@ -1119,7 +1119,8 @@ class EbookFileTests(TestCase):
#test the ask-appender
c.add_ask_to_ebfs()
if settings.AWS_SECRET_ACCESS_KEY:
assert test_pdf(c.work.ebookfiles().filter(asking=True)[0].file.url)
askingpdfurl = c.work.ebookfiles().filter(asking=True)[0].file.url
assert test_pdf(askingpdfurl)
else:
assert test_pdf(c.work.ebookfiles().filter(asking=True)[0].file)
@ -1157,6 +1158,29 @@ class EbookFileTests(TestCase):
self.assertTrue(c.work.ebookfiles().filter(asking=True, ebook__active=True).count() == 0)
self.assertTrue(c.work.ebookfiles().filter(asking=False, ebook__active=True).count() > 0)
def test_bad_ebookfile(self):
w = Work.objects.create(title="Work 3")
e = Edition.objects.create(title=w.title, work=w)
temp = NamedTemporaryFile(delete=False)
test_file_content = "bad text file"
temp.write(test_file_content)
temp.close()
try:
# put the bad file into Django storage
temp_file = open(temp.name)
dj_file = DjangoFile(temp_file)
ebf = EbookFile(format='epub', edition=e, file=dj_file)
ebf.save()
temp_file.close()
ebf.make_mobi()
finally:
# make sure we get rid of temp file
os.remove(temp.name)
self.assertTrue(ebf.mobied < 0)
class MobigenTests(TestCase):
def test_convert_to_mobi(self):
"""

View File

@ -9,10 +9,12 @@ PyYAML==3.11
amqp==1.4.9
anyjson==0.3.3
billiard==3.3.0.23
awscli==1.10.26
boto==2.42.0
awscli==1.15.33
boto==2.48.0
boto3==1.7.33
celery==3.1.23
certifi==2016.2.28
certifi==2018.4.16
chardet==3.0.4
# pip installing pillow seems to delete distribute
# but having distribute in requirements starting to cause problems
# distribute==0.6.28
@ -33,7 +35,7 @@ git+git://github.com/eshellman/django-notification.git@a4620e893e2da220994e0189b
django-registration==2.1.2
django-selectable==0.9.0
django-smtp-ssl==1.0
django-storages==1.4.1
django-storages==1.5.2
django-tastypie==0.13.3
#django-transmeta==0.7.3
git+git://github.com/resulto/django-transmeta.git@ad4d7278ba330dcf8c8446f8ae9b2c769ae8684e
@ -70,7 +72,7 @@ rdflib==4.2.0
rdflib-jsonld==0.3
redis==2.10.3
reportlab==3.4.0
requests==2.10.0
requests==2.18.4
requests-mock==1.2.0
requests-oauthlib==0.6.2
selenium==2.53.1
@ -87,19 +89,19 @@ xhtml2pdf==0.2.2
webencodings==0.5.1
#for urllib3 secure
cffi==1.7.0
cryptography==2.1.4
cryptography==2.2.2
enum34==1.1.6
idna==2.1
idna==2.6
ipaddress==1.0.16
ndg-httpsclient==0.4.2
pyOpenSSL==16.2.0
pyOpenSSL==18.0.0
pyasn1==0.1.9
pycparser==2.14
setuptools==25.0.0
urllib3==1.16
urllib3==1.22
beautifulsoup4==4.6.0
RISparser==0.4.2
# include these 2 for development
libsass==0.14.5
django-compressor==2.2
django-sass-processor==0.7
#libsass==0.13.4
#django-compressor==2.2
django-sass-processor==0.5.6

View File

@ -16,8 +16,8 @@ DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'regluit',
'USER': 'regluit',
'PASSWORD': 'regluit',
'USER': 'root',
'PASSWORD': '',
'HOST': '',
'PORT': '',
'TEST': {
@ -31,12 +31,11 @@ TIME_ZONE = 'America/New_York'
# settings for outbout email
# if you have a gmail account you can use your email address and password
EMAIL_USE_TLS = True
EMAIL_HOST = 'smtp.gmail.com'
EMAIL_HOST_USER = 'me@gmail.com'
EMAIL_HOST_PASSWORD = 'my-password'
EMAIL_PORT = 587
DEFAULT_FROM_EMAIL = 'info@ebookfoundation.org'
EMAIL_BACKEND = 'django_smtp_ssl.SSLEmailBackend'
MAIL_USE_TLS = True
EMAIL_HOST = 'email-smtp.us-east-1.amazonaws.com'
EMAIL_PORT = 465
DEFAULT_FROM_EMAIL = 'notices@gluejar.com'
# formerly of settings/common.py to surface old vars
@ -49,8 +48,22 @@ REDIRECT_IS_HTTPS = False
#BASE_URL = 'http://0.0.0.0/'
BASE_URL_SECURE = 'http://0.0.0.0/'
BROKER_TRANSPORT = "redis"
BROKER_HOST = "localhost"
BROKER_PORT = 6379
BROKER_VHOST = "0"
# Amazon S3 access
AWS_STORAGE_BUCKET_NAME = 'unglueit-testfiles'
SOCIAL_AUTH_TWITTER_KEY = ''
SOCIAL_AUTH_TWITTER_SECRET = ''
SOCIAL_AUTH_FACEBOOK_KEY = ''
SOCIAL_AUTH_FACEBOOK_SECRET = ''
SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = ''
SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = ''
GOOGLE_BOOKS_API_KEY = ''
TEST_INTEGRATION = False
LOCAL_TEST = True
DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
# use database as queuing service in development
BROKER_TRANSPORT = "djkombu.transport.DatabaseTransport"
#INSTALLED_APPS += ("djkombu",)

25
setup.py Normal file
View File

@ -0,0 +1,25 @@
import os
from setuptools import find_packages, setup
with open(os.path.join(os.path.dirname(__file__), 'README.md')) as readme:
README = readme.read()
# allow setup.py to be run from any path
os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir)))
setup(
name='regluit',
version='3.0.0',
description='Web application for Unglue.it',
author='Free Ebook Foundation',
author_email='info@ebookfoundation.org',
url='https://unglue.it',
packages=find_packages(exclude=[
'bookdata',
'deploy'
'logs',
'selenium',
'static',
'vagrant',
]),
)

BIN
test/travis-host.py.enc Normal file

Binary file not shown.

View File

@ -1,51 +1,51 @@
$ANSIBLE_VAULT;1.1;AES256
38353435396232353338653362653465343063393836633332633964663162363636316433396365
3336316136626134653262626630663738333938666336330a626562646335323264626363316233
62383131353536323166353364633437383333663434653037353964646662633865323466306237
3336306364653366320a323464383061353862663932623461346230316364396530653564353762
33363864663730643265396565363430376564386164366163323266393735633830363534353665
31376233393136306365383366333866383962313864376561333964623639623834373931396133
31646665313165376266336132663833663436633165393536316262373635353837306365363964
38656539313935366663336539376334313863633439343664353530393336383064623533336264
34346630643737643938373366383432303139653566306163333463366465623465303862616361
61363139613265636432616533656564386231323066623863663463313336626435323433343337
31626530626132646262366334643261626431643261633334393262636137643935376461643739
38633631626366303934393139356166633536333634396163333164306532373765656138393464
39623636353034343363396361383837663863313465636134616333666563666534653930626331
65633536383230653665356233323330666439643665396335303566366230646366356534663732
30313835323633306130343865626131383032666236663362633733383235643239383765663332
32643332313361353736313035653137323239616635653662386435373939386636653138393036
38396231623635316633333366646339336662313164656634363536376163333631353031373232
37643636303934313664633765353562343666353961643237316461383131343566326466663936
32613933303430633165363339373433363637343733633833356339373965336131666234373737
33653763326461343164313663633361303261333737326266663030646136643234313263376432
33636332376634313364383463643030333462323266383263643535393061313665636637323266
62626334303936626535656431366339393561356461393862633464336534386665623735383862
37336232336662633331386661643131343663356133643838636231396336303364393562336139
30623030316365303937333037386362613065636161336530613637353737323535636138366263
30303533353531396135663139646566646631643533306165383364643764656330666134366633
64353333333065663238353666303637366162346535656534666465336630323764303639616564
63646638646566663033373263653062343466363364333537623661643331666664343065656633
32356239316638613264303939363830373030353263613863356465663538666638333462336234
65653563306562663530383334643763353464643363356462343863653965623634663338333136
31366531383837396564353134333232376630613063613134393038366631616365366439616263
36363231396665626263373830383330393432633934626363393866353538663433343563323631
65346430666666333237366162636363356462313165383233313438353431636539663131386134
39626136613935393265363534336265653435656333386363303735613336633062663131396336
65616339376561383633636362323530323061356134303365303234383063336261643330363363
39396332666532353563363966326266333632313164643965343538633763376466353864383435
34366266666162306334613262663861646538346139343631343139353234333661386234666431
39353038373434363130633435633161623565333062343734626239653434383035653361393865
64646135353236313261303633346161393636323531326334393036663137626365303934346337
33336136366534343238623262313064356634333661356339383462303037383630656631643762
38346439353338616165323562633436623131623532313663323131303737353734336234386164
37343465636332663363396536393266613235333762363538373261316263653135663931333962
31613630303266363265623363393364636437316465646532623264326239346531363165346566
34333831666236643636663738383363616231333166336334663035336639666165303965633064
33306434363730633830346535326565643563306338386235313633363631353666393161313934
63346531653062336666336366356330646566393563356464346436656333363962383562363463
64313466633064316331646130366333326438633662323034383037396363643861353736336333
66663531373230386330393065663762653932646561373962633034343635333265313964343965
33663239316334623636646530363565636235666536646132303533663230396363313732373063
37663333646639383037336333343438643933643238396236653935396663303163363962633237
3038323032383666326138306366633063636366363534393333
66646230643466316437343665656535396535613863396639333265316636633339303663393064
3133353537366632303931353265396364373865373562300a643733396562363463623630623930
63376533663434653439366163333333303131343562313836336465643561613435613938383438
3434346462663264630a313761383664326362346638383639336131393239313338616565663933
39326632333833653331376462393639376561306233663132376565646364363464376464646436
36613238323331396234306561653438303031313965333266336537303532313661336236653962
32393361356334613064663739366164383733666330613565663132306239303864626431653161
32343436376533353339656163646165663639396238303938376662393234313138663261663662
39393465633062306565353630366362376330643236333639343439346330623130353831366563
65653537383233623962396532336134373934656135666539663532363963393738323264666265
62383263643733633662333236623432313630353030306336653931376334633263356638386631
34393935353035366332353132646432383536316137386530336233623161393834636561646437
37653732313233333838313738373031643631323961323332323935356436386430323163373032
31373839346638303731363131623530303939323462643236303830306430343466343365336536
34356165623564333163623061316462306534326132396566323439373364386536323764316435
61623365656335343564366530346562343436313739613263653164343865643333343737643431
31353334643563353762626536346138643833313430323238393231386139653736373964656566
31333565643730616661306234323965336333383130346638343934343063616166623361633566
33383733323230616234313766333463356262333231303830323535343062313439353031326234
34303135613666333031343234356466306161623738366163373939376662366138643838393431
33626436343763363465393330343061303234616338386637313832663664323061646634616432
36653835626664333366633836656430666433363262303934323465313731653333373733326532
63313233633062636363616336646264316162623565643333636535616566623037653832653261
65336636343961643032633836333164383232323932653538366235323166623563663531383033
39633862623437393563633863666338663532363439623736353034666338363565343065323039
63623934613262373233663465346666333930343830386634633164326135386561633735643263
30353139376663346633643938666462353034303737343163623133313236386335633231623735
35306331613330343932646531393566383366656239633232653731366661383232616332316236
31396335313130633938323766326464363862653934343137616131393065303463613838643436
32376265653161306435303835376565643566613162353764373434663030323863323261333763
63353732313161373735343066303965316239343963376232663939626463646466343862626631
64343039623462343561333035626162636239343435636338306239613961663737313039643832
30666539396535653536613264326266323533363831343539653363353033393436656564303439
64316537623436336135643063633337636634323538616364646661346262373430656163623566
63633332346239323362343165366137623264386132616239373636636330613031333466383165
64373037623630643263623030633435623131313466343762643531623063323230613733643262
65373066616333656133613563636337366366386133636533643931663532393266613433646534
62353166326232366166333763653062626533356131623366336537623662393866393162383938
36326232313831346664346635623362313563323661663636353163343065356261313235303330
61346364363066653831643430613566383265613534653635313637623634326366303634313832
33393439366538383235383137633531336434303063396635626230653536613964336361393337
62306330376463393561333735636431333732363862666137313739666339643937663162646536
31653931376637316364393462333433323061656263323233653733313966613132643664326362
66383635303666626137636566353636643539313663396133663235353231393534656637666665
35333235626362396536373064363061366231656337356563386438343362633666323138316631
65363238373532383062316634353763306231653932663132333030326364396661396464333033
37353964313932326265623830353532643766613036623434353236666266316439623535333530
39303766343162346566646333373561626562643037346363343434393762323735663231636331
64373435303436363735356130636565636663366633646364383566303561353837333136653530
6438343339663436356535303434343832393366303263666535