Merge pull request #86 from EbookFoundation/add-travis

WIP: see if we can easily get travis to do tests

.travis.yml

@@ -0,0 +1,26 @@
+language: python
+
+python:
+  - '2.7'
+
+services:
+  - redis-server
+  - mysql
+
+env:
+  global:
+    - DJANGO_SETTINGS_MODULE=regluit.settings.travis
+    - PYTHONPATH=/home/travis/build/EbookFoundation/
+
+before_install:
+  - sudo mkdir /var/log/django
+  - sudo chmod 777 /var/log/django
+  - mkdir ~/build/EbookFoundation/regluit/settings/keys/
+  - cp ~/build/EbookFoundation/regluit/settings/dummy/__init__.py  ~/build/EbookFoundation/regluit/settings/keys/__init__.py
+  - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/EbookFoundation/regluit/test/travis-host.py.enc -out ~/build/EbookFoundation/regluit/settings/keys/host.py -d
+
+install: 
+  - pip install -r requirements_versioned.pip
+
+script: django-admin test
+

booxtream/__init__.py

@@ -1,10 +1,11 @@
-import requests
 import random
-from django.conf import settings
-from urllib import quote
 from functools import partial
+from urllib import quote
 from xml.etree import ElementTree
 
+import requests
+from django.conf import settings
+
 from django.apps import apps
 
 from . exceptions import BooXtreamError
@@ -34,10 +35,10 @@ class BooXtream(object):
         if not apiuser:
             apiuser = settings.BOOXTREAM_API_USER
         self.endpoint = 'https://service.booxtream.com/'
-        self.postrequest = partial(requests.post, timeout=timeout, auth=(apiuser,apikey))
+        self.postrequest = partial(requests.post, timeout=timeout, auth=(apiuser, apikey))
 
 
-    def platform(self,  epubfile=None, epub=True, kf8mobi=False,  **kwargs):
+    def platform(self, epubfile=None, epub=True, kf8mobi=False, **kwargs):
         """ Make an API request to BooXtream
         ``self.apikey``, ``epubfile`` and the supplied ``kwargs``.
         Attempts to deserialize the XML response and return the download link.
@@ -48,25 +49,25 @@ class BooXtream(object):
         Boox = apps.get_model('booxtream', 'Boox')
 
         url = self.endpoint + 'booxtream.xml'
-        kwargs['epub'] =  '1' if epub else '0'
+        kwargs['epub'] = '1' if epub else '0'
         kwargs['kf8mobi'] = '1' if kf8mobi else '0'
         if epubfile:
-            if hasattr(epubfile,'name') and str(epubfile.name).endswith('.epub'):
-                files= {'epubfile': (str(epubfile.name),epubfile)}
+            if hasattr(epubfile, 'name') and str(epubfile.name).endswith('.epub'):
+                files = {'epubfile': (str(epubfile.name), epubfile)}
             else:
                 # give it a random file name so that kindlegen doesn't choke
                 # needed for in-memory (StringIO) epubs
-                files= {'epubfile': ('%012x.epub' % random.randrange(16**12),epubfile)}
+                files= {'epubfile': ('%012x.epub' % random.randrange(16**12), epubfile)}
         else:
-             files={}
+             files = {}
         if settings.LOCAL_TEST:
             # fake it, so you can test other functions without hitting booxtream
             boox = Boox.objects.create(
                 download_link_epub='https://github.com/eshellman/42_ebook/blob/master/download/42.epub?raw=true&extra=download.booxtream.com/',
                 download_link_mobi='https://github.com/eshellman/42_ebook/blob/master/download/42.mobi?raw=true',
-                referenceid= kwargs.get('referenceid'),
-                downloads_remaining= kwargs.get('downloadlimit'),
-                expirydays=kwargs.get('expirydays'),
+                referenceid= kwargs.get('referenceid', '42'),
+                downloads_remaining=kwargs.get('downloadlimit', 10),
+                expirydays=kwargs.get('expirydays', 30),
             )
             return boox
 
@@ -86,9 +87,8 @@ class BooXtream(object):
         boox = Boox.objects.create(
             download_link_epub=download_link_epub,
             download_link_mobi=download_link_mobi,
-            referenceid= kwargs.get('referenceid'),
-            downloads_remaining= kwargs.get('downloadlimit'),
+            referenceid=kwargs.get('referenceid'),
+            downloads_remaining=kwargs.get('downloadlimit'),
             expirydays=kwargs.get('expirydays'),
         )
         return boox
-

booxtream/exceptions.py

@@ -7,7 +7,7 @@ class BooXtreamError(Exception):
         self.errors = errors
 
     def __str__(self):
-        errormsg='BooXtream errors:'
+        errormsg = 'BooXtream errors:'
         for error in self.errors:
-            errormsg += 'Error %s: %s\n'% (error.find('Code').text,error.find('Msg').text)
+            errormsg += 'Error %s: %s\n'% (error.find('Code').text, error.find('Msg').text)
         return errormsg

booxtream/models.py

@@ -6,10 +6,10 @@ class Boox(models.Model):
     """
     keeps a record of a file that's been watermarked
     """
-    download_link_epub =  models.URLField(null=True)
-    download_link_mobi =  models.URLField(null=True)
+    download_link_epub = models.URLField(null=True)
+    download_link_mobi = models.URLField(null=True)
     referenceid = models.CharField(max_length=32)
-    downloads_remaining = models.PositiveSmallIntegerField(default = 0)
+    downloads_remaining = models.PositiveSmallIntegerField(default=0)
     expirydays = models.PositiveSmallIntegerField()
     created = models.DateTimeField(auto_now_add=True)
 
@@ -22,6 +22,5 @@ class Boox(models.Model):
             return self.download_link_epub
         elif  format == 'mobi':
             return self.download_link_mobi
-        else:
-            return None
+        return None
 

booxtream/tests.py

@@ -23,14 +23,16 @@ class TestBooXtream(unittest.TestCase):
         return manager
 
     def test_booxtream_errors(self):
+        if settings.LOCAL_TEST:
+            return
         from .exceptions import BooXtreamError
         inst = self._makeOne()
         if not settings.BOOXTREAM_API_KEY:
             return
         with self.assertRaises(BooXtreamError) as cm:
             inst.platform()
-        self.assertIn( 'expirydays not set',str(cm.exception))
-        params={
+        self.assertIn('expirydays not set', str(cm.exception))
+        params = {
             'customername': 'Jane Test',
             'languagecode':'1043',
             'expirydays': 1,
@@ -47,7 +49,7 @@ class TestBooXtream(unittest.TestCase):
 
     def test_booxtream_good(self):
         inst = self._makeOne()
-        params={
+        params = {
             'customeremailaddress':'jane@example.com',
             'customername': 'Jane Test',
             'languagecode':'1043',
@@ -57,16 +59,16 @@ class TestBooXtream(unittest.TestCase):
             'chapterfooter':1,
             'disclaimer':1,
             }
-        params['referenceid']= 'order'+str(time.time())
-        boox=inst.platform(epubfile=self.epub2file, **params)
-        self.assertRegexpMatches(boox.download_link_epub,'download.booxtream.com/')
+        params['referenceid'] = 'order' + str(time.time())
+        boox = inst.platform(epubfile=self.epub2file, **params)
+        self.assertRegexpMatches(boox.download_link_epub, 'download.booxtream.com/')
         self.assertFalse(boox.expired)
-        self.assertEqual(boox.downloads_remaining,3)
+        self.assertEqual(boox.downloads_remaining, 3)
 
         # make sure it works with an in-memory file
         self.epub2file.seek(0)
         in_mem_epub = StringIO()
         in_mem_epub.write(self.epub2file.read())
         in_mem_epub.seek(0)
-        boox2=inst.platform(epubfile=in_mem_epub, **params)
-        self.assertRegexpMatches(boox2.download_link_epub,'download.booxtream.com/')
+        boox2 = inst.platform(epubfile=in_mem_epub, **params)
+        self.assertRegexpMatches(boox2.download_link_epub, 'download.booxtream.com/')

core/mobi.py

@@ -13,7 +13,8 @@ def convert_to_mobi(input_url, input_format="application/epub+zip"):
     return a string with the output of mobigen computation
     
     """
-
+    if mobigen_url and mobigen_user_id and mobigen_password:
+        print 'settings ok'
     # using verify=False since at the moment, using a self-signed SSL cert.
 
     payload = requests.get(input_url, verify=False).content 
@@ -26,5 +27,6 @@ def convert_to_mobi(input_url, input_format="application/epub+zip"):
     if r.status_code == 200:
         return r.content
     else:
+        print "{0}: {1}".format(r.status_code, r.content)
         raise Exception("{0}: {1}".format(r.status_code, r.content))
 

core/pdf.py

@@ -1,6 +1,7 @@
 """
 Utilities that manipulate pdf files
 """
+import logging
 import requests
 from xhtml2pdf import pisa             # import python module
 from PyPDF2 import PdfFileMerger,PdfFileReader
@@ -9,6 +10,7 @@ from tempfile import NamedTemporaryFile
 from django.template.loader import render_to_string
 from regluit import settings
 
+logger = logging.getLogger(__name__)
 
 # Utility function
 def ask_pdf(context={}):
@@ -35,7 +37,7 @@ def pdf_append( file1, file2, file_out ):
 def test_pdf(pdf_file):
     temp = None
     try:
-        if isinstance(pdf_file , str):
+        if isinstance(pdf_file , (str, unicode)):
             if pdf_file.startswith('http:') or pdf_file.startswith('https:'):
                 temp = NamedTemporaryFile(delete=False)
                 test_file_content = requests.get(pdf_file).content
@@ -53,7 +55,9 @@ def test_pdf(pdf_file):
         except:
             success = False
         return success
-    except:
+    except Exception:
+        pdf_file = unicode(pdf_file)
+        logger.exception('error testing a pdf: %s' % pdf_file[:100])
         return False
 
 def test_test_pdf(self):

core/tests.py

@@ -993,7 +993,7 @@ class DownloadPageTest(TestCase):
         anon_client = Client()
         response = anon_client.get("/work/%s/download/" % w.id, follow=True)
         self.assertContains(response, "/download_ebook/%s/"% eb1.id, count=11) 
-        self.assertContains(response, "/download_ebook/%s/"% eb2.id, count=5)
+        self.assertContains(response, "/download_ebook/%s/"% eb2.id, count=4)
         self.assertTrue(eb1.edition.work.is_free)
         eb1.delete()
         self.assertTrue(eb2.edition.work.is_free)
@@ -1119,7 +1119,8 @@ class EbookFileTests(TestCase):
         #test the ask-appender
         c.add_ask_to_ebfs()
         if settings.AWS_SECRET_ACCESS_KEY:
-            assert test_pdf(c.work.ebookfiles().filter(asking=True)[0].file.url)
+            askingpdfurl = c.work.ebookfiles().filter(asking=True)[0].file.url
+            assert test_pdf(askingpdfurl)
         else:
             assert test_pdf(c.work.ebookfiles().filter(asking=True)[0].file)
 
@@ -1157,6 +1158,29 @@ class EbookFileTests(TestCase):
         self.assertTrue(c.work.ebookfiles().filter(asking=True, ebook__active=True).count() == 0)
         self.assertTrue(c.work.ebookfiles().filter(asking=False, ebook__active=True).count() > 0)
 
+    def test_bad_ebookfile(self):
+        w = Work.objects.create(title="Work 3")
+        e = Edition.objects.create(title=w.title, work=w)
+
+        temp = NamedTemporaryFile(delete=False)
+        test_file_content = "bad text file"
+        temp.write(test_file_content)
+        temp.close()
+
+        try:
+            # put the bad file into Django storage
+            temp_file = open(temp.name)
+            dj_file = DjangoFile(temp_file)
+            ebf = EbookFile(format='epub', edition=e, file=dj_file)
+            ebf.save()
+            temp_file.close()
+            ebf.make_mobi()
+        finally:
+            # make sure we get rid of temp file
+            os.remove(temp.name)
+        self.assertTrue(ebf.mobied < 0)
+
+
 class MobigenTests(TestCase):
     def test_convert_to_mobi(self):
         """

requirements_versioned.pip

@@ -9,10 +9,12 @@ PyYAML==3.11
 amqp==1.4.9
 anyjson==0.3.3
 billiard==3.3.0.23
-awscli==1.10.26
-boto==2.42.0
+awscli==1.15.33
+boto==2.48.0
+boto3==1.7.33
 celery==3.1.23
-certifi==2016.2.28
+certifi==2018.4.16
+chardet==3.0.4
 # pip installing pillow seems to delete distribute
 # but having distribute in requirements starting to cause problems
 # distribute==0.6.28
@@ -33,7 +35,7 @@ git+git://github.com/eshellman/django-notification.git@a4620e893e2da220994e0189b
 django-registration==2.1.2
 django-selectable==0.9.0
 django-smtp-ssl==1.0
-django-storages==1.4.1
+django-storages==1.5.2
 django-tastypie==0.13.3
 #django-transmeta==0.7.3 
 git+git://github.com/resulto/django-transmeta.git@ad4d7278ba330dcf8c8446f8ae9b2c769ae8684e
@@ -70,7 +72,7 @@ rdflib==4.2.0
 rdflib-jsonld==0.3
 redis==2.10.3
 reportlab==3.4.0
-requests==2.10.0
+requests==2.18.4
 requests-mock==1.2.0
 requests-oauthlib==0.6.2
 selenium==2.53.1
@@ -87,19 +89,19 @@ xhtml2pdf==0.2.2
 webencodings==0.5.1
 #for urllib3 secure
 cffi==1.7.0
-cryptography==2.1.4
+cryptography==2.2.2
 enum34==1.1.6
-idna==2.1
+idna==2.6
 ipaddress==1.0.16
 ndg-httpsclient==0.4.2
-pyOpenSSL==16.2.0
+pyOpenSSL==18.0.0
 pyasn1==0.1.9
 pycparser==2.14
 setuptools==25.0.0
-urllib3==1.16
+urllib3==1.22
 beautifulsoup4==4.6.0
 RISparser==0.4.2
 # include these 2 for development
-libsass==0.14.5
-django-compressor==2.2
-django-sass-processor==0.7
+#libsass==0.13.4
+#django-compressor==2.2
+django-sass-processor==0.5.6

settings/travis.py

@@ -16,8 +16,8 @@ DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.mysql',
         'NAME': 'regluit',
-        'USER': 'regluit',
-        'PASSWORD': 'regluit',
+        'USER': 'root',
+        'PASSWORD': '',
         'HOST': '',
         'PORT': '',
         'TEST': {
@@ -31,12 +31,11 @@ TIME_ZONE = 'America/New_York'
 # settings for outbout email
 # if you have a gmail account you can use your email address and password
 
-EMAIL_USE_TLS = True
-EMAIL_HOST = 'smtp.gmail.com'
-EMAIL_HOST_USER = 'me@gmail.com'
-EMAIL_HOST_PASSWORD = 'my-password'
-EMAIL_PORT = 587
-DEFAULT_FROM_EMAIL = 'info@ebookfoundation.org'
+EMAIL_BACKEND = 'django_smtp_ssl.SSLEmailBackend'
+MAIL_USE_TLS = True 
+EMAIL_HOST = 'email-smtp.us-east-1.amazonaws.com'
+EMAIL_PORT = 465
+DEFAULT_FROM_EMAIL = 'notices@gluejar.com'
 
 
 # formerly of settings/common.py to surface old vars
@@ -49,8 +48,22 @@ REDIRECT_IS_HTTPS = False
 #BASE_URL = 'http://0.0.0.0/'
 BASE_URL_SECURE = 'http://0.0.0.0/'
 
+BROKER_TRANSPORT = "redis"
+BROKER_HOST = "localhost"
+BROKER_PORT = 6379
+BROKER_VHOST = "0"
 
+# Amazon  S3 access
+AWS_STORAGE_BUCKET_NAME = 'unglueit-testfiles'
+
+SOCIAL_AUTH_TWITTER_KEY = ''
+SOCIAL_AUTH_TWITTER_SECRET = ''
+SOCIAL_AUTH_FACEBOOK_KEY = ''
+SOCIAL_AUTH_FACEBOOK_SECRET = ''
+SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = ''
+SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = ''
+GOOGLE_BOOKS_API_KEY = ''
+TEST_INTEGRATION = False
+LOCAL_TEST = True
+DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
 
-# use database as queuing service in development
-BROKER_TRANSPORT = "djkombu.transport.DatabaseTransport"
-#INSTALLED_APPS += ("djkombu",)

setup.py

@@ -0,0 +1,25 @@
+import os
+from setuptools import find_packages, setup
+
+with open(os.path.join(os.path.dirname(__file__), 'README.md')) as readme:
+    README = readme.read()
+
+# allow setup.py to be run from any path
+os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir)))
+
+setup(
+    name='regluit',
+    version='3.0.0',
+    description='Web application for Unglue.it',
+    author='Free Ebook Foundation',
+    author_email='info@ebookfoundation.org',
+    url='https://unglue.it',
+    packages=find_packages(exclude=[
+        'bookdata',
+        'deploy'
+        'logs',
+        'selenium',
+        'static',
+        'vagrant',
+    ]),
+)

vagrant/group_vars/all/secrets.yml

@@ -1,51 +1,51 @@
 $ANSIBLE_VAULT;1.1;AES256
-38353435396232353338653362653465343063393836633332633964663162363636316433396365
-3336316136626134653262626630663738333938666336330a626562646335323264626363316233
-62383131353536323166353364633437383333663434653037353964646662633865323466306237
-3336306364653366320a323464383061353862663932623461346230316364396530653564353762
-33363864663730643265396565363430376564386164366163323266393735633830363534353665
-31376233393136306365383366333866383962313864376561333964623639623834373931396133
-31646665313165376266336132663833663436633165393536316262373635353837306365363964
-38656539313935366663336539376334313863633439343664353530393336383064623533336264
-34346630643737643938373366383432303139653566306163333463366465623465303862616361
-61363139613265636432616533656564386231323066623863663463313336626435323433343337
-31626530626132646262366334643261626431643261633334393262636137643935376461643739
-38633631626366303934393139356166633536333634396163333164306532373765656138393464
-39623636353034343363396361383837663863313465636134616333666563666534653930626331
-65633536383230653665356233323330666439643665396335303566366230646366356534663732
-30313835323633306130343865626131383032666236663362633733383235643239383765663332
-32643332313361353736313035653137323239616635653662386435373939386636653138393036
-38396231623635316633333366646339336662313164656634363536376163333631353031373232
-37643636303934313664633765353562343666353961643237316461383131343566326466663936
-32613933303430633165363339373433363637343733633833356339373965336131666234373737
-33653763326461343164313663633361303261333737326266663030646136643234313263376432
-33636332376634313364383463643030333462323266383263643535393061313665636637323266
-62626334303936626535656431366339393561356461393862633464336534386665623735383862
-37336232336662633331386661643131343663356133643838636231396336303364393562336139
-30623030316365303937333037386362613065636161336530613637353737323535636138366263
-30303533353531396135663139646566646631643533306165383364643764656330666134366633
-64353333333065663238353666303637366162346535656534666465336630323764303639616564
-63646638646566663033373263653062343466363364333537623661643331666664343065656633
-32356239316638613264303939363830373030353263613863356465663538666638333462336234
-65653563306562663530383334643763353464643363356462343863653965623634663338333136
-31366531383837396564353134333232376630613063613134393038366631616365366439616263
-36363231396665626263373830383330393432633934626363393866353538663433343563323631
-65346430666666333237366162636363356462313165383233313438353431636539663131386134
-39626136613935393265363534336265653435656333386363303735613336633062663131396336
-65616339376561383633636362323530323061356134303365303234383063336261643330363363
-39396332666532353563363966326266333632313164643965343538633763376466353864383435
-34366266666162306334613262663861646538346139343631343139353234333661386234666431
-39353038373434363130633435633161623565333062343734626239653434383035653361393865
-64646135353236313261303633346161393636323531326334393036663137626365303934346337
-33336136366534343238623262313064356634333661356339383462303037383630656631643762
-38346439353338616165323562633436623131623532313663323131303737353734336234386164
-37343465636332663363396536393266613235333762363538373261316263653135663931333962
-31613630303266363265623363393364636437316465646532623264326239346531363165346566
-34333831666236643636663738383363616231333166336334663035336639666165303965633064
-33306434363730633830346535326565643563306338386235313633363631353666393161313934
-63346531653062336666336366356330646566393563356464346436656333363962383562363463
-64313466633064316331646130366333326438633662323034383037396363643861353736336333
-66663531373230386330393065663762653932646561373962633034343635333265313964343965
-33663239316334623636646530363565636235666536646132303533663230396363313732373063
-37663333646639383037336333343438643933643238396236653935396663303163363962633237
-3038323032383666326138306366633063636366363534393333
+66646230643466316437343665656535396535613863396639333265316636633339303663393064
+3133353537366632303931353265396364373865373562300a643733396562363463623630623930
+63376533663434653439366163333333303131343562313836336465643561613435613938383438
+3434346462663264630a313761383664326362346638383639336131393239313338616565663933
+39326632333833653331376462393639376561306233663132376565646364363464376464646436
+36613238323331396234306561653438303031313965333266336537303532313661336236653962
+32393361356334613064663739366164383733666330613565663132306239303864626431653161
+32343436376533353339656163646165663639396238303938376662393234313138663261663662
+39393465633062306565353630366362376330643236333639343439346330623130353831366563
+65653537383233623962396532336134373934656135666539663532363963393738323264666265
+62383263643733633662333236623432313630353030306336653931376334633263356638386631
+34393935353035366332353132646432383536316137386530336233623161393834636561646437
+37653732313233333838313738373031643631323961323332323935356436386430323163373032
+31373839346638303731363131623530303939323462643236303830306430343466343365336536
+34356165623564333163623061316462306534326132396566323439373364386536323764316435
+61623365656335343564366530346562343436313739613263653164343865643333343737643431
+31353334643563353762626536346138643833313430323238393231386139653736373964656566
+31333565643730616661306234323965336333383130346638343934343063616166623361633566
+33383733323230616234313766333463356262333231303830323535343062313439353031326234
+34303135613666333031343234356466306161623738366163373939376662366138643838393431
+33626436343763363465393330343061303234616338386637313832663664323061646634616432
+36653835626664333366633836656430666433363262303934323465313731653333373733326532
+63313233633062636363616336646264316162623565643333636535616566623037653832653261
+65336636343961643032633836333164383232323932653538366235323166623563663531383033
+39633862623437393563633863666338663532363439623736353034666338363565343065323039
+63623934613262373233663465346666333930343830386634633164326135386561633735643263
+30353139376663346633643938666462353034303737343163623133313236386335633231623735
+35306331613330343932646531393566383366656239633232653731366661383232616332316236
+31396335313130633938323766326464363862653934343137616131393065303463613838643436
+32376265653161306435303835376565643566613162353764373434663030323863323261333763
+63353732313161373735343066303965316239343963376232663939626463646466343862626631
+64343039623462343561333035626162636239343435636338306239613961663737313039643832
+30666539396535653536613264326266323533363831343539653363353033393436656564303439
+64316537623436336135643063633337636634323538616364646661346262373430656163623566
+63633332346239323362343165366137623264386132616239373636636330613031333466383165
+64373037623630643263623030633435623131313466343762643531623063323230613733643262
+65373066616333656133613563636337366366386133636533643931663532393266613433646534
+62353166326232366166333763653062626533356131623366336537623662393866393162383938
+36326232313831346664346635623362313563323661663636353163343065356261313235303330
+61346364363066653831643430613566383265613534653635313637623634326366303634313832
+33393439366538383235383137633531336434303063396635626230653536613964336361393337
+62306330376463393561333735636431333732363862666137313739666339643937663162646536
+31653931376637316364393462333433323061656263323233653733313966613132643664326362
+66383635303666626137636566353636643539313663396133663235353231393534656637666665
+35333235626362396536373064363061366231656337356563386438343362633666323138316631
+65363238373532383062316634353763306231653932663132333030326364396661396464333033
+37353964313932326265623830353532643766613036623434353236666266316439623535333530
+39303766343162346566646333373561626562643037346363343434393762323735663231636331
+64373435303436363735356130636565636663366633646364383566303561353837333136653530
+6438343339663436356535303434343832393366303263666535