From e1553f812185afed1c262c141321fd5f2d301c27 Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 6 Jun 2018 17:29:12 -0400 Subject: [PATCH 01/23] switch to dockerized mobi --- core/mobi.py | 4 +- core/models/__init__.py | 7 +- requirements_versioned.pip | 9 +-- vagrant/group_vars/all/secrets.yml | 100 ++++++++++++++--------------- 4 files changed, 63 insertions(+), 57 deletions(-) diff --git a/core/mobi.py b/core/mobi.py index 87522369..07bbbc9d 100644 --- a/core/mobi.py +++ b/core/mobi.py @@ -16,11 +16,11 @@ def convert_to_mobi(input_url, input_format="application/epub+zip"): # using verify=False since at the moment, using a self-signed SSL cert. - payload = requests.get(input_url, verify=False).content + payload = requests.get(input_url).content headers = {'Content-Type': input_format} r = requests.post(mobigen_url, auth=(mobigen_user_id, mobigen_password), - data=payload, verify=False, headers=headers) + data=payload, headers=headers) # if HTTP reponse code is ok, the output is the mobi file; else error message if r.status_code == 200: diff --git a/core/models/__init__.py b/core/models/__init__.py index 6bcc8f3b..7136fd16 100755 --- a/core/models/__init__.py +++ b/core/models/__init__.py @@ -966,7 +966,12 @@ class Campaign(models.Model): # now make the mobi file new_mobi_ebf = EbookFile.objects.create(edition=edition, format='mobi', asking=True) - new_mobi_ebf.file.save(path_for_file('ebf', None), ContentFile(mobi.convert_to_mobi(new_epub_ebf.file.url))) + try: + new_mobi_file = ContentFile(mobi.convert_to_mobi(new_epub_ebf.file.url)) + except Exception as e: + logger.error("error making mobi for %s" % (new_epub_ebf.file.url)) + raise e + new_mobi_ebf.file.save(path_for_file('ebf', None), new_mobi_file) new_mobi_ebf.save() new_mobi_ebf.version = version new_ebfs.append(new_mobi_ebf) diff --git a/requirements_versioned.pip b/requirements_versioned.pip index b326b50a..0d8673ce 100644 --- a/requirements_versioned.pip +++ b/requirements_versioned.pip @@ -12,7 +12,8 @@ billiard==3.3.0.23 awscli==1.10.26 boto==2.42.0 celery==3.1.23 -certifi==2016.2.28 +certifi==2018.4.16 +chardet==3.0.4 # pip installing pillow seems to delete distribute # but having distribute in requirements starting to cause problems # distribute==0.6.28 @@ -70,7 +71,7 @@ rdflib==4.2.0 rdflib-jsonld==0.3 redis==2.10.3 reportlab==3.4.0 -requests==2.10.0 +requests==2.18.4 requests-mock==1.2.0 requests-oauthlib==0.6.2 selenium==2.53.1 @@ -89,14 +90,14 @@ webencodings==0.5.1 cffi==1.7.0 cryptography==1.4 enum34==1.1.6 -idna==2.1 +idna==2.6 ipaddress==1.0.16 ndg-httpsclient==0.4.2 pyOpenSSL==16.0.0 pyasn1==0.1.9 pycparser==2.14 setuptools==25.0.0 -urllib3==1.16 +urllib3==1.22 beautifulsoup4==4.6.0 RISparser==0.4.2 # include these 2 for development diff --git a/vagrant/group_vars/all/secrets.yml b/vagrant/group_vars/all/secrets.yml index d4a96e78..cfb9284f 100644 --- a/vagrant/group_vars/all/secrets.yml +++ b/vagrant/group_vars/all/secrets.yml @@ -1,51 +1,51 @@ $ANSIBLE_VAULT;1.1;AES256 -63376563643062623466613335633730323031653736643437643030303533383931666434323939 -3266633436313635643463333266356238316638326131610a653336303661326334323561623239 -65623834633736663861343464333764303865663934303533333031343934333635616333643631 -3933383131353062630a633133353430386437626130656632393230636664376438613862303466 -66343337303961303361353030313936626363383465316166316663643830313164646661363366 -66656630323437643031363237633439366434666537396464336364373134383061623364636635 -31393435326337373162333365386465633730323135383364383433366664353938303962313461 -61633835393963646666376562633731626639313438326235306335653639666635386566343033 -36383064653634613065303538346638363666326336643631616636623632376662323037633165 -32633662636531643366343963663439313439636335336435616666633863323433643930666536 -30623234313838616461383534663230616364336131383763316235356265363964636539316637 -61376433306366346362303439303665333961393239373263373365306363336334623536353031 -61636339353561303532383833336164346538396139373835633162323339333732373431613035 -62333739373832623762356364383035343937656337666439323830663235666637326134346531 -66386135323064333732663136396431616563653731343134363262373639333836623033656536 -38653931346337666566336331363631666238666638353237663065653164343936636239343631 -62636339396332663637643239376335343731643764343666356361623239383462313936366363 -32616638653666666638316563656366666361353538303133656433376637663532313666343631 -63633436336137653339653364616532333434623036396439343433613361626231613331373931 -64343232313839383332353036633633383762316531383936393539316531316336636537666439 -33613166613733346336643932363263663833366533623965623039313036373230396135343663 -66653336313437336561333630393365653133636263333031356562623332623733616464333262 -35333535396337616461353361616337613138363239656162323232346436336233633335376666 -63626661373639306665613431393735343937616539336233666235343936653932366337326663 -33353930653965623661383265643837306637363231656435663030393562623438623361353666 -36626635323837363733353365373738646230303436343834636666373165653937333130343637 -31666566643434653663626531383730666238306438656133303862626538613339626437633432 -64393432333834623339393631363932353862646332313062636565326135636138333931333431 -64633937356630326666373939343635323263373934306330393032316630646333363239313137 -65383666663063356430663965336463393537363266636431633362343263666234323463653532 -30376262656335653662656366636336336135626130303835326432366438393962616266373435 -39613234326136623263356131376239313136653131326632343739313137653833316434656265 -30323863383639363462613034353062393964333130656161306633306566646233356537313162 -65376237303961333239303630313831303232643831383036643166343164363934323138353234 -33623130383339343162636239343234393731626530326164643438633239343738306236623433 -37636662353263373866656564316334303736336566336636653937663566343261333335333437 -62313833353235393131326430323263383861366230663639363661383362363438373432346663 -62643166363633633134303836383239386236386361376533353330333930633530333139666462 -63623361356134333337643634393461343635363232313435656137303731326461653330613831 -66613536666635313539643130373132623965333863343930363761613031306664333732393564 -34333937326234383833306233386638323263323430613863346436643030353732356261643834 -62343230633034363839373965646133353238346261373538313833323264616531313965376537 -32646166326135306134356262353461626337663137323335613339646532636663326462393936 -30316265636335336138383930313138306366383766653866613532626366373463613436303638 -31333738306164613031626435373966323361626564666537396539313832613038363865366534 -62656565316466393366303137646431303231626534613031316561336233353137353265623032 -32616134366439363630376666343937633462613731613961343862303639373135373466363663 -34663235333431323764613330303539323164313332313331613761306439623232396436323138 -35623265613335636430323335313230373865633331303633613632306431323339396165626135 -3662393265643861393433333230353233366330646435343361 +64636439626632623833636366633136653733373531316266643663336632613035303563336664 +3163633033613732656536313030613263636331343139630a373432356362376136303463643637 +63393038386266313536356261633133353437363366386161616233353336336237373866633865 +6333613461666564390a346661343365663662613762643061343636383866623835316333663930 +35303839633239326666376437393664613039393235383838613439653239353831313238656530 +66313261313763666562343833663963613037636466623863383636343865626261386533383165 +35633331643361393265656333633330633064306161313566373131376235646531653063643565 +34313032643166636337313866383064383436653466303930663435393336333965333334303963 +39333232376437353765623832346366383833313962373632626232643636326436373233643237 +35353066646130363665313765386631646363663161383461626332616666396139656463376134 +39326234333138303130623963396430313037643263623037633139316362333332313933313035 +65656239333131346461383066656263393365326561316431613266303035636438366232373665 +30313265306332626566313165336437633035343063623630336139313166366136353666623934 +64616339653930333630316236363630356530663465613266643632313431376236356164663937 +62336361613739373030353761336366353764373231303433376435626334623637636637623832 +33373333623666306363646366393663353465396638623134313537303830333639333137343662 +65643865616139356161333239626165303165313730306434326431363032656337316337353739 +38653661316631643930343863326535313336323937633833323565326539376135376339323232 +30616236613034643935653866333465663837373861656536313333346637386632353139666539 +64643966313964393134396630343033633233353534353065363635383636363139303637366338 +64353835386537376363623062313862383637383430336464623566616336373836663835626331 +37633562303933646263646239306437653135633565303530333761326566336565613366303630 +32346436616135326262333930383130323562303465643065663562343832323364303737613764 +30623939303466306636376235613639363933343065376562353030326335663963353766313263 +34346266616239373264646439616536353762656536643738623662396231323334313261316331 +66356566626463363033316162626265353765653132346534663430653135353236333866653031 +37663636373133353465363635386561626666623532343832366536616634313065383531353538 +33633539373366396566333333653939313864663235663165363563326538656335306262393561 +32383831656638666464663462373462663662323833633735333764303363656161616235323464 +38353461306430363566633030386535656264616236393737343662613336353665666564343462 +34383365306166353236346533653333643465353061396135636362336239353231353463633361 +61623965643634396366656265303961623531306163373230626639303132306630356537623934 +63623732663562663231656439623266623034643336643839396534323363306364643534636233 +39353431323166326237663362353137633631333733303231386236356263613163656238326461 +63353162316366343465363662383631333761366262336235383862633630386362363135636232 +35386234646662376434646161633463353163313261333333636162303461623233626637363039 +39356263343632356333666633636437616431393062326230616132336462303231616635326237 +35616231643633386430363335383333386339353833323064393039303839653131316439623036 +37303236303535656466643037326337353766653666633933313330343930636532333666336638 +33376437383064646633366637623638623239663261376639343332363430656232373162356336 +65643331643036316136313737383934633731306466333639626230633465653338393932666636 +35613331323431386438356439613034363364643663623332303435363861353538623238393835 +36376432623361386237666233356638376537393536313335633361353566323963303864343461 +39386464613134666437393734376662386236303837626531613363373161343937333863656232 +63333766636465653666663261353939393363306665353832633938376362613061386665336665 +34373939623633666236326130643662643163666233643936636461613963643836373639336230 +65306337303261376232363239636634386638323238313566326234663331623535353433633431 +37313731663134323839336635366366336462363563373639646161376162313231396365323430 +64383739346161343030343635393935643931323237383030613662636630613935333061613961 +3665636339663830306630376237653762643333346563313939 From c422965bdb3eb43bf572eef2f834e5283af89325 Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 6 Jun 2018 22:26:27 -0400 Subject: [PATCH 02/23] add column to EbookFile to indicate a failed make_mobi so we don't keep trying with a bad epub file --- .../management/commands/make_missing_mobis.py | 2 +- core/migrations/0013_ebookfile_mobied.py | 19 +++++++++++++++ core/models/bibmodels.py | 7 ++++++ core/tests.py | 23 +++++++++++++++++++ 4 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 core/migrations/0013_ebookfile_mobied.py diff --git a/core/management/commands/make_missing_mobis.py b/core/management/commands/make_missing_mobis.py index ee06fe3a..af2d5461 100644 --- a/core/management/commands/make_missing_mobis.py +++ b/core/management/commands/make_missing_mobis.py @@ -21,7 +21,7 @@ class Command(BaseCommand): if not work.ebooks().filter(format="mobi"): for ebook in work.ebooks().filter(format="epub"): ebf = ebook.get_archive_ebf() - if ebf: + if ebf and ebf.mobied >= 0: try: print u'making mobi for {}'.format(work.title) if ebf.make_mobi(): diff --git a/core/migrations/0013_ebookfile_mobied.py b/core/migrations/0013_ebookfile_mobied.py new file mode 100644 index 00000000..18775c0d --- /dev/null +++ b/core/migrations/0013_ebookfile_mobied.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0012_campaign_charitable'), + ] + + operations = [ + migrations.AddField( + model_name='ebookfile', + name='mobied', + field=models.IntegerField(default=0), + ), + ] diff --git a/core/models/bibmodels.py b/core/models/bibmodels.py index 30d17e29..44c59b9c 100644 --- a/core/models/bibmodels.py +++ b/core/models/bibmodels.py @@ -1056,6 +1056,7 @@ class EbookFile(models.Model): asking = models.BooleanField(default=False) ebook = models.ForeignKey('Ebook', related_name='ebook_files', null=True) source = models.URLField(null=True, blank=True) + mobied = models.IntegerField(default=0) #-1 indicates a failed conversion attempt version = None def check_file(self): if self.format == 'epub': @@ -1072,9 +1073,13 @@ class EbookFile(models.Model): def make_mobi(self): if not self.format == 'epub' or not settings.MOBIGEN_URL: return False + if self.mobied < 0: + return False try: mobi_cf = ContentFile(mobi.convert_to_mobi(self.file.url)) except: + self.mobied = -1 + self.save() return False new_mobi_ebf = EbookFile.objects.create( edition=self.edition, @@ -1097,6 +1102,8 @@ class EbookFile(models.Model): ) new_mobi_ebf.ebook = new_ebook new_mobi_ebf.save() + self.mobied = 1 + self.save() return True send_to_kindle_limit = 7492232 diff --git a/core/tests.py b/core/tests.py index 949601f6..ddd6311c 100755 --- a/core/tests.py +++ b/core/tests.py @@ -1157,6 +1157,29 @@ class EbookFileTests(TestCase): self.assertTrue(c.work.ebookfiles().filter(asking=True, ebook__active=True).count() == 0) self.assertTrue(c.work.ebookfiles().filter(asking=False, ebook__active=True).count() > 0) + def test_bad_ebookfile(self): + w = Work.objects.create(title="Work 3") + e = Edition.objects.create(title=w.title, work=w) + + temp = NamedTemporaryFile(delete=False) + test_file_content = "bad text file" + temp.write(test_file_content) + temp.close() + + try: + # put the bad file into Django storage + temp_file = open(temp.name) + dj_file = DjangoFile(temp_file) + ebf = EbookFile(format='epub', edition=e, file=dj_file) + ebf.save() + temp_file.close() + ebf.make_mobi() + finally: + # make sure we get rid of temp file + os.remove(temp.name) + self.assertTrue(ebf.mobied < 0) + + class MobigenTests(TestCase): def test_convert_to_mobi(self): """ From 3e782e636dd69a48c9f7cba10c1b7cd18d0e8b9a Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 6 Jun 2018 23:27:08 -0400 Subject: [PATCH 03/23] files for running tests with travis --- .gitignore | 3 ++- .travis.yml | 23 +++++++++++++++++++ settings/travis.py | 55 +++++++++++++++++++++++++++++++++++++++++++++ test/common.py.enc | Bin 0 -> 544 bytes 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 .travis.yml create mode 100644 settings/travis.py create mode 100644 test/common.py.enc diff --git a/.gitignore b/.gitignore index 4a693c5d..df1ad729 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,8 @@ *.pyc *.log settings/me.* -settings/keys/* +settings/keys/c* +settings/keys/h* *.dot reports ENV diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..25378555 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,23 @@ +language: python + +python: + - '2.7' + +services: + - redis-server + - mysql + +env: + global: + - DJANGO_SETTINGS_MODULE=regluit.settings.travis + - PYTHONPATH=/home/travis/build/EbookFoundation/ + +before_install: + - sudo mkdir /var/log/django + - sudo chmod 777 /var/log/django + - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/EbookFoundation/regluit/test/common.py.enc -out ~/build/EbookFoundation/regluit/settings/keys/common.py -d + +install: + - pip install -r requirements_versioned.pip + +script: django-admin test diff --git a/settings/travis.py b/settings/travis.py new file mode 100644 index 00000000..dec133c2 --- /dev/null +++ b/settings/travis.py @@ -0,0 +1,55 @@ +# coding=utf-8 +from .common import * + +DEBUG = True +TEMPLATES[0]['OPTIONS']['debug'] = DEBUG +IS_PREVIEW = False + +ADMINS = ( + ('Raymond Yee', 'rdhyee+ungluebugs@gluejar.com'), + ('Eric Hellman', 'eric@gluejar.com'), +) + +MANAGERS = ADMINS + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': 'regluit', + 'USER': 'root', + 'PASSWORD': '', + 'HOST': '', + 'PORT': '', + 'TEST_CHARSET': 'utf8', + } +} + +TIME_ZONE = 'America/New_York' + +# settings for outbout email +# if you have a gmail account you can use your email address and password + +EMAIL_BACKEND = 'django_smtp_ssl.SSLEmailBackend' +MAIL_USE_TLS = True +EMAIL_HOST = 'email-smtp.us-east-1.amazonaws.com' +EMAIL_PORT = 465 +DEFAULT_FROM_EMAIL = 'notices@gluejar.com' + + +# formerly of settings/common.py to surface old vars +# TO DO: invalidate before we open source + +# for use with test google account only +GOOGLE_DISPLAY_NAME = 'Unglue.It' +REDIRECT_IS_HTTPS = False + +#BASE_URL = 'http://0.0.0.0/' +BASE_URL_SECURE = 'http://0.0.0.0/' + +BROKER_TRANSPORT = "redis" +BROKER_HOST = "localhost" +BROKER_PORT = 6379 +BROKER_VHOST = "0" + +# Amazon S3 access +AWS_STORAGE_BUCKET_NAME = 'unglueit-testfiles' diff --git a/test/common.py.enc b/test/common.py.enc new file mode 100644 index 0000000000000000000000000000000000000000..9c6f2c2b6a76f76c332b9b737b5f7f387e3424da GIT binary patch literal 544 zcmV+*0^j}km3t@2?&*v5@bBcbz3nIGX1;&`bhq`x+1#;5hb57Wfl)+c!x|7;B!hF7LbqC^K9@Ls&Y#isMsZ$ zI`rZph#`#)RU}$e7j|`;Ja+XiaCpDatLy*fvxYJvXgPacMoZ$0{tW0K78&mJu@4&x z;}$JbrFfGfKH5Ok!NH2f`@>7%#6k>4{{4AE8QtQDZ!sZ*0&c5fZe5WyQ}|->qpxHP z3O;Ke!8zLc09Q?yf}`TZ%o;I%ia;`jq~5Q!!7iRzptF&H-U$L`vKd_pA#`Z@a>NgAVqU$pcBJ7dMHf;T^J7{phl118+UM~vox zND|-PR0pFx6}yjdC53S*b%499-J_r|ls+w)em{%?b Date: Wed, 6 Jun 2018 23:34:06 -0400 Subject: [PATCH 04/23] user is specific to org --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 25378555..6990592c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,12 +10,12 @@ services: env: global: - DJANGO_SETTINGS_MODULE=regluit.settings.travis - - PYTHONPATH=/home/travis/build/EbookFoundation/ + - PYTHONPATH=/home/travis/build/Gluejar/ before_install: - sudo mkdir /var/log/django - sudo chmod 777 /var/log/django - - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/EbookFoundation/regluit/test/common.py.enc -out ~/build/EbookFoundation/regluit/settings/keys/common.py -d + - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/Gluejar/regluit/test/common.py.enc -out ~/build/Gluejar/regluit/settings/keys/common.py -d install: - pip install -r requirements_versioned.pip From 0deb76b678f1aa578f3a8a38edb80dc9477d27f4 Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 7 Jun 2018 09:59:35 -0400 Subject: [PATCH 05/23] make keys a module --- settings/keys/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 settings/keys/__init__.py diff --git a/settings/keys/__init__.py b/settings/keys/__init__.py new file mode 100644 index 00000000..e69de29b From 3028c5f19db5b81b89e017ec2a0c42116da70850 Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 7 Jun 2018 12:17:46 -0400 Subject: [PATCH 06/23] re-decrypt --- test/common.py.enc | Bin 544 -> 544 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/test/common.py.enc b/test/common.py.enc index 9c6f2c2b6a76f76c332b9b737b5f7f387e3424da..9d025ae9bb61ab871fbde619567bc1ce99923aee 100644 GIT binary patch literal 544 zcmV+*0^j|NoHsq>>v6Q!62Df@h=cs2Le&M6z3z|~(u`*aU_RbyKtTxsQB!)(1{lwvz!@!`{oE|n@w%OZ z;hg*1vtNMMom8S3MeW5&!$NyTzof%sKKW{_2^7~_fc#?EIV(%lzA1dMph&{-W?qg= zGQJHGgA!F*bBAq09;<-YO7E@o3HdI zjzH)thhvdyIZ`uYF-0@vtR85sTm>ajDeNF|LZ1dn*f8jMmbDd)Hw!ey+W@C|XAU0i zx=#-j=gj6#l$PkA=$9m(%=JVN8(GtbFdIj(VTJU!>TmKhygNxQ7_2nQt)`P4KjwP6 zx1sHAhq~hO({uI;nzVhkSPag}vV;B|`n4Cg`lCGuA4@T|MC@@tLzCWr$-8jH3832{ zDW8JTQdys$W%+XAWge#sOaVH4K|x$D9s?Y);7jUjy%5$1L{{dEw#0Bs zCBu57h=t|OH)MIjy=KoM(Weq`x+1#;5hb57Wfl)+c!x|7;B!hF7LbqC^K9@Ls&Y#isMsZ$ zI`rZph#`#)RU}$e7j|`;Ja+XiaCpDatLy*fvxYJvXgPacMoZ$0{tW0K78&mJu@4&x z;}$JbrFfGfKH5Ok!NH2f`@>7%#6k>4{{4AE8QtQDZ!sZ*0&c5fZe5WyQ}|->qpxHP z3O;Ke!8zLc09Q?yf}`TZ%o;I%ia;`jq~5Q!!7iRzptF&H-U$L`vKd_pA#`Z@a>NgAVqU$pcBJ7dMHf;T^J7{phl118+UM~vox zND|-PR0pFx6}yjdC53S*b%499-J_r|ls+w)em{%?b Date: Thu, 7 Jun 2018 13:05:25 -0400 Subject: [PATCH 07/23] make sure common.keys overrides common --- settings/common.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/settings/common.py b/settings/common.py index 3a42e422..c300cdfe 100644 --- a/settings/common.py +++ b/settings/common.py @@ -7,11 +7,6 @@ from os.path import dirname, realpath, join import regluit from regluit.payment.parameters import PAYMENT_HOST_PAYPAL, PAYMENT_HOST_AMAZON -try: - from .keys.common import * -except ImportError: - print 'no real key file found, using dummy' - from .dummy.common import * PROJECT_DIR = dirname(dirname(realpath(__file__))) @@ -478,6 +473,11 @@ QUESTIONNAIRE_SHOW_ITEM_RESULTS = False FIREFOX_PATH = '' CHROMEDRIVER_PATH = '' +try: + from .keys.common import * +except ImportError: + print 'no real key file found, using dummy' + from .dummy.common import * try: from .keys.host import * From f75fd07c762315115ba6633bc6bd8a8061639d92 Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 7 Jun 2018 14:39:36 -0400 Subject: [PATCH 08/23] use hosts not common --- .travis.yml | 2 +- settings/travis.py | 8 ++++++++ test/{common.py.enc => travis-host.py.enc} | Bin 3 files changed, 9 insertions(+), 1 deletion(-) rename test/{common.py.enc => travis-host.py.enc} (100%) diff --git a/.travis.yml b/.travis.yml index 6990592c..715396c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ env: before_install: - sudo mkdir /var/log/django - sudo chmod 777 /var/log/django - - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/Gluejar/regluit/test/common.py.enc -out ~/build/Gluejar/regluit/settings/keys/common.py -d + - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/Gluejar/regluit/test/travis-host.py.enc -out ~/build/Gluejar/regluit/settings/keys/host.py -d install: - pip install -r requirements_versioned.pip diff --git a/settings/travis.py b/settings/travis.py index dec133c2..808fcbdf 100644 --- a/settings/travis.py +++ b/settings/travis.py @@ -53,3 +53,11 @@ BROKER_VHOST = "0" # Amazon S3 access AWS_STORAGE_BUCKET_NAME = 'unglueit-testfiles' + +SOCIAL_AUTH_TWITTER_KEY = '' +SOCIAL_AUTH_TWITTER_SECRET = '' +SOCIAL_AUTH_FACEBOOK_KEY = '' +SOCIAL_AUTH_FACEBOOK_SECRET = '' +SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = '' +SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = '' + diff --git a/test/common.py.enc b/test/travis-host.py.enc similarity index 100% rename from test/common.py.enc rename to test/travis-host.py.enc From 4afe792a735a405c5559250bfcb76f145ba3a72b Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 7 Jun 2018 15:08:23 -0400 Subject: [PATCH 09/23] set TEST_INTEGRATION --- settings/travis.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/settings/travis.py b/settings/travis.py index 808fcbdf..1ae01146 100644 --- a/settings/travis.py +++ b/settings/travis.py @@ -60,4 +60,6 @@ SOCIAL_AUTH_FACEBOOK_KEY = '' SOCIAL_AUTH_FACEBOOK_SECRET = '' SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = '' SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = '' +GOOGLE_BOOKS_API_KEY = '' +TEST_INTEGRATION = False From 8aae6808d114b9ea88ccc0ef2de6a09fc2743ce1 Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 7 Jun 2018 16:28:12 -0400 Subject: [PATCH 10/23] thought I already did this --- settings/travis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/settings/travis.py b/settings/travis.py index 1ae01146..57d889a5 100644 --- a/settings/travis.py +++ b/settings/travis.py @@ -62,4 +62,5 @@ SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = '' SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = '' GOOGLE_BOOKS_API_KEY = '' TEST_INTEGRATION = False +DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' From ca5e5bed0c21565abf2368198bb56c6a6e4cfb0d Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 13:55:05 -0400 Subject: [PATCH 11/23] update boto --- requirements_versioned.pip | 7 ++++--- settings/common.py | 2 +- settings/travis.py | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/requirements_versioned.pip b/requirements_versioned.pip index 0d8673ce..bb459b10 100644 --- a/requirements_versioned.pip +++ b/requirements_versioned.pip @@ -9,8 +9,9 @@ PyYAML==3.11 amqp==1.4.9 anyjson==0.3.3 billiard==3.3.0.23 -awscli==1.10.26 -boto==2.42.0 +awscli==1.15.33 +boto==2.48.0 +boto3==1.7.33 celery==3.1.23 certifi==2018.4.16 chardet==3.0.4 @@ -34,7 +35,7 @@ git+git://github.com/eshellman/django-notification.git@a4620e893e2da220994e0189b django-registration==2.1.2 django-selectable==0.9.0 django-smtp-ssl==1.0 -django-storages==1.4.1 +django-storages==1.5.2 django-tastypie==0.13.3 #django-transmeta==0.7.3 git+git://github.com/resulto/django-transmeta.git@ad4d7278ba330dcf8c8446f8ae9b2c769ae8684e diff --git a/settings/common.py b/settings/common.py index c300cdfe..e9d2016b 100644 --- a/settings/common.py +++ b/settings/common.py @@ -488,7 +488,7 @@ except ImportError: LOCAL_TEST = True if AWS_SECRET_ACCESS_KEY: - DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' else: DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' diff --git a/settings/travis.py b/settings/travis.py index 57d889a5..477b4b41 100644 --- a/settings/travis.py +++ b/settings/travis.py @@ -62,5 +62,6 @@ SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = '' SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = '' GOOGLE_BOOKS_API_KEY = '' TEST_INTEGRATION = False -DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' +LOCAL_TEST = True +DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' From ab3c1934d5708aa843495d0eb56ce32e2b141245 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 13:56:07 -0400 Subject: [PATCH 12/23] fix pdf tester boto3 storage returns unicode not str someplace --- core/pdf.py | 8 ++++++-- core/tests.py | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/core/pdf.py b/core/pdf.py index 5f0baf19..0931e46b 100644 --- a/core/pdf.py +++ b/core/pdf.py @@ -1,6 +1,7 @@ """ Utilities that manipulate pdf files """ +import logging import requests from xhtml2pdf import pisa # import python module from PyPDF2 import PdfFileMerger,PdfFileReader @@ -9,6 +10,7 @@ from tempfile import NamedTemporaryFile from django.template.loader import render_to_string from regluit import settings +logger = logging.getLogger(__name__) # Utility function def ask_pdf(context={}): @@ -35,7 +37,7 @@ def pdf_append( file1, file2, file_out ): def test_pdf(pdf_file): temp = None try: - if isinstance(pdf_file , str): + if isinstance(pdf_file , (str, unicode)): if pdf_file.startswith('http:') or pdf_file.startswith('https:'): temp = NamedTemporaryFile(delete=False) test_file_content = requests.get(pdf_file).content @@ -53,7 +55,9 @@ def test_pdf(pdf_file): except: success = False return success - except: + except Exception: + pdf_file = unicode(pdf_file) + logger.exception('error testing a pdf: %s' % pdf_file[:100]) return False def test_test_pdf(self): diff --git a/core/tests.py b/core/tests.py index ddd6311c..8532df09 100755 --- a/core/tests.py +++ b/core/tests.py @@ -1119,7 +1119,8 @@ class EbookFileTests(TestCase): #test the ask-appender c.add_ask_to_ebfs() if settings.AWS_SECRET_ACCESS_KEY: - assert test_pdf(c.work.ebookfiles().filter(asking=True)[0].file.url) + askingpdfurl = c.work.ebookfiles().filter(asking=True)[0].file.url + assert test_pdf(askingpdfurl) else: assert test_pdf(c.work.ebookfiles().filter(asking=True)[0].file) From 8b385d05ede4c1377a107faf81574bfe4c715fc6 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 15:01:22 -0400 Subject: [PATCH 13/23] fix Booxtreem LOCAL_TEST --- booxtream/__init__.py | 6 +++--- booxtream/tests.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/booxtream/__init__.py b/booxtream/__init__.py index 33b7f0cd..a87aeba8 100644 --- a/booxtream/__init__.py +++ b/booxtream/__init__.py @@ -64,9 +64,9 @@ class BooXtream(object): boox = Boox.objects.create( download_link_epub='https://github.com/eshellman/42_ebook/blob/master/download/42.epub?raw=true&extra=download.booxtream.com/', download_link_mobi='https://github.com/eshellman/42_ebook/blob/master/download/42.mobi?raw=true', - referenceid= kwargs.get('referenceid'), - downloads_remaining= kwargs.get('downloadlimit'), - expirydays=kwargs.get('expirydays'), + referenceid= kwargs.get('referenceid', '42'), + downloads_remaining= kwargs.get('downloadlimit', 10), + expirydays=kwargs.get('expirydays', 30), ) return boox diff --git a/booxtream/tests.py b/booxtream/tests.py index 359c9044..e3492546 100644 --- a/booxtream/tests.py +++ b/booxtream/tests.py @@ -23,13 +23,15 @@ class TestBooXtream(unittest.TestCase): return manager def test_booxtream_errors(self): + if settings.LOCAL_TEST: + return from .exceptions import BooXtreamError inst = self._makeOne() if not settings.BOOXTREAM_API_KEY: return with self.assertRaises(BooXtreamError) as cm: inst.platform() - self.assertIn( 'expirydays not set',str(cm.exception)) + self.assertIn('expirydays not set', str(cm.exception)) params={ 'customername': 'Jane Test', 'languagecode':'1043', From 455032ebc35b1064f41ab85802468130eed4be61 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 15:17:39 -0400 Subject: [PATCH 14/23] delint --- booxtream/__init__.py | 30 +++++++++++++++--------------- booxtream/exceptions.py | 6 +++--- booxtream/models.py | 15 +++++++-------- booxtream/tests.py | 20 ++++++++++---------- 4 files changed, 35 insertions(+), 36 deletions(-) diff --git a/booxtream/__init__.py b/booxtream/__init__.py index a87aeba8..4c3d9cca 100644 --- a/booxtream/__init__.py +++ b/booxtream/__init__.py @@ -1,10 +1,11 @@ -import requests import random -from django.conf import settings -from urllib import quote from functools import partial +from urllib import quote from xml.etree import ElementTree +import requests +from django.conf import settings + from django.apps import apps from . exceptions import BooXtreamError @@ -34,10 +35,10 @@ class BooXtream(object): if not apiuser: apiuser = settings.BOOXTREAM_API_USER self.endpoint = 'https://service.booxtream.com/' - self.postrequest = partial(requests.post, timeout=timeout, auth=(apiuser,apikey)) + self.postrequest = partial(requests.post, timeout=timeout, auth=(apiuser, apikey)) - def platform(self, epubfile=None, epub=True, kf8mobi=False, **kwargs): + def platform(self, epubfile=None, epub=True, kf8mobi=False, **kwargs): """ Make an API request to BooXtream ``self.apikey``, ``epubfile`` and the supplied ``kwargs``. Attempts to deserialize the XML response and return the download link. @@ -48,24 +49,24 @@ class BooXtream(object): Boox = apps.get_model('booxtream', 'Boox') url = self.endpoint + 'booxtream.xml' - kwargs['epub'] = '1' if epub else '0' + kwargs['epub'] = '1' if epub else '0' kwargs['kf8mobi'] = '1' if kf8mobi else '0' if epubfile: - if hasattr(epubfile,'name') and str(epubfile.name).endswith('.epub'): - files= {'epubfile': (str(epubfile.name),epubfile)} + if hasattr(epubfile, 'name') and str(epubfile.name).endswith('.epub'): + files = {'epubfile': (str(epubfile.name), epubfile)} else: # give it a random file name so that kindlegen doesn't choke # needed for in-memory (StringIO) epubs - files= {'epubfile': ('%012x.epub' % random.randrange(16**12),epubfile)} + files= {'epubfile': ('%012x.epub' % random.randrange(16**12), epubfile)} else: - files={} + files = {} if settings.LOCAL_TEST: # fake it, so you can test other functions without hitting booxtream boox = Boox.objects.create( download_link_epub='https://github.com/eshellman/42_ebook/blob/master/download/42.epub?raw=true&extra=download.booxtream.com/', download_link_mobi='https://github.com/eshellman/42_ebook/blob/master/download/42.mobi?raw=true', referenceid= kwargs.get('referenceid', '42'), - downloads_remaining= kwargs.get('downloadlimit', 10), + downloads_remaining=kwargs.get('downloadlimit', 10), expirydays=kwargs.get('expirydays', 30), ) return boox @@ -86,9 +87,8 @@ class BooXtream(object): boox = Boox.objects.create( download_link_epub=download_link_epub, download_link_mobi=download_link_mobi, - referenceid= kwargs.get('referenceid'), - downloads_remaining= kwargs.get('downloadlimit'), + referenceid=kwargs.get('referenceid'), + downloads_remaining=kwargs.get('downloadlimit'), expirydays=kwargs.get('expirydays'), ) - return boox - + return boox \ No newline at end of file diff --git a/booxtream/exceptions.py b/booxtream/exceptions.py index 202d1047..12fb6d1f 100644 --- a/booxtream/exceptions.py +++ b/booxtream/exceptions.py @@ -7,7 +7,7 @@ class BooXtreamError(Exception): self.errors = errors def __str__(self): - errormsg='BooXtream errors:' + errormsg = 'BooXtream errors:' for error in self.errors: - errormsg += 'Error %s: %s\n'% (error.find('Code').text,error.find('Msg').text) - return errormsg \ No newline at end of file + errormsg += 'Error %s: %s\n'% (error.find('Code').text, error.find('Msg').text) + return errormsg diff --git a/booxtream/models.py b/booxtream/models.py index 918e2bab..cbf2e606 100644 --- a/booxtream/models.py +++ b/booxtream/models.py @@ -6,22 +6,21 @@ class Boox(models.Model): """ keeps a record of a file that's been watermarked """ - download_link_epub = models.URLField(null=True) - download_link_mobi = models.URLField(null=True) + download_link_epub = models.URLField(null=True) + download_link_mobi = models.URLField(null=True) referenceid = models.CharField(max_length=32) - downloads_remaining = models.PositiveSmallIntegerField(default = 0) + downloads_remaining = models.PositiveSmallIntegerField(default=0) expirydays = models.PositiveSmallIntegerField() created = models.DateTimeField(auto_now_add=True) - + @property def expired(self): return self.created+timedelta(days=self.expirydays) < datetime.now() - + def download_link(self, format): if format == 'epub': return self.download_link_epub elif format == 'mobi': return self.download_link_mobi - else: - return None - + return None + diff --git a/booxtream/tests.py b/booxtream/tests.py index e3492546..c7cf266f 100644 --- a/booxtream/tests.py +++ b/booxtream/tests.py @@ -1,6 +1,6 @@ import unittest import time -import urllib2 +import urllib2 from tempfile import NamedTemporaryFile from StringIO import StringIO from django.conf import settings @@ -32,7 +32,7 @@ class TestBooXtream(unittest.TestCase): with self.assertRaises(BooXtreamError) as cm: inst.platform() self.assertIn('expirydays not set', str(cm.exception)) - params={ + params = { 'customername': 'Jane Test', 'languagecode':'1043', 'expirydays': 1, @@ -49,7 +49,7 @@ class TestBooXtream(unittest.TestCase): def test_booxtream_good(self): inst = self._makeOne() - params={ + params = { 'customeremailaddress':'jane@example.com', 'customername': 'Jane Test', 'languagecode':'1043', @@ -59,16 +59,16 @@ class TestBooXtream(unittest.TestCase): 'chapterfooter':1, 'disclaimer':1, } - params['referenceid']= 'order'+str(time.time()) - boox=inst.platform(epubfile=self.epub2file, **params) - self.assertRegexpMatches(boox.download_link_epub,'download.booxtream.com/') + params['referenceid'] = 'order' + str(time.time()) + boox = inst.platform(epubfile=self.epub2file, **params) + self.assertRegexpMatches(boox.download_link_epub, 'download.booxtream.com/') self.assertFalse(boox.expired) - self.assertEqual(boox.downloads_remaining,3) - + self.assertEqual(boox.downloads_remaining, 3) + # make sure it works with an in-memory file self.epub2file.seek(0) in_mem_epub = StringIO() in_mem_epub.write(self.epub2file.read()) in_mem_epub.seek(0) - boox2=inst.platform(epubfile=in_mem_epub, **params) - self.assertRegexpMatches(boox2.download_link_epub,'download.booxtream.com/') + boox2 = inst.platform(epubfile=in_mem_epub, **params) + self.assertRegexpMatches(boox2.download_link_epub, 'download.booxtream.com/') From cac2866f30d4d38d459f76a437b30b4fcc09cb6c Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 16:39:03 -0400 Subject: [PATCH 15/23] emergency update --- requirements_versioned.pip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_versioned.pip b/requirements_versioned.pip index bb459b10..586333ce 100644 --- a/requirements_versioned.pip +++ b/requirements_versioned.pip @@ -89,7 +89,7 @@ xhtml2pdf==0.2.2 webencodings==0.5.1 #for urllib3 secure cffi==1.7.0 -cryptography==1.4 +cryptography==2.2.2 enum34==1.1.6 idna==2.6 ipaddress==1.0.16 From 55186ec7433eba5ea3408b5da86f63344b5afce8 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 17:25:33 -0400 Subject: [PATCH 16/23] more fixing --- requirements_versioned.pip | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements_versioned.pip b/requirements_versioned.pip index 586333ce..4d348e7d 100644 --- a/requirements_versioned.pip +++ b/requirements_versioned.pip @@ -94,8 +94,8 @@ enum34==1.1.6 idna==2.6 ipaddress==1.0.16 ndg-httpsclient==0.4.2 -pyOpenSSL==16.0.0 -pyasn1==0.1.9 +pyOpenSSL==18.0.0 +pyasn1==0.24.0 pycparser==2.14 setuptools==25.0.0 urllib3==1.22 From 8d6c23e1675947872dc3cc686e618451b1f30137 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 17:31:25 -0400 Subject: [PATCH 17/23] revert --- requirements_versioned.pip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_versioned.pip b/requirements_versioned.pip index 4d348e7d..b668a74d 100644 --- a/requirements_versioned.pip +++ b/requirements_versioned.pip @@ -95,7 +95,7 @@ idna==2.6 ipaddress==1.0.16 ndg-httpsclient==0.4.2 pyOpenSSL==18.0.0 -pyasn1==0.24.0 +pyasn1==0.1.9 pycparser==2.14 setuptools==25.0.0 urllib3==1.22 From 23944c41daeb289f8ff33162a49108a314c3a100 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 17:51:27 -0400 Subject: [PATCH 18/23] causes checkout errors --- settings/keys/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 settings/keys/__init__.py diff --git a/settings/keys/__init__.py b/settings/keys/__init__.py deleted file mode 100644 index e69de29b..00000000 From d5933601afcb66caf7bd0f68fab990f623542ec2 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 17:55:40 -0400 Subject: [PATCH 19/23] another permission issue --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index df1ad729..0eba6b4a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,7 @@ *.pyc *.log settings/me.* -settings/keys/c* -settings/keys/h* +settings/keys/ *.dot reports ENV From 8b709d955bda46ca5c1fa9e453f77f49e2bb384c Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 18:47:09 -0400 Subject: [PATCH 20/23] remove settings/keys --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index df1ad729..4a693c5d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,7 @@ *.pyc *.log settings/me.* -settings/keys/c* -settings/keys/h* +settings/keys/* *.dot reports ENV From 039e2e1a05acfc96af25b8c7b2f41128d5d408c5 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 19:01:42 -0400 Subject: [PATCH 21/23] add keys directory --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 715396c0..9527e3a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,9 @@ env: before_install: - sudo mkdir /var/log/django - sudo chmod 777 /var/log/django + - sudo mkdir ~/build/Gluejar/regluit/settings/keys/ + - sudo chmod 755 ~/build/Gluejar/regluit/settings/keys/ + - sudo cp ~/build/Gluejar/regluit/settings/dummy/__init__.py ~/build/Gluejar/regluit/settings/keys/__init__.py - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/Gluejar/regluit/test/travis-host.py.enc -out ~/build/Gluejar/regluit/settings/keys/host.py -d install: From 15fc49d828238dd834c058c14aa8213f1ade00a3 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 19:12:27 -0400 Subject: [PATCH 22/23] maybe no sudo --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9527e3a9..8f981b21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,9 +15,8 @@ env: before_install: - sudo mkdir /var/log/django - sudo chmod 777 /var/log/django - - sudo mkdir ~/build/Gluejar/regluit/settings/keys/ - - sudo chmod 755 ~/build/Gluejar/regluit/settings/keys/ - - sudo cp ~/build/Gluejar/regluit/settings/dummy/__init__.py ~/build/Gluejar/regluit/settings/keys/__init__.py + - mkdir ~/build/Gluejar/regluit/settings/keys/ + - cp ~/build/Gluejar/regluit/settings/dummy/__init__.py ~/build/Gluejar/regluit/settings/keys/__init__.py - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/Gluejar/regluit/test/travis-host.py.enc -out ~/build/Gluejar/regluit/settings/keys/host.py -d install: From ee954b7f47cc904ef4f66486b4d967fcfe851dd4 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 19:38:43 -0400 Subject: [PATCH 23/23] retire jenkins --- deploy/update-just | 24 ----------------- deploy/update-regluit | 22 ---------------- settings/jenkins.py | 56 ---------------------------------------- sysadmin/playbooks/hosts | 1 - vagrant/Vagrantfile | 4 +-- vagrant/dev.yml | 32 ----------------------- vagrant/just.yml | 31 +--------------------- vagrant/please.yml | 6 ----- 8 files changed, 3 insertions(+), 173 deletions(-) delete mode 100755 deploy/update-just delete mode 100755 deploy/update-regluit delete mode 100644 settings/jenkins.py diff --git a/deploy/update-just b/deploy/update-just deleted file mode 100755 index bc97a438..00000000 --- a/deploy/update-just +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# this script is used by jenkins to remotely update a regluit instance -# for it to work the jenkins user's public ssh key needs to be in the -# authorized key for the machine running the regluit instance -# you can then put something like this in a post build configuration -# ssh ubuntu@please.unglueit.com "/opt/regluit/deploy/update-regluit" - -cd /opt/regluit -find . -name "*.pyc" -delete -find . -type d -empty -delete - -sudo -u ubuntu /usr/bin/git pull -source ENV/bin/activate -pip install --upgrade -r requirements_versioned.pip -#django-admin.py syncdb --migrate --settings regluit.settings.just -django-admin.py migrate --fake-initial --noinput --settings regluit.settings.just -django-admin.py collectstatic --noinput --settings regluit.settings.just - -sudo /etc/init.d/apache2 restart -django-admin.py celeryd_multi restart w1 --settings=regluit.settings.just; -/etc/init.d/celerybeat restart -crontab deploy/crontab_just.txt -touch /opt/regluit/deploy/last-update diff --git a/deploy/update-regluit b/deploy/update-regluit deleted file mode 100755 index 6f12c8a7..00000000 --- a/deploy/update-regluit +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# this script is used by jenkins to remotely update a regluit instance -# for it to work the jenkins user's public ssh key needs to be in the -# authorized key for the machine running the regluit instance -# you can then put something like this in a post build configuration -# ssh ubuntu@please.unglueit.com "/opt/regluit/deploy/update-regluit" - -cd /opt/regluit -find . -name "*.pyc" -delete -find . -type d -empty -delete - -sudo -u ubuntu /usr/bin/git pull -source ENV/bin/activate -#pip install -r requirements.pip -django-admin.py syncdb --migrate --settings regluit.settings.please -django-admin.py collectstatic --noinput --settings regluit.settings.please -sudo /etc/init.d/apache2 restart -django-admin.py celeryd_multi restart w1 --settings=regluit.settings.please; -/etc/init.d/celerybeat restart -crontab deploy/crontab_please.txt -touch /opt/regluit/deploy/last-update \ No newline at end of file diff --git a/settings/jenkins.py b/settings/jenkins.py deleted file mode 100644 index b0282f70..00000000 --- a/settings/jenkins.py +++ /dev/null @@ -1,56 +0,0 @@ -# coding=utf-8 -from .common import * - -DEBUG = True -TEMPLATES[0]['OPTIONS']['debug'] = DEBUG -IS_PREVIEW = False - -ADMINS = ( - ('Raymond Yee', 'rdhyee+ungluebugs@gluejar.com'), - ('Eric Hellman', 'eric@gluejar.com'), -) - -MANAGERS = ADMINS - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': 'regluit', - 'USER': 'regluit', - 'PASSWORD': 'regluit', - 'HOST': '', - 'PORT': '', - 'TEST': { - 'CHARSET': 'utf8', - } - } -} - -TIME_ZONE = 'America/New_York' - -# settings for outbout email -# if you have a gmail account you can use your email address and password - -EMAIL_USE_TLS = True -EMAIL_HOST = 'smtp.gmail.com' -EMAIL_HOST_USER = 'me@gmail.com' -EMAIL_HOST_PASSWORD = 'my-password' -EMAIL_PORT = 587 -DEFAULT_FROM_EMAIL = 'info@ebookfoundation.org' - - -# formerly of settings/common.py to surface old vars -# TO DO: invalidate before we open source - -# for use with test google account only -GOOGLE_DISPLAY_NAME = 'Unglue.It' -REDIRECT_IS_HTTPS = False - -#BASE_URL = 'http://0.0.0.0/' -BASE_URL_SECURE = 'http://0.0.0.0/' - - - -# use database as queuing service in development -BROKER_TRANSPORT = "djkombu.transport.DatabaseTransport" -#INSTALLED_APPS += ("djkombu",) diff --git a/sysadmin/playbooks/hosts b/sysadmin/playbooks/hosts index 5140f856..10a3fc6b 100644 --- a/sysadmin/playbooks/hosts +++ b/sysadmin/playbooks/hosts @@ -2,5 +2,4 @@ just ansible_ssh_port=22 ansible_ssh_host=just.unglue.it ansible_ssh_user=ubuntu web1 ansible_ssh_port=22 ansible_ssh_host=unglue.it ansible_ssh_user=ubuntu gluejar ansible_ssh_port=22 ansible_ssh_host=gluejar.com ansible_ssh_user=ubuntu -jenkins ansible_ssh_port=22 ansible_ssh_host=jenkins.unglueit.com ansible_ssh_user=ubuntu diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index eebb9792..0c14e23b 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -114,7 +114,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/' ansible.raw_arguments = [ "--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory", - "--limit=just,127.0.0.1,jenkins", + "--limit=just,127.0.0.1", "-e vname=just", "-e class=just", "-e hostname=just.unglue.it", @@ -186,7 +186,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/' ansible.raw_arguments = [ "--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory", - "--limit=just2,127.0.0.1,jenkins", + "--limit=just2,127.0.0.1", "-e vname=just2", "-e class=just", "-e hostname=just2.unglue.it", diff --git a/vagrant/dev.yml b/vagrant/dev.yml index 65b6ee7f..7cc3d207 100644 --- a/vagrant/dev.yml +++ b/vagrant/dev.yml @@ -499,13 +499,6 @@ - https://github.com/eshellman.keys sudo: no - - name: add public key from jenkins - authorized_key: > - user={{user}} - key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r" - state=present - when: class in ['just'] - - name: set up script file to load environment for interactive use command: cp "/opt/regluit/deploy/setup-{{class}}.sh" /home/{{user}}/setup.sh sudo: no @@ -549,29 +542,4 @@ service: name=apache2 state=restarted -- name: fix known_hosts on jenkins to match new just - hosts: jenkins - sudo: yes - sudo_user: jenkins -# to run the part of the playbook for jenkins -# PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=true ANSIBLE_HOST_KEY_CHECKING=false ANSIBLE_SSH_ARGS='-o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s' ansible-playbook --private-key=/Users/raymondyee/.ssh/id_rsa --user=ubuntu --connection=ssh --inventory-file=/Users/raymondyee/C/src/Gluejar/regluit/vagrant/.vagrant/provisioners/ansible/inventory --limit='jenkins' just.yml - - tasks: - - #equivalent to - # - #ssh -tt jenkins << EOF - # sudo -i -u jenkins - # ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts - # exit - #exit - #EOF - - - name: make new known_hosts with key from just.unglue.it - raw: ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts - when: class in ['just'] - - - name: add key from github - raw: ssh-keyscan -t rsa github.com >> /var/lib/jenkins/.ssh/known_hosts - when: class in ['just'] diff --git a/vagrant/just.yml b/vagrant/just.yml index f657789b..a158e5a6 100644 --- a/vagrant/just.yml +++ b/vagrant/just.yml @@ -384,41 +384,12 @@ - https://github.com/eshellman.keys sudo: yes - - name: add public key from jenkins - authorized_key: > - user={{user}} - key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r" - state=present - sudo: no + handlers: - name: restart apache2 service: name=apache2 state=restarted -- name: fix known_hosts on jenkins to match new just - hosts: jenkins - sudo: yes - sudo_user: jenkins - -# to run the part of the playbook for jenkins -# PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=true ANSIBLE_HOST_KEY_CHECKING=false ANSIBLE_SSH_ARGS='-o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s' ansible-playbook --private-key=/Users/raymondyee/.ssh/id_rsa --user=ubuntu --connection=ssh --inventory-file=/Users/raymondyee/C/src/Gluejar/regluit/vagrant/.vagrant/provisioners/ansible/inventory --limit='jenkins' just.yml - - tasks: - - #equivalent to - # - #ssh -tt jenkins << EOF - # sudo -i -u jenkins - # ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts - # exit - #exit - #EOF - - - name: make new known_hosts with key from just.unglue.it - raw: ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts - - - name: add key from github - raw: ssh-keyscan -t rsa github.com >> /var/lib/jenkins/.ssh/known_hosts diff --git a/vagrant/please.yml b/vagrant/please.yml index 6f0c0e36..35443e3e 100644 --- a/vagrant/please.yml +++ b/vagrant/please.yml @@ -412,12 +412,6 @@ - https://github.com/eshellman.keys sudo: yes - - name: add public key from jenkins - authorized_key: > - user={{user}} - key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r" - state=present - handlers: - name: restart apache2