From e1553f812185afed1c262c141321fd5f2d301c27 Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 6 Jun 2018 17:29:12 -0400 Subject: [PATCH 01/40] switch to dockerized mobi --- core/mobi.py | 4 +- core/models/__init__.py | 7 +- requirements_versioned.pip | 9 +-- vagrant/group_vars/all/secrets.yml | 100 ++++++++++++++--------------- 4 files changed, 63 insertions(+), 57 deletions(-) diff --git a/core/mobi.py b/core/mobi.py index 87522369..07bbbc9d 100644 --- a/core/mobi.py +++ b/core/mobi.py @@ -16,11 +16,11 @@ def convert_to_mobi(input_url, input_format="application/epub+zip"): # using verify=False since at the moment, using a self-signed SSL cert. - payload = requests.get(input_url, verify=False).content + payload = requests.get(input_url).content headers = {'Content-Type': input_format} r = requests.post(mobigen_url, auth=(mobigen_user_id, mobigen_password), - data=payload, verify=False, headers=headers) + data=payload, headers=headers) # if HTTP reponse code is ok, the output is the mobi file; else error message if r.status_code == 200: diff --git a/core/models/__init__.py b/core/models/__init__.py index 6bcc8f3b..7136fd16 100755 --- a/core/models/__init__.py +++ b/core/models/__init__.py @@ -966,7 +966,12 @@ class Campaign(models.Model): # now make the mobi file new_mobi_ebf = EbookFile.objects.create(edition=edition, format='mobi', asking=True) - new_mobi_ebf.file.save(path_for_file('ebf', None), ContentFile(mobi.convert_to_mobi(new_epub_ebf.file.url))) + try: + new_mobi_file = ContentFile(mobi.convert_to_mobi(new_epub_ebf.file.url)) + except Exception as e: + logger.error("error making mobi for %s" % (new_epub_ebf.file.url)) + raise e + new_mobi_ebf.file.save(path_for_file('ebf', None), new_mobi_file) new_mobi_ebf.save() new_mobi_ebf.version = version new_ebfs.append(new_mobi_ebf) diff --git a/requirements_versioned.pip b/requirements_versioned.pip index b326b50a..0d8673ce 100644 --- a/requirements_versioned.pip +++ b/requirements_versioned.pip @@ -12,7 +12,8 @@ billiard==3.3.0.23 awscli==1.10.26 boto==2.42.0 celery==3.1.23 -certifi==2016.2.28 +certifi==2018.4.16 +chardet==3.0.4 # pip installing pillow seems to delete distribute # but having distribute in requirements starting to cause problems # distribute==0.6.28 @@ -70,7 +71,7 @@ rdflib==4.2.0 rdflib-jsonld==0.3 redis==2.10.3 reportlab==3.4.0 -requests==2.10.0 +requests==2.18.4 requests-mock==1.2.0 requests-oauthlib==0.6.2 selenium==2.53.1 @@ -89,14 +90,14 @@ webencodings==0.5.1 cffi==1.7.0 cryptography==1.4 enum34==1.1.6 -idna==2.1 +idna==2.6 ipaddress==1.0.16 ndg-httpsclient==0.4.2 pyOpenSSL==16.0.0 pyasn1==0.1.9 pycparser==2.14 setuptools==25.0.0 -urllib3==1.16 +urllib3==1.22 beautifulsoup4==4.6.0 RISparser==0.4.2 # include these 2 for development diff --git a/vagrant/group_vars/all/secrets.yml b/vagrant/group_vars/all/secrets.yml index d4a96e78..cfb9284f 100644 --- a/vagrant/group_vars/all/secrets.yml +++ b/vagrant/group_vars/all/secrets.yml @@ -1,51 +1,51 @@ $ANSIBLE_VAULT;1.1;AES256 -63376563643062623466613335633730323031653736643437643030303533383931666434323939 -3266633436313635643463333266356238316638326131610a653336303661326334323561623239 -65623834633736663861343464333764303865663934303533333031343934333635616333643631 -3933383131353062630a633133353430386437626130656632393230636664376438613862303466 -66343337303961303361353030313936626363383465316166316663643830313164646661363366 -66656630323437643031363237633439366434666537396464336364373134383061623364636635 -31393435326337373162333365386465633730323135383364383433366664353938303962313461 -61633835393963646666376562633731626639313438326235306335653639666635386566343033 -36383064653634613065303538346638363666326336643631616636623632376662323037633165 -32633662636531643366343963663439313439636335336435616666633863323433643930666536 -30623234313838616461383534663230616364336131383763316235356265363964636539316637 -61376433306366346362303439303665333961393239373263373365306363336334623536353031 -61636339353561303532383833336164346538396139373835633162323339333732373431613035 -62333739373832623762356364383035343937656337666439323830663235666637326134346531 -66386135323064333732663136396431616563653731343134363262373639333836623033656536 -38653931346337666566336331363631666238666638353237663065653164343936636239343631 -62636339396332663637643239376335343731643764343666356361623239383462313936366363 -32616638653666666638316563656366666361353538303133656433376637663532313666343631 -63633436336137653339653364616532333434623036396439343433613361626231613331373931 -64343232313839383332353036633633383762316531383936393539316531316336636537666439 -33613166613733346336643932363263663833366533623965623039313036373230396135343663 -66653336313437336561333630393365653133636263333031356562623332623733616464333262 -35333535396337616461353361616337613138363239656162323232346436336233633335376666 -63626661373639306665613431393735343937616539336233666235343936653932366337326663 -33353930653965623661383265643837306637363231656435663030393562623438623361353666 -36626635323837363733353365373738646230303436343834636666373165653937333130343637 -31666566643434653663626531383730666238306438656133303862626538613339626437633432 -64393432333834623339393631363932353862646332313062636565326135636138333931333431 -64633937356630326666373939343635323263373934306330393032316630646333363239313137 -65383666663063356430663965336463393537363266636431633362343263666234323463653532 -30376262656335653662656366636336336135626130303835326432366438393962616266373435 -39613234326136623263356131376239313136653131326632343739313137653833316434656265 -30323863383639363462613034353062393964333130656161306633306566646233356537313162 -65376237303961333239303630313831303232643831383036643166343164363934323138353234 -33623130383339343162636239343234393731626530326164643438633239343738306236623433 -37636662353263373866656564316334303736336566336636653937663566343261333335333437 -62313833353235393131326430323263383861366230663639363661383362363438373432346663 -62643166363633633134303836383239386236386361376533353330333930633530333139666462 -63623361356134333337643634393461343635363232313435656137303731326461653330613831 -66613536666635313539643130373132623965333863343930363761613031306664333732393564 -34333937326234383833306233386638323263323430613863346436643030353732356261643834 -62343230633034363839373965646133353238346261373538313833323264616531313965376537 -32646166326135306134356262353461626337663137323335613339646532636663326462393936 -30316265636335336138383930313138306366383766653866613532626366373463613436303638 -31333738306164613031626435373966323361626564666537396539313832613038363865366534 -62656565316466393366303137646431303231626534613031316561336233353137353265623032 -32616134366439363630376666343937633462613731613961343862303639373135373466363663 -34663235333431323764613330303539323164313332313331613761306439623232396436323138 -35623265613335636430323335313230373865633331303633613632306431323339396165626135 -3662393265643861393433333230353233366330646435343361 +64636439626632623833636366633136653733373531316266643663336632613035303563336664 +3163633033613732656536313030613263636331343139630a373432356362376136303463643637 +63393038386266313536356261633133353437363366386161616233353336336237373866633865 +6333613461666564390a346661343365663662613762643061343636383866623835316333663930 +35303839633239326666376437393664613039393235383838613439653239353831313238656530 +66313261313763666562343833663963613037636466623863383636343865626261386533383165 +35633331643361393265656333633330633064306161313566373131376235646531653063643565 +34313032643166636337313866383064383436653466303930663435393336333965333334303963 +39333232376437353765623832346366383833313962373632626232643636326436373233643237 +35353066646130363665313765386631646363663161383461626332616666396139656463376134 +39326234333138303130623963396430313037643263623037633139316362333332313933313035 +65656239333131346461383066656263393365326561316431613266303035636438366232373665 +30313265306332626566313165336437633035343063623630336139313166366136353666623934 +64616339653930333630316236363630356530663465613266643632313431376236356164663937 +62336361613739373030353761336366353764373231303433376435626334623637636637623832 +33373333623666306363646366393663353465396638623134313537303830333639333137343662 +65643865616139356161333239626165303165313730306434326431363032656337316337353739 +38653661316631643930343863326535313336323937633833323565326539376135376339323232 +30616236613034643935653866333465663837373861656536313333346637386632353139666539 +64643966313964393134396630343033633233353534353065363635383636363139303637366338 +64353835386537376363623062313862383637383430336464623566616336373836663835626331 +37633562303933646263646239306437653135633565303530333761326566336565613366303630 +32346436616135326262333930383130323562303465643065663562343832323364303737613764 +30623939303466306636376235613639363933343065376562353030326335663963353766313263 +34346266616239373264646439616536353762656536643738623662396231323334313261316331 +66356566626463363033316162626265353765653132346534663430653135353236333866653031 +37663636373133353465363635386561626666623532343832366536616634313065383531353538 +33633539373366396566333333653939313864663235663165363563326538656335306262393561 +32383831656638666464663462373462663662323833633735333764303363656161616235323464 +38353461306430363566633030386535656264616236393737343662613336353665666564343462 +34383365306166353236346533653333643465353061396135636362336239353231353463633361 +61623965643634396366656265303961623531306163373230626639303132306630356537623934 +63623732663562663231656439623266623034643336643839396534323363306364643534636233 +39353431323166326237663362353137633631333733303231386236356263613163656238326461 +63353162316366343465363662383631333761366262336235383862633630386362363135636232 +35386234646662376434646161633463353163313261333333636162303461623233626637363039 +39356263343632356333666633636437616431393062326230616132336462303231616635326237 +35616231643633386430363335383333386339353833323064393039303839653131316439623036 +37303236303535656466643037326337353766653666633933313330343930636532333666336638 +33376437383064646633366637623638623239663261376639343332363430656232373162356336 +65643331643036316136313737383934633731306466333639626230633465653338393932666636 +35613331323431386438356439613034363364643663623332303435363861353538623238393835 +36376432623361386237666233356638376537393536313335633361353566323963303864343461 +39386464613134666437393734376662386236303837626531613363373161343937333863656232 +63333766636465653666663261353939393363306665353832633938376362613061386665336665 +34373939623633666236326130643662643163666233643936636461613963643836373639336230 +65306337303261376232363239636634386638323238313566326234663331623535353433633431 +37313731663134323839336635366366336462363563373639646161376162313231396365323430 +64383739346161343030343635393935643931323237383030613662636630613935333061613961 +3665636339663830306630376237653762643333346563313939 From c422965bdb3eb43bf572eef2f834e5283af89325 Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 6 Jun 2018 22:26:27 -0400 Subject: [PATCH 02/40] add column to EbookFile to indicate a failed make_mobi so we don't keep trying with a bad epub file --- .../management/commands/make_missing_mobis.py | 2 +- core/migrations/0013_ebookfile_mobied.py | 19 +++++++++++++++ core/models/bibmodels.py | 7 ++++++ core/tests.py | 23 +++++++++++++++++++ 4 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 core/migrations/0013_ebookfile_mobied.py diff --git a/core/management/commands/make_missing_mobis.py b/core/management/commands/make_missing_mobis.py index ee06fe3a..af2d5461 100644 --- a/core/management/commands/make_missing_mobis.py +++ b/core/management/commands/make_missing_mobis.py @@ -21,7 +21,7 @@ class Command(BaseCommand): if not work.ebooks().filter(format="mobi"): for ebook in work.ebooks().filter(format="epub"): ebf = ebook.get_archive_ebf() - if ebf: + if ebf and ebf.mobied >= 0: try: print u'making mobi for {}'.format(work.title) if ebf.make_mobi(): diff --git a/core/migrations/0013_ebookfile_mobied.py b/core/migrations/0013_ebookfile_mobied.py new file mode 100644 index 00000000..18775c0d --- /dev/null +++ b/core/migrations/0013_ebookfile_mobied.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0012_campaign_charitable'), + ] + + operations = [ + migrations.AddField( + model_name='ebookfile', + name='mobied', + field=models.IntegerField(default=0), + ), + ] diff --git a/core/models/bibmodels.py b/core/models/bibmodels.py index 30d17e29..44c59b9c 100644 --- a/core/models/bibmodels.py +++ b/core/models/bibmodels.py @@ -1056,6 +1056,7 @@ class EbookFile(models.Model): asking = models.BooleanField(default=False) ebook = models.ForeignKey('Ebook', related_name='ebook_files', null=True) source = models.URLField(null=True, blank=True) + mobied = models.IntegerField(default=0) #-1 indicates a failed conversion attempt version = None def check_file(self): if self.format == 'epub': @@ -1072,9 +1073,13 @@ class EbookFile(models.Model): def make_mobi(self): if not self.format == 'epub' or not settings.MOBIGEN_URL: return False + if self.mobied < 0: + return False try: mobi_cf = ContentFile(mobi.convert_to_mobi(self.file.url)) except: + self.mobied = -1 + self.save() return False new_mobi_ebf = EbookFile.objects.create( edition=self.edition, @@ -1097,6 +1102,8 @@ class EbookFile(models.Model): ) new_mobi_ebf.ebook = new_ebook new_mobi_ebf.save() + self.mobied = 1 + self.save() return True send_to_kindle_limit = 7492232 diff --git a/core/tests.py b/core/tests.py index 949601f6..ddd6311c 100755 --- a/core/tests.py +++ b/core/tests.py @@ -1157,6 +1157,29 @@ class EbookFileTests(TestCase): self.assertTrue(c.work.ebookfiles().filter(asking=True, ebook__active=True).count() == 0) self.assertTrue(c.work.ebookfiles().filter(asking=False, ebook__active=True).count() > 0) + def test_bad_ebookfile(self): + w = Work.objects.create(title="Work 3") + e = Edition.objects.create(title=w.title, work=w) + + temp = NamedTemporaryFile(delete=False) + test_file_content = "bad text file" + temp.write(test_file_content) + temp.close() + + try: + # put the bad file into Django storage + temp_file = open(temp.name) + dj_file = DjangoFile(temp_file) + ebf = EbookFile(format='epub', edition=e, file=dj_file) + ebf.save() + temp_file.close() + ebf.make_mobi() + finally: + # make sure we get rid of temp file + os.remove(temp.name) + self.assertTrue(ebf.mobied < 0) + + class MobigenTests(TestCase): def test_convert_to_mobi(self): """ From 3e782e636dd69a48c9f7cba10c1b7cd18d0e8b9a Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 6 Jun 2018 23:27:08 -0400 Subject: [PATCH 03/40] files for running tests with travis --- .gitignore | 3 ++- .travis.yml | 23 +++++++++++++++++++ settings/travis.py | 55 +++++++++++++++++++++++++++++++++++++++++++++ test/common.py.enc | Bin 0 -> 544 bytes 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 .travis.yml create mode 100644 settings/travis.py create mode 100644 test/common.py.enc diff --git a/.gitignore b/.gitignore index 4a693c5d..df1ad729 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,8 @@ *.pyc *.log settings/me.* -settings/keys/* +settings/keys/c* +settings/keys/h* *.dot reports ENV diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..25378555 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,23 @@ +language: python + +python: + - '2.7' + +services: + - redis-server + - mysql + +env: + global: + - DJANGO_SETTINGS_MODULE=regluit.settings.travis + - PYTHONPATH=/home/travis/build/EbookFoundation/ + +before_install: + - sudo mkdir /var/log/django + - sudo chmod 777 /var/log/django + - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/EbookFoundation/regluit/test/common.py.enc -out ~/build/EbookFoundation/regluit/settings/keys/common.py -d + +install: + - pip install -r requirements_versioned.pip + +script: django-admin test diff --git a/settings/travis.py b/settings/travis.py new file mode 100644 index 00000000..dec133c2 --- /dev/null +++ b/settings/travis.py @@ -0,0 +1,55 @@ +# coding=utf-8 +from .common import * + +DEBUG = True +TEMPLATES[0]['OPTIONS']['debug'] = DEBUG +IS_PREVIEW = False + +ADMINS = ( + ('Raymond Yee', 'rdhyee+ungluebugs@gluejar.com'), + ('Eric Hellman', 'eric@gluejar.com'), +) + +MANAGERS = ADMINS + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': 'regluit', + 'USER': 'root', + 'PASSWORD': '', + 'HOST': '', + 'PORT': '', + 'TEST_CHARSET': 'utf8', + } +} + +TIME_ZONE = 'America/New_York' + +# settings for outbout email +# if you have a gmail account you can use your email address and password + +EMAIL_BACKEND = 'django_smtp_ssl.SSLEmailBackend' +MAIL_USE_TLS = True +EMAIL_HOST = 'email-smtp.us-east-1.amazonaws.com' +EMAIL_PORT = 465 +DEFAULT_FROM_EMAIL = 'notices@gluejar.com' + + +# formerly of settings/common.py to surface old vars +# TO DO: invalidate before we open source + +# for use with test google account only +GOOGLE_DISPLAY_NAME = 'Unglue.It' +REDIRECT_IS_HTTPS = False + +#BASE_URL = 'http://0.0.0.0/' +BASE_URL_SECURE = 'http://0.0.0.0/' + +BROKER_TRANSPORT = "redis" +BROKER_HOST = "localhost" +BROKER_PORT = 6379 +BROKER_VHOST = "0" + +# Amazon S3 access +AWS_STORAGE_BUCKET_NAME = 'unglueit-testfiles' diff --git a/test/common.py.enc b/test/common.py.enc new file mode 100644 index 0000000000000000000000000000000000000000..9c6f2c2b6a76f76c332b9b737b5f7f387e3424da GIT binary patch literal 544 zcmV+*0^j}km3t@2?&*v5@bBcbz3nIGX1;&`bhq`x+1#;5hb57Wfl)+c!x|7;B!hF7LbqC^K9@Ls&Y#isMsZ$ zI`rZph#`#)RU}$e7j|`;Ja+XiaCpDatLy*fvxYJvXgPacMoZ$0{tW0K78&mJu@4&x z;}$JbrFfGfKH5Ok!NH2f`@>7%#6k>4{{4AE8QtQDZ!sZ*0&c5fZe5WyQ}|->qpxHP z3O;Ke!8zLc09Q?yf}`TZ%o;I%ia;`jq~5Q!!7iRzptF&H-U$L`vKd_pA#`Z@a>NgAVqU$pcBJ7dMHf;T^J7{phl118+UM~vox zND|-PR0pFx6}yjdC53S*b%499-J_r|ls+w)em{%?b Date: Wed, 6 Jun 2018 23:34:06 -0400 Subject: [PATCH 04/40] user is specific to org --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 25378555..6990592c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,12 +10,12 @@ services: env: global: - DJANGO_SETTINGS_MODULE=regluit.settings.travis - - PYTHONPATH=/home/travis/build/EbookFoundation/ + - PYTHONPATH=/home/travis/build/Gluejar/ before_install: - sudo mkdir /var/log/django - sudo chmod 777 /var/log/django - - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/EbookFoundation/regluit/test/common.py.enc -out ~/build/EbookFoundation/regluit/settings/keys/common.py -d + - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/Gluejar/regluit/test/common.py.enc -out ~/build/Gluejar/regluit/settings/keys/common.py -d install: - pip install -r requirements_versioned.pip From 0deb76b678f1aa578f3a8a38edb80dc9477d27f4 Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 7 Jun 2018 09:59:35 -0400 Subject: [PATCH 05/40] make keys a module --- settings/keys/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 settings/keys/__init__.py diff --git a/settings/keys/__init__.py b/settings/keys/__init__.py new file mode 100644 index 00000000..e69de29b From 3028c5f19db5b81b89e017ec2a0c42116da70850 Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 7 Jun 2018 12:17:46 -0400 Subject: [PATCH 06/40] re-decrypt --- test/common.py.enc | Bin 544 -> 544 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/test/common.py.enc b/test/common.py.enc index 9c6f2c2b6a76f76c332b9b737b5f7f387e3424da..9d025ae9bb61ab871fbde619567bc1ce99923aee 100644 GIT binary patch literal 544 zcmV+*0^j|NoHsq>>v6Q!62Df@h=cs2Le&M6z3z|~(u`*aU_RbyKtTxsQB!)(1{lwvz!@!`{oE|n@w%OZ z;hg*1vtNMMom8S3MeW5&!$NyTzof%sKKW{_2^7~_fc#?EIV(%lzA1dMph&{-W?qg= zGQJHGgA!F*bBAq09;<-YO7E@o3HdI zjzH)thhvdyIZ`uYF-0@vtR85sTm>ajDeNF|LZ1dn*f8jMmbDd)Hw!ey+W@C|XAU0i zx=#-j=gj6#l$PkA=$9m(%=JVN8(GtbFdIj(VTJU!>TmKhygNxQ7_2nQt)`P4KjwP6 zx1sHAhq~hO({uI;nzVhkSPag}vV;B|`n4Cg`lCGuA4@T|MC@@tLzCWr$-8jH3832{ zDW8JTQdys$W%+XAWge#sOaVH4K|x$D9s?Y);7jUjy%5$1L{{dEw#0Bs zCBu57h=t|OH)MIjy=KoM(Weq`x+1#;5hb57Wfl)+c!x|7;B!hF7LbqC^K9@Ls&Y#isMsZ$ zI`rZph#`#)RU}$e7j|`;Ja+XiaCpDatLy*fvxYJvXgPacMoZ$0{tW0K78&mJu@4&x z;}$JbrFfGfKH5Ok!NH2f`@>7%#6k>4{{4AE8QtQDZ!sZ*0&c5fZe5WyQ}|->qpxHP z3O;Ke!8zLc09Q?yf}`TZ%o;I%ia;`jq~5Q!!7iRzptF&H-U$L`vKd_pA#`Z@a>NgAVqU$pcBJ7dMHf;T^J7{phl118+UM~vox zND|-PR0pFx6}yjdC53S*b%499-J_r|ls+w)em{%?b Date: Thu, 7 Jun 2018 13:05:25 -0400 Subject: [PATCH 07/40] make sure common.keys overrides common --- settings/common.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/settings/common.py b/settings/common.py index 3a42e422..c300cdfe 100644 --- a/settings/common.py +++ b/settings/common.py @@ -7,11 +7,6 @@ from os.path import dirname, realpath, join import regluit from regluit.payment.parameters import PAYMENT_HOST_PAYPAL, PAYMENT_HOST_AMAZON -try: - from .keys.common import * -except ImportError: - print 'no real key file found, using dummy' - from .dummy.common import * PROJECT_DIR = dirname(dirname(realpath(__file__))) @@ -478,6 +473,11 @@ QUESTIONNAIRE_SHOW_ITEM_RESULTS = False FIREFOX_PATH = '' CHROMEDRIVER_PATH = '' +try: + from .keys.common import * +except ImportError: + print 'no real key file found, using dummy' + from .dummy.common import * try: from .keys.host import * From f75fd07c762315115ba6633bc6bd8a8061639d92 Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 7 Jun 2018 14:39:36 -0400 Subject: [PATCH 08/40] use hosts not common --- .travis.yml | 2 +- settings/travis.py | 8 ++++++++ test/{common.py.enc => travis-host.py.enc} | Bin 3 files changed, 9 insertions(+), 1 deletion(-) rename test/{common.py.enc => travis-host.py.enc} (100%) diff --git a/.travis.yml b/.travis.yml index 6990592c..715396c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ env: before_install: - sudo mkdir /var/log/django - sudo chmod 777 /var/log/django - - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/Gluejar/regluit/test/common.py.enc -out ~/build/Gluejar/regluit/settings/keys/common.py -d + - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/Gluejar/regluit/test/travis-host.py.enc -out ~/build/Gluejar/regluit/settings/keys/host.py -d install: - pip install -r requirements_versioned.pip diff --git a/settings/travis.py b/settings/travis.py index dec133c2..808fcbdf 100644 --- a/settings/travis.py +++ b/settings/travis.py @@ -53,3 +53,11 @@ BROKER_VHOST = "0" # Amazon S3 access AWS_STORAGE_BUCKET_NAME = 'unglueit-testfiles' + +SOCIAL_AUTH_TWITTER_KEY = '' +SOCIAL_AUTH_TWITTER_SECRET = '' +SOCIAL_AUTH_FACEBOOK_KEY = '' +SOCIAL_AUTH_FACEBOOK_SECRET = '' +SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = '' +SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = '' + diff --git a/test/common.py.enc b/test/travis-host.py.enc similarity index 100% rename from test/common.py.enc rename to test/travis-host.py.enc From 4afe792a735a405c5559250bfcb76f145ba3a72b Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 7 Jun 2018 15:08:23 -0400 Subject: [PATCH 09/40] set TEST_INTEGRATION --- settings/travis.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/settings/travis.py b/settings/travis.py index 808fcbdf..1ae01146 100644 --- a/settings/travis.py +++ b/settings/travis.py @@ -60,4 +60,6 @@ SOCIAL_AUTH_FACEBOOK_KEY = '' SOCIAL_AUTH_FACEBOOK_SECRET = '' SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = '' SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = '' +GOOGLE_BOOKS_API_KEY = '' +TEST_INTEGRATION = False From 8aae6808d114b9ea88ccc0ef2de6a09fc2743ce1 Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 7 Jun 2018 16:28:12 -0400 Subject: [PATCH 10/40] thought I already did this --- settings/travis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/settings/travis.py b/settings/travis.py index 1ae01146..57d889a5 100644 --- a/settings/travis.py +++ b/settings/travis.py @@ -62,4 +62,5 @@ SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = '' SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = '' GOOGLE_BOOKS_API_KEY = '' TEST_INTEGRATION = False +DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' From ca5e5bed0c21565abf2368198bb56c6a6e4cfb0d Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 13:55:05 -0400 Subject: [PATCH 11/40] update boto --- requirements_versioned.pip | 7 ++++--- settings/common.py | 2 +- settings/travis.py | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/requirements_versioned.pip b/requirements_versioned.pip index 0d8673ce..bb459b10 100644 --- a/requirements_versioned.pip +++ b/requirements_versioned.pip @@ -9,8 +9,9 @@ PyYAML==3.11 amqp==1.4.9 anyjson==0.3.3 billiard==3.3.0.23 -awscli==1.10.26 -boto==2.42.0 +awscli==1.15.33 +boto==2.48.0 +boto3==1.7.33 celery==3.1.23 certifi==2018.4.16 chardet==3.0.4 @@ -34,7 +35,7 @@ git+git://github.com/eshellman/django-notification.git@a4620e893e2da220994e0189b django-registration==2.1.2 django-selectable==0.9.0 django-smtp-ssl==1.0 -django-storages==1.4.1 +django-storages==1.5.2 django-tastypie==0.13.3 #django-transmeta==0.7.3 git+git://github.com/resulto/django-transmeta.git@ad4d7278ba330dcf8c8446f8ae9b2c769ae8684e diff --git a/settings/common.py b/settings/common.py index c300cdfe..e9d2016b 100644 --- a/settings/common.py +++ b/settings/common.py @@ -488,7 +488,7 @@ except ImportError: LOCAL_TEST = True if AWS_SECRET_ACCESS_KEY: - DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' else: DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' diff --git a/settings/travis.py b/settings/travis.py index 57d889a5..477b4b41 100644 --- a/settings/travis.py +++ b/settings/travis.py @@ -62,5 +62,6 @@ SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = '' SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = '' GOOGLE_BOOKS_API_KEY = '' TEST_INTEGRATION = False -DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' +LOCAL_TEST = True +DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' From ab3c1934d5708aa843495d0eb56ce32e2b141245 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 13:56:07 -0400 Subject: [PATCH 12/40] fix pdf tester boto3 storage returns unicode not str someplace --- core/pdf.py | 8 ++++++-- core/tests.py | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/core/pdf.py b/core/pdf.py index 5f0baf19..0931e46b 100644 --- a/core/pdf.py +++ b/core/pdf.py @@ -1,6 +1,7 @@ """ Utilities that manipulate pdf files """ +import logging import requests from xhtml2pdf import pisa # import python module from PyPDF2 import PdfFileMerger,PdfFileReader @@ -9,6 +10,7 @@ from tempfile import NamedTemporaryFile from django.template.loader import render_to_string from regluit import settings +logger = logging.getLogger(__name__) # Utility function def ask_pdf(context={}): @@ -35,7 +37,7 @@ def pdf_append( file1, file2, file_out ): def test_pdf(pdf_file): temp = None try: - if isinstance(pdf_file , str): + if isinstance(pdf_file , (str, unicode)): if pdf_file.startswith('http:') or pdf_file.startswith('https:'): temp = NamedTemporaryFile(delete=False) test_file_content = requests.get(pdf_file).content @@ -53,7 +55,9 @@ def test_pdf(pdf_file): except: success = False return success - except: + except Exception: + pdf_file = unicode(pdf_file) + logger.exception('error testing a pdf: %s' % pdf_file[:100]) return False def test_test_pdf(self): diff --git a/core/tests.py b/core/tests.py index ddd6311c..8532df09 100755 --- a/core/tests.py +++ b/core/tests.py @@ -1119,7 +1119,8 @@ class EbookFileTests(TestCase): #test the ask-appender c.add_ask_to_ebfs() if settings.AWS_SECRET_ACCESS_KEY: - assert test_pdf(c.work.ebookfiles().filter(asking=True)[0].file.url) + askingpdfurl = c.work.ebookfiles().filter(asking=True)[0].file.url + assert test_pdf(askingpdfurl) else: assert test_pdf(c.work.ebookfiles().filter(asking=True)[0].file) From 8b385d05ede4c1377a107faf81574bfe4c715fc6 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 15:01:22 -0400 Subject: [PATCH 13/40] fix Booxtreem LOCAL_TEST --- booxtream/__init__.py | 6 +++--- booxtream/tests.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/booxtream/__init__.py b/booxtream/__init__.py index 33b7f0cd..a87aeba8 100644 --- a/booxtream/__init__.py +++ b/booxtream/__init__.py @@ -64,9 +64,9 @@ class BooXtream(object): boox = Boox.objects.create( download_link_epub='https://github.com/eshellman/42_ebook/blob/master/download/42.epub?raw=true&extra=download.booxtream.com/', download_link_mobi='https://github.com/eshellman/42_ebook/blob/master/download/42.mobi?raw=true', - referenceid= kwargs.get('referenceid'), - downloads_remaining= kwargs.get('downloadlimit'), - expirydays=kwargs.get('expirydays'), + referenceid= kwargs.get('referenceid', '42'), + downloads_remaining= kwargs.get('downloadlimit', 10), + expirydays=kwargs.get('expirydays', 30), ) return boox diff --git a/booxtream/tests.py b/booxtream/tests.py index 359c9044..e3492546 100644 --- a/booxtream/tests.py +++ b/booxtream/tests.py @@ -23,13 +23,15 @@ class TestBooXtream(unittest.TestCase): return manager def test_booxtream_errors(self): + if settings.LOCAL_TEST: + return from .exceptions import BooXtreamError inst = self._makeOne() if not settings.BOOXTREAM_API_KEY: return with self.assertRaises(BooXtreamError) as cm: inst.platform() - self.assertIn( 'expirydays not set',str(cm.exception)) + self.assertIn('expirydays not set', str(cm.exception)) params={ 'customername': 'Jane Test', 'languagecode':'1043', From 455032ebc35b1064f41ab85802468130eed4be61 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 15:17:39 -0400 Subject: [PATCH 14/40] delint --- booxtream/__init__.py | 30 +++++++++++++++--------------- booxtream/exceptions.py | 6 +++--- booxtream/models.py | 15 +++++++-------- booxtream/tests.py | 20 ++++++++++---------- 4 files changed, 35 insertions(+), 36 deletions(-) diff --git a/booxtream/__init__.py b/booxtream/__init__.py index a87aeba8..4c3d9cca 100644 --- a/booxtream/__init__.py +++ b/booxtream/__init__.py @@ -1,10 +1,11 @@ -import requests import random -from django.conf import settings -from urllib import quote from functools import partial +from urllib import quote from xml.etree import ElementTree +import requests +from django.conf import settings + from django.apps import apps from . exceptions import BooXtreamError @@ -34,10 +35,10 @@ class BooXtream(object): if not apiuser: apiuser = settings.BOOXTREAM_API_USER self.endpoint = 'https://service.booxtream.com/' - self.postrequest = partial(requests.post, timeout=timeout, auth=(apiuser,apikey)) + self.postrequest = partial(requests.post, timeout=timeout, auth=(apiuser, apikey)) - def platform(self, epubfile=None, epub=True, kf8mobi=False, **kwargs): + def platform(self, epubfile=None, epub=True, kf8mobi=False, **kwargs): """ Make an API request to BooXtream ``self.apikey``, ``epubfile`` and the supplied ``kwargs``. Attempts to deserialize the XML response and return the download link. @@ -48,24 +49,24 @@ class BooXtream(object): Boox = apps.get_model('booxtream', 'Boox') url = self.endpoint + 'booxtream.xml' - kwargs['epub'] = '1' if epub else '0' + kwargs['epub'] = '1' if epub else '0' kwargs['kf8mobi'] = '1' if kf8mobi else '0' if epubfile: - if hasattr(epubfile,'name') and str(epubfile.name).endswith('.epub'): - files= {'epubfile': (str(epubfile.name),epubfile)} + if hasattr(epubfile, 'name') and str(epubfile.name).endswith('.epub'): + files = {'epubfile': (str(epubfile.name), epubfile)} else: # give it a random file name so that kindlegen doesn't choke # needed for in-memory (StringIO) epubs - files= {'epubfile': ('%012x.epub' % random.randrange(16**12),epubfile)} + files= {'epubfile': ('%012x.epub' % random.randrange(16**12), epubfile)} else: - files={} + files = {} if settings.LOCAL_TEST: # fake it, so you can test other functions without hitting booxtream boox = Boox.objects.create( download_link_epub='https://github.com/eshellman/42_ebook/blob/master/download/42.epub?raw=true&extra=download.booxtream.com/', download_link_mobi='https://github.com/eshellman/42_ebook/blob/master/download/42.mobi?raw=true', referenceid= kwargs.get('referenceid', '42'), - downloads_remaining= kwargs.get('downloadlimit', 10), + downloads_remaining=kwargs.get('downloadlimit', 10), expirydays=kwargs.get('expirydays', 30), ) return boox @@ -86,9 +87,8 @@ class BooXtream(object): boox = Boox.objects.create( download_link_epub=download_link_epub, download_link_mobi=download_link_mobi, - referenceid= kwargs.get('referenceid'), - downloads_remaining= kwargs.get('downloadlimit'), + referenceid=kwargs.get('referenceid'), + downloads_remaining=kwargs.get('downloadlimit'), expirydays=kwargs.get('expirydays'), ) - return boox - + return boox \ No newline at end of file diff --git a/booxtream/exceptions.py b/booxtream/exceptions.py index 202d1047..12fb6d1f 100644 --- a/booxtream/exceptions.py +++ b/booxtream/exceptions.py @@ -7,7 +7,7 @@ class BooXtreamError(Exception): self.errors = errors def __str__(self): - errormsg='BooXtream errors:' + errormsg = 'BooXtream errors:' for error in self.errors: - errormsg += 'Error %s: %s\n'% (error.find('Code').text,error.find('Msg').text) - return errormsg \ No newline at end of file + errormsg += 'Error %s: %s\n'% (error.find('Code').text, error.find('Msg').text) + return errormsg diff --git a/booxtream/models.py b/booxtream/models.py index 918e2bab..cbf2e606 100644 --- a/booxtream/models.py +++ b/booxtream/models.py @@ -6,22 +6,21 @@ class Boox(models.Model): """ keeps a record of a file that's been watermarked """ - download_link_epub = models.URLField(null=True) - download_link_mobi = models.URLField(null=True) + download_link_epub = models.URLField(null=True) + download_link_mobi = models.URLField(null=True) referenceid = models.CharField(max_length=32) - downloads_remaining = models.PositiveSmallIntegerField(default = 0) + downloads_remaining = models.PositiveSmallIntegerField(default=0) expirydays = models.PositiveSmallIntegerField() created = models.DateTimeField(auto_now_add=True) - + @property def expired(self): return self.created+timedelta(days=self.expirydays) < datetime.now() - + def download_link(self, format): if format == 'epub': return self.download_link_epub elif format == 'mobi': return self.download_link_mobi - else: - return None - + return None + diff --git a/booxtream/tests.py b/booxtream/tests.py index e3492546..c7cf266f 100644 --- a/booxtream/tests.py +++ b/booxtream/tests.py @@ -1,6 +1,6 @@ import unittest import time -import urllib2 +import urllib2 from tempfile import NamedTemporaryFile from StringIO import StringIO from django.conf import settings @@ -32,7 +32,7 @@ class TestBooXtream(unittest.TestCase): with self.assertRaises(BooXtreamError) as cm: inst.platform() self.assertIn('expirydays not set', str(cm.exception)) - params={ + params = { 'customername': 'Jane Test', 'languagecode':'1043', 'expirydays': 1, @@ -49,7 +49,7 @@ class TestBooXtream(unittest.TestCase): def test_booxtream_good(self): inst = self._makeOne() - params={ + params = { 'customeremailaddress':'jane@example.com', 'customername': 'Jane Test', 'languagecode':'1043', @@ -59,16 +59,16 @@ class TestBooXtream(unittest.TestCase): 'chapterfooter':1, 'disclaimer':1, } - params['referenceid']= 'order'+str(time.time()) - boox=inst.platform(epubfile=self.epub2file, **params) - self.assertRegexpMatches(boox.download_link_epub,'download.booxtream.com/') + params['referenceid'] = 'order' + str(time.time()) + boox = inst.platform(epubfile=self.epub2file, **params) + self.assertRegexpMatches(boox.download_link_epub, 'download.booxtream.com/') self.assertFalse(boox.expired) - self.assertEqual(boox.downloads_remaining,3) - + self.assertEqual(boox.downloads_remaining, 3) + # make sure it works with an in-memory file self.epub2file.seek(0) in_mem_epub = StringIO() in_mem_epub.write(self.epub2file.read()) in_mem_epub.seek(0) - boox2=inst.platform(epubfile=in_mem_epub, **params) - self.assertRegexpMatches(boox2.download_link_epub,'download.booxtream.com/') + boox2 = inst.platform(epubfile=in_mem_epub, **params) + self.assertRegexpMatches(boox2.download_link_epub, 'download.booxtream.com/') From cac2866f30d4d38d459f76a437b30b4fcc09cb6c Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 16:39:03 -0400 Subject: [PATCH 15/40] emergency update --- requirements_versioned.pip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_versioned.pip b/requirements_versioned.pip index bb459b10..586333ce 100644 --- a/requirements_versioned.pip +++ b/requirements_versioned.pip @@ -89,7 +89,7 @@ xhtml2pdf==0.2.2 webencodings==0.5.1 #for urllib3 secure cffi==1.7.0 -cryptography==1.4 +cryptography==2.2.2 enum34==1.1.6 idna==2.6 ipaddress==1.0.16 From 55186ec7433eba5ea3408b5da86f63344b5afce8 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 17:25:33 -0400 Subject: [PATCH 16/40] more fixing --- requirements_versioned.pip | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements_versioned.pip b/requirements_versioned.pip index 586333ce..4d348e7d 100644 --- a/requirements_versioned.pip +++ b/requirements_versioned.pip @@ -94,8 +94,8 @@ enum34==1.1.6 idna==2.6 ipaddress==1.0.16 ndg-httpsclient==0.4.2 -pyOpenSSL==16.0.0 -pyasn1==0.1.9 +pyOpenSSL==18.0.0 +pyasn1==0.24.0 pycparser==2.14 setuptools==25.0.0 urllib3==1.22 From 8d6c23e1675947872dc3cc686e618451b1f30137 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 17:31:25 -0400 Subject: [PATCH 17/40] revert --- requirements_versioned.pip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_versioned.pip b/requirements_versioned.pip index 4d348e7d..b668a74d 100644 --- a/requirements_versioned.pip +++ b/requirements_versioned.pip @@ -95,7 +95,7 @@ idna==2.6 ipaddress==1.0.16 ndg-httpsclient==0.4.2 pyOpenSSL==18.0.0 -pyasn1==0.24.0 +pyasn1==0.1.9 pycparser==2.14 setuptools==25.0.0 urllib3==1.22 From 23944c41daeb289f8ff33162a49108a314c3a100 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 17:51:27 -0400 Subject: [PATCH 18/40] causes checkout errors --- settings/keys/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 settings/keys/__init__.py diff --git a/settings/keys/__init__.py b/settings/keys/__init__.py deleted file mode 100644 index e69de29b..00000000 From d5933601afcb66caf7bd0f68fab990f623542ec2 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 17:55:40 -0400 Subject: [PATCH 19/40] another permission issue --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index df1ad729..0eba6b4a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,7 @@ *.pyc *.log settings/me.* -settings/keys/c* -settings/keys/h* +settings/keys/ *.dot reports ENV From 8b709d955bda46ca5c1fa9e453f77f49e2bb384c Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 18:47:09 -0400 Subject: [PATCH 20/40] remove settings/keys --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index df1ad729..4a693c5d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,7 @@ *.pyc *.log settings/me.* -settings/keys/c* -settings/keys/h* +settings/keys/* *.dot reports ENV From 039e2e1a05acfc96af25b8c7b2f41128d5d408c5 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 19:01:42 -0400 Subject: [PATCH 21/40] add keys directory --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 715396c0..9527e3a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,9 @@ env: before_install: - sudo mkdir /var/log/django - sudo chmod 777 /var/log/django + - sudo mkdir ~/build/Gluejar/regluit/settings/keys/ + - sudo chmod 755 ~/build/Gluejar/regluit/settings/keys/ + - sudo cp ~/build/Gluejar/regluit/settings/dummy/__init__.py ~/build/Gluejar/regluit/settings/keys/__init__.py - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/Gluejar/regluit/test/travis-host.py.enc -out ~/build/Gluejar/regluit/settings/keys/host.py -d install: From 15fc49d828238dd834c058c14aa8213f1ade00a3 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 19:12:27 -0400 Subject: [PATCH 22/40] maybe no sudo --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9527e3a9..8f981b21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,9 +15,8 @@ env: before_install: - sudo mkdir /var/log/django - sudo chmod 777 /var/log/django - - sudo mkdir ~/build/Gluejar/regluit/settings/keys/ - - sudo chmod 755 ~/build/Gluejar/regluit/settings/keys/ - - sudo cp ~/build/Gluejar/regluit/settings/dummy/__init__.py ~/build/Gluejar/regluit/settings/keys/__init__.py + - mkdir ~/build/Gluejar/regluit/settings/keys/ + - cp ~/build/Gluejar/regluit/settings/dummy/__init__.py ~/build/Gluejar/regluit/settings/keys/__init__.py - openssl aes-256-cbc -K $encrypted_56eb2b7cc527_key -iv $encrypted_56eb2b7cc527_iv -in ~/build/Gluejar/regluit/test/travis-host.py.enc -out ~/build/Gluejar/regluit/settings/keys/host.py -d install: From ee954b7f47cc904ef4f66486b4d967fcfe851dd4 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 8 Jun 2018 19:38:43 -0400 Subject: [PATCH 23/40] retire jenkins --- deploy/update-just | 24 ----------------- deploy/update-regluit | 22 ---------------- settings/jenkins.py | 56 ---------------------------------------- sysadmin/playbooks/hosts | 1 - vagrant/Vagrantfile | 4 +-- vagrant/dev.yml | 32 ----------------------- vagrant/just.yml | 31 +--------------------- vagrant/please.yml | 6 ----- 8 files changed, 3 insertions(+), 173 deletions(-) delete mode 100755 deploy/update-just delete mode 100755 deploy/update-regluit delete mode 100644 settings/jenkins.py diff --git a/deploy/update-just b/deploy/update-just deleted file mode 100755 index bc97a438..00000000 --- a/deploy/update-just +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# this script is used by jenkins to remotely update a regluit instance -# for it to work the jenkins user's public ssh key needs to be in the -# authorized key for the machine running the regluit instance -# you can then put something like this in a post build configuration -# ssh ubuntu@please.unglueit.com "/opt/regluit/deploy/update-regluit" - -cd /opt/regluit -find . -name "*.pyc" -delete -find . -type d -empty -delete - -sudo -u ubuntu /usr/bin/git pull -source ENV/bin/activate -pip install --upgrade -r requirements_versioned.pip -#django-admin.py syncdb --migrate --settings regluit.settings.just -django-admin.py migrate --fake-initial --noinput --settings regluit.settings.just -django-admin.py collectstatic --noinput --settings regluit.settings.just - -sudo /etc/init.d/apache2 restart -django-admin.py celeryd_multi restart w1 --settings=regluit.settings.just; -/etc/init.d/celerybeat restart -crontab deploy/crontab_just.txt -touch /opt/regluit/deploy/last-update diff --git a/deploy/update-regluit b/deploy/update-regluit deleted file mode 100755 index 6f12c8a7..00000000 --- a/deploy/update-regluit +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# this script is used by jenkins to remotely update a regluit instance -# for it to work the jenkins user's public ssh key needs to be in the -# authorized key for the machine running the regluit instance -# you can then put something like this in a post build configuration -# ssh ubuntu@please.unglueit.com "/opt/regluit/deploy/update-regluit" - -cd /opt/regluit -find . -name "*.pyc" -delete -find . -type d -empty -delete - -sudo -u ubuntu /usr/bin/git pull -source ENV/bin/activate -#pip install -r requirements.pip -django-admin.py syncdb --migrate --settings regluit.settings.please -django-admin.py collectstatic --noinput --settings regluit.settings.please -sudo /etc/init.d/apache2 restart -django-admin.py celeryd_multi restart w1 --settings=regluit.settings.please; -/etc/init.d/celerybeat restart -crontab deploy/crontab_please.txt -touch /opt/regluit/deploy/last-update \ No newline at end of file diff --git a/settings/jenkins.py b/settings/jenkins.py deleted file mode 100644 index b0282f70..00000000 --- a/settings/jenkins.py +++ /dev/null @@ -1,56 +0,0 @@ -# coding=utf-8 -from .common import * - -DEBUG = True -TEMPLATES[0]['OPTIONS']['debug'] = DEBUG -IS_PREVIEW = False - -ADMINS = ( - ('Raymond Yee', 'rdhyee+ungluebugs@gluejar.com'), - ('Eric Hellman', 'eric@gluejar.com'), -) - -MANAGERS = ADMINS - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': 'regluit', - 'USER': 'regluit', - 'PASSWORD': 'regluit', - 'HOST': '', - 'PORT': '', - 'TEST': { - 'CHARSET': 'utf8', - } - } -} - -TIME_ZONE = 'America/New_York' - -# settings for outbout email -# if you have a gmail account you can use your email address and password - -EMAIL_USE_TLS = True -EMAIL_HOST = 'smtp.gmail.com' -EMAIL_HOST_USER = 'me@gmail.com' -EMAIL_HOST_PASSWORD = 'my-password' -EMAIL_PORT = 587 -DEFAULT_FROM_EMAIL = 'info@ebookfoundation.org' - - -# formerly of settings/common.py to surface old vars -# TO DO: invalidate before we open source - -# for use with test google account only -GOOGLE_DISPLAY_NAME = 'Unglue.It' -REDIRECT_IS_HTTPS = False - -#BASE_URL = 'http://0.0.0.0/' -BASE_URL_SECURE = 'http://0.0.0.0/' - - - -# use database as queuing service in development -BROKER_TRANSPORT = "djkombu.transport.DatabaseTransport" -#INSTALLED_APPS += ("djkombu",) diff --git a/sysadmin/playbooks/hosts b/sysadmin/playbooks/hosts index 5140f856..10a3fc6b 100644 --- a/sysadmin/playbooks/hosts +++ b/sysadmin/playbooks/hosts @@ -2,5 +2,4 @@ just ansible_ssh_port=22 ansible_ssh_host=just.unglue.it ansible_ssh_user=ubuntu web1 ansible_ssh_port=22 ansible_ssh_host=unglue.it ansible_ssh_user=ubuntu gluejar ansible_ssh_port=22 ansible_ssh_host=gluejar.com ansible_ssh_user=ubuntu -jenkins ansible_ssh_port=22 ansible_ssh_host=jenkins.unglueit.com ansible_ssh_user=ubuntu diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index eebb9792..0c14e23b 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -114,7 +114,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/' ansible.raw_arguments = [ "--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory", - "--limit=just,127.0.0.1,jenkins", + "--limit=just,127.0.0.1", "-e vname=just", "-e class=just", "-e hostname=just.unglue.it", @@ -186,7 +186,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/' ansible.raw_arguments = [ "--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory", - "--limit=just2,127.0.0.1,jenkins", + "--limit=just2,127.0.0.1", "-e vname=just2", "-e class=just", "-e hostname=just2.unglue.it", diff --git a/vagrant/dev.yml b/vagrant/dev.yml index 65b6ee7f..7cc3d207 100644 --- a/vagrant/dev.yml +++ b/vagrant/dev.yml @@ -499,13 +499,6 @@ - https://github.com/eshellman.keys sudo: no - - name: add public key from jenkins - authorized_key: > - user={{user}} - key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r" - state=present - when: class in ['just'] - - name: set up script file to load environment for interactive use command: cp "/opt/regluit/deploy/setup-{{class}}.sh" /home/{{user}}/setup.sh sudo: no @@ -549,29 +542,4 @@ service: name=apache2 state=restarted -- name: fix known_hosts on jenkins to match new just - hosts: jenkins - sudo: yes - sudo_user: jenkins -# to run the part of the playbook for jenkins -# PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=true ANSIBLE_HOST_KEY_CHECKING=false ANSIBLE_SSH_ARGS='-o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s' ansible-playbook --private-key=/Users/raymondyee/.ssh/id_rsa --user=ubuntu --connection=ssh --inventory-file=/Users/raymondyee/C/src/Gluejar/regluit/vagrant/.vagrant/provisioners/ansible/inventory --limit='jenkins' just.yml - - tasks: - - #equivalent to - # - #ssh -tt jenkins << EOF - # sudo -i -u jenkins - # ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts - # exit - #exit - #EOF - - - name: make new known_hosts with key from just.unglue.it - raw: ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts - when: class in ['just'] - - - name: add key from github - raw: ssh-keyscan -t rsa github.com >> /var/lib/jenkins/.ssh/known_hosts - when: class in ['just'] diff --git a/vagrant/just.yml b/vagrant/just.yml index f657789b..a158e5a6 100644 --- a/vagrant/just.yml +++ b/vagrant/just.yml @@ -384,41 +384,12 @@ - https://github.com/eshellman.keys sudo: yes - - name: add public key from jenkins - authorized_key: > - user={{user}} - key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r" - state=present - sudo: no + handlers: - name: restart apache2 service: name=apache2 state=restarted -- name: fix known_hosts on jenkins to match new just - hosts: jenkins - sudo: yes - sudo_user: jenkins - -# to run the part of the playbook for jenkins -# PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=true ANSIBLE_HOST_KEY_CHECKING=false ANSIBLE_SSH_ARGS='-o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s' ansible-playbook --private-key=/Users/raymondyee/.ssh/id_rsa --user=ubuntu --connection=ssh --inventory-file=/Users/raymondyee/C/src/Gluejar/regluit/vagrant/.vagrant/provisioners/ansible/inventory --limit='jenkins' just.yml - - tasks: - - #equivalent to - # - #ssh -tt jenkins << EOF - # sudo -i -u jenkins - # ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts - # exit - #exit - #EOF - - - name: make new known_hosts with key from just.unglue.it - raw: ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts - - - name: add key from github - raw: ssh-keyscan -t rsa github.com >> /var/lib/jenkins/.ssh/known_hosts diff --git a/vagrant/please.yml b/vagrant/please.yml index 6f0c0e36..35443e3e 100644 --- a/vagrant/please.yml +++ b/vagrant/please.yml @@ -412,12 +412,6 @@ - https://github.com/eshellman.keys sudo: yes - - name: add public key from jenkins - authorized_key: > - user={{user}} - key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r" - state=present - handlers: - name: restart apache2 From 655991b8926bbf4548c403119e94b094c4d4a52e Mon Sep 17 00:00:00 2001 From: eric Date: Mon, 11 Jun 2018 12:42:49 -0400 Subject: [PATCH 24/40] remove stale files these functions have moved into EbookFoundation/regluit-provisioning --- sysadmin/playbooks/README.md | 38 -- sysadmin/playbooks/hosts | 5 - sysadmin/playbooks/roles/common/README.md | 38 -- .../playbooks/roles/common/defaults/main.yml | 2 - .../playbooks/roles/common/handlers/main.yml | 2 - sysadmin/playbooks/roles/common/meta/main.yml | 128 ---- .../playbooks/roles/common/tasks/main.yml | 2 - sysadmin/playbooks/roles/common/vars/main.yml | 2 - sysadmin/playbooks/unattended-upgrade.yml | 19 - vagrant/README.md | 1 - vagrant/Vagrantfile | 382 ------------ vagrant/a-cmd.sh | 2 - vagrant/a-play-aws.sh | 8 - vagrant/a-play.sh | 2 - vagrant/ansible.cfg | 6 - vagrant/create_keys.yml | 9 - vagrant/dev.yml | 545 ------------------ .../files/ssl_cert/STAR_unglue_it.ca-bundle | 210 ------- vagrant/files/ssl_cert/STAR_unglue_it.crt | 100 ---- vagrant/files/ssl_cert/server.key | 89 --- vagrant/files/ssl_cert/unglue.it.wildcard.csr | 58 -- vagrant/group_vars/all/misc.yml | 11 - vagrant/group_vars/all/secrets.yml | 51 -- vagrant/hello.yml | 6 - vagrant/hello2.yml | 13 - vagrant/host_vars/just/secrets.yml | 136 ----- vagrant/host_vars/just2/secrets.yml | 1 - vagrant/host_vars/please/secrets.yml | 93 --- vagrant/host_vars/prod/secrets.yml | 159 ----- vagrant/host_vars/prod2/secrets.yml | 1 - vagrant/host_vars/unglueit.template | 4 - vagrant/just.yml | 395 ------------- vagrant/library/ansible_module_template.py | 82 --- vagrant/library/filetest.py | 36 -- vagrant/library/github_deploy_key.py | 92 --- vagrant/load_data_please.sh | 13 - vagrant/localvm.yml | 280 --------- vagrant/please.yml | 418 -------------- vagrant/roles/common/tasks/main.yml | 72 --- vagrant/roles/hello_world/tasks/main.yml | 6 - vagrant/roles/mysql/tasks/main.yml | 18 - vagrant/run_playbook.py | 41 -- vagrant/setup_django_just.sh | 4 - vagrant/setup_django_localvm.sh | 4 - vagrant/templates/apache.conf.j2 | 71 --- vagrant/templates/common.py.j2 | 6 - vagrant/templates/host.py.j2 | 5 - vagrant/templates/just.wsgi.j2 | 15 - vagrant/templates/please.wsgi.j2 | 14 - vagrant/templates/prod.wsgi.j2 | 13 - vagrant/unglueit.yml | 357 ------------ 51 files changed, 4065 deletions(-) delete mode 100644 sysadmin/playbooks/README.md delete mode 100644 sysadmin/playbooks/hosts delete mode 100644 sysadmin/playbooks/roles/common/README.md delete mode 100644 sysadmin/playbooks/roles/common/defaults/main.yml delete mode 100644 sysadmin/playbooks/roles/common/handlers/main.yml delete mode 100644 sysadmin/playbooks/roles/common/meta/main.yml delete mode 100644 sysadmin/playbooks/roles/common/tasks/main.yml delete mode 100644 sysadmin/playbooks/roles/common/vars/main.yml delete mode 100644 sysadmin/playbooks/unattended-upgrade.yml delete mode 100644 vagrant/README.md delete mode 100644 vagrant/Vagrantfile delete mode 100755 vagrant/a-cmd.sh delete mode 100755 vagrant/a-play-aws.sh delete mode 100755 vagrant/a-play.sh delete mode 100644 vagrant/ansible.cfg delete mode 100644 vagrant/create_keys.yml delete mode 100644 vagrant/dev.yml delete mode 100644 vagrant/files/ssl_cert/STAR_unglue_it.ca-bundle delete mode 100644 vagrant/files/ssl_cert/STAR_unglue_it.crt delete mode 100644 vagrant/files/ssl_cert/server.key delete mode 100644 vagrant/files/ssl_cert/unglue.it.wildcard.csr delete mode 100644 vagrant/group_vars/all/misc.yml delete mode 100644 vagrant/group_vars/all/secrets.yml delete mode 100644 vagrant/hello.yml delete mode 100644 vagrant/hello2.yml delete mode 100644 vagrant/host_vars/just/secrets.yml delete mode 120000 vagrant/host_vars/just2/secrets.yml delete mode 100644 vagrant/host_vars/please/secrets.yml delete mode 100644 vagrant/host_vars/prod/secrets.yml delete mode 120000 vagrant/host_vars/prod2/secrets.yml delete mode 100644 vagrant/host_vars/unglueit.template delete mode 100644 vagrant/just.yml delete mode 100644 vagrant/library/ansible_module_template.py delete mode 100644 vagrant/library/filetest.py delete mode 100644 vagrant/library/github_deploy_key.py delete mode 100644 vagrant/load_data_please.sh delete mode 100644 vagrant/localvm.yml delete mode 100644 vagrant/please.yml delete mode 100644 vagrant/roles/common/tasks/main.yml delete mode 100644 vagrant/roles/hello_world/tasks/main.yml delete mode 100644 vagrant/roles/mysql/tasks/main.yml delete mode 100755 vagrant/run_playbook.py delete mode 100644 vagrant/setup_django_just.sh delete mode 100644 vagrant/setup_django_localvm.sh delete mode 100644 vagrant/templates/apache.conf.j2 delete mode 100644 vagrant/templates/common.py.j2 delete mode 100644 vagrant/templates/host.py.j2 delete mode 100644 vagrant/templates/just.wsgi.j2 delete mode 100644 vagrant/templates/please.wsgi.j2 delete mode 100644 vagrant/templates/prod.wsgi.j2 delete mode 100644 vagrant/unglueit.yml diff --git a/sysadmin/playbooks/README.md b/sysadmin/playbooks/README.md deleted file mode 100644 index c07ad140..00000000 --- a/sysadmin/playbooks/README.md +++ /dev/null @@ -1,38 +0,0 @@ -This directory is for [ansible](http://www.ansible.com/home) playbooks for sysadmin tasks around unglue.it. - -We start with automating tasks such as [applying security upgrades](unattended-upgrade.yml) to the servers. - -Goal is to build playbooks for the configuration of servers. - -# Current status of playbooks - -* [host](hosts) is currently written by hand -- the inventory might be dynamically generated at some point. - - -# Installing ansible - -[installation instructions](http://docs.ansible.com/intro_installation.html). - -On the mac, easiest approach is probably to use [homebrew](http://brew.sh/): - - brew install ansible - -You can also use `pip`: - - pip install ansible - -Note: ansible depends on one having ssh access to the servers. - - -# Running unattended-upgrade - -in `regluit/sysadmin/playbooks`, run - - ansible-playbook -i hosts -e "target=instances" unattended-upgrade.yml - -to run on all the servers. - - ansible-playbook -i hosts unattended-upgrade.yml - -will run only just. - diff --git a/sysadmin/playbooks/hosts b/sysadmin/playbooks/hosts deleted file mode 100644 index 10a3fc6b..00000000 --- a/sysadmin/playbooks/hosts +++ /dev/null @@ -1,5 +0,0 @@ -[instances] -just ansible_ssh_port=22 ansible_ssh_host=just.unglue.it ansible_ssh_user=ubuntu -web1 ansible_ssh_port=22 ansible_ssh_host=unglue.it ansible_ssh_user=ubuntu -gluejar ansible_ssh_port=22 ansible_ssh_host=gluejar.com ansible_ssh_user=ubuntu - diff --git a/sysadmin/playbooks/roles/common/README.md b/sysadmin/playbooks/roles/common/README.md deleted file mode 100644 index 225dd44b..00000000 --- a/sysadmin/playbooks/roles/common/README.md +++ /dev/null @@ -1,38 +0,0 @@ -Role Name -========= - -A brief description of the role goes here. - -Requirements ------------- - -Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. - -Role Variables --------------- - -A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. - -Dependencies ------------- - -A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. - -Example Playbook ----------------- - -Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: - - - hosts: servers - roles: - - { role: username.rolename, x: 42 } - -License -------- - -BSD - -Author Information ------------------- - -An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/sysadmin/playbooks/roles/common/defaults/main.yml b/sysadmin/playbooks/roles/common/defaults/main.yml deleted file mode 100644 index 205c4a1b..00000000 --- a/sysadmin/playbooks/roles/common/defaults/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# defaults file for common diff --git a/sysadmin/playbooks/roles/common/handlers/main.yml b/sysadmin/playbooks/roles/common/handlers/main.yml deleted file mode 100644 index 2662adc4..00000000 --- a/sysadmin/playbooks/roles/common/handlers/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# handlers file for common diff --git a/sysadmin/playbooks/roles/common/meta/main.yml b/sysadmin/playbooks/roles/common/meta/main.yml deleted file mode 100644 index 7b1f0ef0..00000000 --- a/sysadmin/playbooks/roles/common/meta/main.yml +++ /dev/null @@ -1,128 +0,0 @@ ---- -galaxy_info: - author: your name - description: - company: your company (optional) - # Some suggested licenses: - # - BSD (default) - # - MIT - # - GPLv2 - # - GPLv3 - # - Apache - # - CC-BY - license: license (GPLv2, CC-BY, etc) - min_ansible_version: 1.2 - # - # Below are all platforms currently available. Just uncomment - # the ones that apply to your role. If you don't see your - # platform on this list, let us know and we'll get it added! - # - #platforms: - #- name: EL - # versions: - # - all - # - 5 - # - 6 - # - 7 - #- name: GenericUNIX - # versions: - # - all - # - any - #- name: Fedora - # versions: - # - all - # - 16 - # - 17 - # - 18 - # - 19 - # - 20 - #- name: SmartOS - # versions: - # - all - # - any - #- name: opensuse - # versions: - # - all - # - 12.1 - # - 12.2 - # - 12.3 - # - 13.1 - # - 13.2 - #- name: Amazon - # versions: - # - all - # - 2013.03 - # - 2013.09 - #- name: GenericBSD - # versions: - # - all - # - any - #- name: FreeBSD - # versions: - # - all - # - 8.0 - # - 8.1 - # - 8.2 - # - 8.3 - # - 8.4 - # - 9.0 - # - 9.1 - # - 9.1 - # - 9.2 - #- name: Ubuntu - # versions: - # - all - # - lucid - # - maverick - # - natty - # - oneiric - # - precise - # - quantal - # - raring - # - saucy - # - trusty - #- name: SLES - # versions: - # - all - # - 10SP3 - # - 10SP4 - # - 11 - # - 11SP1 - # - 11SP2 - # - 11SP3 - #- name: GenericLinux - # versions: - # - all - # - any - #- name: Debian - # versions: - # - all - # - etch - # - lenny - # - squeeze - # - wheezy - # - # Below are all categories currently available. Just as with - # the platforms above, uncomment those that apply to your role. - # - #categories: - #- cloud - #- cloud:ec2 - #- cloud:gce - #- cloud:rax - #- clustering - #- database - #- database:nosql - #- database:sql - #- development - #- monitoring - #- networking - #- packaging - #- system - #- web -dependencies: [] - # List your role dependencies here, one per line. Only - # dependencies available via galaxy should be listed here. - # Be sure to remove the '[]' above if you add dependencies - # to this list. - diff --git a/sysadmin/playbooks/roles/common/tasks/main.yml b/sysadmin/playbooks/roles/common/tasks/main.yml deleted file mode 100644 index 7b72b3dd..00000000 --- a/sysadmin/playbooks/roles/common/tasks/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# tasks file for common diff --git a/sysadmin/playbooks/roles/common/vars/main.yml b/sysadmin/playbooks/roles/common/vars/main.yml deleted file mode 100644 index fed6035d..00000000 --- a/sysadmin/playbooks/roles/common/vars/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# vars file for common diff --git a/sysadmin/playbooks/unattended-upgrade.yml b/sysadmin/playbooks/unattended-upgrade.yml deleted file mode 100644 index 0de3fc6e..00000000 --- a/sysadmin/playbooks/unattended-upgrade.yml +++ /dev/null @@ -1,19 +0,0 @@ -- name: unattended-upgrade - # by default, run only just - hosts: '{{target | default("just")}}' - sudo: yes - - tasks: - - - name: update apt-get - apt: update_cache=yes - - - name: apply upgrade - command: sudo unattended-upgrade - - - name: check whether reboot needed - stat: path=/var/run/reboot-required - register: reboot_required - - - debug: var=reboot_required.stat.exists - diff --git a/vagrant/README.md b/vagrant/README.md deleted file mode 100644 index 55b784e7..00000000 --- a/vagrant/README.md +++ /dev/null @@ -1 +0,0 @@ -See [vangrant_ansible.md](../docs/vagrant_ansible.md) for documentation. \ No newline at end of file diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile deleted file mode 100644 index 0c14e23b..00000000 --- a/vagrant/Vagrantfile +++ /dev/null @@ -1,382 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - - config.vm.define "please" do |node| - - # Every Vagrant virtual environment requires a box to build off of. - node.vm.box = "ubuntu/trusty64" - - # node.vm.network "forwarded_port", guest: 80, host: 8080 - node.vm.network "private_network", ip: "192.168.33.10" - node.ssh.forward_agent = true - #node.vm.network "private_network", type: "dhcp" - - #node.vm.synced_folder "data", "vagrant_data" - node.vm.synced_folder ".", "/vagrant", disabled: false - - node.vm.provision 'ansible' do |ansible| - ansible.playbook = 'dev.yml' - - ansible.verbose = "vv" - # ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/' - ansible.raw_arguments = [ - "--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory", - "--limit=please,127.0.0.1", - "-e vname=please", - "-e class=please", - "-e hostname=please.unglue.it", - "-e setdns=true", - "-e branch=master", - # "--start-at-task=restart_here" - ] - end - - - # 512MB not enough for compiling lxml: https://stackoverflow.com/a/25916353/7782 - # https://stackoverflow.com/a/26468913/7782 --> for how to get to this setting - node.vm.provider "virtualbox" do |v| - v.memory = 1024 - v.cpus = 2 - end - - node.vm.provider :aws do |aws, override| - aws.access_key_id = ENV.fetch('AWS_ACCESS_KEY_ID') - aws.secret_access_key = ENV.fetch('AWS_SECRET_ACCESS_KEY') - - aws.keypair_name = ENV.fetch('AWS_KEYPAIR_NAME') - - # Ubuntu 12.04 LTS Precise / PV EBS-SSD boot - # alestic 2015.05.05 - - #aws.instance_type="t1.micro" - aws.instance_type="m1.small" - - # aws.region = "us-east-1" - # aws.availability_zone = "us-east-1c" - # 2015.05.05 - # aws.ami = "ami-d8132bb0" - # 2016.03.01 - # aws.ami = "ami-03dcdd69" - # 2017.03.17 - # Trusty 14.04 - # aws.ami = "ami-9fde7f89" - # put into just security group - # aws.security_groups = ["just"] - - - # FEF - - aws.instance_type="m1.small" - aws.region = "us-east-1" - aws.availability_zone = "us-east-1a" - # 2017.03.17 - # Trusty 14.04 - # aws.ami = "ami-9fde7f89" - # 2017.11.22 - # Xenial 16.04 - # hvm:ebs-ssd 20171121.1 ami-aa2ea6d0 - aws.ami = "ami-aa2ea6d0" - # put into just security group - # regluit-pub-a - aws.subnet_id = "subnet-a97777e0" - # SSHAccesss, just_ec2 - aws.associate_public_ip = true - aws.security_groups = ["sg-93aed0ef", "sg-f6a1df8a"] - - - aws.tags = { - 'Name' => 'please_vagrant' - } - - override.vm.box = "dummy" - override.ssh.username = "ubuntu" - override.ssh.private_key_path = ENV['SSH_PRIVATE_KEY_PATH'] || "~/.ssh/id_rsa" - end - - end - - config.vm.define "just" do |node| - - node.vm.box = "ubuntu/trusty64" - node.vm.network "private_network", type: "dhcp" - #node.vm.network "private_network", ip: "192.168.33.10" - - node.ssh.forward_agent = true - - node.vm.provision 'ansible' do |ansible| - ansible.playbook = 'dev.yml' - ansible.verbose = "vv" - # ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/' - ansible.raw_arguments = [ - "--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory", - "--limit=just,127.0.0.1", - "-e vname=just", - "-e class=just", - "-e hostname=just.unglue.it", - "-e setdns=false", - "-e branch=master", - "-e do_migrate=false" - ] - - end - - node.vm.provider "virtualbox" do |v| - v.memory = 1024 - v.cpus = 2 - end - - node.vm.provider :aws do |aws, override| - aws.access_key_id = ENV.fetch('AWS_ACCESS_KEY_ID') - aws.secret_access_key = ENV.fetch('AWS_SECRET_ACCESS_KEY') - - aws.keypair_name = ENV.fetch('AWS_KEYPAIR_NAME') - - # Ubuntu 12.04 LTS Precise / PV EBS-SSD boot - # alestic 2015.05.05 - - aws.instance_type="m1.small" - - aws.region = "us-east-1" - aws.availability_zone = "us-east-1a" - # aws.ami = "ami-d8132bb0" - # 2017.03.17 - # Trusty 14.04 - # aws.ami = "ami-9fde7f89" - # 2017.11.22 - # Xenial 16.04 - # hvm:ebs-ssd 20171121.1 ami-aa2ea6d0 - aws.ami = "ami-aa2ea6d0" - - # aws.security_groups = ["just"] - - # regluit-pub-a - aws.subnet_id = "subnet-a97777e0" - # SSHAccesss, just_ec2 - aws.associate_public_ip = true - aws.security_groups = ["sg-93aed0ef", "sg-e0b1b59f"] - - aws.tags = { - 'Name' => 'just_vagrant' - } - - override.vm.box = "dummy" - override.ssh.username = "ubuntu" - override.ssh.private_key_path = ENV['SSH_PRIVATE_KEY_PATH'] || "~/.ssh/id_rsa" - end - - end - - config.vm.define "just2" do |node| - - node.vm.box = "ubuntu/trusty64" - node.vm.network "private_network", type: "dhcp" - #node.vm.network "private_network", ip: "192.168.33.10" - - - node.ssh.forward_agent = true - - node.vm.provision 'ansible' do |ansible| - ansible.playbook = 'dev.yml' - ansible.verbose = "vv" - # ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/' - ansible.raw_arguments = [ - "--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory", - "--limit=just2,127.0.0.1", - "-e vname=just2", - "-e class=just", - "-e hostname=just2.unglue.it", - "-e setdns=false", - "-e branch=master", - "-e do_migrate=false" - ] - - end - - node.vm.provider "virtualbox" do |v| - v.memory = 1024 - v.cpus = 2 - end - - node.vm.provider :aws do |aws, override| - aws.access_key_id = ENV.fetch('AWS_ACCESS_KEY_ID') - aws.secret_access_key = ENV.fetch('AWS_SECRET_ACCESS_KEY') - - aws.keypair_name = ENV.fetch('AWS_KEYPAIR_NAME') - - # Ubuntu 12.04 LTS Precise / PV EBS-SSD boot - # alestic 2015.05.05 - - aws.instance_type="m1.small" - - aws.region = "us-east-1" - aws.availability_zone = "us-east-1a" - # aws.ami = "ami-d8132bb0" - # 2017.03.17 - # Trusty 14.04 - # aws.ami = "ami-9fde7f89" - # 2017.11.22 - # Xenial 16.04 - # hvm:ebs-ssd 20171121.1 ami-aa2ea6d0 - aws.ami = "ami-aa2ea6d0" - - - # aws.security_groups = ["just"] - # regluit-pub-a - aws.subnet_id = "subnet-a97777e0" - # SSHAccesss, just_ec2 - aws.associate_public_ip = true - aws.security_groups = ["sg-93aed0ef", "sg-e0b1b59f"] - - aws.tags = { - 'Name' => 'just2_vagrant' - } - - override.vm.box = "dummy" - override.ssh.username = "ubuntu" - override.ssh.private_key_path = ENV['SSH_PRIVATE_KEY_PATH'] || "~/.ssh/id_rsa" - end - - end - - config.vm.define "prod" do |node| - - node.vm.box = "ubuntu/trusty64" - node.vm.network "private_network", type: "dhcp" - #node.vm.network "private_network", ip: "192.168.33.10" - - - node.ssh.forward_agent = true - - node.vm.provision 'ansible' do |ansible| - ansible.playbook = 'dev.yml' - ansible.verbose = "vv" - # ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/' - ansible.raw_arguments = [ - "--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory", - "--limit=prod,127.0.0.1", - "-e vname=prod", - "-e class=prod", - "-e hostname=unglue.it", - "-e setdns=false", - "-e do_migrate=false", - "-e branch=production" - ] - - end - - node.vm.provider "virtualbox" do |v| - v.memory = 1024 - v.cpus = 2 - end - - node.vm.provider :aws do |aws, override| - aws.access_key_id = ENV.fetch('AWS_ACCESS_KEY_ID') - aws.secret_access_key = ENV.fetch('AWS_SECRET_ACCESS_KEY') - - aws.keypair_name = ENV.fetch('AWS_KEYPAIR_NAME') - - # Ubuntu 12.04 LTS Precise / PV EBS-SSD boot - # alestic 2015.05.05 - - aws.instance_type="c1.medium" - - aws.region = "us-east-1" - #aws.availability_zone = "us-east-1c" - # 2017.12.15 -- for some reason 1c doesn't hava the capacity - aws.availability_zone = "us-east-1b" - # aws.ami = "ami-d8132bb0" - # 2017.03.17 - # Trusty 14.04 - # aws.ami = "ami-9fde7f89" - # 2017.11.22 - # Xenial 16.04 - # hvm:ebs-ssd 20171121.1 ami-aa2ea6d0 - aws.ami = "ami-aa2ea6d0" - - aws.security_groups = ["web-production"] - - aws.tags = { - 'Name' => 'prod_vagrant' - } - - override.vm.box = "dummy" - override.ssh.username = "ubuntu" - override.ssh.private_key_path = ENV['SSH_PRIVATE_KEY_PATH'] || "~/.ssh/id_rsa" - end - - end - - config.vm.define "prod2" do |node| - - node.vm.box = "ubuntu/trusty64" - node.vm.network "private_network", type: "dhcp" - #node.vm.network "private_network", ip: "192.168.33.10" - - - node.ssh.forward_agent = true - - node.vm.provision 'ansible' do |ansible| - ansible.playbook = 'dev.yml' - ansible.verbose = "vv" - # ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/' - ansible.raw_arguments = [ - "--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory", - "--limit=prod2,127.0.0.1", - "-e vname=prod2", - "-e class=prod", - "-e hostname=unglue.it", - "-e setdns=false", - "-e do_migrate=false", - "-e branch=production" - ] - - end - - node.vm.provider "virtualbox" do |v| - v.memory = 1024 - v.cpus = 2 - end - - node.vm.provider :aws do |aws, override| - aws.access_key_id = ENV.fetch('AWS_ACCESS_KEY_ID') - aws.secret_access_key = ENV.fetch('AWS_SECRET_ACCESS_KEY') - - aws.keypair_name = ENV.fetch('AWS_KEYPAIR_NAME') - - # Ubuntu 12.04 LTS Precise / PV EBS-SSD boot - # alestic 2015.05.05 - - aws.instance_type="c1.medium" - - aws.region = "us-east-1" - aws.availability_zone = "us-east-1c" - # aws.ami = "ami-d8132bb0" - # 2017.03.17 - # Trusty 14.04 - # aws.ami = "ami-9fde7f89" - # 2017.11.22 - # Xenial 16.04 - # hvm:ebs-ssd 20171121.1 ami-aa2ea6d0 - aws.ami = "ami-aa2ea6d0" - - aws.security_groups = ["web-production"] - - aws.tags = { - 'Name' => 'prod2_vagrant' - } - - override.vm.box = "dummy" - override.ssh.username = "ubuntu" - override.ssh.private_key_path = ENV['SSH_PRIVATE_KEY_PATH'] || "~/.ssh/id_rsa" - end - - end - -end - - diff --git a/vagrant/a-cmd.sh b/vagrant/a-cmd.sh deleted file mode 100755 index 27461e6c..00000000 --- a/vagrant/a-cmd.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -ansible -i .vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory --private-key=~/.vagrant.d/insecure_private_key -u vagrant unglueit -a $1 diff --git a/vagrant/a-play-aws.sh b/vagrant/a-play-aws.sh deleted file mode 100755 index 2bf094b2..00000000 --- a/vagrant/a-play-aws.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -ansible-playbook -vvvv \ - -i .vagrant/provisioners/ansible/inventory/ \ - --private-key=/Users/raymondyee/.ssh/id_rsa \ - -e aws_access_key=$AWS_ACCESS_KEY \ - -e aws_secret_key=$AWS_SECRET_ACCESS_KEY \ - -u ubuntu $1 diff --git a/vagrant/a-play.sh b/vagrant/a-play.sh deleted file mode 100755 index e3611358..00000000 --- a/vagrant/a-play.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -ansible-playbook -vvvv -i .vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory --private-key=.vagrant/machines/unglueit/virtualbox/private_key -u vagrant $1 diff --git a/vagrant/ansible.cfg b/vagrant/ansible.cfg deleted file mode 100644 index 5da5b312..00000000 --- a/vagrant/ansible.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[defaults] -library = ./library -inventory = .vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory -private-key = ~/.vagrant.d/insecure_private_key -remote_user = vagrant -timeout = 15 diff --git a/vagrant/create_keys.yml b/vagrant/create_keys.yml deleted file mode 100644 index c4fc4362..00000000 --- a/vagrant/create_keys.yml +++ /dev/null @@ -1,9 +0,0 @@ -- name: test - hosts: localhost - tasks: - - # to run: ansible-playbook create_commonpy.yml - # create settings/common.py (locally) - - name: create settings/keys/common.py locally - template: src=templates/common.py.j2 dest=../settings/keys/common.py - delegate_to: localhost diff --git a/vagrant/dev.yml b/vagrant/dev.yml deleted file mode 100644 index 7cc3d207..00000000 --- a/vagrant/dev.yml +++ /dev/null @@ -1,545 +0,0 @@ -- name: route53 setup - hosts: 127.0.0.1 - vars: - aws_access_key: "{{ lookup('env','AWS_ACCESS_KEY_ID') }}" - aws_secret_key: "{{ lookup('env','AWS_SECRET_ACCESS_KEY') }}" - target: '{{vname}}' - dev_ssh_host: "{{ hostvars[target]['ansible_ssh_host'] }}" - tasks: - - - name: print dev_ssh_host - debug: msg="dev_ssh_host {{hostname}}" - - - name: get DNS record for dev - route53: - command: get - zone: unglue.it - record: "{{hostname}}" - type: A - aws_access_key: "{{aws_access_key}}" - aws_secret_key: "{{aws_secret_key}}" - when: "{{setdns | default('false')}}" - - - name: set DNS record for dev - route53: - command: create - zone: unglue.it - record: "{{hostname}}" - type: A - ttl: 60 - value: "{{dev_ssh_host}}" - overwrite: yes - aws_access_key: "{{aws_access_key}}" - aws_secret_key: "{{aws_secret_key}}" - when: "{{setdns | default('false')}}" - - -- name: dev setup - hosts: '{{vname}}' - vars: - user: "{{ ansible_ssh_user }}" - aws_access_key: "{{ lookup('env','AWS_ACCESS_KEY_ID') }}" - aws_secret_key: "{{ lookup('env','AWS_SECRET_ACCESS_KEY') }}" - target: '{{vname}}' - migrate: "{{do_migrate | default('true')}}" - sudo: yes - - gather_facts: False - pre_tasks: - - name: Install python for Ansible - raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) - register: output - changed_when: output.stdout != "" - tags: always - - setup: # aka gather_facts - - name: check apt last update - stat: path=/var/cache/apt - register: apt_cache_stat - - name: update apt if needed - apt: update_cache=yes - when: ansible_date_time.epoch|float - apt_cache_stat.stat.mtime > 60*60*12 - - - tasks: - - # add repo to get latest version of python 2.7 - # - name: add-apt-repository ppa:fkrull/deadsnakes-python2.7 - # apt_repository: repo='ppa:fkrull/deadsnakes-python2.7' state=present update_cache=true - # when: class in ['please', 'just', 'prod'] - - - name: do apt-get update --fix-missing - command: apt-get update --fix-missing - - - name: installing dependencies - apt: pkg={{ item }} update_cache=yes state=present - with_items: - - python2.7 - - python-pip - - git-core - - apache2 - - cronolog - - libapache2-mod-wsgi - - mysql-client - - python-virtualenv - - python-mysqldb - - redis-server - - python-lxml - - python-dev - - libjpeg-dev - - libmysqlclient-dev - - libxml2-dev - - libxslt1-dev - - python-setuptools - - python-dev - - postfix - - mailutils - - libffi-dev - - build-essential - - libssl-dev - tags: install - - - name: make {{user}} group - group: name={{user}} - - - name: make {{user}} user - user: name={{user}} shell=/bin/bash group={{user}} generate_ssh_key=yes - - # create celery user and group - # also put {{user}} into celery group - - - name: make celery group - group: name=celery - - - name: create celery user - user: > - name=celery - createhome=no - group=celery - generate_ssh_key=no - - - name: add {{user}} to celery, www-data groups - user: name={{user}} groups=celery,www-data append=yes - - # - name: add www-data to {{user}} group - # user: name=www-data groups={{user}} append=yes - - - name: install some python modules to use - #pip: name={{item}} virtualenv=/home/{{user}}/venv - pip: name={{item}} - with_items: - - PyGithub - - - name: create /opt/regluit - file: path=/opt/regluit state=directory owner={{user}} group={{user}} mode=0745 - - - name: git config - command: "{{item}}" - with_items: - - git config --global user.name "Raymond Yee" - - git config --global user.email "rdhyee@gluejar.com" - - - name: ssh-keygen - #command: pwd - command: ssh-keygen -b 2048 -t rsa -f /home/{{user}}/.ssh/id_rsa -P "" - sudo: no - args: - creates: /home/{{user}}/.ssh/id_rsa - - - name: create deploy key for repo - action: github_deploy_key - sudo: no - args: - github_auth_key: "{{github_auth_key}}" - repo_name: Gluejar/regluit - key_name: "{{hostname}} {{ ansible_date_time.iso8601 }}" - key_path: /home/{{user}}/.ssh/id_rsa.pub - - - name: postfix install - raw: DEBIAN_FRONTEND='noninteractive' apt-get install -y -q --force-yes postfix - - - name: clone the regluit git repo into /opt/regluit - sudo: no - git: repo=ssh://git@github.com/Gluejar/regluit.git dest=/opt/regluit accept_hostkey=True force=yes version={{branch | default("master")}} - - - # installing mysql - # https://github.com/bennojoy/mysql --> probably the right way - # how do you make use of other people's playbooks in the right way? - # https://stackoverflow.com/a/7740571/7782 - - - name: mysql setup - raw: debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password password {{mysql_root_pw}}' - args: - executable: /bin/bash - when: class == 'please' - - - raw: debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password_again password {{mysql_root_pw}}' - args: - executable: /bin/bash - when: class == 'please' - - - raw: apt-get -y install mysql-server - when: class == 'please' - - - name: Create regluit database - mysql_db: db=regluit state=present encoding=utf8 collation=utf8_bin login_user=root login_password={{mysql_root_pw}} - when: class == 'please' - - # GRANT ALL PRIVILEGES ON regluit.* TO 'regluit'@'localhost' WITH GRANT OPTION; (covered?) - - name: Create database user - mysql_user: > - user={{SECRET_KEYS.DATABASE_USER}} - password={{SECRET_KEYS.DATABASE_PASSWORD}} - host={{SECRET_KEYS.DATABASE_HOST}} - priv=*.*:ALL - state=present - login_user=root - login_password={{mysql_root_pw}} - when: class == 'please' - - - # running stuff within a virtualenv - # https://stackoverflow.com/a/20572360 - # https://stackoverflow.com/questions/20575084/best-way-to-always-run-ansible-inside-a-virtualenv-on-remote-machines?rq=1 - - - #sudo("ln -s /opt/regluit/deploy/please.conf /etc/apache2/sites-available/please") - - - name: create apache conf for sites-available from template - template: src=templates/apache.conf.j2 dest="/etc/apache2/sites-available/{{class}}.conf" owner={{user}} group={{user}} mode=0664 - - #run('pip install -r requirements_versioned.pip') - - - name: upgrade pip - pip: > - name={{item}} - virtualenv=/opt/regluit/ENV - virtualenv_command=virtualenv - extra_args="--upgrade" - with_items: - - pip - sudo: no - - - name: pip requirements - pip: > - requirements=/opt/regluit/requirements_versioned.pip - virtualenv=/opt/regluit/ENV - virtualenv_command=virtualenv - virtualenv_site_packages=yes - sudo: no - - - #run('echo "/opt/regluit/" > ENV/lib/python2.7/site-packages/regluit.pth') - #run('echo "/opt/" > ENV/lib/python2.7/site-packages/opt.pth') - - - name: establish regluit.pth - lineinfile: create=yes dest=/opt/regluit/ENV/lib/python2.7/site-packages/regluit.pth line="/opt/regluit/" - sudo: no - - - name: establish opt.pth - lineinfile: create=yes dest=/opt/regluit/ENV/lib/python2.7/site-packages/regluit.pth line="/opt/" - sudo: no - - #sudo('mkdir /var/www/static') - #sudo('chown ubuntu:ubuntu /var/www/static') - - - name: create /var/www/static - file: path=/var/www/static state=directory owner={{user}} group={{user}} mode=0755 - - # - #run('django-admin.py syncdb --migrate --noinput --settings regluit.settings.please') - - # provide a directory for django log file - - name: make /var/log/regluit - file: path=/var/log/regluit state=directory owner={{user}} group=www-data mode=2775 - - - # create the wsgi script from the appropriate template - - name: create the wsgi script from the appropriate template - template: src=templates/{{class}}.wsgi.j2 dest=/opt/regluit/deploy/{{class}}.wsgi owner={{user}} group={{user}} mode=0664 - when: class in ['please', 'just', 'prod'] - - - name: restart_here - debug: msg="provision restart here" - - - name: Create /settings/keys/ - file: path=/opt/regluit/settings/keys/ state=directory mode=0755 - - # create settings/keys/common.py - - name: create settings/keys/common.py - template: src=templates/common.py.j2 dest=/opt/regluit/settings/keys/common.py owner={{user}} group={{user}} mode=0755 - - # create settings/keys/host.py - - name: create settings/keys/host.py - template: src=templates/host.py.j2 dest=/opt/regluit/settings/keys/host.py owner={{user}} group={{user}} mode=0755 - when: class in ['please', 'just', 'prod'] - - - name: create empty settings/keys/__init__.py - copy: - content: "" - dest: /opt/regluit/settings/keys/__init__.py - force: no - group: "{{user}}" - owner: "{{user}}" - mode: 0755 - - #Run syncdb on the application - # TO DO: syncdb might be deprecated - # https://stackoverflow.com/a/29683785 - - - name: django syncdb - django_manage: > - command=syncdb - app_path=/opt/regluit/ - settings="regluit.settings.{{class}}" - virtualenv=/opt/regluit/ENV - sudo: no - when: migrate - notify: - - restart apache2 - - - name: django migrations - django_manage: > - command=migrate - app_path=/opt/regluit/ - settings="regluit.settings.{{class}}" - virtualenv=/opt/regluit/ENV - sudo: no - when: migrate - notify: - - restart apache2 - - #run('django-admin.py collectstatic --noinput --settings regluit.settings.please') - - - name: django collectstatic - django_manage: > - command=collectstatic - app_path=/opt/regluit/ - settings="regluit.settings.{{class}}" - virtualenv=/opt/regluit/ENV - sudo: no - notify: - - restart apache2 - - - name: copy STAR_unglue_it.crt - copy: > - src=files/ssl_cert/STAR_unglue_it.crt - dest=/etc/ssl/certs/server.crt - owner={{user}} - group={{user}} - mode=0644 - notify: - - restart apache2 - - - name: copy server.key - copy: > - src=files/ssl_cert/server.key - dest=/etc/ssl/private/server.key - owner={{user}} - group={{user}} - mode=0600 - notify: - - restart apache2 - - - name: copy STAR_unglue_it.ca-bundle - copy: > - src=files/ssl_cert/STAR_unglue_it.ca-bundle - dest=/etc/ssl/certs/STAR_unglue_it.ca-bundle - owner={{user}} - group={{user}} - mode=0600 - notify: - - restart apache2 - - - name: remove /etc/logrotate.d/apache2 - file: path=/etc/logrotate.d/apache2 state=absent - notify: - - restart apache2 - - - name: a2dissite 000-default - command: a2dissite 000-default - notify: - - restart apache2 - - - name: a2ensite dev - command: a2ensite "{{class}}" - notify: - - restart apache2 - - - name: a2enmod ssl rewrite headers - command: a2enmod ssl rewrite headers - notify: - - restart apache2 - - # - name: show django_secret_key - # debug: msg="{{django_secret_key}}" - - # out-dated: no more injecting secret key into local.py - #- name: insert SECRET_KEY into /opt/regluit/settings/local.py - # lineinfile: create=yes dest=/opt/regluit/settings/local.py line="SECRET_KEY=u'{{django_secret_key}}'" - # notify: - # - restart apache2 - # sudo: no - - # - #sudo ("/etc/init.d/apache2 restart") - # - - - name: turn on ports 22, 80, 443 - ufw: rule=allow port={{ item }} proto=tcp - with_items: - - 22 - - 80 - - 443 - - - name: enable ufw - ufw: state=enabled - - # create directories for celery - # /var/log/celery and /var/run/celery - - - name: create /var/log/celery - file: path=/var/log/celery state=directory owner=celery group=celery mode=0775 - - - name: create /var/run/celery - file: path=/var/run/celery state=directory owner=celery group=celery mode=0775 - - # set up celeryd - - - name: set up /etc/init.d/celeryd (from deploy/celeryd) - command: cp /opt/regluit/deploy/celeryd /etc/init.d/celeryd - - # still need? - - name: set mode on /etc/init.d/celeryd - file: path=/etc/init.d/celeryd mode=0755 - - - name: copy deploy/celeryd.conf - command: cp "/opt/regluit/deploy/celeryd_{{class}}.conf" /etc/default/celeryd - - - name: set mode on /etc/default/celeryd - file: path=/etc/default/celeryd mode=0644 - - # - name: just before launching celeryd - # pause: prompt='Press return to continue. Press Ctrl+c and then "a" to abort' - - # start up celeryd - - # sudo ("/etc/init.d/celeryd start") - # old way with root - # - name: start celeryd - # command: /etc/init.d/celeryd start - - # - name: start celery queue with celery multi - # command: /opt/regluit/ENV/bin/django-admin.py celeryd_multi restart w1 - # sudo: no - - - name: celeryd_multi - django_manage: > - command="celeryd_multi restart w1" - app_path=/opt/regluit/ - settings="regluit.settings.{{class}}" - virtualenv=/opt/regluit/ENV - sudo: no - - # - name: just after attempt to launch celeryd - # pause: prompt='Press return to continue. Press Ctrl+c and then "a" to abort' - - # sudo ("cp deploy/celerybeat /etc/init.d/celerybeat") - # sudo ("chmod 755 /etc/init.d/celerybeat") - # https://stackoverflow.com/questions/24162996/how-to-move-rename-a-file-using-an-ansible-task-on-a-remote-system - - - # set up celerybeat - - - name: copy deploy/celerybeat - command: cp /opt/regluit/deploy/celerybeat /etc/init.d/celerybeat - - - name: set mode on /etc/init.d/celerybeat - file: path=/etc/init.d/celerybeat mode=0775 - - - name: copy deploy/celerybeat,conf to /etc/default/celerybeat - command: cp "/opt/regluit/deploy/celerybeat_{{class}}.conf" /etc/default/celerybeat - - - name: set mode on /etc/default/celerybeat - file: path=/etc/default/celerybeat mode=0775 - - - name: create /var/log/celerybeat - file: path=/var/log/celerybeat state=directory owner=celery group=celery mode=0775 - - # - name: just before launching celerybeat - # pause: prompt='Press return to continue. Press Ctrl+c and then "a" to abort' - - - name: start celerybeat - command: /etc/init.d/celerybeat start - sudo: no - - # - name: just after attempt to launch celerybloeat - # pause: prompt='Press return to continue. Press Ctrl+c and then "a" to abort' - - # run data loading script - - name: run data loading script - script: "load_data_{{class}}.sh" - when: class in ['please'] - - # set up crontab - - name: crontab - command: crontab "/opt/regluit/deploy/crontab_{{class}}.txt" - sudo: no - - - name: add ssh keys from /opt/regluit/deploy/public_keys/ - authorized_key: user="{{user}}" key={{ lookup('file', item) }} state=present - with_fileglob: - - "../deploy/public_keys/*.pub" - sudo: no - - - name: add ssh keys from public_key directory - authorized_key: user="{{user}}" key="{{item}}" state=present - with_items: - - https://github.com/rdhyee.keys - - https://github.com/eshellman.keys - sudo: no - - - name: set up script file to load environment for interactive use - command: cp "/opt/regluit/deploy/setup-{{class}}.sh" /home/{{user}}/setup.sh - sudo: no - - - name: set up script to dump database - command: cp "/opt/regluit/deploy/dump_db_{{class}}.sh" "/home/{{user}}/dump.sh" - when: class in ['prod'] - sudo: no - - - name: chmod +x dump.sh - file: path="/home/{{user}}/dump.sh" state=file owner="{{user}}" group="{{user}}" mode=0745 - when: class in ['prod'] - - - name: put an empty file in main dir to help identify this instance - command: touch "/home/{{user}}/{{class}}_{{ ansible_date_time.iso8601 }}" - sudo: no - - - name: apply upgrade - command: sudo unattended-upgrade - - - name: check whether reboot needed - stat: path=/var/run/reboot-required - register: reboot_required - - - name: restart machine - shell: sleep 2 && shutdown -r now "Ansible updates triggered" - async: 1 - poll: 0 - sudo: true - ignore_errors: true - when: reboot_required - - - name: waiting for server to come back - local_action: wait_for host="{{ inventory_hostname }}" state=started delay=30 timeout=300 - sudo: false - when: reboot_required - - - handlers: - - name: restart apache2 - service: name=apache2 state=restarted - - - diff --git a/vagrant/files/ssl_cert/STAR_unglue_it.ca-bundle b/vagrant/files/ssl_cert/STAR_unglue_it.ca-bundle deleted file mode 100644 index 9258132a..00000000 --- a/vagrant/files/ssl_cert/STAR_unglue_it.ca-bundle +++ /dev/null @@ -1,210 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -30636262636630653738383536613136363733643931316362326266656433376333386239373962 -6635343138326335313430623566656130663136393531310a343562653666623235336539316561 -31386163663632383436343735376266356134386335363637383536613531323231626535313236 -6163326630306530360a663636393463303062316331623630353235383137333831333239393134 -64353830366463376130633831633835313034643930636166396436373063623039316465613964 -61333538656264353735613034356533353331313735393965363164616331306538383231303932 -31373838363862343931643034643863313738636238376634363430613365356136323237613530 -34323339323337383539363462303863663163373465373262636161346662333738663532316363 -65313464333363643461353539306662353364643663373032346531633362373230653061646436 -34393462326137336163316361363937323738663632386662643338626433626433633235323838 -65653036643961626131383264393138316263366534656663386264376638393639636164323661 -61646166376637396131356339313162343762636234643236316363643063326638366536656534 -31616164313164373336666634356639653831623966333739356462656137363161663635353136 -30653665353334646165663333643863643539373331613263386561353030363862666264353933 -64643439336236386631393934623839626163373463653134656539376262316233626130363663 -65636534303431616631633534643030346534633635393563366266356138383938636264383632 -33616365333863393733646565643236613064616462633661303630613963666236663265636361 -61376336346262346632313833633735356364636239366439393265613866316439643732326330 -38396332363666656636646164646262303162656331636661393738626263383563646261323036 -63363634616362323965616639373961333966303366363937333233356631393938383339613262 -39393461316563383131666539333938313437336462376233353464303739356463323265366665 -64316161333431353264336236373466613230363237316633386661303538666536366462323666 -32373164626330313665653133356563383436663765666437636138343163663533646331346631 -31643638373537653533353036653766663935663636663561343239383666333661343830353133 -34363363333466393761643863393436386231626134663236326335336438633338313963646363 -30623262386437313733353038353231353163326133643139633963303864633739383935363664 -61336265656666643037316139306534323135333763383230633261336139636362396539643539 -34303937656634653862383665356530643064383832363163343331383538623236366535393061 -38373738393064376539343837356335326132363135323935333964383635383965656462383636 -63663661626632396663383838353633663839303936646537363266346661373033663064326435 -62646130376163623363346232346530376638613934623831303636366365313331613436626337 -61346262333933363366653166326330663638643230373364653161333263323035366263623330 -66636532616234636263323366626432363337326564663531323663356166303930653661313030 -32363133313162313331623138356432653630663263623935303739663363323130646133613735 -66333933393266663265333036616465353338656664313230643761303439363035663566393930 -31303633396665316330623962643439313132633531656362333630653537393061316663356566 -39376230643830333035316439346336313464623238373739623636313236316162356337313339 -62376533663138613962306331313066383439373039636631633431316232643266326361666134 -64383437636561323964346134663032663466333831663836393039353562663065643337303330 -62356666353662386439353563643735633730613463666263666539323333613130633765393834 -32343439306636306131386537373164363636386330623534373535323631396464626661323361 -38353935623566373834643661616631373333303735393835643737626234643337653130396365 -65666332323665356330396238633866653761346531303439666236653264323661626135353238 -36643562623266636333623831303439336366646133653332313130336436386537653234666435 -31343863376233623335366333363232653966316230363836613231613130613166643864373565 -35356264383335623539643630323534623265313631383130643739306533663330396434653236 -64376166373639363731663437663331346338373539363637353933386239373738373565353030 -64353261366232643061356165323961383537626239623732393936623461393961353837633733 -34356166393938346232396532353166373534623530646335633338633636336636646630356561 -32363061356632313661386535306339636135623834333231303433343037353633613365326333 -37666235323239383661323536353830383833326565346564663631336465313664346533323337 -38613961313233623033386166336235383362393463393832646538373139333533313266303937 -62346532356632316136633533373665373839623336383435353163386531366466646664326565 -30303639393933373361623333326432366332396137396238336464376131323631343637306531 -37343933373139333333623233386334636436356432303764653235353133366566383031633136 -34616133336663623664616432336265323534373163316338386561623864623861343835633062 -34343738616134366164356337663566326462303933386139643734376332313066613230326363 -66623837323238356366333836386662366435626236333835346662623631373636663033623733 -38316639343638313937353636616638396261333561313032383130393538663366636630376539 -63353566613835343964356635303939343764396631393738646432313765666564316363626431 -65346637656137363266346266616666316361383231343666643063353131386163356638346636 -30383932633264373031373935616230323933373863663564363233313232363066326662303165 -63346532313837396365353365636663313334626433303732326164363330303438636333636163 -35656537623165323666633435386465303063633534656630306664343637336164626433373361 -65376530333361303337623634353434373164626133626263613561613436303161626432626239 -37343536313531626233343530633634613166633735613233393962323165623166366161356336 -35386635366631326266343432386161663033323962313763343163663137643631393166333031 -39346564656665343835616464663231333562616661643234326663633563303964386131656363 -63666363356562383830646633396238623537336439326162376135633665653165326536353037 -37313362666531333837633439643165356534383265643637363933366463343839333961376235 -63333063643634623735363233646334633235373132646139313365303336646264633061643131 -32616465633131356130396465643235353530326564373136313133613064313562306333313965 -38396237653430353533353462373130303262663564653361313465353934353864656235653135 -34613639613633633234386639633230653537393965323363623338656465326531346432333637 -64343436373436356535333865643865373539393036626137626538626635653637666362356162 -30313632396539623962346562646563316264613232343730643565386135346664646363333330 -66656661636235633730393063343634636138383264336535326533356231626530613565336161 -35626232343436616535373230653039303065343432343738616363626635383364373266323333 -33376364666638313564366237356635653731623566626661373961663736666638343061646134 -35313438356531383433636565326365313063383834613837666330396331353738323237363333 -63363963666465313137386537356462663636323737306362326362363266343062626130616230 -64363433343738343632636633373330386464396364383031343531663432363563653032336332 -65366364333133386139636165303964366165326336316534323636633462356335633066643165 -34613030323234383965396233373333393336653237613062323530353762633931343333323865 -30663862373132633834353039653032373737313564333638333762303335636265646162313931 -66393662653933363832623061633634626661356437663331373937396631356131333462656666 -32303166396533643261646130306161326561326533626531663038643565376439633339333432 -64333437366566666366303230616539663564333463666631646163356235636333363633663639 -37666332346161376239656137306430623463653466626663636539666532373565373731613165 -62663532383165656432313762376235333331343738646532373265353066363262333261626137 -31396332383963353066336666366133336138653363613863653833653964663164313831333065 -35333738383232656634356564643634346633313137333863363766643433363631333630643731 -65363161303235323830376638303135613630663362653566363965313436323033643366306630 -39303935636639333961666135633962653362666536663333623430356164376262346662643635 -33323663333832316633373035386334623134326536366236636561373635353366386237323933 -62393435363235396262653065656562383063663236323134313833363833396131643533666235 -37386366343933376338336533316239383430343566626434303537653431626261363438336334 -35363961656332336230386535386531363761303766313563333832633033646461393635376339 -39633630336366373663666239623030326331346361323132363565366366616236346230636632 -38376534363933383965386338313835356135663035326536343532643930383466633538316538 -31353038656533336335376265626139333364633537633662653362393666346163386431326261 -35393261353230373034633666663638306134643066643533323761666264326162343630376661 -65613733343866306364333665353639646263323863663562333965636335613966323564373535 -39363939383031393265613035376635663962386530663639356331393830383130626439346666 -31393532316431393666616535363430636637366134366636643730636431343034336539616135 -38633063333063383766386265336363386362383136653536616663376463333935356162343930 -36383831666537313936393033363738666335653739646431363039373939626538343037633631 -39383439373336336561663431626265666434646565356438336530643863656464323034353565 -36366563323836336162373465313938646239373637613066333238323533346338353834636631 -30333262376638336537623466643736653562336336663436333963366464666131303862313335 -61353862656435653939343362393934303633303333366231646431656234656638613033383133 -31353262353231663633633063663537353536383138366132646364353661363135336333323133 -64613231626662666364313232333633386265353935363065333933666165633862393965326637 -63623962616164626237303466623565623032666431383633323233633861326132346263386631 -64386364653633363965356664636434393031383663393137343733343732363239633436663636 -64346237653966363538303632363461663030396237623661356438653562396232313935626164 -64326162363264323831663965383639393937613931663239316663323164306537326363376339 -33333063373464363065336132613230643831303937373163386263646562313361306133306639 -36636264373638326663613236353439383532323032393431373333626330313235623362643137 -30636462643536363638613936383931363663343964363038373834343164633062343265666137 -30373734396232343936326330623362316636616535643136313765356439323934386364643631 -34316630366332366432373739363637623437363132646330326361363366313633343634383132 -34626633343163613136343665373931363065306232313336356461373366623934306563383935 -35373930633334343864306165653034386133393636386662303737303136613432333531313461 -62323065343037386231656335326364363761396532346661343664636637633334383131613833 -34306465643133383239306534383035303363353766613631643464383264656536323866326464 -38666665613166653638633039666261326135363833393832373564333765353134636138343266 -64373536313666353332613535633061363835363839323430333430643863343461643236333937 -62346262396331613536343539653766323561343639643766653535353335383736333464613331 -61336337613737356236636333393236653263663065353431633537666161373638663861363863 -36393364616566343561393364313531356134626431646261346363343939616332313631636331 -31393435373265313836653533323237613336333061306330636665626233613537383932316235 -35646633653664313834376466306335653465613866393939323265316436333062646662656462 -36313861306533326163306266353538396436386365393464613335646432366461326331623664 -35616665666337663163663631393539663731376333393339373065643363326535643563383532 -63656466623939376632636433316661613730386363363930613830386638653837323938373030 -34653565386564333866336337316164663931396630333634336130383465643061633239656638 -66666166303835326365353433636130653038643938653365333431646561306536623037666231 -30333830376637653932613961353931363537336462643030376537386539623636303038363862 -34373036393931303465626563623664663532666166336334653234346561303538313036343565 -61363734333737393131643661326331383036333363653333373536373134306436646139636561 -61386462323832653061313034616532643265343636313366346537356164346461313436373637 -34316663356135636231613330323431316566323361363636633436653864633564366366383339 -61633266343336643430623138326436383361636161326264346365643536343838333138646563 -65383230373361663461383236313832366334303632313935313263646266393030313238613334 -66626631323033383334636561313666653165356135366435613438323662353135633234346662 -33333336646166643862396361383636306431353439333562633533303265303266613662306162 -34383062333166393739656365303738373361373436343064323636336634353232383232356334 -35633561383735316663383333303334633631346339323562353634396533343636653963653462 -61393039303366626563393737306336376437666138393562373933333335633435396632333232 -39666435383832663462626563646162666565393530393164373963386161666436343263316639 -30343064663133616135613164653937366636353062303139366261313165373362643365316663 -33653630633732623466343236646136663730663938333234613163663835393036363161386332 -62383833396164313132633535323530633937626437383462663866346163623234663564306533 -65326362636433333930623663653031313438363930373462663639366434633665656138303866 -35626333336532346663366530303061343835376239643466326361613030613730326130633165 -39333465393466336231333335616662386238346537633164666636333133306266613064343165 -39653963346133333234353266323065656237303233623264376233616334623665323433623632 -63316262306337386535653233336635616438356565343433393662653036663736663663666435 -33333433343566623331333961613662306236386439386135653138333638366566313534383934 -62626261303530306237653637363933393634663330336462393932353230313730336362376134 -35343530363431323366613361643636663961366238636537616237633965666639633265376437 -63363930353064346664323435313136316132373237356332383537623032633831353563326230 -65626166336166363164613130393165356139353439353436666537346362326635386136313464 -34383633336636376137643733313637373535643966376435353339363535653139313739313362 -65343434343364666639306663663665356533323635316534316561313263663662623764326138 -38386434316535653262616530643164663335323436386365313361656430616535346463643765 -64613231623333363835386337643334333039363835343731316165346438363234666134343263 -63663036663566373734656661613432623761393964313166393532396664326230363230303161 -36373364663137373234383134363966633935666365303430343539663461623561383236626137 -66623964333766323534316439643666323666336539393566316331363862666561656634326166 -64383236633638663163666236633531346330643066353566336337646265623361326261373862 -32626538316631303465396436643236376161613232666339373537383438633630316365363065 -31653433326333613431303936333137323632383163323561313636396331376664656365326536 -39323632623861333466333530643864616166623261346237356431326335343431373164633366 -34363932656234613431306135323434353164323461346262656337336137616430386533313933 -65306566373634363933616233316562643930666161643234323864336631343032326462623365 -65363036633061643463343637313933396436636333663861393333353732383736633534303864 -62376339343938383639626237373961623931616264353264353337333133396564623831643338 -39653962623766643239393061643938643066386564613932656565306564303066316165336230 -62363330653137616666343333323636663462643264656364396138626265623964663730636561 -31356339323934366637623165623039323439333434653336623962643734653436376462386232 -64393863353164373734313262383638393961623466653034626666353361393230623933373336 -35323661333330393835363064396564613766626665623866346334643463663865323264386661 -33663561616131616134333564343334343765373061393131376231653536353163643435613564 -32376466363230363930393734663433313133646238396366356361663636343362313237303766 -63613538383836646333366664313637333537303333373537646130393631656636623461383533 -31333661376236323736666136393062643566313131383165663965656637303634366330666539 -61336265356662336236626432643738343038396263393665396337633837303761613332623436 -33313235323938333333666437613731376562376233353664313362356433383938376436373863 -32633131386462333838323536333439323733393266376632386530366363633337323963353437 -37653334626434616433643930363765366439613236373731343339613237633962356531303433 -38333261613062323963366131633738613463363364653531663639626234383263633963336635 -65343330636330353363356134363138333031393838623635343335393462343036316638623861 -30323638373032653331616361383338333761386264623435656561623337303863323536633932 -61613964363538313435363231613834323333336364353962346265646233613935313632343738 -65303732376561643763336162333962353136663136613466666631323063646432323130616266 -65663431626562646537366661393638346538336538666466343261396431383232356238353062 -37653031653766333931393763346366356563393261363438363535303163303962306630306665 -62346633356463653465363965396164323437346132396138316562316237303033396233636336 -30623039326132393534633937623363656666663366643130633161646566386135303939383762 -33356334636565643761323638346162373636323433306436303331336432333830623739643237 -64646432366466326139306663356438613433366534616462383938646239643262373836363032 -61623735666238626435316530363833366135633563663530643631623539333832343561323138 -32383138303464393265373665333736303861646464373861613730653861353463323837383737 -30356437653833376631303636393739636663663936656663393130616637303332393636363431 -38663837393231326536643033376663316138363234306637613135396365616636656236646335 -33633962626135343564633337663936393031316133646334303635653130626131646531396134 -38343631613562393435326139366330373064393839636138323466646533303538 diff --git a/vagrant/files/ssl_cert/STAR_unglue_it.crt b/vagrant/files/ssl_cert/STAR_unglue_it.crt deleted file mode 100644 index 3dd020d3..00000000 --- a/vagrant/files/ssl_cert/STAR_unglue_it.crt +++ /dev/null @@ -1,100 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -66366138323934613133623237633539616236326462373461303832393739313466373236323765 -3461353265343631356335643139363335356262346238360a383136303237393662303762393766 -61346362386338663037396631633932373834303265383662356539323766393466656564633465 -3835623862356266390a646463633764653431353537643265363764653934656430646363373335 -32343135303266623365656532633061373564623664616564343638636138623433363964303366 -63323939396365363539366562626135626662336236313963613233633637313731313465363636 -39386237393034383263346164623036646535623463306330663034383632373836333661666435 -61656662353762653266363036373430343333653835646365613835303935396230363032636164 -64613730663339316261313664633636613763313631653839313465336562633663306563633561 -38613661333766366633333463376162306365323330356339613266326331353866316638363237 -33646165313865313431666163613234366133616263396630303362333336323638373131396334 -39656534616331643530646530316335633730323166373830353262366465306631636339316266 -32303162313438373531616439356563303030383136613531353561316234353632646232396433 -38643463333836326435303733373239626562363264653532323334356262346133633361373963 -64396564313630393164323231313937643435613234376436386563623435633666616331643530 -39366536616165303562663638313739353763656134316132616162303161623130616263306366 -65613264623935333134383733353637653336636532383165646338303633353330623231383239 -39343834643038656539623162353561646364393162323839643533333363616437393239313034 -61356134643031356536653262663833336361653632626364336565616163376238326334663661 -35623338393730636237616161383032353762383965633962343330353235643363346633313462 -36653365346263323062653239373132323734626363623337393635373738663931656363383136 -62613836356562363866623436393131323130306636356235343035333534326331343337383431 -38613565383365666632326238363165313631373262336234666434313065363363346636653339 -36363432306639303266366665643934346562663934666665343030396233666534633438396332 -31323662363130616338333233373961316639633436313737353530646135373533613433353034 -62346432623261346334623738663835666639616564373961643439336432316365666665393135 -38623936616361313634333339353133633165663936616332323938396533393235636435376439 -62633537633136386366313934373263383730633334343636373035316638343334356530613530 -63396438343139383439666539383531386437313865303864316437363563663065643266353138 -66323133646463653066323466653662653736306162636565326663386362366332303761653564 -31383133396131663563343339623563363239363763643530383833353263646264363565656535 -64343737663261653530623836356430666462633964353832663964396462363430623336646336 -65356566653938376132663230333033376261376538306266643565613561373636383532616139 -30396564613564333964303262656162313839663435666539393734376639653562643735393037 -34613864353764303661653561383466663730333932663139616164333239633961326632346230 -30316138626562346434643033303333333234396533626633333437316636323062643035623664 -38626434366261326663393839343765363133623339373738313563653736373565336164356435 -37623837343437623833623137373662653934363133633366636436663831653737376631376431 -34363461323535323337343632653430333961343165633864346132343938313361373864363565 -32663734626631323930613638323133396135313562643536343038366233373136653330626239 -65313933636136303365353466646533353236343934356330356161343433643139383764393134 -32356331653831633832376262356238336634353362373837646563653835613634356463653466 -30383361356631616538333565646435656361313839346165376231643633643634313863663137 -61333236313436383464663439366637643663356535613861343831663737623364303339643733 -36373461323130333639613235373961313736303436666361613134323265386165616237393164 -34366530363034376437393866333861646636396434636631343033633565396164663833623331 -32666161636163333266393361383838653333326437303235326565306663356366613430303237 -65343937653439353334343834303234643136656565366461393838653739336233613234616437 -62643065356462306565363964663332663564313734336239633833306135613662633535666662 -36616639323332353864383733393561666464313466393535373961613831313463306461663266 -37653230663732393061616365656638623830346536373932636461663532343532366566343436 -63376365643061623839393139333661643439376363396564363237346461393264663131616162 -32396464363231613231363561373334646362636536343732396364653132663664366434313538 -39396262333232383934396335653461346338316564303563626363646136633266393937343434 -34396234363362336232653136666639613466663833303062356264633461643932613162303031 -33663539366332643538353632336433393564366565323034393531626236353432623531383033 -64613435623761623562363563323364396166353265363039663932626431666339376632353539 -63656330626438653366396635353236663762666439613237623638663731306331313564333239 -37363663366134303037376433323735366266613831356635663932626162343639306366386439 -61643831336139303133336366366165303934646362303665643135386462353565303932383930 -33393334353235306534366339613066633434393038636331366265336163306334323638393264 -64313538316632373430656532623532313063613337626165353365303832373566316131666336 -62353062343565633836306235343134323433303633346130363362633263343564616535396263 -65666666646231336636616364646565303538343361653932356135393161316335343333653439 -61373266633033653931623631643430613137393633393063353833306463663630333434616462 -33653230613639616531666335336131373636323065616239313733323664623138313964303930 -35366665626362303161396562306639613435396331366363616138613735383030366362396337 -66356666643131393433343237346533383335376462383566643035616438396366633737393133 -66323630656539363237646663363764363536373266383036636166333834366663613863663533 -62393235666666653433613665376431306439616138633533656362323436613231303264303764 -30336665336661393866343935313465626264303139353632356137663439373232326339393136 -64306365353639626565653965666133346364613538333135653831663032613135653263353965 -63373532323062653630646363323063383065646562313539373637623463313333313535323866 -32363631613035653637663935313535626333333433623735663439383239373231613037303736 -39633436383361356633373037333362363861346263313038373131653938663930666538306432 -34386163316163623064376132653062333535396631393363393265303964356431663439636234 -64636138333031656364656565346163663533353333353666373466373734633263626439316230 -65313031396564613163333364376637303539343563613133613133376239623066303139393866 -32623135396134396439653061386561666433613536336566613530656530333433343537623866 -31333364636631336264636531343365633264633433643661353164653831343836323763343033 -37386131306563626437343034333330623932303639646631613239303331646364323465616562 -62353030633765393237653161636165363465336136333264373832616537356531313066643263 -38346331383061373133353132646562313630643335376565396662653830656538356165356137 -31396232326666613236313961636334383962373533613566333930313866373334613064376561 -65313565303630326531613131316463386636623830346131656532363632613032303466623334 -33353633663730306638623566326533363065663363373537353130393938313936383763663966 -34336535663565633162313732376237336463343833303939353965653665356563326534643033 -34343638666463363462363235343731346461336235306264643866366235663961363461373930 -63326334323363656263616531373861633936636339623835323936373661373364336233316631 -66366462643663316162383438376162663065633333353138363836353331343162396636623130 -62383461626534386664633166323764303631373731366133633930643232303934666535393237 -35373462313561353561643866623761313839316662373134363431373562373062313430613462 -37333635313037656433656363313864653037313732656232323639383838326263323564643263 -38373337353635353836366136326334343661393064316139373431313330653966323763306337 -62333066323961623535336232303234636138306630313732373364613363663434633734346231 -64663930633162313937343130343961356437353366306338663636366631326366663938363531 -63353863306137653237656330656237356563346337343262363664313339323863616438373537 -63643235306137363264383238316434313331336337356364366261313835656562643566666130 -633536646634326262663866316566376638 diff --git a/vagrant/files/ssl_cert/server.key b/vagrant/files/ssl_cert/server.key deleted file mode 100644 index 2e3e98f8..00000000 --- a/vagrant/files/ssl_cert/server.key +++ /dev/null @@ -1,89 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -32373237336239343336393066383464393861636235623430343533636365306430323738643234 -3538336532623734333038333832393735343363363566340a643638636634653735656635336338 -39396461646661353061373636313032333065373562373036336232346133313334333862353034 -6337653431323235650a663536386561366531623363303963353539326339393034313961393362 -30306235623461326365646365373835333235303965336635363361366336303236333237383539 -61383438316464336338633163366532323435616563343463623266346632393930353332626137 -34396538653134373433333461373337333639393530383738373764353731363264373835626364 -35393163333462343562303136313264653764333038343838656562653133353934336666643534 -63366264393462643065323865303033636133663164366334313564333262633736626263363338 -39376164343330666332333534313634633137653432343232656464363066356336623166303030 -34653432366563663432343164636665373139346435396135303736653030653930393963663538 -36333636623832633630386235623365363065343936363031613165383534353837353231373164 -37373038363361633330653633396562323738663739316634666137306161313233633561383064 -30323236643635663766393862343363653866303431346535626565356162353433623132363031 -62656332383835386136656534636135346234326561623037396239303034353961396134396334 -35303463366439363336363666623064623564323032646337366332666536373064353962373461 -63306166616535356231396566316433633062376164303434323639376339643731393461393536 -65633363616235343532386435323736313138666661666434643935623266323133336463326532 -33326364346364626165666262396133343736656335353732666362613234343163343465326361 -33383136636364383466363666316131383366623665323561636130343337613539383134646535 -31383131623237636433313133336162633537653533383435336461616439343035336164333363 -31623638333465306538313637636436626132656534313064383663393230653132333938343537 -65343339333037613332316534336133373864666130303166393038373531396266303832383632 -65343531616661303363303966303230353630356437336164383038366563363166363763396464 -37386330653563613063323438363737313565363130353264613236346565323837326631363935 -31346365653938653566646437623333326533646235343966383031656132396539616661393461 -65333235306432666539623964393132616436653430383236353033333366336363323539333636 -64616563376464646534343838616437306232393262313065353936356235373231613561323866 -61383536636361363039656138306131383732326464613931363837376234326433376533643861 -32616434383536373532383238663263643862373738333338343332303735663863346331663037 -35643637356531323639623533313865656132353139323436363661643263323633363161653235 -37636635623865303536353264313263373938366165623364643337313831653161623231343764 -33386162623531396234623937633465653930636637623837653138373365346132323731346130 -64613338386662653833653034313066636566333839306539666161333333616261653537313363 -64656632333462633232396236373666366464623464653434383832333365376532626531313735 -30313335353664643566313437393834393262333665666366326463663761626135326434363931 -36306439376138313838613532343663383938366333316432303930326239646232623633373564 -62316632383131626432333461323330366165366331303735333330363335366265316330663563 -64393231393136626162386335626433336337373765623933346237643532366662616434366163 -37373832303664303836373434343032313731323362353031346438643335323131366538653334 -33643432646339383865386238666134383861616437663238366365663737623663323137613865 -63633965323933366332363131313834653564636336303966656663366361353731316336633233 -32623339343838373639343534326365346135393137303736346634303863333664376332636634 -30303938353536636365353338393932353435383635326435666133353430323464333531356563 -31353733323331656538373932643332316332386437373938373635633832383662316537346132 -35316339343538343462363237666332376461363262623438623738623732383337373738346235 -64333335396336333337643235616436373736623339366236313938653433343661366332646265 -34636530666636363764366533643739653561626134393936616632616264663230616164376132 -30656336633230323535653766636361626562306262326265626166343036643034303730306164 -63356131333739333338353835363430363864333063363238303861323033643534366264623433 -30653639633862376232393065626636396137343635343030316630313863323365363764616630 -38346336663932626430613437656437313462633039393934653532386535323836366465383761 -39323732666339386565653836323461623238316564323333363461396161643338633137646335 -66373461633233333337663332323064373662636535376433613232653137353834373630663366 -61343066303366666362353837343337386631653461646230656533663965626135383131346635 -31623131653264353861353630646662303835643738323565383233336334356264353939373933 -31303964346563636437633532383762383234303130366331313864386330653130653437356466 -38373135333066336139336332643666373932666565303062636465326236623963323337633634 -66646633366532646232326435663065393035386438343030323038663033356639356264663532 -61396130643135666266366664323234353065303031343531366532633532363534646536646235 -62383531393832643137633332353364333230383361663264623564366539323132623639623939 -61323263373866343933653734323661383634636139383833323837303236326364666466386336 -34316465346261366131316535303563623238346530643264326539653432633237656133613532 -30663738313139333461346163626130343535306538623763343134623832346132656666393761 -35316164646635613032326565353163656134616366386364306436303134386339346462653262 -61383961656334653564623262366265316364366538353362313233343239396265383235303534 -33666366323466303534333165653334343437316435613566326338666336326664616133313062 -66333339346335626663323335373939303163623331303133643937653362666430323930623462 -30393239633763666463643066393161353235386461633066306235376261346139363330663235 -31643539616335396331656364383265346333323661333066353732653336616330303931376164 -37643931313337646337333236323939613333303834356662373836633631393736616462363338 -31623064303936306330343361313763336239316362633732326564653566313265616566336135 -61616538653365313666613366313064623232653563383465386535636530643831633735323433 -30353239333166343366643738363834613230316463636339666434633961386335623238353030 -34356536376164613338376534353438383039383930316439653732643339653531316530393534 -35326165666439396537396464656339333366333535366530353064396436663966333465616630 -33666233313537363165333738393362656631386564376365616266393137633931643833343232 -62393663663836303563386235383363623966316635366133353165346635373063383666373833 -35633264666139616463333339653733376431363761653433653138356364383865633937363433 -33313361653536633066656164616331343033373339336334666635353630636532323632323261 -39303937626633613434653835376538356164396631326163346465663337386230656139396237 -36613762346462626135323233393537616539646234663866353433396530383966613333376137 -37373461333561313966303239633834663837656230613830303433353639643431323930633238 -63383161313938663737396539663163303161633732393130363737303732313166623534303339 -65323162386430393639303435653436656239303032663761626462633565626161656632626162 -62386631316434383239303631343536363034626663626430633635353933313533316337613532 -33383335326637626535336664626663376332316236633735663339373131373630643665356133 -31393664663737326638 diff --git a/vagrant/files/ssl_cert/unglue.it.wildcard.csr b/vagrant/files/ssl_cert/unglue.it.wildcard.csr deleted file mode 100644 index 7328e869..00000000 --- a/vagrant/files/ssl_cert/unglue.it.wildcard.csr +++ /dev/null @@ -1,58 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -65656166313438643163366632316530386139623538636439633638636566383566646239306131 -3938333565313734613639376539323233353932373635640a313831613637383734636539666463 -64306331656564626233383831633862623861346364366632313733623263306464636264363965 -3933326435396164390a646439366537323232633232373339343538616431343137373338326166 -65373365613465616631623463313364396332366539386462333737623737343330393662646533 -62353663306531663935623765353838333138346630313333373066303835323461383630306662 -34316363316661613865616163393133626139663162393336356665373465353766656661633731 -37373563396437366661373134623664393430616532353530303861336238666539306562343662 -32613431393331306234643362323961343261376238353635366132623638663463343730343730 -37323733313739636238376538323264333238643532306439366463626233613237313933383333 -31373064343036616239616562373264653163396265613935656638613662663364633866393638 -62303563363763356433376634626234663234323636366232616431646534383730346233333963 -31616263353462653932343537343863326337643037326265666433623965646264303066316139 -31653864383331626231313736623563316330393563376562303933313237383939396438623231 -65383666336565393236326433646561346635356661356136306165323432383531646264323064 -31353738663861306135636131333261333964663338636637306665333663313164393537396663 -36666663616634323137333835626133383737666632316336313661656434376239616437346636 -62643035376637373031636330646638356562643962313336653964616337643064346562366531 -33613935656630633736386632616162663535366466663637313339323939646136636165656166 -38346562346331323738393934396362316361643538653163333935656139356535363432313532 -38633336646332386661643464666263646432373134306639306561663134373164326632613531 -64313162303263333534666436333634663262623632366631633865336433303564363138353566 -36303862656531343765643965666531613231343035636537373030613932303039623131363830 -37346263666637373639643930303438316430663462333430613335613131396566303533343864 -62373439663236636232353230343433336137613930623133303936323562313061336637306566 -34383336646336663263373334323131316262613136353239616630303264633238383266323464 -37623864646331363735306132393936306162393463313736366166656439343039626434353739 -37613532393830323431393964333437306366313335393434323463346133343237303732343962 -33373164363135363335323435636666386231363463373864663933373430656239366539643164 -30323038323739373965616630336166356362653833303132336230323661373737666232346630 -38393933303736306537383364356162656339613461333262656137636662303532663737636237 -36333661663734643363636661353337616162396666356663393432633361313534333537393161 -66666165363635346162623965333061386563663166643736653134303638613332313637333064 -65333438333661343664303361316631643763323164386330636139353932646635393963656336 -36343362323332336130633533663234343236393639323062663962633336626435653137306539 -63633866616438623764616631663533363233346362336132303861653039386133323965643863 -30633932646132633862666266323761656438343935636332326263653437333366306338636137 -65303538613661373538313437646466393536313665623666396235353962306634386261396337 -61376439636531383631383234313230303731653435393964393835393236653739316634623363 -61363730333632323039326438366633336462656135366164663434383561356337393636353661 -64616435386238316265386565633938623935663332633062316163356337303765366262313534 -61396361326333616136366435666662346637663864326561653634313030303033303132366132 -64353635383932633733303333636137323237393764396136313661353463306137316563303232 -32363063626630343230333065376638626236653532383766363161313862616336396334303764 -38393262613531343134653130313237343036383464393739353631636235343930653537373935 -63306431643132643362656433313061303838306164643236316166303331313636376431326239 -66386635333631653232383664343864383739663364363131373062623835363635393930326639 -33303733353761336638363935316232336533366339396166383539363963373536346232663865 -39613463386138363364316365373161313130316638616635313235363235396166323935353664 -39646163306161323536366532333231373530636234643764366134306164333431336636333533 -32333966393634393135333031356533666531613262313432346366373262373565323962323939 -36343764303866653638336134653263616561396332643030356434383531343166356466646633 -37336564613131633534313064303933633134393131656536383439393633366164376636353634 -34663236663163633338303838303733643262363834633064313532626631646239306564313361 -62616331306330636366306633366430353964376466313032356434346532366362356136376536 -33613962316539376138323033623736366531643261386535653733303465333331383131666530 -38336561353633303639616635656566323130616561656235393135616631356334 diff --git a/vagrant/group_vars/all/misc.yml b/vagrant/group_vars/all/misc.yml deleted file mode 100644 index c08529f2..00000000 --- a/vagrant/group_vars/all/misc.yml +++ /dev/null @@ -1,11 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -63613831616639376633626266303462633762313365343031313561363336333536303436623033 -6431323039346335366131353366616466663932363365320a343964663464313835663930386236 -64313434633233613764393834646431666632373033636561373833623931363033396235653031 -6237326633343230330a303432623634363363343036363534396162303532326335633532663065 -39336337653362346134396266353639643530643430363963663633623163373336633966343038 -66633563356332663136636434323635633164313232343337346335623337633337316533313531 -38306332303562383663303033653237386563653339653230313939623435663166653364653862 -65373863306531633963653639303033343733653339656563656264633362663238663136643733 -39373565353639383830333462393935353165633236346238613334643863336637326161636166 -6162303863363232643039313835313036363865663531363230 diff --git a/vagrant/group_vars/all/secrets.yml b/vagrant/group_vars/all/secrets.yml deleted file mode 100644 index cfb9284f..00000000 --- a/vagrant/group_vars/all/secrets.yml +++ /dev/null @@ -1,51 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -64636439626632623833636366633136653733373531316266643663336632613035303563336664 -3163633033613732656536313030613263636331343139630a373432356362376136303463643637 -63393038386266313536356261633133353437363366386161616233353336336237373866633865 -6333613461666564390a346661343365663662613762643061343636383866623835316333663930 -35303839633239326666376437393664613039393235383838613439653239353831313238656530 -66313261313763666562343833663963613037636466623863383636343865626261386533383165 -35633331643361393265656333633330633064306161313566373131376235646531653063643565 -34313032643166636337313866383064383436653466303930663435393336333965333334303963 -39333232376437353765623832346366383833313962373632626232643636326436373233643237 -35353066646130363665313765386631646363663161383461626332616666396139656463376134 -39326234333138303130623963396430313037643263623037633139316362333332313933313035 -65656239333131346461383066656263393365326561316431613266303035636438366232373665 -30313265306332626566313165336437633035343063623630336139313166366136353666623934 -64616339653930333630316236363630356530663465613266643632313431376236356164663937 -62336361613739373030353761336366353764373231303433376435626334623637636637623832 -33373333623666306363646366393663353465396638623134313537303830333639333137343662 -65643865616139356161333239626165303165313730306434326431363032656337316337353739 -38653661316631643930343863326535313336323937633833323565326539376135376339323232 -30616236613034643935653866333465663837373861656536313333346637386632353139666539 -64643966313964393134396630343033633233353534353065363635383636363139303637366338 -64353835386537376363623062313862383637383430336464623566616336373836663835626331 -37633562303933646263646239306437653135633565303530333761326566336565613366303630 -32346436616135326262333930383130323562303465643065663562343832323364303737613764 -30623939303466306636376235613639363933343065376562353030326335663963353766313263 -34346266616239373264646439616536353762656536643738623662396231323334313261316331 -66356566626463363033316162626265353765653132346534663430653135353236333866653031 -37663636373133353465363635386561626666623532343832366536616634313065383531353538 -33633539373366396566333333653939313864663235663165363563326538656335306262393561 -32383831656638666464663462373462663662323833633735333764303363656161616235323464 -38353461306430363566633030386535656264616236393737343662613336353665666564343462 -34383365306166353236346533653333643465353061396135636362336239353231353463633361 -61623965643634396366656265303961623531306163373230626639303132306630356537623934 -63623732663562663231656439623266623034643336643839396534323363306364643534636233 -39353431323166326237663362353137633631333733303231386236356263613163656238326461 -63353162316366343465363662383631333761366262336235383862633630386362363135636232 -35386234646662376434646161633463353163313261333333636162303461623233626637363039 -39356263343632356333666633636437616431393062326230616132336462303231616635326237 -35616231643633386430363335383333386339353833323064393039303839653131316439623036 -37303236303535656466643037326337353766653666633933313330343930636532333666336638 -33376437383064646633366637623638623239663261376639343332363430656232373162356336 -65643331643036316136313737383934633731306466333639626230633465653338393932666636 -35613331323431386438356439613034363364643663623332303435363861353538623238393835 -36376432623361386237666233356638376537393536313335633361353566323963303864343461 -39386464613134666437393734376662386236303837626531613363373161343937333863656232 -63333766636465653666663261353939393363306665353832633938376362613061386665336665 -34373939623633666236326130643662643163666233643936636461613963643836373639336230 -65306337303261376232363239636634386638323238313566326234663331623535353433633431 -37313731663134323839336635366366336462363563373639646161376162313231396365323430 -64383739346161343030343635393935643931323237383030613662636630613935333061613961 -3665636339663830306630376237653762643333346563313939 diff --git a/vagrant/hello.yml b/vagrant/hello.yml deleted file mode 100644 index 44a17500..00000000 --- a/vagrant/hello.yml +++ /dev/null @@ -1,6 +0,0 @@ -- name: test - hosts: 127.0.0.1 - sudo: no - - roles: - - hello_world \ No newline at end of file diff --git a/vagrant/hello2.yml b/vagrant/hello2.yml deleted file mode 100644 index 8a67bb5b..00000000 --- a/vagrant/hello2.yml +++ /dev/null @@ -1,13 +0,0 @@ -- name: Hello Ansible - hosts: localvm - vars: - user: "{{ ansible_ssh_user }}" - sudo: yes - - pre_tasks: - - name: Hello server - shell: date > now.txt - - roles: - - role: mysql - diff --git a/vagrant/host_vars/just/secrets.yml b/vagrant/host_vars/just/secrets.yml deleted file mode 100644 index fb20c325..00000000 --- a/vagrant/host_vars/just/secrets.yml +++ /dev/null @@ -1,136 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -32636230316261363062326338363338386265623431633263313431373432616230646334313062 -3836363231306431333533326538386636383561343565660a663539313233313537383136613434 -35303864346335386338353633366338383134623139306338363065316232323463626264636335 -6134346134386531370a353234353061386364306232663231646362653838383539393931346466 -31353966373761653365363464636336326464623736343630343136353066353331393731646236 -38646663653862363435343338623731373532303261313661393538643761346638376238616530 -63363238613961316332363733653837343662376433636261336565316231393533383038653936 -31323664363839656561346431333166623263383737363964396564663164393439613634663636 -31346566626332363963656434333634353233356533323863623939383730333061613639383539 -39303530373033366635303531353662366339373937313663623138323836353064326637613737 -36313430363465656538666534333364366663316537323639363239353561306530316130303133 -32356636326130346630376134393264316461383464336261313734656333653036636534303961 -35663631643132646334393539663365326664343636326163653161653137646663663466363035 -63666530353438636231383332333235626431636531643036373935666132633433643838633132 -35386561343366336535396538343630613632623938323963366562613733626238356334383432 -32656536666539313462643738656531373438623631393230646661613731616466613439313038 -63333538643662343837666239376430333562373065316266336131343532646239346634663263 -33323235663965326439343534396461376236373962666530396466616265346266343935633566 -39396432343163303666343762643934323461313935343531626432353762623236326162643939 -64663831653962626633356430633232393737343237323134616333393436313337666165366634 -32623861626630303137656331643738643562323531336564383466666532303636383538313630 -30623763333430343762393164353131373733336230323432313266356536383361333537343937 -32636532613134303234313531646436666633613431616338663263316337303635376361306338 -39333764376136363664303165393938646366333832333261313337623337303138616661663362 -61343663376135613036313736653964343963616439636239663261393037323439323061366339 -30323130333431323534643635663531366266393062363362623930633139366164326236636232 -65363562393063666334613136306166616232313333633337373965383631386165613261363662 -65316466613465313332626139363365383262653731633162366639323239346239363930363761 -30663839663830363462663136336432346233383934383538343334623335313033386462393135 -64326633643965643366666331336362623137626166616237303464323135643634653662303335 -30386237313338333532666236393637663131313939393939666265303264666466363962653166 -62626231616463346134346337323132316331313862336665613261626532666130356433336265 -64633136666433653532333964393861316235646434623733653961343034356165663065666330 -39376239343462396361323731376632323936396332393461646238623531323332623937323862 -30373932656463333161306336323166393964376637366236393137323432383432313736323464 -61666634306564343865646438346664363563643531623064346234316533306431303735323866 -32623762643230366631396431393862623363343331626136336131386636336439376661376235 -31643761336332303933613936393365343035396332376663656632343739343264646565646466 -37353031366661333134313134333137343530623235616239386439643539346636656362616333 -31646535323362306139356332333536653839646234316231346535343538336432363261353534 -36373634346131346132306366336136623163623039623231363739653730346135613631346335 -36303036383335623635383363383733353135653665643239613939333931386539386161623762 -33613133303763303431666564633031323162336230653661383730336664353233353530323237 -33663466303163363165326234306561383634316130386637363866323732623233396363343065 -37303862383764616231356234666633366337363961326339613031346135636633616566393337 -65353339386439393965313866363533633633656463643364393134653339656135353436316139 -36313663386566666133626466396539366566643130363439333732663465316162633132646234 -62363139666331363962366430383233333731383031633437623166386337653436376639616435 -35363561373162396362613136353537623935613136373633303965393533336461626366646366 -61373036663837323839356163613538613032346437316433663534396333313966616231623964 -37633735306561303861666237616438663232653866623637616561323365373966326530623161 -31633139386533333661373734333232623961613464646130323735643232333666633832336334 -38373066306238353866346230376265633839323831383563346438393436633430316636333434 -37653937333365633332613735363032376536666230643462646435613533363166303930653566 -65643634633936653530313662363139303133383933333866366237646435323131613131663564 -32616532633965666234383766613534613830396465333933393963393238663535316366646630 -31316234333934353035623661636230376566313338653832353661646135663037666333303464 -33623732353136376136616131343430336665636331663665373134353634343233383738393066 -66313635666538663861633563323032356465343465393933666631643230663661626430383630 -62326436343831666462613031333266663139333838366630346135303633386632306235643165 -34353439646163613966656434313930366136366132653039346663343963333135363965663664 -64376462356163336430313565643538383732623134633135396636656161323465616238376163 -38313931333062666137376338613732353634393063396438363239316662623535663533376664 -63343262653232343039373266346330623466326462636632336565383536323862323661663936 -31663466353339313339343862653134323135333339646634303633346436323930396137343162 -36333336313331366432373165376633623137303535356462353866373865363533383065656238 -31383039643833313837373139366366663737656530633164393838313339323665386265306566 -63383632393337613434393338346463363835363164393630653964383832653466303164306263 -30633462316231323533633364623962343361663235626236373738356133623461316665366332 -65366166333066366638323732303362633735333439396661306162646534396564386639613533 -39303735316234356533383733383435613766326639336533663238333465653437323638623537 -35363962636433616130666537616664343462313863313735343237663034323564353961383662 -30386461386532323065373663363631613134663038303437373635333632643261666330613230 -65613335303335326163396432313837323866623639616632383437613939623132613864396432 -30656631313466656665623433643038366332656165383532343162386465313565303532396233 -36646430623935303264346466376661386130323063303630323662306565313730663537333436 -61356235343939386337643065636238346531653931373531376464373862316236656161396632 -66663331343365636266623661373637383262383264313931343162666231353732366231646634 -32333166353231346662373062663633383961376633316633396464386331663962313435666131 -39633835333035636330376239616130666434393735393561623263646638303966636139633732 -39363165613732393535623365316461303934343233663064306163316130326161663636373861 -35353638633833613730336565383862633232386436356462313466323835326530666430663630 -39666164376666663638643133343439336136613432323861653135613330333661366539316262 -31386664623664383334633033303439313663323832383862656333366337323564613532616336 -64313565333638633733613132393565316463393130336331383865643061313435656135613362 -66316664623938616266323738633565386561396366346461653532613039316635303133303737 -64303037653431626631366361373364343834306231393066343736386662633935393162646530 -38653236353937393139613330643066386638303536373333306530343933366335336263636666 -32653138616363666630613431386239313030306161653565633638373632313139323832393434 -65333137636663373532646130313932306436666363366335316330396431323363346437613561 -33343036333136616330343364643833313465326432346535313232373066633763656161306138 -38383038353037376330643934393665663966396461633462643735643233323165373830633439 -30383233616466613563333666393734326366373462336533356165636464376432326464643738 -35336664313161663232303130303334353431363134303936623335323863633038373861363931 -65363764316339333065656466313337373135626430326266353432633637346364393330306331 -37376464653333636135363761346430336238666261323866353739376231643934336238623266 -66303962313166363133643164316466396662623233363539363332633433373762313237613532 -65623264323961346661326639366665303939363261363436633266313165323361333135353530 -34376266656364323633396533326564613331333731326563343735343463303731303939613538 -62626361616265643563643232356139316633363537373863646166653166626536353630323734 -37313131656435313866333239393531313162333263366233373930333263643035643061666332 -63386639653737336262653234346466326137376234323539393466363363663231636131656262 -65323034613235326664356436343339633832383966336436323038383335323861356638306235 -36376163383364323834356130393664356566313464326566336461656566326135326163376533 -38343365323862346139336336326630376330393236636363653761626337313132393739323165 -62313462666232326562366264666165613234393036653231636639663061326266303839306236 -38313331386536656334323732396532303565383431313631666632373934623634313463306533 -65306638616466336266646464323539376637336232363738613462646264626430323266663830 -37303162656666383336633264393233306365316164366265626637303961646262663066306164 -36306534326461313866316532633561623862383863626162396564316265323034643661363066 -33663166393866303839393361623031613665636637613035366463643166313662373635333737 -32363230346234313965353833373339646463313166343736313563323434316134666132646566 -63663764663764393865636365653336323664356166343032373138356630326461346632386230 -63373864643565323430666462386464653239666362363134663332636233656631386232366638 -62316139343266353065613632393437393138623337366661643362323235316332666261313830 -63393961636336653639656236643464333765646261393834383865346233643361666461623437 -65316137633833613666626561313630393233376666626330353264333536313932646232633938 -37613163616531653830326334353236353062376563343961646163313766663033643234633135 -35666531353432666662386663613534396136666434363034666361383837316532393432616639 -36316232353565396539646636326638383164356531363136666365313335313038343237643964 -30336137616436323332383035323432333563373732613762643137656532323035646338343430 -35333539333433353539306236626131353330663466313564376231646632663361653335343439 -34353334653531663634656665346536623861656564353938313236383434346137396364316665 -36316434363830373337343530386135623234363435353964346534643637346430616336383561 -64353332353263613530323363366163616633323232333561663038656338643934303839633161 -61356630353463393138343038303732653461346535383462643066323733373863343763346337 -39333763333836356531373065376161396431396663663037636462616564323138366566333633 -39643437653637303333316634386462323533363862393463623133626530396239396533313133 -36663266303065373234636231383162373231623936363863363466636331653961336639373337 -34616635613337666235376133346233623462626331323533643230646665366161653334636565 -31313738663838366235633966373665383865393536396333353066333930646335376561383732 -66393466653664393933303066373765383730666235326539653032303935373734663934353531 -64356466613035363464643463386135316438353131383935303637333636383164633563666637 -63333266343234656437363436313762316364623831376135653737316431363864363437366665 -323963393638666636623065313162323939 diff --git a/vagrant/host_vars/just2/secrets.yml b/vagrant/host_vars/just2/secrets.yml deleted file mode 120000 index a71bea0b..00000000 --- a/vagrant/host_vars/just2/secrets.yml +++ /dev/null @@ -1 +0,0 @@ -../just/secrets.yml \ No newline at end of file diff --git a/vagrant/host_vars/please/secrets.yml b/vagrant/host_vars/please/secrets.yml deleted file mode 100644 index 1c3921a0..00000000 --- a/vagrant/host_vars/please/secrets.yml +++ /dev/null @@ -1,93 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -38386461333437313133643631373538633664303964383463643764363236343164383636653035 -3966626566393633313332623565383763643434383930360a323663363833646139343039323139 -34333837383866646565346561313032393334363966383564666662326366633532623132626435 -6263306636633738300a613362393561363931356634333430366164616539353737646438363962 -31356266643864393332343962653364363836363235343932623839363562313966353337366463 -32333832663536633731393666383966663365646662663664366330326330343564316238346462 -64316231373230393530306563323136626631633137323636383432366235306437623930663666 -39613830326339353335653537356262313332383765313438623236326131316466326631366336 -31653338646666313533646365303537636439313438623266383231376231613534303630383338 -37363461373262623732613536323362343036373238313434393666313033353032633830663532 -62353365343937333133306138626337626536386465633536613331343635343931356431303964 -63636136643938646332633833623733343835366133323832393236343330373632626663656231 -37653537663739336432313563643865663238303535636463623339313161626138313961346239 -36316631353935613265616137363962333039663138366439346461373365633464333265393435 -30633436633533323038386664666439313939373033333134396437386335366633306164363831 -31363561333633343464646566643361366365306264613536363334623663626236306562393664 -37383163373538306632616330326433393461323961353565373561393361656661333962663136 -64343761613134363637316466393161326264396263346264656263613931373465633030306265 -61353961356230316361316566643537333031343635663837323238356231356234313839306331 -61383863323766656461653737363533336333333635386662343563333233386131623734316139 -61666661323962303631646365363735363135343733653265646532613166313962663230386638 -30343563313237373039623933313161313465343337366331303761626664306163353632626465 -66643964323638356462316635323264376235323734633732393538643032343461646262663235 -35313934303362386335306264346231353431333437383038643233346336396465336333326364 -63656135363764616538383666366131323464623337343939356665376366363161626330393131 -34336463336566646138336562633931353531366635326632306335303865363232373964663664 -31363965386431363961343962666538633837663933623362653137363365623037623631626537 -65366534643533346630393837383130613961653636613835643665346337633363613732383234 -36626164626461356238646135353338393463333839323964643966643463353264393362616466 -32336338373936343733393436333536383562613365323538633163656634333536383930643532 -64383834396166343931346166646234343231643665623836346131663238663365613063623161 -34343131323063366434613164383761366265623061646665363964346230313038306664363561 -33316664633861313632343161613236343934386166626334666632356266353966663163303731 -30373166363637663131613564303634373337626538626665303139656362383935383535663366 -66663437323039343262333636623363366231313366626434383039336231323938666633303663 -63326264373065613539323433646163653334613733323536633862343535306165633731343361 -64366136663535306433636338653763653030643837633830343034363361363132666132386633 -63393331396165366239313637303836313737326531646635613036373436346239383333336336 -31623434333336643863303334313539336533636439393738353766313338313161336239386363 -64353032633934613533646532613961346531353664333230613036393737333966373437363863 -36626139333030326134386164653931393432323731373562343735303737376532613534396463 -32346432313361643235653165376630316631616463336238316234626231396364303364643136 -35393165333439373336323630636138333730316266336261316637613738383639343661373239 -64336535306465333161346637346636643034316236353664653464323061323834366633653561 -37663730353531373630613034643464333933663331393337303335313631383134353330323238 -63363030386236323738383661633366316265663566613331386166633436666564303738356263 -34356166656465656230336166636533643866666565646336626338643135633236353033626230 -66646230303262626635653665336333383831313232353565313638396462353035376137666230 -36313931626665316234346335343838363533623838353263313732633735653861633936633239 -37363139666137623665376131666636306233363861363931333034383063656335663333656134 -37323461373261343563663239623533313532386539646562333864633337393966613532336265 -62646161633830303630356636356138383962633935386335336432663461613039333333666263 -31303431336266616463393238663662643961656536353238613965646138316630663934313235 -32353261346337346438623732383036393961356332363838316266313566623437336263393836 -37353939626536326538643962363731373833356532366232323034393137373463383630646165 -35633439613330663065356630353631326136363633666233343837633433626136623332303263 -33306435643734633865316565363832376137616231666562306335643637663132316435383765 -34666638343031323762653363333062363034626134626463636331383331353939613865366633 -37333639626136373964366234393939633664656535306539646362656437636539643036363364 -32343939323763303966343230616136643132623132333230613663363931363832326661356364 -39363231643036633464643539346362376566343933393861643335626338656335306434393464 -63633537656166653638333532333633626439306436333736316465646538343137326430613934 -63313538373435353535323734616562323631393763313065336461363237393331616330393435 -62366238373532313132396132323561643866396365373335326462303163613339666536396630 -63646564373431333734633832316633316531623730356462663232636662353131646265376435 -65353562613066353830623239333265346362653861316432636361613134613366633735666138 -31346265383164623062363431306664383436653265343434326366653066306630323635633136 -64366163383161303735613935653231356635316338333132363836313165333932653635643062 -63633530636631643565333765666438343231326439303239303530306436323365646639346433 -66373161643164613861323661303730653635306662366234323132346533643164303338353136 -37353664636239663066323265313330303631396562646636623136663836643236333139656434 -66313531373161333038393161383663313062356439323839663139393736633637386332633938 -63313737393166316361623762303237623835336534353764373139666539343766333461373437 -39356438313039306333396262373330323561343065653765646566653831336363646638333637 -61326662353765326566663266646365336533636437383862326561663032366230613764313136 -64303232396534303136613931376633323937376464373133646636336263643462646365333164 -32623833363264376233316136633666366435303635386330636238343139316136306231396533 -35316438623539393362366363353635363732323839373063636531363462383334623365656436 -36383335333561326333626532653132663766363231353966396633303333343165653838663262 -66376666626135376564393235616130393436613064653130616362613031383532313633313936 -38333532376638383461643332353837363061656632353430376163393431623032336530643234 -62313331323561343166653636366461613834326534396231326163303462303235663162626462 -63333766363330633066386466346235366335383266386561643336306536363034396231373137 -31376261663937393439386462613531616366613362623863326235326531663337316531623831 -64666636313035323235323236393238346261343765613639346163363430386130326365313331 -38613365623438383339613664643037316431636632393066363531316631336264623930346532 -31636333616139636236326464323333376236393862646333656464653063656263303433623164 -35636161623463353363636334383266356236656163633064303231663430343430306131333662 -63633131653262346636633632646638356263306238616135393938393238303461303965313361 -39656138373639373335316362336266313838396661343933633162376237616635363764623630 -36336666333531353237323263356664303932616136373736326361623733336331343136303637 -31336630613031373564 diff --git a/vagrant/host_vars/prod/secrets.yml b/vagrant/host_vars/prod/secrets.yml deleted file mode 100644 index e079aa4a..00000000 --- a/vagrant/host_vars/prod/secrets.yml +++ /dev/null @@ -1,159 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -62366135653638303662356566373038626433373934373433333037333139613066383437336137 -3931393964383335613238653435356430316439343532610a626534363233643963383438323864 -39323737613966356661383331303531666161333235616461613033663464636564373336613635 -6431643733343335360a363334623837343065346335656663383236346436326665323830633633 -65363434616637623561333036346634643465343233376538366534363239666537303866376232 -30613434646662373663613232613037346362373363326661323362373264616461366364373963 -64666237326165353261636232636336353139333163626261386433313336306438386262626131 -31376632383836343666623461383966373039316336613961393430653561363131613637666665 -62643538643535313437383031303964663734396363373134613535373831396438633634393466 -65356566623634633931313538326336346535393330353837663433306661386364333639363361 -32373963313535623632353232333637366266636134653834643937313262366331366534313266 -66616631353039613739366237653461343766326233373565306435353135613836656364623362 -31373137363938393765616439633161373336616134373538323437663731356437323639363164 -66663732346563306262373066633531323862656161366631313332663534343732366666626333 -32643365313031316337663965376239343762303366313630333939366238643436613064353034 -32643234663135633637303964386164356330343930613265623439626339393161633530666661 -33323030346434613535353564643738303163363830326331363362343538326663376339623862 -38383832636565623662653263623136626439326233336466373937343163336332663637613461 -33373137613136333338353834376164653561373361346534633237613762343133323734636664 -35396565643538663737333138383536373966303730383937346264383766363666323439623531 -64653662366131646161346434663564333932663437643833373432356532343935373135303836 -61313161393432643732626562623734363164393464376239633232353334616535306639323862 -34313736333533326533323466383066613232366661316531613963613332363839633636373832 -38663430323630633437653639326134653039636538326234316465626161643630356563313462 -33623833666231323233373734333062323837336663303433333562376332343637363831616165 -63656164333835393931643931613930643964643564616232653331643435633135303233373364 -65633764396261616637343437316337376464386139623038636133663162393064363565623135 -31323530386537336431663438326331363863313234666634383531616537653739363931336561 -38613333616133346262363566623066363730346536636333633439353431313461646561613363 -63613536616164616461313664303461373665616464643730636264633034383163666235616666 -36366239386236333737313939623738316435626362363262626663346635643536346632396637 -30353461326465656431313136373531613931313064316364323435393030623162643366303337 -33303436666433393130663339346636306361346463366233313333616239666161616165323261 -30316433383435396431313664326330626263643932623936363365393763393930313237343635 -37356437663933653739316632613239303766623561636364663666353565636634633663643133 -34386138646465366634633139356533313933393663373662623261656666633239393338656434 -38313164396238653934363939636337343737303264306534623266666136623034373334646565 -63666236333061643439336533356637343637366431373830393466303464383231356531373461 -64616366393231643432386531363231666164653566383939306161643239323537336137336263 -30346130386334323438356437656236616330613633343735396438373437333037393865633130 -34646634383235323335663766376537353062383534346161326166326332326535303239653630 -35633038303634363863323333383236376138356236393362663433633234386631333561316632 -37336664336234376336383232653862646434643463303535663965363061313965323937393963 -32373766306234396432633766346137646466663661656465656261666132383938386437643233 -63366362663230373736393762326437396135366433633037653466393063363663633430653365 -30376364326166353061616333396661376634363863333534386162333862346166333632636330 -38643733316232366264633233373261343861313931616461336333313139363364336566613236 -36323836343761393430633430653034366361633735633737326137616138373265393066353563 -30626664323436373337333864616333386132393437366163663732613933373637636539613031 -30336561376365343239353436613165633530313864323839336564346562376262646130653335 -63633338326330623237373335613839366435663336623038656231373366336433323231386334 -61336633656539363937366134636461376131313938326439393764653030613136636139333465 -36366639383763383333626335346438613431336233633438616136333638376636643038623761 -38316633653336666266393765633262613539353764616236373431383663323166356662373937 -35316130623533623966363130326136323237653066333730643734303462626232666664313464 -66613139366562303935666538386161613864376665616662353736653530383932356136346665 -34396266386661326439336631333764316563383831623035643432653933313736633262393762 -37363836623738633732616434653732353864326361373031313165343430333031636265363063 -34613864333138363133346239636532653662303431336665666466346533386136656137333936 -65303663363136666333653731313631383765363736323262393333326362356631333636353637 -38636364313031323436356335653436666236666435633062353938613939616465643038646563 -32356639623961396234623336373238393764383363613362313163636432366265373230306165 -61306630313264376534336232383030363139663331373935656132343431636464303862343166 -35616338616138363439343030633934356666336134623063343662316238376435346664643763 -32323337626534393039333530363231383866646332643439383361346137313634663430623333 -66356461303362323630313538333630653366613638646365393236386366653063396365623334 -36363233376435333735363939646334643762386436646339366137656566613164616363353539 -34666537653934373936366337616663626338356136376464653531303464383736646532623565 -66313830333263363239376534376239333537663565626239326634343239323736343833346634 -63336561323131616439386535306365646437643666313236303063663839373666616639653637 -38636264646261666337303736363536663965333632356130393638326530373861396566306133 -39393333386133663330343062666534353230653431636638656163396165616637303566356338 -61663861393837623638373638663030343438353362343262653765303335613433383632333133 -61393262313930656535303132653466613530653834393561323632363662653031356330396562 -35393334646235666233363830346432313135623436343161313432396138613734366564666538 -31633863663037643636386636313731353730623039383765373330653962323466386436393933 -61353865393964346561643334643064323630353561363734353563313164326132623433356630 -64653966303463616638613163343230623539306233613535376433313764623863353534313736 -65663338393535663162356165373533393361343332616434653663396238303036396538306563 -35383933306662373733653839663538353831316664363336656236646365663535386264306430 -30663666636236633632353964666331663733373265333563623962616237616138386636326539 -37306661646337313463343466303862356130363561356631336666336535353065656438386638 -34303738633335376561373033336435653035646561363131623332336561383066333838363761 -61313735303964316264323263633865643635393732653762623336643637373137666230643866 -34393439616666383661346638353962313535623137656534306233653966316134633131323065 -63626361363930393962626631626330396164383436643531393161633931333939623831623165 -36383035393138653336633363633533653366303164613233636564343439653563616265383736 -66636437643538643965363239656463316136396436366465663837306433653436323732313539 -65333534633038653637393534376639343765613433623162633464623236363139393265353434 -62666134616539353561363131323733356262633537636361373134363735663534373962343365 -64373631343636333132653738613861666137633138663866333939376430626234643432336630 -30666531336465323434363432386438356562343666306563343164666437323738623730336630 -66353764623434623366623961326366626339653065323230373337346535633833353837333065 -37633933643938383261346339643533323132616163356539343435666233303231356431346138 -36396165303463313233633432316139353264613365333065376235363264616563343263643431 -39333934336462646330356239623266303035336330633332373962633637373537636362333663 -35303438356130383636656438653163343163623036376435613736346230663136356236663037 -32323737306137656632373137323136393230636465386338333764623039353831373863616233 -33303861333934663636663535633338636631343362316139383264653566396233616662343234 -65653262333236623763626638353338383364613333363035613232363437353434613363303834 -63326266356261383936396462343164626331623138366334613762313832616462623834346537 -30333762333035303462663633666563333931306437663462376231373039306439346135343662 -36376463343538333464306561356138343930663562373662323439333135616533656337303666 -39643534366438663630643639626263666661643737373133343539333762323630346264653464 -32323330323465353063333635633931366436653061333430336662646434626634386633316335 -33336533313762643637316335376337653131656463386533343733306336326534323039666262 -61313933613961613463313137653864383464336132616534343839616537633363633239383663 -63306462366664363531323164393330366664306666306637353138353332633233363863623465 -64663238393337633864653530313365393261313733333164623035326465386463653938313562 -37373330333635633936316562383932613033353431366332623032313539363239646661663063 -36303966313039396333396465663564646638376334363335303935663138616461623062356431 -63326235643866343639393731323862396263646635373537313262313937316266323931303632 -64643131643735646661653561346231626665393831323335323034353563303739356661653131 -32393131653234613734643162653235613535626165646462613832366662323133303637393036 -62633836336432623633646566373737373861613933633634663061313266373734646431326162 -30333763616365396233386637393738303362303838656537656434306162396532313339383238 -36393038346561636535656437333937356232363531303566633364663834316163653937393732 -63626164366237393035613237376333326231363663353637616633323565663139373732343937 -34303837633132646538393932616664313834663237366637613439356133623338316635366635 -38646637383830646238303139326639393031306430363161303635333461363930643033313830 -64323339653938393738303135666439303235316635353262343234333361326666393435373238 -34633735336664646462616131376533646233663030643836616139336539356234393265333037 -66303164316432353866626533363263303637353133343239616136316165623934353731643061 -30373232313535326466663434316537386230633264346134663434333062343965626232303163 -62323633333862616635363630333732303766363761653434623561656265396430343564633161 -38396566383933393035393835613737643966653633323034323036383331363934633433303839 -33616266343531383661643738333961666234326535623931666362666239666264303362386238 -65313937373133373966353165643038613464393335323263376230666639313531386361663965 -33666631396637336336376237623337313464613632343763623662653863356530343135616338 -62333538386231663162666566303133303636646461383438646163666163366666333365353131 -61656365323963646430616339663132653962623435666433656366633764343436656431376334 -62396231663738306430326361623961353063326230386336666366616235623866376630633465 -32663061643165313737633732386638303535636133643866343037643861316536656462633633 -36396230343338363431386131646632613231623339656534343361313263666233363265366537 -31646630633338616434303134666137383133373762636564623535633236346265316235373630 -61396663626131643162386338643632366137323731333661316433326131633434353534623130 -35303237336464363063303964663162336137306638346131373466396138363264316335353730 -32313734386234366438636361616665346362386664633434656466323336316661386666343563 -63643538306232373161363932663232663131366564633638626535373933336236666439373961 -34626539396234366538636137346565653031343931396564386166623865343063653166643332 -35393537383331323465386130363336343835396233366230326662643961623263666436616266 -63326630303533336333336665343438623965316130633330373634323635363466323933623631 -62616337386535333839336461643463326636636463656239373537303135393239663335666263 -35396162306564663066643233333630373835313831396331623137643762633931623039303139 -38383865623062616236323566313733323665366566323337306430376664633233326631356336 -37623737346235346235346238623135323032663137623434616266613262333034313664626135 -38656339636466353635393363363233346635396235663135366466653036343764623639323765 -66336230376531633936663562363661613665383232646261653635383363386139326235346336 -39306664376532363933366630646135353836343161386631386439343136323138326233363061 -33396561326163333331363161356537396161353831636437376333616538643435656465316634 -66306334643366363132653736303161623833313761366131333830316134353536336361366131 -39383761346334313038303465343661336661666636363535613831663537633932306162303866 -61643233616437346461393734313864663734366430303563616163643138383032636466633034 -34383832656131626262663439363930333938326431343230306539633731386632393364386165 -66386634643331313739303934353736303264393566633233363533323138316137306661373061 -34643164363163616565656634663437646164353462643633346332336236653537323037636161 -61303961353964343131336331383563323964383433313163303663396434356137656166386166 -6231323331346633313965323433306630643837306263613966 diff --git a/vagrant/host_vars/prod2/secrets.yml b/vagrant/host_vars/prod2/secrets.yml deleted file mode 120000 index 3d81269d..00000000 --- a/vagrant/host_vars/prod2/secrets.yml +++ /dev/null @@ -1 +0,0 @@ -../prod/secrets.yml \ No newline at end of file diff --git a/vagrant/host_vars/unglueit.template b/vagrant/host_vars/unglueit.template deleted file mode 100644 index 519bf6fa..00000000 --- a/vagrant/host_vars/unglueit.template +++ /dev/null @@ -1,4 +0,0 @@ -github_auth_key: -mysql_root_pw: unglueit_pw_123 -mysql_regluit_pw: regluit - diff --git a/vagrant/just.yml b/vagrant/just.yml deleted file mode 100644 index a158e5a6..00000000 --- a/vagrant/just.yml +++ /dev/null @@ -1,395 +0,0 @@ -- name: just setup - hosts: just - vars: - user: "{{ ansible_ssh_user }}" - sudo: yes - - pre_tasks: - - name: check apt last update - stat: path=/var/cache/apt - register: apt_cache_stat - - name: update apt if needed - apt: update_cache=yes - when: ansible_date_time.epoch|float - apt_cache_stat.stat.mtime > 60*60*12 - - tasks: - - - name: set DNS record for just.unglue.it - route53: - command: create - zone: unglue.it - record: just.unglue.it - type: A - ttl: 60 - value: "{{ansible_ssh_host}}" - overwrite: yes - aws_access_key: "{{ lookup('env','AWS_ACCESS_KEY_ID') }}" - aws_secret_key: "{{ lookup('env','AWS_SECRET_ACCESS_KEY') }}" - - # TEMPORARILY TURN OFF - # sudo add repo to get latest version of python 2.7 - #- name: add-apt-repository ppa:fkrull/deadsnakes-python2.7 - # apt_repository: repo='ppa:fkrull/deadsnakes-python2.7' state=present update_cache=true - - - name: do apt-get update --fix-missing - command: apt-get update --fix-missing - - - name: installing dependencies - apt: pkg={{ item }} update_cache=yes state=present - with_items: - - python2.7 - - git-core - - apache2 - - cronolog - - libapache2-mod-wsgi - - mysql-client - - python-virtualenv - - python-mysqldb - - redis-server - - python-lxml - - python-dev - - libjpeg-dev - - libmysqlclient-dev - - libxml2-dev - - libxslt1-dev - - python-setuptools - - postfix - tags: install - - - name: make {{user}} group - group: name={{user}} - - - name: make {{user}} user - user: name={{user}} shell=/bin/bash group={{user}} generate_ssh_key=yes - - - name: install some python modules to use - #pip: name={{item}} virtualenv=/home/{{user}}/venv - pip: name={{item}} - with_items: - - PyGithub - - - name: create /opt/regluit - file: path=/opt/regluit state=directory owner={{user}} group={{user}} mode=0745 - - - name: git config - command: "{{item}}" - with_items: - - git config --global user.name "Raymond Yee" - - git config --global user.email "rdhyee@gluejar.com" - - - name: ssh-keygen - #command: pwd - command: ssh-keygen -b 2048 -t rsa -f /home/{{user}}/.ssh/id_rsa -P "" - sudo: no - args: - creates: /home/{{user}}/.ssh/id_rsa - - - name: create deploy key for repo - action: github_deploy_key - sudo: no - args: - github_auth_key: "{{github_auth_key}}" - repo_name: Gluejar/regluit - key_name: vagrant_ansible_test - key_path: /home/{{user}}/.ssh/id_rsa.pub - - - name: postfix install - raw: DEBIAN_FRONTEND='noninteractive' apt-get install -y -q --force-yes postfix - - - name: clone the regluit git repo into /opt/regluit - sudo: no - git: repo=ssh://git@github.com/Gluejar/regluit.git dest=/opt/regluit accept_hostkey=True force=yes version=sysadmin - - ## installing mysql - ## https://github.com/bennojoy/mysql --> probably the right way - ## how do you make use of other people's playbooks in the right way? - ## https://stackoverflow.com/a/7740571/7782 - # - #- name: mysql setup - # raw: debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password password {{mysql_root_pw}}' - #- raw: debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password_again password {{mysql_root_pw}}' - #- raw: apt-get -y install mysql-server - # - #- name: Create regluit database - # mysql_db: db=regluit state=present encoding=utf8 collation=utf8_bin login_user=root login_password={{mysql_root_pw}} - # - # # GRANT ALL PRIVILEGES ON regluit.* TO 'regluit'@'localhost' WITH GRANT OPTION; (covered?) - #- name: Create database user - # mysql_user: > - # user=regluit - # password={{mysql_regluit_pw}} - # host=localhost - # priv=*.*:ALL - # state=present - # login_user=root - # login_password={{mysql_root_pw}} - - - # running stuff within a virtualenv - # https://stackoverflow.com/a/20572360 - # https://stackoverflow.com/questions/20575084/best-way-to-always-run-ansible-inside-a-virtualenv-on-remote-machines?rq=1 - - - ## hard coding of just - #sudo("ln -s /opt/regluit/deploy/just.conf /etc/apache2/sites-available/just") - - - name: link just.conf into sites-available - file: src=/opt/regluit/deploy/just.conf dest=/etc/apache2/sites-available/just state=link - - #run('pip install -r requirements_versioned.pip') - - - name: pip install requests (to see whether in right place) - pip: > - executable=/opt/regluit/ENV/bin/pip - name={{item}} - virtualenv=/opt/regluit/ENV - virtualenv_command=virtualenv - with_items: - - requests - - census - sudo: no - - - - name: pip requirements - pip: > - executable=/opt/regluit/ENV/bin/pip - requirements=/opt/regluit/requirements_versioned.pip - virtualenv=/opt/regluit/ENV - virtualenv_command=virtualenv - virtualenv_site_packages=yes - sudo: no - - - #run('echo "/opt/regluit/" > ENV/lib/python2.7/site-packages/regluit.pth') - #run('echo "/opt/" > ENV/lib/python2.7/site-packages/opt.pth') - - - name: establish regluit.pth - lineinfile: create=yes dest=/opt/regluit/ENV/lib/python2.7/site-packages/regluit.pth line="/opt/regluit/" - - - name: establish opt.pth - lineinfile: create=yes dest=/opt/regluit/ENV/lib/python2.7/site-packages/regluit.pth line="/opt/" - - #sudo('mkdir /var/www/static') - #sudo('chown ubuntu:ubuntu /var/www/static') - - - name: create /var/www/static - file: path=/var/www/static state=directory owner={{user}} group={{user}} mode=0755 - - # - #run('django-admin.py syncdb --migrate --noinput --settings regluit.settings.just') - - #Run syncdb on the application - - - name: django syncdb - django_manage: > - command=syncdb - app_path=/opt/regluit/ - settings="regluit.settings.just" - virtualenv=/opt/regluit/ENV - sudo: no - notify: - - restart apache2 - - - - name: django migrations - django_manage: > - command=migrate - app_path=/opt/regluit/ - settings="regluit.settings.just" - virtualenv=/opt/regluit/ENV - sudo: no - notify: - - restart apache2 - - #run('django-admin.py collectstatic --noinput --settings regluit.settings.just') - - - name: django collectstatic - django_manage: > - command=collectstatic - app_path=/opt/regluit/ - settings="regluit.settings.just" - virtualenv=/opt/regluit/ENV - sudo: no - notify: - - restart apache2 - - - name: copy STAR_unglue_it.crt - copy: > - src=/Volumes/ryvault1/gluejar/other_keys/unglue_it/STAR_unglue_it.crt - dest=/etc/ssl/certs/server.crt - owner={{user}} - group={{user}} - mode=0644 - notify: - - restart apache2 - - - name: copy server.key - copy: > - src=/Volumes/ryvault1/gluejar/other_keys/unglue_it/server.key - dest=/etc/ssl/private/server.key - owner={{user}} - group={{user}} - mode=0600 - notify: - - restart apache2 - - - name: copy STAR_unglue_it.ca-bundle - copy: > - src=/Volumes/ryvault1/gluejar/other_keys/unglue_it/STAR_unglue_it.ca-bundle - dest=/etc/ssl/certs/STAR_unglue_it.ca-bundle - owner={{user}} - group={{user}} - mode=0600 - notify: - - restart apache2 - - - name: remove /etc/logrotate.d/apache2 - file: path=/etc/logrotate.d/apache2 state=absent - notify: - - restart apache2 - - - name: a2dissite default - command: a2dissite default - notify: - - restart apache2 - - - name: a2ensite just - command: a2ensite just - notify: - - restart apache2 - - - name: a2enmod ssl rewrite headers - command: a2enmod ssl rewrite headers - notify: - - restart apache2 - - # - #sudo ("/etc/init.d/apache2 restart") - # - - - name: turn on ports 22, 80, 443 - ufw: rule=allow port={{ item }} proto=tcp - with_items: - - 22 - - 80 - - 443 - - - name: enable ufw - ufw: state=enabled - - #with cd("/opt/regluit"): - # - # sudo ("yes | adduser --no-create-home celery --disabled-password --disabled-login") - - - name: make celery group - group: name=celery - - - name: create celery user - user: > - name=celery - createhome=no - group=celery - generate_ssh_key=no - - # sudo ("cp deploy/celeryd /etc/init.d/celeryd") - # sudo ("chmod 755 /etc/init.d/celeryd") - - - name: copy deploy/celeryd - command: cp /opt/regluit/deploy/celeryd /etc/init.d/celeryd - - - name: set mode on /etc/init.d/celeryd - file: path=/etc/init.d/celeryd mode=0755 - - # sudo ("cp deploy/celeryd.conf /etc/default/celeryd") - - - name: copy deploy/celeryd_just.conf - command: cp /opt/regluit/deploy/celeryd_just.conf /etc/default/celeryd - - - name: set mode on /etc/default/celeryd - file: path=/etc/default/celeryd mode=0644 - - # sudo ("mkdir /var/log/celery") - - name: make /var/log/celery - file: path=/var/log/celery state=directory owner=celery group=celery mode=0755 - - # sudo ("mkdir /var/run/celery") - # sudo ("chown celery:celery /var/log/celery /var/run/celery") - - - name: make /var/run/celery - file: path=/var/run/celery state=directory owner=celery group=celery mode=0755 - - # sudo ("/etc/init.d/celeryd start") - - - name: start celeryd - command: /etc/init.d/celeryd start - - # sudo ("cp deploy/celerybeat /etc/init.d/celerybeat") - # sudo ("chmod 755 /etc/init.d/celerybeat") - # https://stackoverflow.com/questions/24162996/how-to-move-rename-a-file-using-an-ansible-task-on-a-remote-system - - - name: copy deploy/celerybeat - command: cp /opt/regluit/deploy/celerybeat /etc/init.d/celerybeat - - - name: set mode on /etc/init.d/celerybeat - file: path=/etc/init.d/celerybeat mode=0755 - - # sudo ("cp deploy/celerybeat.conf /etc/default/celerybeat") - - - name: copy deploy/celerybeat_just.conf - command: cp /opt/regluit/deploy/celerybeat_just.conf /etc/default/celerybeat - - - name: set mode on /etc/default/celerybeat - file: path=/etc/default/celerybeat mode=0755 - - # sudo ("mkdir /var/log/celerybeat") - # sudo ("chown celery:celery /var/log/celerybeat") - - - name: make /var/log/celerybeat - file: path=/var/log/celerybeat state=directory owner=celery group=celery mode=0755 - - # sudo ("/etc/init.d/celerybeat start") - - name: start celerybeat - command: /etc/init.d/celerybeat start - - # run data loading script (not on just) - #- name: run data loading script - # script: load_data_just.sh - - # add setup_django.sh script to root dir - - - name: add setup_django.sh script to root dir - command: cp /opt/regluit/vagrant/setup_django_just.sh /home/{{user}}/setup_django.sh - sudo: no - - # set up crontab - - name: crontab for just - command: crontab /opt/regluit/deploy/crontab_just.txt - sudo: no - - # deal with SSH keys - - #- name: add RY ssh key - # authorized_key: user={{user}} key="{{ lookup('file', '/Users/raymondyee/.ssh/id_rsa.pub') }}" state=present - - - name: add ssh keys from public_key directory - authorized_key: user={{user}} key={{item}} state=present - with_fileglob: - - /opt/regluit/deploy/public_keys/* - sudo: no - - - name: add ssh keys from github (redundant?) - authorized_key: user={{user}} key="{{item}}" state=present - with_items: - - https://github.com/rdhyee.keys - - https://github.com/eshellman.keys - sudo: yes - - - - - handlers: - - name: restart apache2 - service: name=apache2 state=restarted - - - diff --git a/vagrant/library/ansible_module_template.py b/vagrant/library/ansible_module_template.py deleted file mode 100644 index 846bb0ce..00000000 --- a/vagrant/library/ansible_module_template.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/Users/raymondyee/anaconda/bin/python -# -*- coding: utf-8 -*- - -# (c) 2015, Raymond Yee - - -import json -import base64 - -DOCUMENTATION = ''' ---- -module: MODULE_NAME -short_description: MODULE_SHORT_DESCRIPTION -description: - - Adds service hooks and removes service hooks that have an error status. -version_added: "1.4" -options: - user: - description: - - Github username. - required: true - oauthkey: - description: - - The oauth key provided by github. It can be found/generated on github under "Edit Your Profile" >> "Applications" >> "Personal Access Tokens" - required: true - - - - -author: Raymond Yee, raymond.yee@gmail.com -''' - -EXAMPLES = ''' -# Example creating a new service hook. It ignores duplicates. -- github_hooks: action=create hookurl=http://11.111.111.111:2222 user={{ gituser }} oauthkey={{ oauthkey }} repo=https://api.github.com/repos/pcgentry/Github-Auto-Deploy - -# Cleaning all hooks for this repo that had an error on the last update. Since this works for all hooks in a repo it is probably best that this would be called from a handler. -- local_action: github_hooks action=cleanall user={{ gituser }} oauthkey={{ oauthkey }} repo={{ repo }} -''' - - -def main(): - module = AnsibleModule( - argument_spec=dict( - oauthkey=dict(required=True), - repo=dict(required=True), - user=dict(required=True), - validate_certs=dict(default='yes', type='bool'), - content_type=dict(default='json', choices=['json', 'form']), - ) - ) - - action = module.params['action'] - hookurl = module.params['hookurl'] - oauthkey = module.params['oauthkey'] - repo = module.params['repo'] - user = module.params['user'] - content_type = module.params['content_type'] - - if action == "list": - (rc, out) = _list(module, hookurl, oauthkey, repo, user) - - if action == "clean504": - (rc, out) = _clean504(module, hookurl, oauthkey, repo, user) - - if action == "cleanall": - (rc, out) = _cleanall(module, hookurl, oauthkey, repo, user) - - if action == "create": - (rc, out) = _create(module, hookurl, oauthkey, repo, user, content_type) - - if rc != 0: - module.fail_json(msg="failed", result=out) - - module.exit_json(msg="success", result=out) - - -# import module snippets -from ansible.module_utils.basic import * -from ansible.module_utils.urls import * - -main() \ No newline at end of file diff --git a/vagrant/library/filetest.py b/vagrant/library/filetest.py deleted file mode 100644 index fb40a0e2..00000000 --- a/vagrant/library/filetest.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -# (c) 2015, Raymond Yee - -import os -import json - -# import module snippets -from ansible.module_utils.basic import * -from ansible.module_utils.urls import * - - -DOCUMENTATION = ''' ---- -module: filetest -short_description: tests for existence of a path -description: - - Checks on the existence of a given path -#options: -# path: -# description: -# - path to test. -# required: true - - -author: Raymond Yee, raymond.yee@gmail.com -''' - -def main(): - cwd = str(os.getcwd()) - print ( json.dumps({ - "cwd" : cwd -})) - -main() \ No newline at end of file diff --git a/vagrant/library/github_deploy_key.py b/vagrant/library/github_deploy_key.py deleted file mode 100644 index e23498ab..00000000 --- a/vagrant/library/github_deploy_key.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -# (c) 2015, Raymond Yee - -import os -import json -import base64 -from StringIO import StringIO - -from github import Github - -DOCUMENTATION = ''' ---- -module: github_deploy_key -short_description: create a deploy key to a github repository -description: - - create a deploy key to a github repository -options: - repo_name: - description: - - the repository to write key to - required: true - github_auth_key: - description: - - The oauth key provided by github. It can be found/generated on github under "Edit Your Profile" >> "Applications" >> "Personal Access Tokens" - required: true - key_path: - description: - - location of the key to upload - key_name: - description: - - name for the key - - -author: Raymond Yee, raymond.yee@gmail.com - -dependency: pygithub -''' - - -def main(): - module = AnsibleModule( - argument_spec=dict( - github_auth_key=dict(required=True), - repo_name=dict(required=True), - key_path=dict(required=True), - key_name=dict(required=True), - ) - ) - - github_auth_key = module.params['github_auth_key'] - repo_name = module.params['repo_name'] - key_path = module.params['key_path'] - key_name = module.params['key_name'] - - failed = True - - try: - g = Github(github_auth_key) - s = open(key_path).read() - repo = g.get_repo(repo_name) - key = repo.create_key(key_name, s) - - failed = False - except Exception, e: - failed = False - - # error handling and what to return with success? - - if not failed: - msg = "None:success" - else: - msg = str(e) + " " + str(e.message) + " " + str(e.get(args)) - - - module.exit_json( - changed = True, - github_auth_key = github_auth_key, - repo_name = repo_name, - key_path = key_path, - key_name = key_name, - msg = msg, - failed = failed - ) - - -# import module snippets -from ansible.module_utils.basic import * -from ansible.module_utils.urls import * - -main() \ No newline at end of file diff --git a/vagrant/load_data_please.sh b/vagrant/load_data_please.sh deleted file mode 100644 index 7913d267..00000000 --- a/vagrant/load_data_please.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -cd /opt/regluit -source ENV/bin/activate -export DJANGO_SETTINGS_MODULE=regluit.settings.please - -# sysadmin/drop_tables.sh | django-admin.py dbshell -# cat test/campaign_starter.sql | django-admin.py dbshell - -# django-admin.py makemigrations -# django-admin.py migrate --fake-initial --noinput -# load Pride and Prejudice -echo -e "django-admin.py shell_plus << EOF\nfrom regluit.core import bookloader; w = bookloader.add_by_isbn('9781909621657')\nEOF\n" | sh diff --git a/vagrant/localvm.yml b/vagrant/localvm.yml deleted file mode 100644 index 024211c5..00000000 --- a/vagrant/localvm.yml +++ /dev/null @@ -1,280 +0,0 @@ -- name: localvm setup - hosts: localvm - vars: - user: "{{ ansible_ssh_user }}" - config_name: localvm - sudo: yes - - pre_tasks: - - name: check apt last update - stat: path=/var/cache/apt - register: apt_cache_stat - - name: update apt if needed - apt: update_cache=yes - when: ansible_date_time.epoch|float - apt_cache_stat.stat.mtime > 60*60*12 - - roles: - - # base setup - - role: common - - role: mysql - - # installing mysql - # https://github.com/bennojoy/mysql --> probably the right way - # how do you make use of other people's playbooks in the right way? - # https://stackoverflow.com/a/7740571/7782 - - post_tasks: - - # running stuff within a virtualenv - # https://stackoverflow.com/a/20572360 - # https://stackoverflow.com/questions/20575084/best-way-to-always-run-ansible-inside-a-virtualenv-on-remote-machines?rq=1 - - - ## hard coding of {{config_name}} - #sudo("ln -s /opt/regluit/deploy/{{config_name}}.conf /etc/apache2/sites-available/{{config_name}}") - - - name: link {{config_name}}.conf into sites-available - file: src=/opt/regluit/deploy/{{config_name}}.conf dest=/etc/apache2/sites-available/{{config_name}} state=link - - - name: link {{config_name}}.conf into sites-available (with .conf) - file: src=/opt/regluit/deploy/{{config_name}}.conf dest=/etc/apache2/sites-available/{{config_name}}.conf state=link - - - #run('pip install -r requirements_versioned.pip') - - - name: pip install requests (to see whether in right place) - pip: > - executable=/opt/regluit/ENV/bin/pip - name={{item}} - virtualenv=/opt/regluit/ENV - virtualenv_command=virtualenv - with_items: - - requests - - census - sudo: no - - - - name: pip requirements - pip: > - executable=/opt/regluit/ENV/bin/pip - requirements=/opt/regluit/requirements_versioned.pip - virtualenv=/opt/regluit/ENV - virtualenv_command=virtualenv - virtualenv_site_packages=yes - sudo: no - - - #run('echo "/opt/regluit/" > ENV/lib/python2.7/site-packages/regluit.pth') - #run('echo "/opt/" > ENV/lib/python2.7/site-packages/opt.pth') - - - name: establish regluit.pth - lineinfile: create=yes dest=/opt/regluit/ENV/lib/python2.7/site-packages/regluit.pth line="/opt/regluit/" - - - name: establish opt.pth - lineinfile: create=yes dest=/opt/regluit/ENV/lib/python2.7/site-packages/regluit.pth line="/opt/" - - #sudo('mkdir /var/www/static') - #sudo('chown ubuntu:ubuntu /var/www/static') - - - name: create /var/www/static - file: path=/var/www/static state=directory owner={{user}} group={{user}} mode=0755 - - # - #run('django-admin.py syncdb --migrate --noinput --settings regluit.settings') - - #Run syncdb on the application - - - name: django_syncdb - django_manage: > - command=syncdb - app_path=/opt/regluit/ - settings="regluit.settings.{{config_name}}" - virtualenv=/opt/regluit/ENV - sudo: no - notify: - - restart apache2 - - - - name: django migrations - django_manage: > - command=migrate - app_path=/opt/regluit/ - settings="regluit.settings.{{config_name}}" - virtualenv=/opt/regluit/ENV - sudo: no - notify: - - restart apache2 - - #run('django-admin.py collectstatic --noinput --settings regluit.settings.{{config_name}}') - - - name: django collectstatic - django_manage: > - command=collectstatic - app_path=/opt/regluit/ - settings="regluit.settings.{{config_name}}" - virtualenv=/opt/regluit/ENV - sudo: no - notify: - - restart apache2 - - - name: create self-signed SSL cert - command: openssl req -new -nodes -x509 -subj "/C=US/ST=NJ/L=Montclair/O=Gluejar Inc./CN=localhost" -days 365 -keyout /etc/ssl/private/server.key -out /etc/ssl/certs/server.crt creates=/etc/ssl/certs/server.crt - sudo: yes - notify: - - restart apache2 - - - name: set mode on /etc/ssl/certs/server.crt - file: path=/etc/ssl/certs/server.crt mode=0644 - notify: - - restart apache2 - - - name: set mode on /etc/ssl/private/server.key - file: path=/etc/ssl/private/server.key mode=0600 - notify: - - restart apache2 - - - name: remove all default enabled sites - shell: rm /etc/apache2/sites-enabled/* - sudo: yes - ignore_errors: yes - notify: - - restart apache2 - - - name: a2ensite {{config_name}} - command: a2ensite {{config_name}} - notify: - - restart apache2 - - - name: a2enmod ssl rewrite headers - command: a2enmod ssl rewrite headers - notify: - - restart apache2 - - # - #sudo ("/etc/init.d/apache2 restart") - # - - - name: turn on ports 22, 80, 443 - ufw: rule=allow port={{ item }} proto=tcp - with_items: - - 22 - - 80 - - 443 - - - name: enable ufw - ufw: state=enabled - - #with cd("/opt/regluit"): - # - # sudo ("yes | adduser --no-create-home celery --disabled-password --disabled-login") - - - name: make celery group - group: name=celery - - - name: create celery user - user: > - name=celery - createhome=no - group=celery - generate_ssh_key=no - - # sudo ("cp deploy/celeryd /etc/init.d/celeryd") - # sudo ("chmod 755 /etc/init.d/celeryd") - - - name: copy deploy/celeryd - command: cp /opt/regluit/deploy/celeryd /etc/init.d/celeryd - - - name: set mode on /etc/init.d/celeryd - file: path=/etc/init.d/celeryd mode=0755 - - # sudo ("cp deploy/celeryd.conf /etc/default/celeryd") - - - name: copy deploy/celeryd_{{config_name}}.conf - command: cp /opt/regluit/deploy/celeryd_{{config_name}}.conf /etc/default/celeryd - - - name: set mode on /etc/default/celeryd - file: path=/etc/default/celeryd mode=0644 - - # sudo ("mkdir /var/log/celery") - - name: make /var/log/celery - file: path=/var/log/celery state=directory owner=celery group=celery mode=0755 - - # sudo ("mkdir /var/run/celery") - # sudo ("chown celery:celery /var/log/celery /var/run/celery") - - - name: make /var/run/celery - file: path=/var/run/celery state=directory owner=celery group=celery mode=0755 - - # sudo ("/etc/init.d/celeryd start") - - - name: start celeryd - command: /etc/init.d/celeryd start - - # sudo ("cp deploy/celerybeat /etc/init.d/celerybeat") - # sudo ("chmod 755 /etc/init.d/celerybeat") - # https://stackoverflow.com/questions/24162996/how-to-move-rename-a-file-using-an-ansible-task-on-a-remote-system - - - name: copy deploy/celerybeat - command: cp /opt/regluit/deploy/celerybeat /etc/init.d/celerybeat - - - name: set mode on /etc/init.d/celerybeat - file: path=/etc/init.d/celerybeat mode=0755 - - # sudo ("cp deploy/celerybeat.conf /etc/default/celerybeat") - - - name: copy deploy/celerybeat_{{config_name}}.conf - command: cp /opt/regluit/deploy/celerybeat_{{config_name}}.conf /etc/default/celerybeat - - - name: set mode on /etc/default/celerybeat - file: path=/etc/default/celerybeat mode=0755 - - # sudo ("mkdir /var/log/celerybeat") - # sudo ("chown celery:celery /var/log/celerybeat") - - - name: make /var/log/celerybeat - file: path=/var/log/celerybeat state=directory owner=celery group=celery mode=0755 - - # sudo ("/etc/init.d/celerybeat start") - - name: start celerybeat - command: /etc/init.d/celerybeat start - - # run data loading script - - name: run data loading script - script: load_data_{{config_name}}.sh - - # add setup_django.sh script to root dir - - - name: add setup_django.sh script to root dir - command: cp /opt/regluit/vagrant/setup_django_{{config_name}}.sh /home/{{user}}/setup_django.sh - sudo: no - - # set up crontab - - name: crontab for {{config_name}} - command: crontab /opt/regluit/deploy/crontab_{{config_name}}.txt - sudo: no - - # deal with SSH keys - - #- name: add RY ssh key - # authorized_key: user={{user}} key="{{ lookup('file', '/Users/raymondyee/.ssh/id_rsa.pub') }}" state=present - - #- name: add ssh keys from public_key directory - # authorized_key: user={{user}} key={{item}} state=present - # with_fileglob: - # - /opt/regluit/deploy/public_keys/* - # sudo: no - # - - name: add ssh keys from public_key directory - authorized_key: user={{user}} key="{{item}}" state=present - with_items: - - https://github.com/rdhyee.keys - - https://github.com/eshellman.keys - sudo: yes - - handlers: - - name: restart apache2 - service: name=apache2 state=restarted - - - diff --git a/vagrant/please.yml b/vagrant/please.yml deleted file mode 100644 index 35443e3e..00000000 --- a/vagrant/please.yml +++ /dev/null @@ -1,418 +0,0 @@ -- name: route53 setup - hosts: 127.0.0.1 - vars: - aws_access_key: "{{ lookup('env','AWS_ACCESS_KEY_ID') }}" - aws_secret_key: "{{ lookup('env','AWS_SECRET_ACCESS_KEY') }}" - please_ssh_host: "{{ hostvars['please']['ansible_ssh_host'] }}" - - tasks: - - - name: print please ansible_ssh_host - debug: msg="please.ansible_ssh_host {{please_ssh_host}}" - - - - name: get DNS record for please.unglue.it - route53: - command: get - zone: unglue.it - record: please.unglue.it - type: A - aws_access_key: "{{aws_access_key}}" - aws_secret_key: "{{aws_secret_key}}" - - - name: set DNS record for please.unglue.it - route53: - command: create - zone: unglue.it - record: please.unglue.it - type: A - ttl: 60 - value: "{{please_ssh_host}}" - overwrite: yes - aws_access_key: "{{aws_access_key}}" - aws_secret_key: "{{aws_secret_key}}" - - -- name: please setup - hosts: please - vars: - user: "{{ ansible_ssh_user }}" - aws_access_key: "{{ lookup('env','AWS_ACCESS_KEY_ID') }}" - aws_secret_key: "{{ lookup('env','AWS_SECRET_ACCESS_KEY') }}" - sudo: yes - - pre_tasks: - - name: check apt last update - stat: path=/var/cache/apt - register: apt_cache_stat - - name: update apt if needed - apt: update_cache=yes - when: ansible_date_time.epoch|float - apt_cache_stat.stat.mtime > 60*60*12 - - tasks: - - # sudo add repo to get latest version of python 2.7 - - name: add-apt-repository ppa:fkrull/deadsnakes-python2.7 - apt_repository: repo='ppa:fkrull/deadsnakes-python2.7' state=present update_cache=true - - - name: do apt-get update --fix-missing - command: apt-get update --fix-missing - - - name: installing dependencies - apt: pkg={{ item }} update_cache=yes state=present - with_items: - - python2.7 - - git-core - - apache2 - - cronolog - - libapache2-mod-wsgi - - mysql-client - - python-virtualenv - - python-mysqldb - - redis-server - - python-lxml - - python-dev - - libjpeg-dev - - libmysqlclient-dev - - libxml2-dev - - libxslt1-dev - - python-setuptools - - python-dev - - postfix - tags: install - - - name: make {{user}} group - group: name={{user}} - - - name: make {{user}} user - user: name={{user}} shell=/bin/bash group={{user}} generate_ssh_key=yes - - - name: install some python modules to use - #pip: name={{item}} virtualenv=/home/{{user}}/venv - pip: name={{item}} - with_items: - - PyGithub - - - name: create /opt/regluit - file: path=/opt/regluit state=directory owner={{user}} group={{user}} mode=0745 - - - name: git config - command: "{{item}}" - with_items: - - git config --global user.name "Raymond Yee" - - git config --global user.email "rdhyee@gluejar.com" - - - name: ssh-keygen - #command: pwd - command: ssh-keygen -b 2048 -t rsa -f /home/{{user}}/.ssh/id_rsa -P "" - sudo: no - args: - creates: /home/{{user}}/.ssh/id_rsa - - - name: create deploy key for repo - action: github_deploy_key - sudo: no - args: - github_auth_key: "{{github_auth_key}}" - repo_name: Gluejar/regluit - key_name: vagrant_ansible_test - key_path: /home/{{user}}/.ssh/id_rsa.pub - - - name: postfix install - raw: DEBIAN_FRONTEND='noninteractive' apt-get install -y -q --force-yes postfix - - - name: clone the regluit git repo into /opt/regluit - sudo: no - git: repo=ssh://git@github.com/Gluejar/regluit.git dest=/opt/regluit accept_hostkey=True force=yes version=dj16ry - - - # installing mysql - # https://github.com/bennojoy/mysql --> probably the right way - # how do you make use of other people's playbooks in the right way? - # https://stackoverflow.com/a/7740571/7782 - - - name: mysql setup - raw: debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password password {{mysql_root_pw}}' - args: - executable: /bin/bash - - - raw: debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password_again password {{mysql_root_pw}}' - args: - executable: /bin/bash - - - raw: apt-get -y install mysql-server - - - name: Create regluit database - mysql_db: db=regluit state=present encoding=utf8 collation=utf8_bin login_user=root login_password={{mysql_root_pw}} - - # GRANT ALL PRIVILEGES ON regluit.* TO 'regluit'@'localhost' WITH GRANT OPTION; (covered?) - - name: Create database user - mysql_user: > - user=regluit - password={{mysql_regluit_pw}} - host=localhost - priv=*.*:ALL - state=present - login_user=root - login_password={{mysql_root_pw}} - - - # running stuff within a virtualenv - # https://stackoverflow.com/a/20572360 - # https://stackoverflow.com/questions/20575084/best-way-to-always-run-ansible-inside-a-virtualenv-on-remote-machines?rq=1 - - - ## hard coding of please - #sudo("ln -s /opt/regluit/deploy/please.conf /etc/apache2/sites-available/please") - - - name: link please.conf into sites-available - file: src=/opt/regluit/deploy/please.conf dest=/etc/apache2/sites-available/please state=link - - #run('pip install -r requirements_versioned.pip') - - - name: upgrade pip - pip: > - executable=/opt/regluit/ENV/bin/pip - name={{item}} - virtualenv=/opt/regluit/ENV - virtualenv_command=virtualenv - extra_args="--upgrade" - with_items: - - pip - sudo: no - - # - name: pip install requests (to see whether in right place) - # pip: > - # executable=/opt/regluit/ENV/bin/pip - # name={{item}} - # virtualenv=/opt/regluit/ENV - # virtualenv_command=virtualenv - # with_items: - # - requests - # - census - # sudo: no - - - - name: pip requirments - pip: > - executable=/opt/regluit/ENV/bin/pip - requirements=/opt/regluit/requirements_versioned.pip - virtualenv=/opt/regluit/ENV - virtualenv_command=virtualenv - virtualenv_site_packages=yes - sudo: no - - - #run('echo "/opt/regluit/" > ENV/lib/python2.7/site-packages/regluit.pth') - #run('echo "/opt/" > ENV/lib/python2.7/site-packages/opt.pth') - - - name: establish regluit.pth - lineinfile: create=yes dest=/opt/regluit/ENV/lib/python2.7/site-packages/regluit.pth line="/opt/regluit/" - - - name: establish opt.pth - lineinfile: create=yes dest=/opt/regluit/ENV/lib/python2.7/site-packages/regluit.pth line="/opt/" - - #sudo('mkdir /var/www/static') - #sudo('chown ubuntu:ubuntu /var/www/static') - - - name: create /var/www/static - file: path=/var/www/static state=directory owner={{user}} group={{user}} mode=0755 - - # - #run('django-admin.py syncdb --migrate --noinput --settings regluit.settings.please') - - #Run syncdb on the application - - - name: django syncdb - django_manage: > - command=syncdb - app_path=/opt/regluit/ - settings="regluit.settings.please" - virtualenv=/opt/regluit/ENV - notify: - - restart apache2 - - - - name: django migrations - django_manage: > - command=migrate - app_path=/opt/regluit/ - settings="regluit.settings.please" - virtualenv=/opt/regluit/ENV - notify: - - restart apache2 - - #run('django-admin.py collectstatic --noinput --settings regluit.settings.please') - - - name: django collectstatic - django_manage: > - command=collectstatic - app_path=/opt/regluit/ - settings="regluit.settings.please" - virtualenv=/opt/regluit/ENV - notify: - - restart apache2 - - - name: copy STAR_unglue_it.crt - copy: > - src=/Volumes/ryvault1/gluejar/other_keys/unglue_it/STAR_unglue_it.crt - dest=/etc/ssl/certs/server.crt - owner={{user}} - group={{user}} - mode=0644 - notify: - - restart apache2 - - - name: copy server.key - copy: > - src=/Volumes/ryvault1/gluejar/other_keys/unglue_it/server.key - dest=/etc/ssl/private/server.key - owner={{user}} - group={{user}} - mode=0600 - notify: - - restart apache2 - - - name: copy STAR_unglue_it.ca-bundle - copy: > - src=/Volumes/ryvault1/gluejar/other_keys/unglue_it/STAR_unglue_it.ca-bundle - dest=/etc/ssl/certs/STAR_unglue_it.ca-bundle - owner={{user}} - group={{user}} - mode=0600 - notify: - - restart apache2 - - - name: remove /etc/logrotate.d/apache2 - file: path=/etc/logrotate.d/apache2 state=absent - notify: - - restart apache2 - - - name: a2dissite default - command: a2dissite default - notify: - - restart apache2 - - - name: a2ensite please - command: a2ensite please - notify: - - restart apache2 - - - name: a2enmod ssl rewrite headers - command: a2enmod ssl rewrite headers - notify: - - restart apache2 - - # - #sudo ("/etc/init.d/apache2 restart") - # - - - name: turn on ports 22, 80, 443 - ufw: rule=allow port={{ item }} proto=tcp - with_items: - - 22 - - 80 - - 443 - - - name: enable ufw - ufw: state=enabled - - #with cd("/opt/regluit"): - # - # sudo ("yes | adduser --no-create-home celery --disabled-password --disabled-login") - - - name: make celery group - group: name=celery - - - name: create celery user - user: > - name=celery - createhome=no - group=celery - generate_ssh_key=no - - # sudo ("cp deploy/celeryd /etc/init.d/celeryd") - # sudo ("chmod 755 /etc/init.d/celeryd") - - - name: copy deploy/celeryd - command: cp /opt/regluit/deploy/celeryd /etc/init.d/celeryd - - - name: set mode on /etc/init.d/celeryd - file: path=/etc/init.d/celeryd mode=0755 - - # sudo ("cp deploy/celeryd.conf /etc/default/celeryd") - - - name: copy deploy/celeryd_please.conf - command: cp /opt/regluit/deploy/celeryd_please.conf /etc/default/celeryd - - - name: set mode on /etc/default/celeryd - file: path=/etc/default/celeryd mode=0644 - - # sudo ("mkdir /var/log/celery") - - name: make /var/log/celery - file: path=/var/log/celery state=directory owner=celery group=celery mode=0755 - - # sudo ("mkdir /var/run/celery") - # sudo ("chown celery:celery /var/log/celery /var/run/celery") - - - name: make /var/run/celery - file: path=/var/run/celery state=directory owner=celery group=celery mode=0755 - - # sudo ("/etc/init.d/celeryd start") - - - name: start celeryd - command: /etc/init.d/celeryd start - - # sudo ("cp deploy/celerybeat /etc/init.d/celerybeat") - # sudo ("chmod 755 /etc/init.d/celerybeat") - # https://stackoverflow.com/questions/24162996/how-to-move-rename-a-file-using-an-ansible-task-on-a-remote-system - - - name: copy deploy/celerybeat - command: cp /opt/regluit/deploy/celerybeat /etc/init.d/celerybeat - - - name: set mode on /etc/init.d/celerybeat - file: path=/etc/init.d/celerybeat mode=0755 - - # sudo ("cp deploy/celerybeat.conf /etc/default/celerybeat") - - - name: copy deploy/celerybeat_please.conf - command: cp /opt/regluit/deploy/celerybeat_please.conf /etc/default/celerybeat - - - name: set mode on /etc/default/celerybeat - file: path=/etc/default/celerybeat mode=0755 - - # sudo ("mkdir /var/log/celerybeat") - # sudo ("chown celery:celery /var/log/celerybeat") - - - name: make /var/log/celerybeat - file: path=/var/log/celerybeat state=directory owner=celery group=celery mode=0755 - - # sudo ("/etc/init.d/celerybeat start") - - name: start celerybeat - command: /etc/init.d/celerybeat start - - # run data loading script - - name: run data loading script - script: load_data_please.sh - - # set up crontab - - name: crontab for please - command: crontab /opt/regluit/deploy/crontab_please.txt - - #- name: add ssh keys from public_key directory - # authorized_key: user={{user}} key={{item}} state=present - # with_fileglob: - # - /opt/regluit/deploy/public_keys/* - # sudo: no - - - name: add ssh keys from public_key directory - authorized_key: user={{user}} key="{{item}}" state=present - with_items: - - https://github.com/rdhyee.keys - - https://github.com/eshellman.keys - sudo: yes - - - handlers: - - name: restart apache2 - service: name=apache2 state=restarted diff --git a/vagrant/roles/common/tasks/main.yml b/vagrant/roles/common/tasks/main.yml deleted file mode 100644 index b2c1459e..00000000 --- a/vagrant/roles/common/tasks/main.yml +++ /dev/null @@ -1,72 +0,0 @@ - # sudo add repo to get latest version of python 2.7 - - name: add-apt-repository ppa:fkrull/deadsnakes-python2.7 - apt_repository: repo='ppa:fkrull/deadsnakes-python2.7' state=present update_cache=true - - - name: do apt-get update --fix-missing - command: apt-get update --fix-missing - - - name: installing dependencies - apt: pkg={{ item }} update_cache=yes state=present - with_items: - - python2.7 - - git-core - - apache2 - - libapache2-mod-wsgi - - mysql-client - - python-virtualenv - - python-mysqldb - - redis-server - - python-lxml - - python-dev - - libjpeg-dev - - libmysqlclient-dev - - libxml2-dev - - libxslt1-dev - - python-setuptools - - postfix - tags: install - - - name: make {{user}} group - group: name={{user}} - - - name: make {{user}} user - user: name={{user}} shell=/bin/bash group={{user}} generate_ssh_key=yes - - - name: install some python modules to use - #pip: name={{item}} virtualenv=/home/{{user}}/venv - pip: name={{item}} - with_items: - - PyGithub - - - name: create /opt/regluit - file: path=/opt/regluit state=directory owner={{user}} group={{user}} mode=0745 - - - name: git config - command: "{{item}}" - with_items: - - git config --global user.name "Raymond Yee" - - git config --global user.email "rdhyee@gluejar.com" - - - name: ssh-keygen - #command: pwd - command: ssh-keygen -b 2048 -t rsa -f /home/{{user}}/.ssh/id_rsa -P "" - sudo: no - args: - creates: /home/{{user}}/.ssh/id_rsa - - - name: create deploy key for repo - action: github_deploy_key - sudo: no - args: - github_auth_key: "{{github_auth_key}}" - repo_name: Gluejar/regluit - key_name: localvm - key_path: /home/{{user}}/.ssh/id_rsa.pub - - - name: postfix install - raw: DEBIAN_FRONTEND='noninteractive' apt-get install -y -q --force-yes postfix - - - name: clone the regluit git repo into /opt/regluit - sudo: no - git: repo=ssh://git@github.com/Gluejar/regluit.git dest=/opt/regluit accept_hostkey=True force=yes version=sysadmin - \ No newline at end of file diff --git a/vagrant/roles/hello_world/tasks/main.yml b/vagrant/roles/hello_world/tasks/main.yml deleted file mode 100644 index 76bef515..00000000 --- a/vagrant/roles/hello_world/tasks/main.yml +++ /dev/null @@ -1,6 +0,0 @@ -- name: calculate date - shell: date - register: date_result - -- name: show date_result - debug: msg="date_result {{date_result.stdout_lines[0]}}" diff --git a/vagrant/roles/mysql/tasks/main.yml b/vagrant/roles/mysql/tasks/main.yml deleted file mode 100644 index 9ce95514..00000000 --- a/vagrant/roles/mysql/tasks/main.yml +++ /dev/null @@ -1,18 +0,0 @@ - - name: mysql setup - raw: debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password password {{mysql_root_pw}}' - - raw: debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password_again password {{mysql_root_pw}}' - - raw: apt-get -y install mysql-server - - - name: Create regluit database - mysql_db: db=regluit state=present encoding=utf8 collation=utf8_bin login_user=root login_password={{mysql_root_pw}} - - # GRANT ALL PRIVILEGES ON regluit.* TO 'regluit'@'localhost' WITH GRANT OPTION; (covered?) - - name: Create database user - mysql_user: > - user=regluit - password={{mysql_regluit_pw}} - host=localhost - priv=*.*:ALL - state=present - login_user=root - login_password={{mysql_root_pw}} \ No newline at end of file diff --git a/vagrant/run_playbook.py b/vagrant/run_playbook.py deleted file mode 100755 index 30f2236a..00000000 --- a/vagrant/run_playbook.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function -import sh -import argparse -import vagrant - -# compute: inventory, private key, user and pass along all the other parameters to ansible-playbook - -parser = argparse.ArgumentParser(description='Run the ansible playbook, using vagrant ssh-config parameters') - -parser.add_argument('node', metavar='n', type=str, nargs='?', - help='node to run playbook on') -parser.add_argument('playbook_path', metavar='f', type=str, nargs='?', - help='path for the ansible playbook') - -(args, unknown) = parser.parse_known_args() -print (args.node, args.playbook_path) - -# for now if not multimachine, ignore node parameter -v = vagrant.Vagrant() - -if len(v.status()) > 1: - multimachine = True - private_key = v.keyfile(args.node) - user = v.user(args.node) -else: - multimachine = False - private_key = v.keyfile() - user = v.user() - -params = ["=".join(p) for p in ( - ("--private-key", private_key), - ("--user", user), - ("--inventory-file",".vagrant/provisioners/ansible/inventory") - )] + unknown + [args.playbook_path] - - -for line in sh.ansible_playbook(*params, _cwd=".", _iter=True): - print(line, end="") - diff --git a/vagrant/setup_django_just.sh b/vagrant/setup_django_just.sh deleted file mode 100644 index 5767945d..00000000 --- a/vagrant/setup_django_just.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -cd /opt/regluit -source ENV/bin/activate -export DJANGO_SETTINGS_MODULE=regluit.settings.just \ No newline at end of file diff --git a/vagrant/setup_django_localvm.sh b/vagrant/setup_django_localvm.sh deleted file mode 100644 index e8d42cc0..00000000 --- a/vagrant/setup_django_localvm.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -cd /opt/regluit -source ENV/bin/activate -export DJANGO_SETTINGS_MODULE=regluit.settings.localvm \ No newline at end of file diff --git a/vagrant/templates/apache.conf.j2 b/vagrant/templates/apache.conf.j2 deleted file mode 100644 index e0136c4a..00000000 --- a/vagrant/templates/apache.conf.j2 +++ /dev/null @@ -1,71 +0,0 @@ -WSGIPythonHome /opt/regluit/ENV -WSGIPythonPath /opt/regluit/ENV/bin/python -WSGISocketPrefix /opt/regluit - - - - -ServerName {% if class == 'prod' %}unglue.it{% else %}{{class}}.unglue.it{% endif %} - -ServerAdmin info@ebookfoundation.org - -Redirect permanent / {% if class == 'prod' %}https://unglue.it{% else %}https://{{class}}.unglue.it{% endif %} - - - - - - -ServerName {% if class == 'prod' %}unglue.it:443{% else %}{{class}}.unglue.it:443{% endif %} - -ServerAdmin info@ebookfoundation.org - -SSLEngine on -SSLProtocol All -SSLv2 -SSLv3 - -SSLCertificateFile /etc/ssl/certs/server.crt -SSLCertificateKeyFile /etc/ssl/private/server.key -SSLCertificateChainFile /etc/ssl/certs/STAR_unglue_it.ca-bundle - -#SSLCertificateChainFile /etc/ssl/certs/gd_bundle.crt - -WSGIDaemonProcess regluit processes=4 threads=4 python-eggs=/tmp/regluit-python-eggs -WSGIScriptAlias / /opt/regluit/deploy/{{class}}.wsgi - -# generated using https://mozilla.github.io/server-side-tls/ssl-config-generator/ -# intermediate mode -# 2015.03.04 (with Apache v 2.2.22 and OpenSSL 1.0.1 and HSTS enabled) - -SSLProtocol all -SSLv2 -SSLv3 -SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA -SSLHonorCipherOrder on - -# HSTS (mod_headers is required) (15768000 seconds = 6 months) -Header always add Strict-Transport-Security "max-age=15768000" - - - - Require all granted - - - - - Options Indexes FollowSymLinks - AllowOverride None - - Require all granted - - -Alias /static /var/www/static - -BrowserMatch "MSIE [2-6]" \ - nokeepalive ssl-unclean-shutdown \ - downgrade-1.0 force-response-1.0 -# MSIE 7 and newer should be able to use keepalive -BrowserMatch "MSIE [17-9]" ssl-unclean-shutdown - -ErrorLog "|/usr/bin/cronolog /var/log/apache2/%Y%m%d_error.log" -LogLevel warn -CustomLog "|/usr/bin/cronolog /var/log/apache2/%Y%m%d_access.log" combined - - \ No newline at end of file diff --git a/vagrant/templates/common.py.j2 b/vagrant/templates/common.py.j2 deleted file mode 100644 index 015baff2..00000000 --- a/vagrant/templates/common.py.j2 +++ /dev/null @@ -1,6 +0,0 @@ -import os - -# all the COMMON_KEYS -{% for key in COMMON_KEYS %} -{{key}} = os.environ.get('{{key}}', '{{COMMON_KEYS[key]}}') -{% endfor %} diff --git a/vagrant/templates/host.py.j2 b/vagrant/templates/host.py.j2 deleted file mode 100644 index 8a43ee0f..00000000 --- a/vagrant/templates/host.py.j2 +++ /dev/null @@ -1,5 +0,0 @@ -import os -# all the SECRET_KEYS -{% for key in SECRET_KEYS %} -{{key}} = os.environ.get('{{key}}', '{{SECRET_KEYS[key]}}') -{% endfor %} diff --git a/vagrant/templates/just.wsgi.j2 b/vagrant/templates/just.wsgi.j2 deleted file mode 100644 index c1341a18..00000000 --- a/vagrant/templates/just.wsgi.j2 +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python - -import os - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "regluit.settings.just") -os.environ['CELERY_LOADER'] = 'django' - - -{% for key in SECRET_KEYS %} -os.environ['{{key}}'] = '{{SECRET_KEYS[key]}}' -{% endfor %} - - -from django.core.wsgi import get_wsgi_application -application = get_wsgi_application() diff --git a/vagrant/templates/please.wsgi.j2 b/vagrant/templates/please.wsgi.j2 deleted file mode 100644 index a5d405a4..00000000 --- a/vagrant/templates/please.wsgi.j2 +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python - -import os - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "regluit.settings.please") -os.environ['CELERY_LOADER'] = 'django' - - -{% for key in SECRET_KEYS %} -os.environ['{{key}}'] = '{{SECRET_KEYS[key]}}' -{% endfor %} - -from django.core.wsgi import get_wsgi_application -application = get_wsgi_application() \ No newline at end of file diff --git a/vagrant/templates/prod.wsgi.j2 b/vagrant/templates/prod.wsgi.j2 deleted file mode 100644 index 30daa000..00000000 --- a/vagrant/templates/prod.wsgi.j2 +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python - -import os - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "regluit.settings.prod") -os.environ['CELERY_LOADER'] = 'django' - -{% for key in SECRET_KEYS %} -os.environ['{{key}}'] = '{{SECRET_KEYS[key]}}' -{% endfor %} - -from django.core.wsgi import get_wsgi_application -application = get_wsgi_application() \ No newline at end of file diff --git a/vagrant/unglueit.yml b/vagrant/unglueit.yml deleted file mode 100644 index d46488dd..00000000 --- a/vagrant/unglueit.yml +++ /dev/null @@ -1,357 +0,0 @@ -- name: unglueit setup - hosts: please - vars: - user: "{{ ansible_ssh_user }}" - sudo: yes - - pre_tasks: - - name: check apt last update - stat: path=/var/cache/apt - register: apt_cache_stat - - name: update apt if needed - apt: update_cache=yes - when: ansible_date_time.epoch|float - apt_cache_stat.stat.mtime > 60*60*12 - - tasks: - - - name: set DNS record for please.unglue.it - route53: - command: create - zone: unglue.it - record: please.unglue.it - type: A - ttl: 60 - value: "{{ansible_ssh_host}}" - overwrite: yes - aws_access_key: "{{aws_access_key}}" - aws_secret_key: "{{aws_secret_key}}" - - # sudo add repo to get latest version of python 2.7 - - name: add-apt-repository ppa:fkrull/deadsnakes-python2.7 - apt_repository: repo='ppa:fkrull/deadsnakes-python2.7' state=present update_cache=true - - - name: do apt-get update --fix-missing - command: apt-get update --fix-missing - - - name: installing dependencies - apt: pkg={{ item }} update_cache=yes state=present - with_items: - - python2.7 - - git-core - - apache2 - - libapache2-mod-wsgi - - mysql-client - - python-virtualenv - - python-mysqldb - - redis-server - - python-lxml - - python-dev - - libjpeg-dev - - libmysqlclient-dev - - libxml2-dev - - libxslt1-dev - - python-setuptools - - python-dev - - postfix - tags: install - - - name: make {{user}} group - group: name={{user}} - - - name: make {{user}} user - user: name={{user}} shell=/bin/bash group={{user}} generate_ssh_key=yes - - - name: install some python modules to use - #pip: name={{item}} virtualenv=/home/{{user}}/venv - pip: name={{item}} - with_items: - - PyGithub - - - name: create /opt/regluit - file: path=/opt/regluit state=directory owner={{user}} group={{user}} mode=0745 - - - name: git config - command: "{{item}}" - with_items: - - git config --global user.name "Raymond Yee" - - git config --global user.email "rdhyee@gluejar.com" - - - name: ssh-keygen - #command: pwd - command: ssh-keygen -b 2048 -t rsa -f /home/{{user}}/.ssh/id_rsa -P "" - sudo: no - args: - creates: /home/{{user}}/.ssh/id_rsa - - - name: create deploy key for repo - action: github_deploy_key - sudo: no - args: - github_auth_key: "{{github_auth_key}}" - repo_name: Gluejar/regluit - key_name: vagrant_ansible_test - key_path: /home/{{user}}/.ssh/id_rsa.pub - - - name: postfix install - raw: DEBIAN_FRONTEND='noninteractive' apt-get install -y -q --force-yes postfix - - - name: clone the regluit git repo into /opt/regluit - sudo: no - git: repo=ssh://git@github.com/Gluejar/regluit.git dest=/opt/regluit accept_hostkey=True force=yes version=sysadmin - - - # installing mysql - # https://github.com/bennojoy/mysql --> probably the right way - # how do you make use of other people's playbooks in the right way? - # https://stackoverflow.com/a/7740571/7782 - - - name: mysql setup - raw: debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password password {{mysql_root_pw}}' - - raw: debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password_again password {{mysql_root_pw}}' - - raw: apt-get -y install mysql-server - - - name: Create regluit database - mysql_db: db=regluit state=present encoding=utf8 collation=utf8_bin login_user=root login_password={{mysql_root_pw}} - - # GRANT ALL PRIVILEGES ON regluit.* TO 'regluit'@'localhost' WITH GRANT OPTION; (covered?) - - name: Create database user - mysql_user: > - user=regluit - password={{mysql_regluit_pw}} - host=localhost - priv=*.*:ALL - state=present - login_user=root - login_password={{mysql_root_pw}} - - - # running stuff within a virtualenv - # https://stackoverflow.com/a/20572360 - # https://stackoverflow.com/questions/20575084/best-way-to-always-run-ansible-inside-a-virtualenv-on-remote-machines?rq=1 - - - ## hard coding of please - #sudo("ln -s /opt/regluit/deploy/please.conf /etc/apache2/sites-available/please") - - - name: link please.conf into sites-available - file: src=/opt/regluit/deploy/please.conf dest=/etc/apache2/sites-available/please state=link - - #run('pip install -r requirements_versioned.pip') - - - name: pip install requests (to see whether in right place) - pip: > - executable=/opt/regluit/ENV/bin/pip - name={{item}} - virtualenv=/opt/regluit/ENV - virtualenv_command=virtualenv - with_items: - - requests - - census - sudo: no - - - - name: pip requirments - pip: > - executable=/opt/regluit/ENV/bin/pip - requirements=/opt/regluit/requirements_versioned.pip - virtualenv=/opt/regluit/ENV - virtualenv_command=virtualenv - virtualenv_site_packages=yes - sudo: no - - - #run('echo "/opt/regluit/" > ENV/lib/python2.7/site-packages/regluit.pth') - #run('echo "/opt/" > ENV/lib/python2.7/site-packages/opt.pth') - - - name: establish regluit.pth - lineinfile: create=yes dest=/opt/regluit/ENV/lib/python2.7/site-packages/regluit.pth line="/opt/regluit/" - - - name: establish opt.pth - lineinfile: create=yes dest=/opt/regluit/ENV/lib/python2.7/site-packages/regluit.pth line="/opt/" - - #sudo('mkdir /var/www/static') - #sudo('chown ubuntu:ubuntu /var/www/static') - - - name: create /var/www/static - file: path=/var/www/static state=directory owner={{user}} group={{user}} mode=0755 - - # - #run('django-admin.py syncdb --migrate --noinput --settings regluit.settings.please') - - #Run syncdb on the application - - - name: django syncdb - django_manage: > - command=syncdb - app_path=/opt/regluit/ - settings="regluit.settings.please" - virtualenv=/opt/regluit/ENV - notify: - - restart apache2 - - - - name: django migrations - django_manage: > - command=migrate - app_path=/opt/regluit/ - settings="regluit.settings.please" - virtualenv=/opt/regluit/ENV - notify: - - restart apache2 - - #run('django-admin.py collectstatic --noinput --settings regluit.settings.please') - - - name: django collectstatic - django_manage: > - command=collectstatic - app_path=/opt/regluit/ - settings="regluit.settings.please" - virtualenv=/opt/regluit/ENV - notify: - - restart apache2 - - - name: copy STAR_unglue_it.crt - copy: > - src=/Volumes/ryvault1/gluejar/other_keys/unglue_it/STAR_unglue_it.crt - dest=/etc/ssl/certs/server.crt - owner={{user}} - group={{user}} - mode=0644 - notify: - - restart apache2 - - - name: copy server.key - copy: > - src=/Volumes/ryvault1/gluejar/other_keys/unglue_it/server.key - dest=/etc/ssl/private/server.key - owner={{user}} - group={{user}} - mode=0600 - notify: - - restart apache2 - - - name: copy STAR_unglue_it.ca-bundle - copy: > - src=/Volumes/ryvault1/gluejar/other_keys/unglue_it/STAR_unglue_it.ca-bundle - dest=/etc/ssl/certs/STAR_unglue_it.ca-bundle - owner={{user}} - group={{user}} - mode=0600 - notify: - - restart apache2 - - - name: a2dissite default - command: a2dissite default - notify: - - restart apache2 - - - name: a2ensite please - command: a2ensite please - notify: - - restart apache2 - - - name: a2enmod ssl rewrite - command: a2enmod ssl rewrite - notify: - - restart apache2 - - # - #sudo ("/etc/init.d/apache2 restart") - # - - - name: turn on ports 22, 80, 443 - ufw: rule=allow port={{ item }} proto=tcp - with_items: - - 22 - - 80 - - 443 - - - name: enable ufw - ufw: state=enabled - - #with cd("/opt/regluit"): - # - # sudo ("yes | adduser --no-create-home celery --disabled-password --disabled-login") - - - name: make celery group - group: name=celery - - - name: create celery user - user: > - name=celery - createhome=no - group=celery - generate_ssh_key=no - - # sudo ("cp deploy/celeryd /etc/init.d/celeryd") - # sudo ("chmod 755 /etc/init.d/celeryd") - - - name: copy deploy/celeryd - command: cp /opt/regluit/deploy/celeryd /etc/init.d/celeryd - - - name: set mode on /etc/init.d/celeryd - file: path=/etc/init.d/celeryd mode=0755 - - # sudo ("cp deploy/celeryd.conf /etc/default/celeryd") - - - name: copy deploy/celeryd_please.conf - command: cp /opt/regluit/deploy/celeryd_please.conf /etc/default/celeryd - - - name: set mode on /etc/default/celeryd - file: path=/etc/default/celeryd mode=0644 - - # sudo ("mkdir /var/log/celery") - - name: make /var/log/celery - file: path=/var/log/celery state=directory owner=celery group=celery mode=0755 - - # sudo ("mkdir /var/run/celery") - # sudo ("chown celery:celery /var/log/celery /var/run/celery") - - - name: make /var/run/celery - file: path=/var/run/celery state=directory owner=celery group=celery mode=0755 - - # sudo ("/etc/init.d/celeryd start") - - - name: start celeryd - command: /etc/init.d/celeryd start - - # sudo ("cp deploy/celerybeat /etc/init.d/celerybeat") - # sudo ("chmod 755 /etc/init.d/celerybeat") - # https://stackoverflow.com/questions/24162996/how-to-move-rename-a-file-using-an-ansible-task-on-a-remote-system - - - name: copy deploy/celerybeat - command: cp /opt/regluit/deploy/celerybeat /etc/init.d/celerybeat - - - name: set mode on /etc/init.d/celerybeat - file: path=/etc/init.d/celerybeat mode=0755 - - # sudo ("cp deploy/celerybeat.conf /etc/default/celerybeat") - - - name: copy deploy/celerybeat_please.conf - command: cp /opt/regluit/deploy/celerybeat_please.conf /etc/default/celerybeat - - - name: set mode on /etc/default/celerybeat - file: path=/etc/default/celerybeat mode=0755 - - # sudo ("mkdir /var/log/celerybeat") - # sudo ("chown celery:celery /var/log/celerybeat") - - - name: make /var/log/celerybeat - file: path=/var/log/celerybeat state=directory owner=celery group=celery mode=0755 - - # sudo ("/etc/init.d/celerybeat start") - - name: start celerybeat - command: /etc/init.d/celerybeat start - - # run data loading script - - name: run data loading script - script: load_data_please.sh - - # set up crontab - - name: crontab for please - command: crontab /opt/regluit/deploy/crontab_please.txt - - - handlers: - - name: restart apache2 - service: name=apache2 state=restarted From ea278c3f41239754744d2832cac8eec19a439de5 Mon Sep 17 00:00:00 2001 From: eric Date: Mon, 11 Jun 2018 14:43:56 -0400 Subject: [PATCH 25/40] add reset option --- .../management/commands/make_missing_mobis.py | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/core/management/commands/make_missing_mobis.py b/core/management/commands/make_missing_mobis.py index af2d5461..562aa9cc 100644 --- a/core/management/commands/make_missing_mobis.py +++ b/core/management/commands/make_missing_mobis.py @@ -1,22 +1,27 @@ from django.core.management.base import BaseCommand -from regluit.core.models import Work +from regluit.core.models import Work, EbookFile class Command(BaseCommand): help = "generate mobi ebooks where needed and possible." - args = "" - + + def add_arguments(self, parser): + parser.add_argument('max', nargs='?', type=int, default=1, help="maximum mobis to make") + parser.add_argument('--reset', '-r', action='store_true', help="reset failed mobi conversions") + + def handle(self, max=None, **options): - if max: - try: - max = int(max) - except ValueError: - max = 1 - else: - max = 1 + maxbad = 10 + if options['reset']: + bads = EbookFile.objects.filter(mobied__lt=0) + for bad in bads: + bad.mobied = 0 + bad.save() + epubs = Work.objects.filter(editions__ebooks__format='epub').distinct().order_by('-id') i = 0 + n_bad = 0 for work in epubs: if not work.ebooks().filter(format="mobi"): for ebook in work.ebooks().filter(format="epub"): @@ -26,11 +31,14 @@ class Command(BaseCommand): print u'making mobi for {}'.format(work.title) if ebf.make_mobi(): print 'made mobi' - i = i + 1 + i += 1 break else: - print 'failed to make mobi' + self.stdout.write('failed to make mobi') + n_bad += 1 + except: - print 'failed to make mobi' - if i >= max: + self.stdout.write('failed to make mobi') + n_bad += 1 + if i >= max or n_bad >= maxbad: break From bade8e7f4dc1b1a020ffcef7697e3331cb890068 Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 15 Jun 2018 10:34:23 -0400 Subject: [PATCH 26/40] handle records without downloads --- core/loaders/doab.py | 14 ++++++++++++++ core/loaders/doab_utils.py | 2 ++ 2 files changed, 16 insertions(+) diff --git a/core/loaders/doab.py b/core/loaders/doab.py index 14f90c43..a053f0df 100644 --- a/core/loaders/doab.py +++ b/core/loaders/doab.py @@ -393,6 +393,20 @@ def add_by_doab(doab_id, record=None): url_to_provider(dl_url) if dl_url else None, **metadata ) + else: + if 'format' in metadata: + del metadata['format'] + edition = load_doab_edition( + title, + doab_id, + '', + '', + license, + language, + isbns, + None, + **metadata + ) return edition except IdDoesNotExistError: return None diff --git a/core/loaders/doab_utils.py b/core/loaders/doab_utils.py index e03c2348..ceef8bb7 100644 --- a/core/loaders/doab_utils.py +++ b/core/loaders/doab_utils.py @@ -103,6 +103,8 @@ FRONTIERSIN = re.compile(r'frontiersin.org/books/[^/]+/(\d+)') def online_to_download(url): urls = [] + if not url: + return urls if url.find(u'mdpi.com/books/pdfview/book/') >= 0: doc = get_soup(url) if doc: From 7593944dc0641c61a10eff0080a4093d22784d8c Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 15 Jun 2018 15:30:04 -0400 Subject: [PATCH 27/40] reset default to 15 days --- core/loaders/doab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/loaders/doab.py b/core/loaders/doab.py index a053f0df..22e337d1 100644 --- a/core/loaders/doab.py +++ b/core/loaders/doab.py @@ -425,8 +425,8 @@ def load_doab_oai(from_year=None, limit=100000): if from_year: from_ = datetime.datetime(year=from_year, month=1, day=1) else: - # last 45 days - from_ = datetime.datetime.now() - datetime.timedelta(days=45) + # last 15 days + from_ = datetime.datetime.now() - datetime.timedelta(days=15) doab_ids = [] for record in doab_client.listRecords(metadataPrefix='oai_dc', from_=from_): if not record[1]: From 3bc7d5c003a6ecf17f226ce47a7b6f9f8d3b0ab5 Mon Sep 17 00:00:00 2001 From: eric Date: Mon, 18 Jun 2018 17:03:41 -0400 Subject: [PATCH 28/40] fix loader tests --- core/loaders/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/loaders/tests.py b/core/loaders/tests.py index 6a8f485e..f94e1ad3 100644 --- a/core/loaders/tests.py +++ b/core/loaders/tests.py @@ -19,10 +19,10 @@ class LoaderTests(TestCase): dropbox_url = 'https://www.dropbox.com/s/h5jzpb4vknk8n7w/Jakobsson_The_Troll_Inside_You_EBook.pdf?dl=0' dropbox_ebook = Ebook.objects.create(format='online', url=dropbox_url, edition=edition) - dropbox_ebf = dl_online(dropbox_ebook) + dropbox_ebf, new_ebf = dl_online(dropbox_ebook) self.assertTrue(dropbox_ebf.ebook.filesize) jbe_url = 'http://www.jbe-platform.com/content/books/9789027295958' jbe_ebook = Ebook.objects.create(format='online', url=jbe_url, edition=edition) - jbe_ebf = dl_online(jbe_ebook) + jbe_ebf, new_ebf = dl_online(jbe_ebook) self.assertTrue(jbe_ebf.ebook.filesize) From 53a0791da028c0b5a97b2674725e91395c7e0765 Mon Sep 17 00:00:00 2001 From: eric Date: Mon, 18 Jun 2018 17:04:19 -0400 Subject: [PATCH 29/40] handle series properly --- core/bookloader.py | 14 ++++++++++++++ core/migrations/0014_auto_20180618_1646.py | 19 +++++++++++++++++++ core/parameters.py | 2 +- core/tests.py | 8 +++++++- 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 core/migrations/0014_auto_20180618_1646.py diff --git a/core/bookloader.py b/core/bookloader.py index 64208755..d16fe3c0 100755 --- a/core/bookloader.py +++ b/core/bookloader.py @@ -521,6 +521,20 @@ def merge_works(w1, w2, user=None): #(for example, when w2 has already been deleted) if w1 is None or w2 is None or w1.id == w2.id or w1.id is None or w2.id is None: return w1 + + #don't merge if the works are related. + if w2 in w1.works_related_to.all() or w1 in w2.works_related_to.all(): + return w1 + + # check if one of the works is a series with parts (that have their own isbn) + if w1.works_related_from.filter(relation='part'): + models.WorkRelation.objects.get_or_create(to_work=w2, from_work=w1, relation='part') + return w1 + if w2.works_related_from.filter(relation='part'): + models.WorkRelation.objects.get_or_create(to_work=w1, from_work=w2, relation='part') + return w1 + + if w2.selected_edition is not None and w1.selected_edition is None: #the merge should be reversed temp = w1 diff --git a/core/migrations/0014_auto_20180618_1646.py b/core/migrations/0014_auto_20180618_1646.py new file mode 100644 index 00000000..756c895f --- /dev/null +++ b/core/migrations/0014_auto_20180618_1646.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0013_ebookfile_mobied'), + ] + + operations = [ + migrations.AlterField( + model_name='workrelation', + name='relation', + field=models.CharField(max_length=15, choices=[(b'translation', b'translation'), (b'revision', b'revision'), (b'sequel', b'sequel'), (b'part', b'part')]), + ), + ] diff --git a/core/parameters.py b/core/parameters.py index 9db6a10c..a7fffae8 100644 --- a/core/parameters.py +++ b/core/parameters.py @@ -20,7 +20,7 @@ TEXT_RELATION_CHOICES = ( ('translation', 'translation'), ('revision', 'revision'), ('sequel', 'sequel'), - ('compilation', 'compilation') + ('part', 'part') ) ID_CHOICES = ( diff --git a/core/tests.py b/core/tests.py index 8532df09..d684a9ef 100755 --- a/core/tests.py +++ b/core/tests.py @@ -298,7 +298,13 @@ class BookLoaderTests(TestCase): # first try to merge work 1 into itself -- should not do anything bookloader.merge_works(w1, w1) self.assertEqual(models.Work.objects.count(), before + 2) - + + # first try to merge related works -- should not do anything + rel, created = models.WorkRelation.objects.get_or_create(to_work=w1, from_work=w2, relation='part') + bookloader.merge_works(w1, w2) + self.assertEqual(models.Work.objects.count(), before + 2) + rel.delete() + # merge the second work into the first bookloader.merge_works(e1.work, e2.work) self.assertEqual(models.Work.objects.count(), before + 1) From 2f9dda8432899fa26caefeaa60c8113c49a23d8e Mon Sep 17 00:00:00 2001 From: eric Date: Mon, 18 Jun 2018 17:04:40 -0400 Subject: [PATCH 30/40] less agressive merging in doab --- core/loaders/doab.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/loaders/doab.py b/core/loaders/doab.py index 22e337d1..6d364328 100644 --- a/core/loaders/doab.py +++ b/core/loaders/doab.py @@ -142,10 +142,11 @@ def add_all_isbns(isbns, work, language=None, title=None): if edition: first_edition = first_edition if first_edition else edition if work and (edition.work_id != work.id): - if work.created < edition.work.created: - work = merge_works(work, edition.work) - else: - work = merge_works(edition.work, work) + if work.doab and edition.work.doab and work.doab != edition.work.doab: + if work.created < edition.work.created: + work = merge_works(work, edition.work) + else: + work = merge_works(edition.work, work) else: work = edition.work return work, first_edition From 604260363faa90c069f729bef3c35dc082480cca Mon Sep 17 00:00:00 2001 From: eric Date: Mon, 18 Jun 2018 22:07:29 -0400 Subject: [PATCH 31/40] deleting works should delete attached objects I think the only place this should get invoked is via admin --- core/bookloader.py | 2 +- core/models/bibmodels.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/core/bookloader.py b/core/bookloader.py index d16fe3c0..a55133cc 100755 --- a/core/bookloader.py +++ b/core/bookloader.py @@ -597,7 +597,7 @@ def merge_works(w1, w2, user=None): for work_relation in w2.works_related_from.all(): work_relation.from_work = w1 work_relation.save() - w2.delete() + w2.delete(cascade=False) return w1 def detach_edition(e): diff --git a/core/models/bibmodels.py b/core/models/bibmodels.py index 44c59b9c..12530d1b 100644 --- a/core/models/bibmodels.py +++ b/core/models/bibmodels.py @@ -22,6 +22,8 @@ from django.db.models import F from django.db.models.signals import post_save, pre_delete from django.utils.timezone import now +from django_comments.models import Comment + import regluit from regluit.marc.models import MARCRecord as NewMARC from questionnaire.models import Landing @@ -131,13 +133,39 @@ class Work(models.Model): class Meta: ordering = ['title'] + def __unicode__(self): return self.title def __init__(self, *args, **kwargs): self._last_campaign = None super(Work, self).__init__(*args, **kwargs) - + + def delete(self, *args, **kwargs): + if kwargs.get('cascade', True): + if self.offers.all() or self.claim.all() or self.campaigns.all() or self.acqs.all() \ + or self.holds.all() or self.landings.all(): + return + for wishlist in self.wishlists.all(): + wishlist.remove_work(self) + for userprofile in self.contributors.all(): + userprofile.works.remove(self) + for identifier in self.identifiers.all(): + identifier.delete() + for comment in Comment.objects.for_model(self): + comment.delete() + for edition in self.editions.all(): + for ebook in edition.ebooks.all(): + ebook.delete() + for ebookfile in edition.ebook_files.all(): + ebookfile.delete() + edition.delete() + for work_relation in self.works_related_to.all(): + work_relation.delete() + for work_relation in self.works_related_from.all(): + work_relation.delete() + super(Work, self).delete(*args, **kwargs) # Call the "real" save() method. + def id_for(self, type): return id_for(self, type) From d8171088a62c48607855484ff74817fb5790912d Mon Sep 17 00:00:00 2001 From: eric Date: Tue, 19 Jun 2018 10:42:02 -0400 Subject: [PATCH 32/40] support lists of doabids --- core/management/commands/load_by_doab.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/management/commands/load_by_doab.py b/core/management/commands/load_by_doab.py index 611f18ed..beb32483 100644 --- a/core/management/commands/load_by_doab.py +++ b/core/management/commands/load_by_doab.py @@ -4,7 +4,10 @@ from regluit.core.loaders import doab class Command(BaseCommand): help = "load doab books by doab_id via oai" - args = "" + + def add_arguments(self, parser): + parser.add_argument('doab_ids', nargs='+', type=int, default=1, help="doab ids to add") - def handle(self, doab_id, **options): - doab.add_by_doab(doab_id) + def handle(self, doab_ids, **options): + for doab_id in doab_ids: + doab.add_by_doab(doab_id) From b709ad93f83dcfed5074c6c4842a334055066b89 Mon Sep 17 00:00:00 2001 From: eric Date: Tue, 19 Jun 2018 10:43:42 -0400 Subject: [PATCH 33/40] fix handling of cascade argument --- core/models/bibmodels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/models/bibmodels.py b/core/models/bibmodels.py index 12530d1b..4347c6f5 100644 --- a/core/models/bibmodels.py +++ b/core/models/bibmodels.py @@ -141,8 +141,8 @@ class Work(models.Model): self._last_campaign = None super(Work, self).__init__(*args, **kwargs) - def delete(self, *args, **kwargs): - if kwargs.get('cascade', True): + def delete(self, cascade=True, *args, **kwargs): + if cascade: if self.offers.all() or self.claim.all() or self.campaigns.all() or self.acqs.all() \ or self.holds.all() or self.landings.all(): return From e49fbd18261d30911aa6732148b97ff76b224cde Mon Sep 17 00:00:00 2001 From: eric Date: Tue, 19 Jun 2018 10:44:15 -0400 Subject: [PATCH 34/40] correct handling of google books returning multiple items --- core/bookloader.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/core/bookloader.py b/core/bookloader.py index a55133cc..af92876d 100755 --- a/core/bookloader.py +++ b/core/bookloader.py @@ -242,6 +242,17 @@ def update_edition(edition): return edition +def get_isbn_item(items, isbn): + # handle case where google sends back several items + for item in items: + volumeInfo = item.get('volumeInfo', {}) + industryIdentifiers = volumeInfo.get('industryIdentifiers', []) + for ident in industryIdentifiers: + if ident['identifier'] == isbn: + return item + else: + return None # no items + return item def add_by_isbn_from_google(isbn, work=None): """add a book to the UnglueIt database from google based on ISBN. The work parameter @@ -263,11 +274,12 @@ def add_by_isbn_from_google(isbn, work=None): logger.info(u"adding new book by isbn %s", isbn) results = get_google_isbn_results(isbn) if results: + item = get_isbn_item(results['items'], isbn) try: return add_by_googlebooks_id( - results['items'][0]['id'], + item['id'], work=work, - results=results['items'][0], + results=item, isbn=isbn ) except LookupFailure, e: From 59b49e2c2e7bc50a8c053e0c08bb9923772324c0 Mon Sep 17 00:00:00 2001 From: eric Date: Tue, 19 Jun 2018 13:34:29 -0400 Subject: [PATCH 35/40] fix exception when there's no google result --- core/bookloader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/bookloader.py b/core/bookloader.py index af92876d..72987b2f 100755 --- a/core/bookloader.py +++ b/core/bookloader.py @@ -273,8 +273,11 @@ def add_by_isbn_from_google(isbn, work=None): logger.info(u"adding new book by isbn %s", isbn) results = get_google_isbn_results(isbn) - if results: + if results and items in results: item = get_isbn_item(results['items'], isbn) + if not item: + logger.exception(u"no items for %s", isbn) + return None try: return add_by_googlebooks_id( item['id'], From 042613b0ae3aeaa412f981cbdbc22f0047c8f7a8 Mon Sep 17 00:00:00 2001 From: eric Date: Tue, 19 Jun 2018 14:14:10 -0400 Subject: [PATCH 36/40] wip --- core/models/__init__.py | 5 +++-- requirements_versioned.pip | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/models/__init__.py b/core/models/__init__.py index 7136fd16..215ad4fd 100755 --- a/core/models/__init__.py +++ b/core/models/__init__.py @@ -13,7 +13,8 @@ from tempfile import SpooledTemporaryFile import requests from ckeditor.fields import RichTextField from notification import models as notification -from postmonkey import PostMonkey, MailChimpException +#from postmonkey import PostMonkey, MailChimpException +from mailchimp3 import MailChimp #django imports from django.apps import apps @@ -95,7 +96,7 @@ from .bibmodels import ( ) from .rh_models import Claim, RightsHolder -pm = PostMonkey(settings.MAILCHIMP_API_KEY) +pm = MailChimp(mc_api=settings.MAILCHIMP_API_KEY) logger = logging.getLogger(__name__) diff --git a/requirements_versioned.pip b/requirements_versioned.pip index b668a74d..f1e0f02d 100644 --- a/requirements_versioned.pip +++ b/requirements_versioned.pip @@ -58,7 +58,7 @@ oauth2==1.5.211 oauthlib==1.1.2 pandas==0.19.1 paramiko==1.14.1 -postmonkey==1.0b +mailchimp3==3.0.4 pycrypto==2.6 pymarc==3.0.2 pyoai==2.5.0 From fadba3ce1bc2f2aef25069b5db3b7e46ac7bbd22 Mon Sep 17 00:00:00 2001 From: eric Date: Tue, 19 Jun 2018 14:15:53 -0400 Subject: [PATCH 37/40] syntax --- core/bookloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/bookloader.py b/core/bookloader.py index 72987b2f..a9fed0c2 100755 --- a/core/bookloader.py +++ b/core/bookloader.py @@ -273,7 +273,7 @@ def add_by_isbn_from_google(isbn, work=None): logger.info(u"adding new book by isbn %s", isbn) results = get_google_isbn_results(isbn) - if results and items in results: + if results and 'items' in results: item = get_isbn_item(results['items'], isbn) if not item: logger.exception(u"no items for %s", isbn) From 1d83e7af784f9984132e9ed443f031d44f18a7e3 Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 20 Jun 2018 10:03:48 -0400 Subject: [PATCH 38/40] update to mailchimp 3.0 api --- core/models/__init__.py | 28 +++++++++++++++++++++------- core/tasks.py | 16 ++++++++++++---- core/tests.py | 7 ++++--- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/core/models/__init__.py b/core/models/__init__.py index 215ad4fd..f3795655 100755 --- a/core/models/__init__.py +++ b/core/models/__init__.py @@ -13,8 +13,8 @@ from tempfile import SpooledTemporaryFile import requests from ckeditor.fields import RichTextField from notification import models as notification -#from postmonkey import PostMonkey, MailChimpException from mailchimp3 import MailChimp +from mailchimp3.mailchimpclient import MailChimpError #django imports from django.apps import apps @@ -96,7 +96,7 @@ from .bibmodels import ( ) from .rh_models import Claim, RightsHolder -pm = MailChimp(mc_api=settings.MAILCHIMP_API_KEY) +mc_client = MailChimp(mc_api=settings.MAILCHIMP_API_KEY) logger = logging.getLogger(__name__) @@ -1257,10 +1257,17 @@ class UserProfile(models.Model): # use @example.org email addresses for testing! return False try: - return settings.MAILCHIMP_NEWS_ID in pm.listsForEmail(email_address=self.user.email) - except MailChimpException, e: - if e.code != 215: # don't log case where user is not on a list + member = mc_client.lists.members.get( + list_id=settings.MAILCHIMP_NEWS_ID, + subscriber_hash=self.user.email + ) + if member['status'] == 'subscribed': + return 'True' + except MailChimpError, e: + if e[0]['status'] != 404: # don't log case where user is not on a list logger.error("error getting mailchimp status %s" % (e)) + except ValueError, e: + logger.error("bad email address %s" % (self.user.email)) except Exception, e: logger.error("error getting mailchimp status %s" % (e)) return False @@ -1268,7 +1275,7 @@ class UserProfile(models.Model): def ml_subscribe(self, **kwargs): if "@example.org" in self.user.email: # use @example.org email addresses for testing! - return True + return from regluit.core.tasks import ml_subscribe_task ml_subscribe_task.delay(self, **kwargs) @@ -1277,7 +1284,14 @@ class UserProfile(models.Model): # use @example.org email addresses for testing! return True try: - return pm.listUnsubscribe(id=settings.MAILCHIMP_NEWS_ID, email_address=self.user.email) + mc_client.lists.members.delete( + list_id=settings.MAILCHIMP_NEWS_ID, + subscriber_hash=self.user.email, + ) + return True + except MailChimpError, e: + if e[0]['status'] != 404: # don't log case where user is not on a list + logger.error("error getting mailchimp status %s" % (e)) except Exception, e: logger.error("error unsubscribing from mailchimp list %s" % (e)) return False diff --git a/core/tasks.py b/core/tasks.py index 8a10b2f1..7095d5d6 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -17,6 +17,10 @@ from django.utils.timezone import now from notification.engine import send_all from notification import models as notification +from mailchimp3 import MailChimp +from mailchimp3.mailchimpclient import MailChimpError + + """ regluit imports """ @@ -33,6 +37,7 @@ from regluit.core.parameters import RESERVE, REWARDS, THANKS from regluit.utils.localdatetime import date_today logger = logging.getLogger(__name__) +mc_client = MailChimp(mc_api=settings.MAILCHIMP_API_KEY) @task def populate_edition(isbn): @@ -183,14 +188,17 @@ def convert_to_mobi(input_url, input_format="application/epub+zip"): def generate_mobi_ebook_for_edition(edition): return mobigen.generate_mobi_ebook_for_edition(edition) -from postmonkey import PostMonkey, MailChimpException -pm = PostMonkey(settings.MAILCHIMP_API_KEY) - @task def ml_subscribe_task(profile, **kwargs): try: if not profile.on_ml: - return pm.listSubscribe(id=settings.MAILCHIMP_NEWS_ID, email_address=profile.user.email, **kwargs) + data = {"email_address": profile.user.email, "status_if_new": "pending"} + mc_client.lists.members.create_or_update( + list_id=settings.MAILCHIMP_NEWS_ID, + subscriber_hash=profile.user.email, + data=data, + ) + return True except Exception, e: logger.error("error subscribing to mailchimp list %s" % (e)) return False diff --git a/core/tests.py b/core/tests.py index d684a9ef..2b15e5e7 100755 --- a/core/tests.py +++ b/core/tests.py @@ -1010,10 +1010,11 @@ class MailingListTests(TestCase): #mostly to check that MailChimp account is setp correctly def test_mailchimp(self): - from postmonkey import PostMonkey - pm = PostMonkey(settings.MAILCHIMP_API_KEY) + from mailchimp3 import MailChimp + mc_client = MailChimp(settings.MAILCHIMP_API_KEY) if settings.TEST_INTEGRATION: - self.assertEqual(pm.ping(), "Everything's Chimpy!") + root = mc_client.root.get() + self.assertEqual(root[u'account_id'], u'15472878790f9faa11317e085') self.user = User.objects.create_user('chimp_test', 'eric@gluejar.com', 'chimp_test') self.assertTrue(self.user.profile.on_ml) From 68ddec61c8ec3afb9501293b39abecbc18bcf4b1 Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 20 Jun 2018 11:36:15 -0400 Subject: [PATCH 39/40] fix regression in 5598b154c5705fc0f611a6fae37e315dd22ddea8 --- frontend/templates/claim.html | 2 +- frontend/views/rh_views.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/frontend/templates/claim.html b/frontend/templates/claim.html index 2bdc4ce2..9ff84225 100644 --- a/frontend/templates/claim.html +++ b/frontend/templates/claim.html @@ -44,6 +44,6 @@ {% endif %} {% else %} -Please find a work to claim. +It appears you have reached this page in the wrong context. Please see information for rights holders. {% endif %} {% endblock %} \ No newline at end of file diff --git a/frontend/views/rh_views.py b/frontend/views/rh_views.py index 02e5990f..03410506 100644 --- a/frontend/views/rh_views.py +++ b/frontend/views/rh_views.py @@ -1,5 +1,6 @@ from datetime import timedelta from decimal import Decimal as D +import logging from django.conf import settings from django.core.urlresolvers import reverse, reverse_lazy @@ -23,6 +24,8 @@ from regluit.frontend.forms import ( ) from regluit.utils.localdatetime import date_today +logger = logging.getLogger(__name__) + class RHAgree(CreateView): template_name = "rh_agree.html" form_class = RightsHolderForm @@ -74,7 +77,7 @@ class ClaimView(CreateView): return UserClaimForm(self.request.user, data=self.request.POST, prefix='claim') def form_valid(self, form): - print form.cleaned_data + logger.info(form.cleaned_data) work = form.cleaned_data['work'] rights_holder = form.cleaned_data['rights_holder'] if not rights_holder.approved: @@ -88,9 +91,10 @@ class ClaimView(CreateView): return HttpResponseRedirect(reverse('rightsholders')) def get_context_data(self, form): - if not form.is_valid(): - return {'form': form} - work = form.cleaned_data['work'] + try: + work = form.cleaned_data['work'] + except AttributeError: + return {} rights_holder = form.cleaned_data['rights_holder'] active_claims = work.claim.exclude(status = 'release') return { From e82c38923617ff2eb64cc524bfafb17eeb9c7d82 Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 20 Jun 2018 15:57:34 -0400 Subject: [PATCH 40/40] make sure to queue the notification emmisions so that we don't need the emit_notices cron job --- core/models/__init__.py | 10 ++++++++-- core/models/rh_models.py | 2 ++ core/signals.py | 5 ++--- core/tasks.py | 2 +- frontend/views/__init__.py | 2 ++ 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/core/models/__init__.py b/core/models/__init__.py index f3795655..662ddca3 100755 --- a/core/models/__init__.py +++ b/core/models/__init__.py @@ -319,14 +319,17 @@ class Acq(models.Model): self.expire_in(timedelta(days=14)) self.user.wishlist.add_work(self.work, "borrow") notification.send([self.user], "library_borrow", {'acq':self}) - return self + result = self elif self.borrowable and user: user.wishlist.add_work(self.work, "borrow") borrowed = Acq.objects.create(user=user, work=self.work, license=BORROWED, lib_acq=self) from regluit.core.tasks import watermark_acq notification.send([user], "library_borrow", {'acq':borrowed}) watermark_acq.delay(borrowed) - return borrowed + result = borrowed + from regluit.core.tasks import emit_notifications + emit_notifications.delay() + return result @property def borrowable(self): @@ -1372,6 +1375,9 @@ class Gift(models.Model): self.used = now() self.save() notification.send([self.giver], "purchase_got_gift", {'gift': self}, True) + from regluit.core.tasks import emit_notifications + emit_notifications.delay() + # this was causing a circular import problem and we do not seem to be using diff --git a/core/models/rh_models.py b/core/models/rh_models.py index 792e729b..72ca471b 100644 --- a/core/models/rh_models.py +++ b/core/models/rh_models.py @@ -109,5 +109,7 @@ def notify_rh(sender, created, instance, **kwargs): for claim in instance.claim.filter(status='pending'): claim.status = 'active' claim.save() + from regluit.core.tasks import emit_notifications + emit_notifications.delay() post_save.connect(notify_rh, sender=RightsHolder) diff --git a/core/signals.py b/core/signals.py index ea17ac4f..96f94cf1 100644 --- a/core/signals.py +++ b/core/signals.py @@ -212,9 +212,8 @@ def handle_transaction_charged(sender,transaction=None, **kwargs): from regluit.core.tasks import send_mail_task message = render_to_string("notification/purchase_complete/full.txt", context ) send_mail_task.delay('unglue.it transaction confirmation', message, 'notices@gluejar.com', [transaction.receipt]) - if transaction.user: - from regluit.core.tasks import emit_notifications - emit_notifications.delay() + from regluit.core.tasks import emit_notifications + emit_notifications.delay() transaction_charged.connect(handle_transaction_charged) diff --git a/core/tasks.py b/core/tasks.py index 7095d5d6..36dc8f2b 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -173,7 +173,7 @@ def refresh_acqs(): # notify the user with the hold if 'example.org' not in reserve_acq.user.email: - notification.send([reserve_acq.user], "library_reserve", {'acq':reserve_acq}) + notification.send_now([reserve_acq.user], "library_reserve", {'acq':reserve_acq}) # delete the hold hold.delete() break diff --git a/frontend/views/__init__.py b/frontend/views/__init__.py index 4b0b65db..84e764bc 100755 --- a/frontend/views/__init__.py +++ b/frontend/views/__init__.py @@ -2876,6 +2876,8 @@ def receive_gift(request, nonce): gift.acq.expire_in(0) gift.use() notification.send([giftee], "purchase_gift", context, True) + from regluit.core.tasks import emit_notifications + emit_notifications.delay() return render(request, 'gift_duplicate.html', context) context['form'] = RegiftForm() return render(request, 'gift_duplicate.html', context)