From 88a50073de19a62bb50a23a8d21aa51c97e04304 Mon Sep 17 00:00:00 2001 From: eric Date: Sat, 9 Jun 2018 15:54:14 -0400 Subject: [PATCH] Merge remote-tracking branch 'Gluejar/master' --- .../management/commands/make_missing_mobis.py | 2 +- core/migrations/0013_ebookfile_mobied.py | 19 +++++++++++ core/mobi.py | 4 +-- core/models/__init__.py | 7 +++- core/models/bibmodels.py | 7 ++++ deploy/update-just | 24 -------------- deploy/update-regluit | 22 ------------- settings/common.py | 12 +++---- settings/travis.py | 4 +-- sysadmin/playbooks/hosts | 1 - vagrant/Vagrantfile | 4 +-- vagrant/dev.yml | 32 ------------------- vagrant/just.yml | 31 +----------------- vagrant/please.yml | 6 ---- 14 files changed, 45 insertions(+), 130 deletions(-) create mode 100644 core/migrations/0013_ebookfile_mobied.py delete mode 100755 deploy/update-just delete mode 100755 deploy/update-regluit diff --git a/core/management/commands/make_missing_mobis.py b/core/management/commands/make_missing_mobis.py index ee06fe3a..af2d5461 100644 --- a/core/management/commands/make_missing_mobis.py +++ b/core/management/commands/make_missing_mobis.py @@ -21,7 +21,7 @@ class Command(BaseCommand): if not work.ebooks().filter(format="mobi"): for ebook in work.ebooks().filter(format="epub"): ebf = ebook.get_archive_ebf() - if ebf: + if ebf and ebf.mobied >= 0: try: print u'making mobi for {}'.format(work.title) if ebf.make_mobi(): diff --git a/core/migrations/0013_ebookfile_mobied.py b/core/migrations/0013_ebookfile_mobied.py new file mode 100644 index 00000000..18775c0d --- /dev/null +++ b/core/migrations/0013_ebookfile_mobied.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0012_campaign_charitable'), + ] + + operations = [ + migrations.AddField( + model_name='ebookfile', + name='mobied', + field=models.IntegerField(default=0), + ), + ] diff --git a/core/mobi.py b/core/mobi.py index 53f696e3..b26d43e6 100644 --- a/core/mobi.py +++ b/core/mobi.py @@ -17,11 +17,11 @@ def convert_to_mobi(input_url, input_format="application/epub+zip"): print 'settings ok' # using verify=False since at the moment, using a self-signed SSL cert. - payload = requests.get(input_url, verify=False).content + payload = requests.get(input_url).content headers = {'Content-Type': input_format} r = requests.post(mobigen_url, auth=(mobigen_user_id, mobigen_password), - data=payload, verify=False, headers=headers) + data=payload, headers=headers) # if HTTP reponse code is ok, the output is the mobi file; else error message if r.status_code == 200: diff --git a/core/models/__init__.py b/core/models/__init__.py index 6f331828..420405d3 100755 --- a/core/models/__init__.py +++ b/core/models/__init__.py @@ -967,7 +967,12 @@ class Campaign(models.Model): # now make the mobi file new_mobi_ebf = EbookFile.objects.create(edition=edition, format='mobi', asking=True) - new_mobi_ebf.file.save(path_for_file('ebf', None), ContentFile(mobi.convert_to_mobi(new_epub_ebf.file.url))) + try: + new_mobi_file = ContentFile(mobi.convert_to_mobi(new_epub_ebf.file.url)) + except Exception as e: + logger.error("error making mobi for %s" % (new_epub_ebf.file.url)) + raise e + new_mobi_ebf.file.save(path_for_file('ebf', None), new_mobi_file) new_mobi_ebf.save() new_mobi_ebf.version = version new_ebfs.append(new_mobi_ebf) diff --git a/core/models/bibmodels.py b/core/models/bibmodels.py index 5a53f5e5..d8e16d64 100644 --- a/core/models/bibmodels.py +++ b/core/models/bibmodels.py @@ -1056,6 +1056,7 @@ class EbookFile(models.Model): asking = models.BooleanField(default=False) ebook = models.ForeignKey('Ebook', related_name='ebook_files', null=True) source = models.URLField(null=True, blank=True) + mobied = models.IntegerField(default=0) #-1 indicates a failed conversion attempt version = None def check_file(self): if self.format == 'epub': @@ -1072,9 +1073,13 @@ class EbookFile(models.Model): def make_mobi(self): if not self.format == 'epub' or not settings.MOBIGEN_URL: return False + if self.mobied < 0: + return False try: mobi_cf = ContentFile(mobi.convert_to_mobi(self.file.url)) except: + self.mobied = -1 + self.save() return False new_mobi_ebf = EbookFile.objects.create( edition=self.edition, @@ -1096,6 +1101,8 @@ class EbookFile(models.Model): ) new_mobi_ebf.ebook = new_ebook new_mobi_ebf.save() + self.mobied = 1 + self.save() return True send_to_kindle_limit = 7492232 diff --git a/deploy/update-just b/deploy/update-just deleted file mode 100755 index bc97a438..00000000 --- a/deploy/update-just +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# this script is used by jenkins to remotely update a regluit instance -# for it to work the jenkins user's public ssh key needs to be in the -# authorized key for the machine running the regluit instance -# you can then put something like this in a post build configuration -# ssh ubuntu@please.unglueit.com "/opt/regluit/deploy/update-regluit" - -cd /opt/regluit -find . -name "*.pyc" -delete -find . -type d -empty -delete - -sudo -u ubuntu /usr/bin/git pull -source ENV/bin/activate -pip install --upgrade -r requirements_versioned.pip -#django-admin.py syncdb --migrate --settings regluit.settings.just -django-admin.py migrate --fake-initial --noinput --settings regluit.settings.just -django-admin.py collectstatic --noinput --settings regluit.settings.just - -sudo /etc/init.d/apache2 restart -django-admin.py celeryd_multi restart w1 --settings=regluit.settings.just; -/etc/init.d/celerybeat restart -crontab deploy/crontab_just.txt -touch /opt/regluit/deploy/last-update diff --git a/deploy/update-regluit b/deploy/update-regluit deleted file mode 100755 index 6f12c8a7..00000000 --- a/deploy/update-regluit +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# this script is used by jenkins to remotely update a regluit instance -# for it to work the jenkins user's public ssh key needs to be in the -# authorized key for the machine running the regluit instance -# you can then put something like this in a post build configuration -# ssh ubuntu@please.unglueit.com "/opt/regluit/deploy/update-regluit" - -cd /opt/regluit -find . -name "*.pyc" -delete -find . -type d -empty -delete - -sudo -u ubuntu /usr/bin/git pull -source ENV/bin/activate -#pip install -r requirements.pip -django-admin.py syncdb --migrate --settings regluit.settings.please -django-admin.py collectstatic --noinput --settings regluit.settings.please -sudo /etc/init.d/apache2 restart -django-admin.py celeryd_multi restart w1 --settings=regluit.settings.please; -/etc/init.d/celerybeat restart -crontab deploy/crontab_please.txt -touch /opt/regluit/deploy/last-update \ No newline at end of file diff --git a/settings/common.py b/settings/common.py index 1b918e6a..5d746c37 100644 --- a/settings/common.py +++ b/settings/common.py @@ -7,11 +7,6 @@ from os.path import dirname, realpath, join import regluit from regluit.payment.parameters import PAYMENT_HOST_PAYPAL, PAYMENT_HOST_AMAZON -try: - from .keys.common import * -except ImportError: - print 'no real key file found, using dummy' - from .dummy.common import * PROJECT_DIR = dirname(dirname(realpath(__file__))) @@ -482,6 +477,11 @@ QUESTIONNAIRE_SHOW_ITEM_RESULTS = False FIREFOX_PATH = '' CHROMEDRIVER_PATH = '' +try: + from .keys.common import * +except ImportError: + print 'no real key file found, using dummy' + from .dummy.common import * try: from .keys.host import * @@ -492,6 +492,6 @@ except ImportError: LOCAL_TEST = True if AWS_SECRET_ACCESS_KEY: - DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' else: DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' diff --git a/settings/travis.py b/settings/travis.py index 00b28f4f..477b4b41 100644 --- a/settings/travis.py +++ b/settings/travis.py @@ -20,9 +20,7 @@ DATABASES = { 'PASSWORD': '', 'HOST': '', 'PORT': '', - 'TEST': { - 'CHARSET': 'utf8', - } + 'TEST_CHARSET': 'utf8', } } diff --git a/sysadmin/playbooks/hosts b/sysadmin/playbooks/hosts index 5140f856..10a3fc6b 100644 --- a/sysadmin/playbooks/hosts +++ b/sysadmin/playbooks/hosts @@ -2,5 +2,4 @@ just ansible_ssh_port=22 ansible_ssh_host=just.unglue.it ansible_ssh_user=ubuntu web1 ansible_ssh_port=22 ansible_ssh_host=unglue.it ansible_ssh_user=ubuntu gluejar ansible_ssh_port=22 ansible_ssh_host=gluejar.com ansible_ssh_user=ubuntu -jenkins ansible_ssh_port=22 ansible_ssh_host=jenkins.unglueit.com ansible_ssh_user=ubuntu diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index eebb9792..0c14e23b 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -114,7 +114,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/' ansible.raw_arguments = [ "--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory", - "--limit=just,127.0.0.1,jenkins", + "--limit=just,127.0.0.1", "-e vname=just", "-e class=just", "-e hostname=just.unglue.it", @@ -186,7 +186,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/' ansible.raw_arguments = [ "--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory", - "--limit=just2,127.0.0.1,jenkins", + "--limit=just2,127.0.0.1", "-e vname=just2", "-e class=just", "-e hostname=just2.unglue.it", diff --git a/vagrant/dev.yml b/vagrant/dev.yml index 65b6ee7f..7cc3d207 100644 --- a/vagrant/dev.yml +++ b/vagrant/dev.yml @@ -499,13 +499,6 @@ - https://github.com/eshellman.keys sudo: no - - name: add public key from jenkins - authorized_key: > - user={{user}} - key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r" - state=present - when: class in ['just'] - - name: set up script file to load environment for interactive use command: cp "/opt/regluit/deploy/setup-{{class}}.sh" /home/{{user}}/setup.sh sudo: no @@ -549,29 +542,4 @@ service: name=apache2 state=restarted -- name: fix known_hosts on jenkins to match new just - hosts: jenkins - sudo: yes - sudo_user: jenkins -# to run the part of the playbook for jenkins -# PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=true ANSIBLE_HOST_KEY_CHECKING=false ANSIBLE_SSH_ARGS='-o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s' ansible-playbook --private-key=/Users/raymondyee/.ssh/id_rsa --user=ubuntu --connection=ssh --inventory-file=/Users/raymondyee/C/src/Gluejar/regluit/vagrant/.vagrant/provisioners/ansible/inventory --limit='jenkins' just.yml - - tasks: - - #equivalent to - # - #ssh -tt jenkins << EOF - # sudo -i -u jenkins - # ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts - # exit - #exit - #EOF - - - name: make new known_hosts with key from just.unglue.it - raw: ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts - when: class in ['just'] - - - name: add key from github - raw: ssh-keyscan -t rsa github.com >> /var/lib/jenkins/.ssh/known_hosts - when: class in ['just'] diff --git a/vagrant/just.yml b/vagrant/just.yml index f657789b..a158e5a6 100644 --- a/vagrant/just.yml +++ b/vagrant/just.yml @@ -384,41 +384,12 @@ - https://github.com/eshellman.keys sudo: yes - - name: add public key from jenkins - authorized_key: > - user={{user}} - key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r" - state=present - sudo: no + handlers: - name: restart apache2 service: name=apache2 state=restarted -- name: fix known_hosts on jenkins to match new just - hosts: jenkins - sudo: yes - sudo_user: jenkins - -# to run the part of the playbook for jenkins -# PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=true ANSIBLE_HOST_KEY_CHECKING=false ANSIBLE_SSH_ARGS='-o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s' ansible-playbook --private-key=/Users/raymondyee/.ssh/id_rsa --user=ubuntu --connection=ssh --inventory-file=/Users/raymondyee/C/src/Gluejar/regluit/vagrant/.vagrant/provisioners/ansible/inventory --limit='jenkins' just.yml - - tasks: - - #equivalent to - # - #ssh -tt jenkins << EOF - # sudo -i -u jenkins - # ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts - # exit - #exit - #EOF - - - name: make new known_hosts with key from just.unglue.it - raw: ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts - - - name: add key from github - raw: ssh-keyscan -t rsa github.com >> /var/lib/jenkins/.ssh/known_hosts diff --git a/vagrant/please.yml b/vagrant/please.yml index 6f0c0e36..35443e3e 100644 --- a/vagrant/please.yml +++ b/vagrant/please.yml @@ -412,12 +412,6 @@ - https://github.com/eshellman.keys sudo: yes - - name: add public key from jenkins - authorized_key: > - user={{user}} - key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r" - state=present - handlers: - name: restart apache2