Merge pull request #87 from EbookFoundation/catch-up

Merge remote-tracking branch 'Gluejar/master'

core/management/commands/make_missing_mobis.py

@@ -21,7 +21,7 @@ class Command(BaseCommand):
             if not work.ebooks().filter(format="mobi"):
                 for ebook in work.ebooks().filter(format="epub"):
                     ebf = ebook.get_archive_ebf()
-                    if ebf:
+                    if ebf and ebf.mobied >= 0:
                         try:
                             print u'making mobi for {}'.format(work.title)
                             if ebf.make_mobi():

core/migrations/0013_ebookfile_mobied.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0012_campaign_charitable'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='ebookfile',
+            name='mobied',
+            field=models.IntegerField(default=0),
+        ),
+    ]

core/mobi.py

@@ -17,11 +17,11 @@ def convert_to_mobi(input_url, input_format="application/epub+zip"):
         print 'settings ok'
     # using verify=False since at the moment, using a self-signed SSL cert.
 
-    payload = requests.get(input_url, verify=False).content 
+    payload = requests.get(input_url).content 
 
     headers = {'Content-Type': input_format}
     r = requests.post(mobigen_url, auth=(mobigen_user_id, mobigen_password),
-                      data=payload, verify=False, headers=headers)
+                      data=payload, headers=headers)
 
     # if HTTP reponse code is ok, the output is the mobi file; else error message
     if r.status_code == 200:

core/models/__init__.py

@@ -967,7 +967,12 @@ class Campaign(models.Model):
                     
                     # now make the mobi file
                     new_mobi_ebf = EbookFile.objects.create(edition=edition, format='mobi', asking=True)
-                    new_mobi_ebf.file.save(path_for_file('ebf', None), ContentFile(mobi.convert_to_mobi(new_epub_ebf.file.url)))
+                    try:
+                        new_mobi_file = ContentFile(mobi.convert_to_mobi(new_epub_ebf.file.url))
+                    except Exception as e:
+                        logger.error("error making mobi  for %s" % (new_epub_ebf.file.url))
+                        raise e
+                    new_mobi_ebf.file.save(path_for_file('ebf', None), new_mobi_file)
                     new_mobi_ebf.save()
                     new_mobi_ebf.version = version
                     new_ebfs.append(new_mobi_ebf)

core/models/bibmodels.py

@@ -1056,6 +1056,7 @@ class EbookFile(models.Model):
     asking = models.BooleanField(default=False)
     ebook = models.ForeignKey('Ebook', related_name='ebook_files', null=True)
     source = models.URLField(null=True, blank=True)
+    mobied = models.IntegerField(default=0) #-1 indicates a failed conversion attempt
     version = None
     def check_file(self):
         if self.format == 'epub':
@@ -1072,9 +1073,13 @@ class EbookFile(models.Model):
     def make_mobi(self):
         if not self.format == 'epub' or not settings.MOBIGEN_URL:
             return False
+        if self.mobied < 0:
+            return False
         try:
             mobi_cf = ContentFile(mobi.convert_to_mobi(self.file.url))
         except:
+            self.mobied = -1
+            self.save()
             return False
         new_mobi_ebf = EbookFile.objects.create(
             edition=self.edition,
@@ -1096,6 +1101,8 @@ class EbookFile(models.Model):
             )
             new_mobi_ebf.ebook = new_ebook
         new_mobi_ebf.save()
+        self.mobied = 1
+        self.save()
         return True
 
 send_to_kindle_limit = 7492232

deploy/update-just

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-# this script is used by jenkins to remotely update a regluit instance
-# for it to work the jenkins user's public ssh key needs to be in the
-# authorized key for the machine running the regluit instance
-# you can then put something like this in a post build configuration
-# ssh ubuntu@please.unglueit.com "/opt/regluit/deploy/update-regluit"
-
-cd /opt/regluit
-find . -name "*.pyc" -delete
-find . -type d -empty -delete
-
-sudo -u ubuntu /usr/bin/git pull
-source ENV/bin/activate
-pip install --upgrade -r requirements_versioned.pip
-#django-admin.py syncdb --migrate --settings regluit.settings.just
-django-admin.py migrate --fake-initial --noinput --settings regluit.settings.just
-django-admin.py collectstatic --noinput --settings regluit.settings.just
-
-sudo /etc/init.d/apache2 restart
-django-admin.py celeryd_multi restart w1 --settings=regluit.settings.just;
-/etc/init.d/celerybeat restart
-crontab deploy/crontab_just.txt
-touch /opt/regluit/deploy/last-update

deploy/update-regluit

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-# this script is used by jenkins to remotely update a regluit instance
-# for it to work the jenkins user's public ssh key needs to be in the
-# authorized key for the machine running the regluit instance
-# you can then put something like this in a post build configuration
-# ssh ubuntu@please.unglueit.com "/opt/regluit/deploy/update-regluit"
-
-cd /opt/regluit
-find . -name "*.pyc" -delete
-find . -type d -empty -delete
-
-sudo -u ubuntu /usr/bin/git pull
-source ENV/bin/activate
-#pip install -r requirements.pip
-django-admin.py syncdb --migrate --settings regluit.settings.please
-django-admin.py collectstatic --noinput --settings regluit.settings.please
-sudo /etc/init.d/apache2 restart
-django-admin.py celeryd_multi restart w1 --settings=regluit.settings.please;
-/etc/init.d/celerybeat restart
-crontab deploy/crontab_please.txt
-touch /opt/regluit/deploy/last-update

settings/common.py

@@ -7,11 +7,6 @@ from os.path import dirname, realpath, join
 import regluit
 from regluit.payment.parameters import PAYMENT_HOST_PAYPAL, PAYMENT_HOST_AMAZON
 
-try:
-    from .keys.common import *
-except ImportError:
-    print 'no real key file found, using dummy'
-    from .dummy.common import *
 
 PROJECT_DIR = dirname(dirname(realpath(__file__)))
 
@@ -482,6 +477,11 @@ QUESTIONNAIRE_SHOW_ITEM_RESULTS = False
 FIREFOX_PATH = ''
 CHROMEDRIVER_PATH = ''
 
+try:
+    from .keys.common import *
+except ImportError:
+    print 'no real key file found, using dummy'
+    from .dummy.common import *
 
 try:
     from .keys.host import *
@@ -492,6 +492,6 @@ except ImportError:
     LOCAL_TEST = True
 
 if AWS_SECRET_ACCESS_KEY:
-    DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage'
+    DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
 else:
     DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'

settings/travis.py

@@ -20,9 +20,7 @@ DATABASES = {
         'PASSWORD': '',
         'HOST': '',
         'PORT': '',
-        'TEST': {
+        'TEST_CHARSET': 'utf8',
-            'CHARSET': 'utf8',
-        }
     }
 }
 

sysadmin/playbooks/hosts

@@ -2,5 +2,4 @@
 just ansible_ssh_port=22 ansible_ssh_host=just.unglue.it ansible_ssh_user=ubuntu
 web1 ansible_ssh_port=22 ansible_ssh_host=unglue.it ansible_ssh_user=ubuntu
 gluejar ansible_ssh_port=22 ansible_ssh_host=gluejar.com ansible_ssh_user=ubuntu
-jenkins ansible_ssh_port=22 ansible_ssh_host=jenkins.unglueit.com ansible_ssh_user=ubuntu
 

vagrant/Vagrantfile

@@ -114,7 +114,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       # ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/'
       ansible.raw_arguments  = [
           "--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory",
-          "--limit=just,127.0.0.1,jenkins",
+          "--limit=just,127.0.0.1",
           "-e vname=just",
           "-e class=just",
           "-e hostname=just.unglue.it",
@@ -186,7 +186,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       # ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/'
       ansible.raw_arguments  = [
           "--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory",
-          "--limit=just2,127.0.0.1,jenkins",
+          "--limit=just2,127.0.0.1",
           "-e vname=just2",
           "-e class=just",
           "-e hostname=just2.unglue.it",

vagrant/dev.yml

@@ -499,13 +499,6 @@
         - https://github.com/eshellman.keys
       sudo: no
       
-    - name: add public key from jenkins
-      authorized_key: >
-        user={{user}}
-        key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r"
-        state=present
-      when: class in ['just']
-
     - name: set up script file to load environment for interactive use
       command: cp "/opt/regluit/deploy/setup-{{class}}.sh" /home/{{user}}/setup.sh
       sudo: no
@@ -549,29 +542,4 @@
       service: name=apache2 state=restarted
 
 
-- name: fix known_hosts on jenkins to match new just
-  hosts: jenkins
-  sudo: yes
-  sudo_user: jenkins
 
-# to run the part of the playbook for jenkins
-# PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=true ANSIBLE_HOST_KEY_CHECKING=false ANSIBLE_SSH_ARGS='-o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s' ansible-playbook --private-key=/Users/raymondyee/.ssh/id_rsa  --user=ubuntu --connection=ssh --inventory-file=/Users/raymondyee/C/src/Gluejar/regluit/vagrant/.vagrant/provisioners/ansible/inventory --limit='jenkins'  just.yml
-
-  tasks:
-
-    #equivalent to
-    #
-    #ssh -tt jenkins << EOF
-    #  sudo -i -u jenkins
-    #  ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts
-    # exit
-    #exit
-    #EOF
-
-    - name: make new known_hosts with key from just.unglue.it
-      raw: ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts
-      when: class in ['just']
-
-    - name: add key from github
-      raw: ssh-keyscan -t rsa github.com >> /var/lib/jenkins/.ssh/known_hosts
-      when: class in ['just']

vagrant/just.yml

@@ -384,41 +384,12 @@
         - https://github.com/eshellman.keys
       sudo: yes     
       
-    - name: add public key from jenkins
+
-      authorized_key: >
-        user={{user}}
-        key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r"
-        state=present
-      sudo: no
       
 
   handlers:
     - name: restart apache2
       service: name=apache2 state=restarted    
           
-- name: fix known_hosts on jenkins to match new just
-  hosts: jenkins
-  sudo: yes
-  sudo_user: jenkins
-
-# to run the part of the playbook for jenkins
-# PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=true ANSIBLE_HOST_KEY_CHECKING=false ANSIBLE_SSH_ARGS='-o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s' ansible-playbook --private-key=/Users/raymondyee/.ssh/id_rsa  --user=ubuntu --connection=ssh --inventory-file=/Users/raymondyee/C/src/Gluejar/regluit/vagrant/.vagrant/provisioners/ansible/inventory --limit='jenkins'  just.yml
-
-  tasks:
-
-    #equivalent to
-    #
-    #ssh -tt jenkins << EOF
-    #  sudo -i -u jenkins
-    #  ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts
-    # exit
-    #exit
-    #EOF
-
-    - name: make new known_hosts with key from just.unglue.it
-      raw: ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts
-
-    - name: add key from github
-      raw: ssh-keyscan -t rsa github.com >> /var/lib/jenkins/.ssh/known_hosts
             
        

vagrant/please.yml

@@ -412,12 +412,6 @@
         - https://github.com/eshellman.keys
       sudo: yes     
       
-    - name: add public key from jenkins
-      authorized_key: >
-        user={{user}}
-        key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r"
-        state=present
-
 
   handlers:
     - name: restart apache2