EbookFoundation
/
regluit
mirror of https://github.com/EbookFoundation/regluit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'Gluejar/master'

pull/87/head
eric 2018-06-09 15:54:14 -04:00
parent d7751af8f9
commit 88a50073de
14 changed files with 45 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/management/commands/make_missing_mobis.py
View File

@ -21,7 +21,7 @@ class Command(BaseCommand):
if not work.ebooks().filter(format="mobi"):
for ebook in work.ebooks().filter(format="epub"):
ebf = ebook.get_archive_ebf()
if ebf:
if ebf and ebf.mobied >= 0:
try:
print u'making mobi for {}'.format(work.title)
if ebf.make_mobi():

19
core/migrations/0013_ebookfile_mobied.py Normal file
View File

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0012_campaign_charitable'),
]
operations = [
migrations.AddField(
model_name='ebookfile',
name='mobied',
field=models.IntegerField(default=0),
),
]

4
core/mobi.py
View File

@ -17,11 +17,11 @@ def convert_to_mobi(input_url, input_format="application/epub+zip"):
print 'settings ok'
# using verify=False since at the moment, using a self-signed SSL cert.
payload = requests.get(input_url, verify=False).content
payload = requests.get(input_url).content
headers = {'Content-Type': input_format}
r = requests.post(mobigen_url, auth=(mobigen_user_id, mobigen_password),
data=payload, verify=False, headers=headers)
data=payload, headers=headers)
# if HTTP reponse code is ok, the output is the mobi file; else error message
if r.status_code == 200:

7
core/models/__init__.py
View File

@ -967,7 +967,12 @@ class Campaign(models.Model):
# now make the mobi file
new_mobi_ebf = EbookFile.objects.create(edition=edition, format='mobi', asking=True)
new_mobi_ebf.file.save(path_for_file('ebf', None), ContentFile(mobi.convert_to_mobi(new_epub_ebf.file.url)))
try:
new_mobi_file = ContentFile(mobi.convert_to_mobi(new_epub_ebf.file.url))
except Exception as e:
logger.error("error making mobi for %s" % (new_epub_ebf.file.url))
raise e
new_mobi_ebf.file.save(path_for_file('ebf', None), new_mobi_file)
new_mobi_ebf.save()
new_mobi_ebf.version = version
new_ebfs.append(new_mobi_ebf)

7
core/models/bibmodels.py
View File

@ -1056,6 +1056,7 @@ class EbookFile(models.Model):
asking = models.BooleanField(default=False)
ebook = models.ForeignKey('Ebook', related_name='ebook_files', null=True)
source = models.URLField(null=True, blank=True)
mobied = models.IntegerField(default=0) #-1 indicates a failed conversion attempt
version = None
def check_file(self):
if self.format == 'epub':
@ -1072,9 +1073,13 @@ class EbookFile(models.Model):
def make_mobi(self):
if not self.format == 'epub' or not settings.MOBIGEN_URL:
return False
if self.mobied < 0:
return False
try:
mobi_cf = ContentFile(mobi.convert_to_mobi(self.file.url))
except:
self.mobied = -1
self.save()
return False
new_mobi_ebf = EbookFile.objects.create(
edition=self.edition,
@ -1096,6 +1101,8 @@ class EbookFile(models.Model):
)
new_mobi_ebf.ebook = new_ebook
new_mobi_ebf.save()
self.mobied = 1
self.save()
return True
send_to_kindle_limit = 7492232

24
deploy/update-just
View File

@ -1,24 +0,0 @@
#!/bin/bash
# this script is used by jenkins to remotely update a regluit instance
# for it to work the jenkins user's public ssh key needs to be in the
# authorized key for the machine running the regluit instance
# you can then put something like this in a post build configuration
# ssh ubuntu@please.unglueit.com "/opt/regluit/deploy/update-regluit"
cd /opt/regluit
find . -name "*.pyc" -delete
find . -type d -empty -delete
sudo -u ubuntu /usr/bin/git pull
source ENV/bin/activate
pip install --upgrade -r requirements_versioned.pip
#django-admin.py syncdb --migrate --settings regluit.settings.just
django-admin.py migrate --fake-initial --noinput --settings regluit.settings.just
django-admin.py collectstatic --noinput --settings regluit.settings.just
sudo /etc/init.d/apache2 restart
django-admin.py celeryd_multi restart w1 --settings=regluit.settings.just;
/etc/init.d/celerybeat restart
crontab deploy/crontab_just.txt
touch /opt/regluit/deploy/last-update

22
deploy/update-regluit
View File

@ -1,22 +0,0 @@
#!/bin/bash
# this script is used by jenkins to remotely update a regluit instance
# for it to work the jenkins user's public ssh key needs to be in the
# authorized key for the machine running the regluit instance
# you can then put something like this in a post build configuration
# ssh ubuntu@please.unglueit.com "/opt/regluit/deploy/update-regluit"
cd /opt/regluit
find . -name "*.pyc" -delete
find . -type d -empty -delete
sudo -u ubuntu /usr/bin/git pull
source ENV/bin/activate
#pip install -r requirements.pip
django-admin.py syncdb --migrate --settings regluit.settings.please
django-admin.py collectstatic --noinput --settings regluit.settings.please
sudo /etc/init.d/apache2 restart
django-admin.py celeryd_multi restart w1 --settings=regluit.settings.please;
/etc/init.d/celerybeat restart
crontab deploy/crontab_please.txt
touch /opt/regluit/deploy/last-update

12
settings/common.py
View File

@ -7,11 +7,6 @@ from os.path import dirname, realpath, join
import regluit
from regluit.payment.parameters import PAYMENT_HOST_PAYPAL, PAYMENT_HOST_AMAZON
try:
from .keys.common import *
except ImportError:
print 'no real key file found, using dummy'
from .dummy.common import *
PROJECT_DIR = dirname(dirname(realpath(__file__)))
@ -482,6 +477,11 @@ QUESTIONNAIRE_SHOW_ITEM_RESULTS = False
FIREFOX_PATH = ''
CHROMEDRIVER_PATH = ''
try:
from .keys.common import *
except ImportError:
print 'no real key file found, using dummy'
from .dummy.common import *
try:
from .keys.host import *
@ -492,6 +492,6 @@ except ImportError:
LOCAL_TEST = True
if AWS_SECRET_ACCESS_KEY:
DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage'
DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
else:
DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'

4
settings/travis.py
View File

@ -20,9 +20,7 @@ DATABASES = {
'PASSWORD': '',
'HOST': '',
'PORT': '',
'TEST': {
'CHARSET': 'utf8',
}
'TEST_CHARSET': 'utf8',
}
}

1
sysadmin/playbooks/hosts
View File

@ -2,5 +2,4 @@
just ansible_ssh_port=22 ansible_ssh_host=just.unglue.it ansible_ssh_user=ubuntu
web1 ansible_ssh_port=22 ansible_ssh_host=unglue.it ansible_ssh_user=ubuntu
gluejar ansible_ssh_port=22 ansible_ssh_host=gluejar.com ansible_ssh_user=ubuntu
jenkins ansible_ssh_port=22 ansible_ssh_host=jenkins.unglueit.com ansible_ssh_user=ubuntu

4
vagrant/Vagrantfile vendored
View File

@ -114,7 +114,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
# ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/'
ansible.raw_arguments = [
"--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory",
"--limit=just,127.0.0.1,jenkins",
"--limit=just,127.0.0.1",
"-e vname=just",
"-e class=just",
"-e hostname=just.unglue.it",
@ -186,7 +186,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
# ansible.inventory_path = '.vagrant/provisioners/ansible/inventory/'
ansible.raw_arguments = [
"--inventory-file=.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory",
"--limit=just2,127.0.0.1,jenkins",
"--limit=just2,127.0.0.1",
"-e vname=just2",
"-e class=just",
"-e hostname=just2.unglue.it",

32
vagrant/dev.yml
View File

@ -499,13 +499,6 @@
- https://github.com/eshellman.keys
sudo: no
- name: add public key from jenkins
authorized_key: >
user={{user}}
key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r"
state=present
when: class in ['just']
- name: set up script file to load environment for interactive use
command: cp "/opt/regluit/deploy/setup-{{class}}.sh" /home/{{user}}/setup.sh
sudo: no
@ -549,29 +542,4 @@
service: name=apache2 state=restarted
- name: fix known_hosts on jenkins to match new just
hosts: jenkins
sudo: yes
sudo_user: jenkins
# to run the part of the playbook for jenkins
# PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=true ANSIBLE_HOST_KEY_CHECKING=false ANSIBLE_SSH_ARGS='-o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s' ansible-playbook --private-key=/Users/raymondyee/.ssh/id_rsa --user=ubuntu --connection=ssh --inventory-file=/Users/raymondyee/C/src/Gluejar/regluit/vagrant/.vagrant/provisioners/ansible/inventory --limit='jenkins' just.yml
tasks:
#equivalent to
#
#ssh -tt jenkins << EOF
# sudo -i -u jenkins
# ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts
# exit
#exit
#EOF
- name: make new known_hosts with key from just.unglue.it
raw: ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts
when: class in ['just']
- name: add key from github
raw: ssh-keyscan -t rsa github.com >> /var/lib/jenkins/.ssh/known_hosts
when: class in ['just']

31
vagrant/just.yml
View File

@ -384,41 +384,12 @@
- https://github.com/eshellman.keys
sudo: yes
- name: add public key from jenkins
authorized_key: >
user={{user}}
key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r"
state=present
sudo: no
handlers:
- name: restart apache2
service: name=apache2 state=restarted
- name: fix known_hosts on jenkins to match new just
hosts: jenkins
sudo: yes
sudo_user: jenkins
# to run the part of the playbook for jenkins
# PYTHONUNBUFFERED=1 ANSIBLE_FORCE_COLOR=true ANSIBLE_HOST_KEY_CHECKING=false ANSIBLE_SSH_ARGS='-o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s' ansible-playbook --private-key=/Users/raymondyee/.ssh/id_rsa --user=ubuntu --connection=ssh --inventory-file=/Users/raymondyee/C/src/Gluejar/regluit/vagrant/.vagrant/provisioners/ansible/inventory --limit='jenkins' just.yml
tasks:
#equivalent to
#
#ssh -tt jenkins << EOF
# sudo -i -u jenkins
# ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts
# exit
#exit
#EOF
- name: make new known_hosts with key from just.unglue.it
raw: ssh-keyscan -t rsa just.unglue.it > /var/lib/jenkins/.ssh/known_hosts
- name: add key from github
raw: ssh-keyscan -t rsa github.com >> /var/lib/jenkins/.ssh/known_hosts

6
vagrant/please.yml
View File

@ -412,12 +412,6 @@
- https://github.com/eshellman.keys
sudo: yes
- name: add public key from jenkins
authorized_key: >
user={{user}}
key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYSiXESHXEdugNLGxFABXpVSawDCU/BK05Ef2qUa7oxxhU7fXNqWaSTqowevVruF7kfzMQ7epIxN5XFFjbXf/tsSn1995H9BEhmHLXLuEB5VaPU2HTLqu0DscyPtRbk/WjqPj3jWXs2yHgKcJIXwd5EfSwJuCe1Ut6pMe9E/NUq9QztnydRTt0sGywXpkIpKeBkiQl4SWlPTHcoU6PDbEuMVii8GzRAQlpEQTJwzWJTToR1SZ7o1uusDSxIDfJSvAa5IiuII8CdKbqa/JSx1+4LqlT0yf+2yb67MR5q6+XFM4TeCf5z+4SW+IT/wd2tpbd0DjAdXJlAgBULwhd1L7r"
state=present
handlers:
- name: restart apache2