EbookFoundation
/
readthedocs.org
mirror of https://github.com/EbookFoundation/readthedocs.org.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove unmaintained salt deploy code as well

privacy-backends
Eric Holscher 2014-08-21 17:26:51 -07:00
parent a13b79f62f
commit d119a4a80e
21 changed files with 0 additions and 842 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
deploy/scripts/nginx-smoke-test.py → deploy/nginx-smoke-test.py
View File

16
deploy/salt/memcached/init.sls
View File

@ -1,16 +0,0 @@
# Memcached config
memcached:
pkg:
- installed
service.running:
- enable: True
- require:
- pkg: memcached
- watch:
- file: /etc/memcached.conf
/etc/memcached.conf:
file.managed:
- source: salt://memcached/memcached.conf
- mode: 0640

24
deploy/salt/memcached/memcached.conf
View File

@ -1,24 +0,0 @@
# Run memcached as a daemon. This command is implied, and is not needed for the
# daemon to run. See the README.Debian that comes with this package for more
# information.
-d
# Log memcached's output to /var/log/memcached
logfile /var/log/memcached.log
# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default
# Note that the daemon will grow to this size, but does not start out holding this much
# memory
-m 64
# Default connection port is 11211
-p 11211
# Run the daemon as root. The start-memcached will default to running as root if no
# -u command is present in this config file
-u nobody
# Specify which IP address to listen on. The default is to listen on all IP addresses
# This parameter is one of the only security measures that memcached has, so make sure
# it's listening on a firewalled interface.
-l 0.0.0.0

2
deploy/salt/minion
View File

@ -1,2 +0,0 @@
id: vagrant
file_client: local

23
deploy/salt/nginx/init.sls
View File

@ -1,23 +0,0 @@
# Nginx
nginx:
pkg.installed:
- name: nginx-extras
service.running:
- enable: True
- require:
- pkg: nginx
/etc/nginx/nginx.conf:
file.managed:
- source: salt://nginx/nginx.conf
- mode: 0640
/etc/nginx/sites-enabled/default:
file:
- absent
/usr/share/nginx/perl:
file.directory:
- require:
- pkg: nginx

24
deploy/salt/nginx/nginx.conf
View File

@ -1,24 +0,0 @@
user www-data;
worker_processes 1;
error_log /var/log/nginx/error.log;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
log_format host '$host $remote_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
include /etc/nginx/mime.types;
access_log /var/log/nginx/access.log;
sendfile on;
keepalive_timeout 65;
tcp_nodelay on;
gzip on;
gzip_disable "MSIE [1-6]\.(?!.*SV1)";
gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript;
include /etc/nginx/sites-enabled/*;
}

101
deploy/salt/nginx/perl/lib/ReadTheDocs.pm
View File

@ -1,101 +0,0 @@
package ReadTheDocs;
use strict;
use warnings;
use JSON qw//;
use I18N::AcceptLanguage;
=head2 redirect_home
Redirect project subdomain home to correct build path, bypassing Django. This
requires the project have a C<metadata.json> that is updated when a project is
built or settings are saved. The metadata can include the following keys:
=over
=item version
Default project version, will default to C<latest>
=item language
Default project language, will default to C<en>
=item languages
Languages from linked translation projects. The default language will be added
to this list as the primary language. A lookup on the C<Accept-Language> header
will redirect to the correct language page, as long as there is a translation
for the project.
=back
Inside the Nginx config, the variable C<rtd_metadata> should be set to the path
of the C<metadata.json> path:
location ~ /$ {
set $rtd_metadata /home/docs/sites/readthedocs.org/checkouts/readthedocs.org/user_builds/$domain/metadata.json;
perl ReadTheDocs::redirect_home;
}
=cut
sub redirect_home {
my $req = shift;
my $version = 'latest';
my $lang = 'en';
my $metadata_file = $req->variable('rtd_metadata');
if (defined $metadata_file) {
my $metadata = project_metadata($metadata_file);
# Version
$version = $metadata->{version} || 'latest';
# Language, add default as the primary language
$lang = $metadata->{language} || 'en';
my $languages = $metadata->{languages};
unshift(@{$languages}, $lang);
my $header = $req->header_in('Accept-Language');
my $accept = I18N::AcceptLanguage->new();
$accept->defaultLanguage($lang);
$lang = $accept->accepts($header, $languages);
}
# Return redirect, no body
$req->header_out('Location', sprintf('/%s/%s/', $lang, $version));
$req->header_out('X-Perl-Redirect', 'True');
return 302;
}
=head2 project_metadata($project)
Return parsed metadata from JSON metadata file
=cut
sub project_metadata {
my $filename = shift;
my $file = project_metadata_read($filename);
my $metadata = {};
eval {
$metadata = JSON->new->utf8->decode($file);
};
return $metadata;
}
sub project_metadata_read {
my $filename = shift;
my $file = "";
if (-e $filename) {
open(my $fh, '<', $filename);
$file = <$fh>;
}
return $file;
}
1;

127
deploy/salt/nginx/perl/t/10-redirect.t
View File

@ -1,127 +0,0 @@
use Test::More;
use ReadTheDocs;
plan tests => 46;
no strict 'refs';
no warnings 'redefine';
our $Metadata_Fixture = {
'/tmp/foobar' => {
name => 'foobar',
version => '1.1',
languages => ['es', 'en', 'fr']
},
'/tmp/test' => {
name => 'test',
version => '2.2',
language => 'es',
languages => ['fr', 'ru']
},
'/tmp/lang' => {
name => 'lang',
version => '3.3',
language => 'ru',
languages => ['en']
}
};
{
package ReadTheDocs::Request;
sub new {
my $class = shift;
my %args = @_;
bless {
%args,
output => []
}, $class;
}
sub variable {
my $self = shift;
my $name = shift;
return $self->{variable}->{$name};
}
sub header_in {
my $self = shift;
my $name = shift;
return $self->{header}->{$name};
}
sub header_out {
my ($self, $name, $value) = @_;
push(@{$self->{output}}, sprintf("%s: %s", $name, $value));
}
sub send_http_header {
my $self = shift;
return;
}
sub print {
my $self = shift;
push(@{$self->{output}}, shift);
}
}
{
local *ReadTheDocs::project_metadata = sub {
my $file = shift;
return $Metadata_Fixture->{$file};
};
# Test metadata
my $metadata = ReadTheDocs::project_metadata('/tmp/foobar');
is($metadata->{name}, 'foobar');
is($metadata->{version}, '1.1');
# Mock redirection for project 'foobar'
my $test_cb = sub {
my ($lang, $metadata, $url) = @_;
my $req = ReadTheDocs::Request->new(
variable => {rtd_metadata => $metadata},
header => {'Accept-Language' => $lang}
);
is(ReadTheDocs::redirect_home($req), 302);
is(
pop(@{$req->{output}}),
sprintf('Location: %s', $url),
sprintf('Testing %s -> %s', $metadata, $url)
);
};
# Languages: ['es', 'en', 'fr']
$test_cb->('es,en', '/tmp/foobar', '/es/1.1/');
$test_cb->('du', '/tmp/foobar', '/en/1.1/');
$test_cb->('en;q=0.8, es', '/tmp/foobar', '/es/1.1/');
$test_cb->('ru', '/tmp/foobar', '/en/1.1/');
$test_cb->('en-gb;q=0.8, en;q=0.7, da', '/tmp/foobar', '/en/1.1/');
# Languages: ['es', 'fr', 'ru']
$test_cb->('es,en', '/tmp/test', '/es/2.2/');
$test_cb->('du', '/tmp/test', '/es/2.2/');
$test_cb->('en;q=0.8, es', '/tmp/test', '/es/2.2/');
$test_cb->('ru', '/tmp/test', '/ru/2.2/');
$test_cb->('en-gb;q=0.8, en;q=0.7, da', '/tmp/test', '/es/2.2/');
# Languages: ['ru', 'en']
$test_cb->('es,en', '/tmp/lang', '/en/3.3/');
$test_cb->('du', '/tmp/lang', '/ru/3.3/');
$test_cb->('en;q=0.8, es', '/tmp/lang', '/en/3.3/');
$test_cb->('ru', '/tmp/lang', '/ru/3.3/');
$test_cb->('ru', '/tmp/lang', '/ru/3.3/');
$test_cb->('en-gb;q=0.8, en;q=0.7, da', '/tmp/lang', '/en/3.3/');
# Languages: ['en']
$test_cb->('es,en', '/tmp/nonexistant', '/en/latest/');
$test_cb->('du', '/tmp/nonexistant', '/en/latest/');
$test_cb->('en;q=0.8, es', '/tmp/nonexistant', '/en/latest/');
$test_cb->('ru', '/tmp/nonexistant', '/en/latest/');
$test_cb->('ru', '/tmp/nonexistant', '/en/latest/');
$test_cb->('en-gb;q=0.8, en;q=0.7, da', '/tmp/nonexistant', '/en/latest/');
}
1;

43
deploy/salt/nginx/perl/t/10-settings.t
View File

@ -1,43 +0,0 @@
use Test::More;
use ReadTheDocs;
plan tests => 6;
no strict 'refs';
no warnings 'redefine';
our $Metadata_JSON = {};
$Metadata_JSON->{'/tmp/foo'} = qq/
{"name": "foo-name", "version": "foo-version"}
/;
$Metadata_JSON->{'/tmp/bar'} = qq/
{"name": "bar-name", "nothing": "something"}
/;
$Metadata_JSON->{'/tmp/fail'} = qq/
{"name": "fail", non of this matters}
/;
{
local *ReadTheDocs::project_metadata_read = sub {
my $file = shift;
return $Metadata_JSON->{$file};
};
my $metadata = ReadTheDocs::project_metadata('/tmp/foo');
is($metadata->{name}, 'foo-name');
is($metadata->{version}, 'foo-version');
undef $metadata;
my $metadata = ReadTheDocs::project_metadata('/tmp/bar');
is($metadata->{name}, 'bar-name');
is($metadata->{version}, undef);
undef $metadata;
my $metadata = ReadTheDocs::project_metadata('/tmp/test');
is($metadata->{name}, undef);
is($metadata->{version}, undef);
undef $metadata;
}
1;

148
deploy/salt/nginx/sites/readthedocs
View File

@ -1,148 +0,0 @@
perl_modules perl;
perl_require ReadTheDocs.pm;
server {
listen 8000;
server_name media.readthedocs.org;
access_log /var/log/nginx/rtdmedia.log;
location / {
expires 60m;
root /home/docs/sites/readthedocs.org/checkouts/readthedocs.org/media;
add_header X-Deity {{ grains['host'] }};
}
}
server {
index index.html index.htm;
listen 80;
listen 8000 default;
server_name readthedocs.org;
access_log /var/log/nginx/readthedocs.log host;
location /favicon.ico {
root /home/docs/sites/readthedocs.org/checkouts/readthedocs.org/media/images;
break;
}
location /robots.txt {
root /home/docs/sites/readthedocs.org/checkouts/readthedocs.org/media;
break;
}
location ~* /docs/(.+)/en/(.+)/(.*) {
alias /home/docs/sites/readthedocs.org/checkouts/readthedocs.org/user_builds/$1/rtd-builds/$2/$3;
error_page 404 = @fallback;
error_page 500 = @fallback;
add_header X-Served Nginx;
add_header X-Deity {{ grains['host'] }};
}
location ~* /en/(.+)/(.*) {
alias /home/docs/sites/readthedocs.org/checkouts/readthedocs.org/user_builds/$domain/rtd-builds/$1/$2;
error_page 404 = @fallback;
error_page 500 = @fallback;
add_header X-Served Nginx;
add_header X-Deity {{ grains['host'] }};
}
location ~* /docs/(?P<project>.+)/en/(?P<version>.+) {
rewrite .* http://readthedocs.org/docs/$project/en/$version/;
}
location / {
proxy_pass http://127.0.0.1:8888;
proxy_buffering off;
proxy_set_header Host $host:$server_port;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
add_header X-Deity {{ grains['host'] }};
}
location @fallback {
proxy_pass http://127.0.0.1:8888;
proxy_buffering off;
proxy_set_header Host $host:$server_port;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
add_header X-Deity {{ grains['host'] }};
}
location /user_builds/ {
internal;
root /home/docs/sites/readthedocs.org/checkouts/readthedocs.org;
error_page 404 = @fallback;
error_page 500 = @fallback;
}
}
server {
listen 80;
listen 8000;
server_name ~^(?P<domain>.+).readthedocs.org;
access_log /var/log/nginx/readthedocs.log host;
location /favicon.ico {
root /home/docs/sites/readthedocs.org/checkouts/readthedocs.org/media/images;
break;
}
location /robots.txt {
root /home/docs/sites/readthedocs.org/checkouts/readthedocs.org/media;
break;
}
location ~* /en/(?<version>.+)(?:/(.*)|$) {
alias /home/docs/sites/readthedocs.org/checkouts/readthedocs.org/user_builds/$domain/rtd-builds/$version/$1;
error_page 404 = @fallback;
error_page 500 = @fallback;
add_header X-Served Nginx;
add_header X-Deity {{ grains['host'] }};
}
location ~* /(?<lang>\w\w)/(?<version>.+)(?:/(.*)|$) {
alias /home/docs/sites/readthedocs.org/checkouts/readthedocs.org/user_builds/$domain/translations/$lang/$version/$1;
error_page 404 = @fallback;
error_page 500 = @fallback;
add_header X-Served Nginx;
add_header X-Deity {{ grains['host'] }};
}
location ~ /$ {
set $rtd_metadata /home/docs/sites/readthedocs.org/checkouts/readthedocs.org/user_builds/$domain/metadata.json;
perl ReadTheDocs::redirect_home;
}
location @fallback {
proxy_pass http://127.0.0.1:8888;
proxy_buffering off;
proxy_set_header Host $host:$server_port;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
add_header X-Deity {{ grains['host'] }};
}
}
server {
listen 8000;
server_name sphinxdocs.com www.readthedocs.com readthedocs.com www.readthedocs.org djangowoodies.com;
rewrite ^(.*) http://readthedocs.org$1;
}
server {
listen 8000;
server_name ~^(?P<domain>.+).rtfd.org;
location / {
proxy_pass http://127.0.0.1:8888;
proxy_buffering off;
proxy_set_header Host $host:$server_port;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
add_header X-Deity {{ grains['host'] }};
}
}
server {
listen 8000;
server_name rtfd.org;
rewrite ^(.*)$ http://$domain.readthedocs.org/;
}

28
deploy/salt/postgres/init.sls
View File

@ -1,28 +0,0 @@
# PostgreSQL
postgresql:
pkg:
- installed
service.running:
- enable: True
- require:
- pkg: postgresql
- file: /etc/postgresql/9.1/main/pg_hba.conf
/etc/postgresql/9.1/main/pg_hba.conf:
file.managed:
- source: salt://postgres/pg_hba.conf
- mode: 0755
postgresql-docs:
postgres_user.present:
- name: docs
- password: docs
- login: True
- require:
- service: postgresql
postgres_database.present:
- name: docs
- owner: docs
- require:
- postgres_user: postgresql-docs

6
deploy/salt/postgres/pg_hba.conf
View File

@ -1,6 +0,0 @@
local all postgres trust
local all all trust
# IPv4 local connections:
host all all 127.0.0.1/32 trust
# IPv6 local connections:
host all all ::1/128 trust

26
deploy/salt/python/base.sls
View File

@ -1,26 +0,0 @@
# Python environment requires
python-dev:
pkg:
- installed
python-setuptools:
pkg:
- installed
# Install pip, virtualenv
python-pip:
cmd.run:
- cwd: /
- name: easy_install --script-dir=/usr/bin -U pip==1.5.1
- unless: which pip && pip --version | grep -oP 'pip 1.5.1'
- require:
- pkg: python-setuptools
# Required to rehash fresh pip
- reload_modules: True
python-virtualenv:
pip.installed:
- name: virtualenv==1.11.1
- require:
- cmd: python-pip

175
deploy/salt/readthedocs/site.sls
View File

@ -1,175 +0,0 @@
# RtD env setup
# Virtualenv and checkout
{% set site_path = '/home/docs/sites/readthedocs.org' %}
{{ site_path }}:
file.directory:
- user: docs
- group: docs
- makedirs: True
- require:
- user: docs
virtualenv.managed:
- user: docs
- require:
- pip: python-virtualenv
- file: {{ site_path }}
- reload_modules: True
{% for path in ['run', 'checkouts'] %}
{{ site_path }}/{{ path }}:
file.directory:
- user: docs
- group: docs
- require:
- user: docs
- file: {{ site_path }}
{% endfor %}
{{ site_path }}/checkouts/readthedocs.org:
file.directory:
- require:
- user: docs
- file: {{ site_path }}/checkouts
- pkg: rtd-vcs-pkgs
# Use a command here, instead of pip.installed -- pip 1.5 uses additional flags
# for external dependencies and pip.installed doesn't respect the cwd, making
# requirements such as -r pip_requirements.txt fail
rtd-deps:
cmd.wait:
- name:
{{ site_path }}/bin/pip install
--timeout 120
--allow-all-external
--allow-unverified bzr
--allow-unverified launchpadlib
--allow-unverified lazr.authentication
-r deploy_requirements.txt
- cwd: {{ site_path }}/checkouts/readthedocs.org
- watch:
- file: {{ site_path }}/checkouts/readthedocs.org
- pkg: rtd-build-pkgs
- require:
- virtualenv: {{ site_path }}
# Site data, load once on successful pip install
{% set venv_python = site_path + '/bin/python' %}
rtd-db-sync:
cmd.wait:
- name:
{{ venv_python }} manage.py syncdb
--settings=readthedocs.settings.vagrant
--noinput
- cwd: {{ site_path }}/checkouts/readthedocs.org
- user: docs
- watch:
- cmd: rtd-deps
- require:
- postgres_database: postgresql-docs
rtd-db-migrate:
cmd.wait:
- name:
{{ venv_python }} manage.py migrate
--settings=readthedocs.settings.vagrant
- cwd: {{ site_path }}/checkouts/readthedocs.org
- user: docs
- watch:
- cmd: rtd-db-sync
- require:
- postgres_database: postgresql-docs
{% if grains['id'] == 'vagrant' %}
rtd-db-loaduser:
cmd.wait:
- name:
{{ venv_python }} manage.py loaddata test_auth
--settings=readthedocs.settings.vagrant
- cwd: {{ site_path }}/checkouts/readthedocs.org
- user: docs
- watch:
- cmd: rtd-db-migrate
rtd-db-loaddata:
cmd.wait:
- name:
{{ venv_python }} manage.py loaddata test_data
--settings=readthedocs.settings.vagrant
- cwd: {{ site_path }}/checkouts/readthedocs.org
- user: docs
- watch:
- cmd: rtd-db-loaduser
{% endif %}
# Site build requirements
rtd-vcs-pkgs:
pkg.installed:
- pkgs:
- git-core
- subversion
- bzr
- require:
- pkg: python-dev
rtd-build-pkgs:
pkg.installed:
- pkgs:
- ipython
- graphviz
- libpq-dev
- libxml2-dev
- libxslt1-dev
# Services
{% for service in ['gunicorn', 'celery'] %}
/etc/init/readthedocs-{{ service }}.conf:
file.managed:
- source: salt://upstart/readthedocs-{{ service }}.conf
- reload_modules: True
- require:
- cmd: rtd-deps
- watch:
- cmd: rtd-db-migrate
readthedocs-{{ service }}:
service.running:
- enable: True
- watch:
- file: /etc/init/readthedocs-{{ service }}.conf
- require:
- file: {{ site_path }}/run
- service: redis-server
{% endfor %}
/etc/nginx/sites-enabled/readthedocs:
file.managed:
- source: salt://nginx/sites/readthedocs
- mode: 0640
- template: jinja
- watch_in:
- service: nginx
- require:
- file: /etc/nginx/nginx.conf
- file: /usr/share/nginx/perl/ReadTheDocs.pm
- service: readthedocs-gunicorn
/usr/share/nginx/perl/ReadTheDocs.pm:
file.managed:
- source: salt://nginx/perl/lib/ReadTheDocs.pm
- watch_in:
- service: nginx
- require:
- file: /usr/share/nginx/perl
- pkg: libi18n-acceptlanguage-perl
- pkg: libjson-perl
libi18n-acceptlanguage-perl:
pkg:
- installed
libjson-perl:
pkg:
- installed

40
deploy/salt/readthedocs/user.sls
View File

@ -1,40 +0,0 @@
# User setup
docs:
user.present:
- fullname: Docs User
- uid: 1005
- gid: 205
- home: /home/docs
- shell: /bin/bash
- createhome: True
- require:
- group: docs
group.present:
- gid: 205
/home/docs/.bash_profile:
file.managed:
- source: salt://readthedocs/user/bash_profile
- user: docs
- group: docs
- require:
- user: docs
/home/docs/.ssh:
file.directory:
- user: docs
- group: docs
- mode: 0700
- require:
- user: docs
# TODO configure key based on host name
/home/docs/.ssh/id_rsa:
file.managed:
- source: salt://readthedocs/user/private_key
- user: docs
- group: docs
- mode: 0400
- require:
- user: docs

15
deploy/salt/readthedocs/user/bash_profile
View File

@ -1,15 +0,0 @@
. .bashrc
export PIP_DOWNLOAD_CACHE=/tmp/pip
export DJANGO_SETTINGS_MODULE=settings
export PYTHONPATH=$PYTHONPATH:~/sites/readthedocs.org/checkouts/readthedocs.org/readthedocs
export EDITOR=vim
. sites/readthedocs.org/bin/activate
cd ~/sites/readthedocs.org/
alias chk='cd /home/docs/sites/readthedocs.org/checkouts/readthedocs.org'
alias run='cd /home/docs/sites/readthedocs.org/run'

0
deploy/salt/readthedocs/user/private_key
View File

9
deploy/salt/redis/init.sls
View File

@ -1,9 +0,0 @@
# Redis
redis-server:
pkg:
- installed
service.running:
- enable: True
- require:
- pkg: redis-server

9
deploy/salt/top.sls
View File

@ -1,9 +0,0 @@
base:
'vagrant':
- postgres
- memcached
- nginx
- redis
- python.base
- readthedocs.user
- readthedocs.site

12
deploy/salt/upstart/readthedocs-celery.conf
View File

@ -1,12 +0,0 @@
description "Celery for ReadTheDocs"
start on runlevel [2345]
stop on runlevel [!2345]
#Send KILL after 20 seconds
kill timeout 20
script
exec sudo -i -u docs django-admin.py celeryd --settings=settings.vagrant -f /home/docs/sites/readthedocs.org/run/celery.log -c 3 -E -B
end script
respawn

14
deploy/salt/upstart/readthedocs-gunicorn.conf
View File

@ -1,14 +0,0 @@
description "Gunicorn for ReadTheDocs"
start on runlevel [2345]
stop on runlevel [!2345]
#Send KILL after 20 seconds
kill timeout 5
respawn
env VENV="/home/docs/sites/readthedocs.org"
#Serve Gunicorn on localhost, since we run nginx locally as well.
script
exec sudo -iu docs $VENV/bin/gunicorn_django --preload -w 2 --log-level debug --log-file $VENV/run/gunicorn.log -p $VENV/run/gunicorn.pid -b 127.0.0.1:8888 $VENV/checkouts/readthedocs.org/readthedocs/settings/vagrant.py
end script