From 38e151224de2c84c8ea04b9ba54b251a28b44ffe Mon Sep 17 00:00:00 2001 From: John Kelly Date: Sun, 13 May 2018 18:55:29 -0400 Subject: [PATCH] Add ansible role to provision production server Add new ansible role which provisions production instance based on newfoundation branch. --- .gitignore | 1 + README.md | 27 +- Vagrantfile | 56 + core/models/__init__.py | 2 +- provisioning/ec2.ini | 219 +++ provisioning/ec2.py | 1708 +++++++++++++++++ provisioning/group_vars/production/vars.yml | 60 + provisioning/group_vars/production/vault.yml | 92 + provisioning/hosts | 2 + .../roles/regluit_common/defaults/main.yml | 21 + .../roles/regluit_common/files/celerybeat | 154 ++ .../roles/regluit_common/files/celeryd | 217 +++ .../roles/regluit_common/tasks/celery.yml | 37 + .../roles/regluit_common/tasks/main.yml | 85 + .../roles/regluit_common/tasks/mysql.yml | 12 + .../roles/regluit_common/tasks/redis.yml | 13 + .../templates/celery/celerybeat.j2 | 35 + .../templates/celery/celeryd.j2 | 9 + .../regluit_common/templates/common.py.j2 | 13 + .../roles/regluit_common/templates/host.py.j2 | 47 + .../roles/regluit_common/templates/me.py.j2 | 90 + .../roles/regluit_common/templates/opt.pth.j2 | 1 + .../regluit_common/templates/regluit.pth.j2 | 1 + .../roles/regluit_dev/defaults/main.yml | 5 + provisioning/roles/regluit_dev/tasks/main.yml | 69 + .../regluit_dev/templates/activate_venv.sh.j2 | 7 + .../roles/regluit_prod/files/celerybeat | 154 ++ provisioning/roles/regluit_prod/files/celeryd | 217 +++ .../files/certs/STAR_unglue_it.ca-bundle | 210 ++ .../files/certs/STAR_unglue_it.crt | 100 + .../roles/regluit_prod/files/certs/server.key | 89 + .../files/certs/unglue.it.wildcard.csr | 58 + .../roles/regluit_prod/handlers/main.yml | 6 + .../roles/regluit_prod/tasks/apache.yml | 79 + .../roles/regluit_prod/tasks/celery.yml | 67 + .../roles/regluit_prod/tasks/certs.yml | 39 + .../roles/regluit_prod/tasks/main.yml | 120 ++ .../roles/regluit_prod/tasks/mysql.yml | 48 + .../roles/regluit_prod/tasks/redis.yml | 13 + .../regluit_prod/templates/apache.conf.j2 | 71 + .../templates/celery/celerybeat.j2 | 35 + .../regluit_prod/templates/celery/celeryd.j2 | 9 + .../regluit_prod/templates/common-old.py.j2 | 13 + .../roles/regluit_prod/templates/common.py.j2 | 5 + .../regluit_prod/templates/host-old.py.j2 | 47 + .../roles/regluit_prod/templates/host.py.j2 | 5 + .../roles/regluit_prod/templates/opt.pth.j2 | 1 + .../roles/regluit_prod/templates/prod.py.j2 | 135 ++ .../roles/regluit_prod/templates/prod.wsgi.j2 | 13 + .../regluit_prod/templates/regluit.pth.j2 | 1 + provisioning/setup-prod.retry | 1 + provisioning/setup-prod.yml | 18 + provisioning/setup-regluit.yml | 6 + utils/{crypto.py => encryption.py} | 0 vagrant/group_vars/all/misc.yml | 20 +- vagrant/group_vars/all/secrets.yml | 100 +- vagrant/host_vars/please/secrets.yml | 184 +- vagrant/host_vars/prod/secrets.yml | 316 +-- 58 files changed, 4851 insertions(+), 312 deletions(-) create mode 100644 Vagrantfile create mode 100644 provisioning/ec2.ini create mode 100644 provisioning/ec2.py create mode 100644 provisioning/group_vars/production/vars.yml create mode 100644 provisioning/group_vars/production/vault.yml create mode 100644 provisioning/hosts create mode 100644 provisioning/roles/regluit_common/defaults/main.yml create mode 100644 provisioning/roles/regluit_common/files/celerybeat create mode 100644 provisioning/roles/regluit_common/files/celeryd create mode 100644 provisioning/roles/regluit_common/tasks/celery.yml create mode 100644 provisioning/roles/regluit_common/tasks/main.yml create mode 100644 provisioning/roles/regluit_common/tasks/mysql.yml create mode 100644 provisioning/roles/regluit_common/tasks/redis.yml create mode 100644 provisioning/roles/regluit_common/templates/celery/celerybeat.j2 create mode 100644 provisioning/roles/regluit_common/templates/celery/celeryd.j2 create mode 100644 provisioning/roles/regluit_common/templates/common.py.j2 create mode 100644 provisioning/roles/regluit_common/templates/host.py.j2 create mode 100644 provisioning/roles/regluit_common/templates/me.py.j2 create mode 100644 provisioning/roles/regluit_common/templates/opt.pth.j2 create mode 100644 provisioning/roles/regluit_common/templates/regluit.pth.j2 create mode 100644 provisioning/roles/regluit_dev/defaults/main.yml create mode 100644 provisioning/roles/regluit_dev/tasks/main.yml create mode 100644 provisioning/roles/regluit_dev/templates/activate_venv.sh.j2 create mode 100644 provisioning/roles/regluit_prod/files/celerybeat create mode 100644 provisioning/roles/regluit_prod/files/celeryd create mode 100644 provisioning/roles/regluit_prod/files/certs/STAR_unglue_it.ca-bundle create mode 100644 provisioning/roles/regluit_prod/files/certs/STAR_unglue_it.crt create mode 100644 provisioning/roles/regluit_prod/files/certs/server.key create mode 100644 provisioning/roles/regluit_prod/files/certs/unglue.it.wildcard.csr create mode 100644 provisioning/roles/regluit_prod/handlers/main.yml create mode 100644 provisioning/roles/regluit_prod/tasks/apache.yml create mode 100644 provisioning/roles/regluit_prod/tasks/celery.yml create mode 100644 provisioning/roles/regluit_prod/tasks/certs.yml create mode 100644 provisioning/roles/regluit_prod/tasks/main.yml create mode 100644 provisioning/roles/regluit_prod/tasks/mysql.yml create mode 100644 provisioning/roles/regluit_prod/tasks/redis.yml create mode 100644 provisioning/roles/regluit_prod/templates/apache.conf.j2 create mode 100644 provisioning/roles/regluit_prod/templates/celery/celerybeat.j2 create mode 100644 provisioning/roles/regluit_prod/templates/celery/celeryd.j2 create mode 100644 provisioning/roles/regluit_prod/templates/common-old.py.j2 create mode 100644 provisioning/roles/regluit_prod/templates/common.py.j2 create mode 100644 provisioning/roles/regluit_prod/templates/host-old.py.j2 create mode 100644 provisioning/roles/regluit_prod/templates/host.py.j2 create mode 100644 provisioning/roles/regluit_prod/templates/opt.pth.j2 create mode 100644 provisioning/roles/regluit_prod/templates/prod.py.j2 create mode 100644 provisioning/roles/regluit_prod/templates/prod.wsgi.j2 create mode 100644 provisioning/roles/regluit_prod/templates/regluit.pth.j2 create mode 100644 provisioning/setup-prod.retry create mode 100644 provisioning/setup-prod.yml create mode 100644 provisioning/setup-regluit.yml rename utils/{crypto.py => encryption.py} (100%) diff --git a/.gitignore b/.gitignore index 26b8f268..1736d8cb 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ settings/keys/* *.dot reports ENV +venv .DS_Store build deploy/last-update diff --git a/README.md b/README.md index 5eaf79d2..66fc4a29 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,32 @@ The partitioning between these modules is not as clean as would be ideal. `payme regluit was originally developed on Django 1.3 (python 2.7) and currently runs on Django 1.8. -Develop +Development (Vagrant + Virtualbox) +------- + +The recommended method for local development is to create a virtual machine with [Vagrant](https://www.vagrantup.com/) and [Virtualbox](https://www.virtualbox.org/wiki/Downloads). +With this method, the only requirements on the host machine are `virtualbox` and `vagrant`. +Vagrant will use the `ansible-local` provisioner, therefore installing python and ansible on the host machine is not necessary. + +__Instructions for Ubuntu 16:__ +1. Install virtualbox: `sudo apt-get install virtualbox` +2. Install vagrant: `sudo apt-get install vagrant` +3. Clone the `EbookFoundation/regluit` repository. +4. Navigate to the base directory of the cloned repo (where `Vagrantfile` is located). +5. Run `vagrant up` to create the VM, install dependencies, and start necessary services. + * Note: This step may take up to 15 minutes to complete. +6. Once the VM has been created, run `vagrant ssh` to log in to the virtual machine you just created. If provisioning was successful, you should see a success message upon login. + * If virtualenv doesn't activate upon login, you can do it manually by running `cd /opt/regluit && source venv/bin/activate` +7. Within the VM, run `./manage.py runserver 0.0.0.0:8000` to start the Django development server. +8. On your host machine, open your web browser of choice and navigate to `http://127.0.0.1:8000` + +__Instructions for other platforms (Windows/OSX):__ +* Steps are essentially the same, except for the installation of Vagrant and Virtualbox. Refer to each package's documentation for specific installation instructions. + +_NOTE:_ If running Windows on your host machine, ensure you are running `vagrant up` from an elevated command prompt, e.g. right click on Command Prompt -> Run As Administrator. + + +Development (Host Machine) ------- Here are some instructions for setting up regluit for development on diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 00000000..0ad9cdbc --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,56 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure("2") do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + # Every Vagrant development environment requires a box. You can search for + # boxes at https://vagrantcloud.com/search. + config.vm.box = "ubuntu/xenial64" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + config.vm.box_check_update = false + + # Setup specific for local machine + config.vm.define "regluit-local", primary: true do |local| + # Create a private network + local.vm.network "private_network", type: "dhcp" + local.vm.hostname = "regluit-local" + + # VirtuaLBox provider settings for running locally with Oracle VirtualBox + # --uartmode1 disconnected is necessary to disable serial interface, which + # is known to cause issues with Ubuntu 16 VM's + local.vm.provider "virtualbox" do |vb| + vb.name = "regluit-local" + vb.memory = 1024 + vb.cpus = 2 + vb.customize [ "modifyvm", :id, "--uartmode1", "disconnected" ] + end + + end + + config.vm.synced_folder ".", "/vagrant", disabled: true + config.vm.synced_folder ".", "/opt/regluit" + + config.vm.network "forwarded_port", guest: 8000, host: 8000 + + # Provision node with Ansible running on the Vagrant host + # This requires you have Ansible installed locally + # Vagrant autogenerates an ansible inventory file to use + config.vm.provision "ansible_local" do |ansible| + ansible.playbook = "/opt/regluit/provisioning/setup-regluit.yml" + ansible.provisioning_path = "/opt/regluit" + ansible.verbose = true + ansible.install = true + end + + config.vm.post_up_message = "Successfully created regluit-local VM. Run 'vagrant ssh' to log in and start the development server." + +end diff --git a/core/models/__init__.py b/core/models/__init__.py index 37917b8b..7f0ddf40 100755 --- a/core/models/__init__.py +++ b/core/models/__init__.py @@ -45,7 +45,7 @@ from regluit.payment.parameters import ( TRANSACTION_STATUS_FAILED, TRANSACTION_STATUS_INCOMPLETE ) -from regluit.utils import crypto +from regluit.utils import encryption as crypto from regluit.utils.localdatetime import now, date_today from regluit.core.parameters import ( diff --git a/provisioning/ec2.ini b/provisioning/ec2.ini new file mode 100644 index 00000000..c5ec6720 --- /dev/null +++ b/provisioning/ec2.ini @@ -0,0 +1,219 @@ +# Ansible EC2 external inventory script settings +# + +[ec2] + +# to talk to a private eucalyptus instance uncomment these lines +# and edit edit eucalyptus_host to be the host name of your cloud controller +#eucalyptus = True +#eucalyptus_host = clc.cloud.domain.org + +# AWS regions to make calls to. Set this to 'all' to make request to all regions +# in AWS and merge the results together. Alternatively, set this to a comma +# separated list of regions. E.g. 'us-east-1,us-west-1,us-west-2' and do not +# provide the 'regions_exclude' option. If this is set to 'auto', AWS_REGION or +# AWS_DEFAULT_REGION environment variable will be read to determine the region. +regions = all +regions_exclude = us-gov-west-1, cn-north-1 + +# When generating inventory, Ansible needs to know how to address a server. +# Each EC2 instance has a lot of variables associated with it. Here is the list: +# http://docs.pythonboto.org/en/latest/ref/ec2.html#module-boto.ec2.instance +# Below are 2 variables that are used as the address of a server: +# - destination_variable +# - vpc_destination_variable + +# This is the normal destination variable to use. If you are running Ansible +# from outside EC2, then 'public_dns_name' makes the most sense. If you are +# running Ansible from within EC2, then perhaps you want to use the internal +# address, and should set this to 'private_dns_name'. The key of an EC2 tag +# may optionally be used; however the boto instance variables hold precedence +# in the event of a collision. +destination_variable = public_dns_name + +# This allows you to override the inventory_name with an ec2 variable, instead +# of using the destination_variable above. Addressing (aka ansible_ssh_host) +# will still use destination_variable. Tags should be written as 'tag_TAGNAME'. +#hostname_variable = tag_Name + +# For server inside a VPC, using DNS names may not make sense. When an instance +# has 'subnet_id' set, this variable is used. If the subnet is public, setting +# this to 'ip_address' will return the public IP address. For instances in a +# private subnet, this should be set to 'private_ip_address', and Ansible must +# be run from within EC2. The key of an EC2 tag may optionally be used; however +# the boto instance variables hold precedence in the event of a collision. +# WARNING: - instances that are in the private vpc, _without_ public ip address +# will not be listed in the inventory until You set: +# vpc_destination_variable = private_ip_address +vpc_destination_variable = ip_address + +# The following two settings allow flexible ansible host naming based on a +# python format string and a comma-separated list of ec2 tags. Note that: +# +# 1) If the tags referenced are not present for some instances, empty strings +# will be substituted in the format string. +# 2) This overrides both destination_variable and vpc_destination_variable. +# +#destination_format = {0}.{1}.example.com +#destination_format_tags = Name,environment + +# To tag instances on EC2 with the resource records that point to them from +# Route53, set 'route53' to True. +route53 = False + +# To use Route53 records as the inventory hostnames, uncomment and set +# to equal the domain name you wish to use. You must also have 'route53' (above) +# set to True. +# route53_hostnames = .example.com + +# To exclude RDS instances from the inventory, uncomment and set to False. +#rds = False + +# To exclude ElastiCache instances from the inventory, uncomment and set to False. +#elasticache = False + +# Additionally, you can specify the list of zones to exclude looking up in +# 'route53_excluded_zones' as a comma-separated list. +# route53_excluded_zones = samplezone1.com, samplezone2.com + +# By default, only EC2 instances in the 'running' state are returned. Set +# 'all_instances' to True to return all instances regardless of state. +all_instances = False + +# By default, only EC2 instances in the 'running' state are returned. Specify +# EC2 instance states to return as a comma-separated list. This +# option is overridden when 'all_instances' is True. +# instance_states = pending, running, shutting-down, terminated, stopping, stopped + +# By default, only RDS instances in the 'available' state are returned. Set +# 'all_rds_instances' to True return all RDS instances regardless of state. +all_rds_instances = False + +# Include RDS cluster information (Aurora etc.) +include_rds_clusters = False + +# By default, only ElastiCache clusters and nodes in the 'available' state +# are returned. Set 'all_elasticache_clusters' and/or 'all_elastic_nodes' +# to True return all ElastiCache clusters and nodes, regardless of state. +# +# Note that all_elasticache_nodes only applies to listed clusters. That means +# if you set all_elastic_clusters to false, no node will be return from +# unavailable clusters, regardless of the state and to what you set for +# all_elasticache_nodes. +all_elasticache_replication_groups = False +all_elasticache_clusters = False +all_elasticache_nodes = False + +# API calls to EC2 are slow. For this reason, we cache the results of an API +# call. Set this to the path you want cache files to be written to. Two files +# will be written to this directory: +# - ansible-ec2.cache +# - ansible-ec2.index +cache_path = ~/.ansible/tmp + +# The number of seconds a cache file is considered valid. After this many +# seconds, a new API call will be made, and the cache file will be updated. +# To disable the cache, set this value to 0 +cache_max_age = 300 + +# Organize groups into a nested/hierarchy instead of a flat namespace. +nested_groups = False + +# Replace - tags when creating groups to avoid issues with ansible +replace_dash_in_groups = True + +# If set to true, any tag of the form "a,b,c" is expanded into a list +# and the results are used to create additional tag_* inventory groups. +expand_csv_tags = False + +# The EC2 inventory output can become very large. To manage its size, +# configure which groups should be created. +group_by_instance_id = True +group_by_region = True +group_by_availability_zone = True +group_by_aws_account = False +group_by_ami_id = True +group_by_instance_type = True +group_by_instance_state = False +group_by_platform = True +group_by_key_pair = True +group_by_vpc_id = True +group_by_security_group = True +group_by_tag_keys = True +group_by_tag_none = True +group_by_route53_names = True +group_by_rds_engine = True +group_by_rds_parameter_group = True +group_by_elasticache_engine = True +group_by_elasticache_cluster = True +group_by_elasticache_parameter_group = True +group_by_elasticache_replication_group = True + +# If you only want to include hosts that match a certain regular expression +# pattern_include = staging-* + +# If you want to exclude any hosts that match a certain regular expression +# pattern_exclude = staging-* + +# Instance filters can be used to control which instances are retrieved for +# inventory. For the full list of possible filters, please read the EC2 API +# docs: http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeInstances.html#query-DescribeInstances-filters +# Filters are key/value pairs separated by '=', to list multiple filters use +# a list separated by commas. To "AND" criteria together, use "&". Note that +# the "AND" is not useful along with stack_filters and so such usage is not allowed. +# See examples below. + +# If you want to apply multiple filters simultaneously, set stack_filters to +# True. Default behaviour is to combine the results of all filters. Stacking +# allows the use of multiple conditions to filter down, for example by +# environment and type of host. +stack_filters = False + +# Retrieve only instances with (key=value) env=staging tag +# instance_filters = tag:env=staging + +# Retrieve only instances with role=webservers OR role=dbservers tag +# instance_filters = tag:role=webservers,tag:role=dbservers + +# Retrieve only t1.micro instances OR instances with tag env=staging +# instance_filters = instance-type=t1.micro,tag:env=staging + +# You can use wildcards in filter values also. Below will list instances which +# tag Name value matches webservers1* +# (ex. webservers15, webservers1a, webservers123 etc) +# instance_filters = tag:Name=webservers1* + +# Retrieve only instances of type t1.micro that also have tag env=stage +# instance_filters = instance-type=t1.micro&tag:env=stage + +# Retrieve instances of type t1.micro AND tag env=stage, as well as any instance +# that are of type m3.large, regardless of env tag +# instance_filters = instance-type=t1.micro&tag:env=stage,instance-type=m3.large + +# An IAM role can be assumed, so all requests are run as that role. +# This can be useful for connecting across different accounts, or to limit user +# access +# iam_role = role-arn + +# A boto configuration profile may be used to separate out credentials +# see http://boto.readthedocs.org/en/latest/boto_config_tut.html +# boto_profile = some-boto-profile-name + + +[credentials] + +# The AWS credentials can optionally be specified here. Credentials specified +# here are ignored if the environment variable AWS_ACCESS_KEY_ID or +# AWS_PROFILE is set, or if the boto_profile property above is set. +# +# Supplying AWS credentials here is not recommended, as it introduces +# non-trivial security concerns. When going down this route, please make sure +# to set access permissions for this file correctly, e.g. handle it the same +# way as you would a private SSH key. +# +# Unlike the boto and AWS configure files, this section does not support +# profiles. +# +# aws_access_key_id = AXXXXXXXXXXXXXX +# aws_secret_access_key = XXXXXXXXXXXXXXXXXXX +# aws_security_token = XXXXXXXXXXXXXXXXXXXXXXXXXXXX \ No newline at end of file diff --git a/provisioning/ec2.py b/provisioning/ec2.py new file mode 100644 index 00000000..a3db570f --- /dev/null +++ b/provisioning/ec2.py @@ -0,0 +1,1708 @@ +#!/usr/bin/env python + +''' +EC2 external inventory script +================================= + +Generates inventory that Ansible can understand by making API request to +AWS EC2 using the Boto library. + +NOTE: This script assumes Ansible is being executed where the environment +variables needed for Boto have already been set: + export AWS_ACCESS_KEY_ID='AK123' + export AWS_SECRET_ACCESS_KEY='abc123' + +Optional region environment variable if region is 'auto' + +This script also assumes that there is an ec2.ini file alongside it. To specify a +different path to ec2.ini, define the EC2_INI_PATH environment variable: + + export EC2_INI_PATH=/path/to/my_ec2.ini + +If you're using eucalyptus you need to set the above variables and +you need to define: + + export EC2_URL=http://hostname_of_your_cc:port/services/Eucalyptus + +If you're using boto profiles (requires boto>=2.24.0) you can choose a profile +using the --boto-profile command line argument (e.g. ec2.py --boto-profile prod) or using +the AWS_PROFILE variable: + + AWS_PROFILE=prod ansible-playbook -i ec2.py myplaybook.yml + +For more details, see: http://docs.pythonboto.org/en/latest/boto_config_tut.html + +You can filter for specific EC2 instances by creating an environment variable +named EC2_INSTANCE_FILTERS, which has the same format as the instance_filters +entry documented in ec2.ini. For example, to find all hosts whose name begins +with 'webserver', one might use: + + export EC2_INSTANCE_FILTERS='tag:Name=webserver*' + +When run against a specific host, this script returns the following variables: + - ec2_ami_launch_index + - ec2_architecture + - ec2_association + - ec2_attachTime + - ec2_attachment + - ec2_attachmentId + - ec2_block_devices + - ec2_client_token + - ec2_deleteOnTermination + - ec2_description + - ec2_deviceIndex + - ec2_dns_name + - ec2_eventsSet + - ec2_group_name + - ec2_hypervisor + - ec2_id + - ec2_image_id + - ec2_instanceState + - ec2_instance_type + - ec2_ipOwnerId + - ec2_ip_address + - ec2_item + - ec2_kernel + - ec2_key_name + - ec2_launch_time + - ec2_monitored + - ec2_monitoring + - ec2_networkInterfaceId + - ec2_ownerId + - ec2_persistent + - ec2_placement + - ec2_platform + - ec2_previous_state + - ec2_private_dns_name + - ec2_private_ip_address + - ec2_publicIp + - ec2_public_dns_name + - ec2_ramdisk + - ec2_reason + - ec2_region + - ec2_requester_id + - ec2_root_device_name + - ec2_root_device_type + - ec2_security_group_ids + - ec2_security_group_names + - ec2_shutdown_state + - ec2_sourceDestCheck + - ec2_spot_instance_request_id + - ec2_state + - ec2_state_code + - ec2_state_reason + - ec2_status + - ec2_subnet_id + - ec2_tenancy + - ec2_virtualization_type + - ec2_vpc_id + +These variables are pulled out of a boto.ec2.instance object. There is a lack of +consistency with variable spellings (camelCase and underscores) since this +just loops through all variables the object exposes. It is preferred to use the +ones with underscores when multiple exist. + +In addition, if an instance has AWS tags associated with it, each tag is a new +variable named: + - ec2_tag_[Key] = [Value] + +Security groups are comma-separated in 'ec2_security_group_ids' and +'ec2_security_group_names'. + +When destination_format and destination_format_tags are specified +the destination_format can be built from the instance tags and attributes. +The behavior will first check the user defined tags, then proceed to +check instance attributes, and finally if neither are found 'nil' will +be used instead. + +'my_instance': { + 'region': 'us-east-1', # attribute + 'availability_zone': 'us-east-1a', # attribute + 'private_dns_name': '172.31.0.1', # attribute + 'ec2_tag_deployment': 'blue', # tag + 'ec2_tag_clusterid': 'ansible', # tag + 'ec2_tag_Name': 'webserver', # tag + ... +} + +Inside of the ec2.ini file the following settings are specified: +... +destination_format: {0}-{1}-{2}-{3} +destination_format_tags: Name,clusterid,deployment,private_dns_name +... + +These settings would produce a destination_format as the following: +'webserver-ansible-blue-172.31.0.1' +''' + +# (c) 2012, Peter Sankauskas +# +# This file is part of Ansible, +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +###################################################################### + +import sys +import os +import argparse +import re +from time import time +import boto +from boto import ec2 +from boto import rds +from boto import elasticache +from boto import route53 +from boto import sts +import six + +from ansible.module_utils import ec2 as ec2_utils + +HAS_BOTO3 = False +try: + import boto3 # noqa + HAS_BOTO3 = True +except ImportError: + pass + +from six.moves import configparser +from collections import defaultdict + +try: + import json +except ImportError: + import simplejson as json + +DEFAULTS = { + 'all_elasticache_clusters': 'False', + 'all_elasticache_nodes': 'False', + 'all_elasticache_replication_groups': 'False', + 'all_instances': 'False', + 'all_rds_instances': 'False', + 'aws_access_key_id': None, + 'aws_secret_access_key': None, + 'aws_security_token': None, + 'boto_profile': None, + 'cache_max_age': '300', + 'cache_path': '~/.ansible/tmp', + 'destination_variable': 'public_dns_name', + 'elasticache': 'True', + 'eucalyptus': 'False', + 'eucalyptus_host': None, + 'expand_csv_tags': 'False', + 'group_by_ami_id': 'True', + 'group_by_availability_zone': 'True', + 'group_by_aws_account': 'False', + 'group_by_elasticache_cluster': 'True', + 'group_by_elasticache_engine': 'True', + 'group_by_elasticache_parameter_group': 'True', + 'group_by_elasticache_replication_group': 'True', + 'group_by_instance_id': 'True', + 'group_by_instance_state': 'False', + 'group_by_instance_type': 'True', + 'group_by_key_pair': 'True', + 'group_by_platform': 'True', + 'group_by_rds_engine': 'True', + 'group_by_rds_parameter_group': 'True', + 'group_by_region': 'True', + 'group_by_route53_names': 'True', + 'group_by_security_group': 'True', + 'group_by_tag_keys': 'True', + 'group_by_tag_none': 'True', + 'group_by_vpc_id': 'True', + 'hostname_variable': None, + 'iam_role': None, + 'include_rds_clusters': 'False', + 'nested_groups': 'False', + 'pattern_exclude': None, + 'pattern_include': None, + 'rds': 'False', + 'regions': 'all', + 'regions_exclude': 'us-gov-west-1, cn-north-1', + 'replace_dash_in_groups': 'True', + 'route53': 'False', + 'route53_excluded_zones': '', + 'route53_hostnames': None, + 'stack_filters': 'False', + 'vpc_destination_variable': 'ip_address' +} + + +class Ec2Inventory(object): + + def _empty_inventory(self): + return {"_meta": {"hostvars": {}}} + + def __init__(self): + ''' Main execution path ''' + + # Inventory grouped by instance IDs, tags, security groups, regions, + # and availability zones + self.inventory = self._empty_inventory() + + self.aws_account_id = None + + # Index of hostname (address) to instance ID + self.index = {} + + # Boto profile to use (if any) + self.boto_profile = None + + # AWS credentials. + self.credentials = {} + + # Read settings and parse CLI arguments + self.parse_cli_args() + self.read_settings() + + # Make sure that profile_name is not passed at all if not set + # as pre 2.24 boto will fall over otherwise + if self.boto_profile: + if not hasattr(boto.ec2.EC2Connection, 'profile_name'): + self.fail_with_error("boto version must be >= 2.24 to use profile") + + # Cache + if self.args.refresh_cache: + self.do_api_calls_update_cache() + elif not self.is_cache_valid(): + self.do_api_calls_update_cache() + + # Data to print + if self.args.host: + data_to_print = self.get_host_info() + + elif self.args.list: + # Display list of instances for inventory + if self.inventory == self._empty_inventory(): + data_to_print = self.get_inventory_from_cache() + else: + data_to_print = self.json_format_dict(self.inventory, True) + + print(data_to_print) + + def is_cache_valid(self): + ''' Determines if the cache files have expired, or if it is still valid ''' + + if os.path.isfile(self.cache_path_cache): + mod_time = os.path.getmtime(self.cache_path_cache) + current_time = time() + if (mod_time + self.cache_max_age) > current_time: + if os.path.isfile(self.cache_path_index): + return True + + return False + + def read_settings(self): + ''' Reads the settings from the ec2.ini file ''' + + scriptbasename = __file__ + scriptbasename = os.path.basename(scriptbasename) + scriptbasename = scriptbasename.replace('.py', '') + + defaults = { + 'ec2': { + 'ini_fallback': os.path.join(os.path.dirname(__file__), 'ec2.ini'), + 'ini_path': os.path.join(os.path.dirname(__file__), '%s.ini' % scriptbasename) + } + } + + if six.PY3: + config = configparser.ConfigParser(DEFAULTS) + else: + config = configparser.SafeConfigParser(DEFAULTS) + ec2_ini_path = os.environ.get('EC2_INI_PATH', defaults['ec2']['ini_path']) + ec2_ini_path = os.path.expanduser(os.path.expandvars(ec2_ini_path)) + + if not os.path.isfile(ec2_ini_path): + ec2_ini_path = os.path.expanduser(defaults['ec2']['ini_fallback']) + + if os.path.isfile(ec2_ini_path): + config.read(ec2_ini_path) + + # Add empty sections if they don't exist + try: + config.add_section('ec2') + except configparser.DuplicateSectionError: + pass + + try: + config.add_section('credentials') + except configparser.DuplicateSectionError: + pass + + # is eucalyptus? + self.eucalyptus = config.getboolean('ec2', 'eucalyptus') + self.eucalyptus_host = config.get('ec2', 'eucalyptus_host') + + # Regions + self.regions = [] + config_regions = config.get('ec2', 'regions') + if (config_regions == 'all'): + if self.eucalyptus_host: + self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name, **self.credentials) + else: + config_regions_exclude = config.get('ec2', 'regions_exclude') + + for region_info in ec2.regions(): + if region_info.name not in config_regions_exclude: + self.regions.append(region_info.name) + else: + self.regions = config_regions.split(",") + if 'auto' in self.regions: + env_region = os.environ.get('AWS_REGION') + if env_region is None: + env_region = os.environ.get('AWS_DEFAULT_REGION') + self.regions = [env_region] + + # Destination addresses + self.destination_variable = config.get('ec2', 'destination_variable') + self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable') + self.hostname_variable = config.get('ec2', 'hostname_variable') + + if config.has_option('ec2', 'destination_format') and \ + config.has_option('ec2', 'destination_format_tags'): + self.destination_format = config.get('ec2', 'destination_format') + self.destination_format_tags = config.get('ec2', 'destination_format_tags').split(',') + else: + self.destination_format = None + self.destination_format_tags = None + + # Route53 + self.route53_enabled = config.getboolean('ec2', 'route53') + self.route53_hostnames = config.get('ec2', 'route53_hostnames') + + self.route53_excluded_zones = [] + self.route53_excluded_zones = [a for a in config.get('ec2', 'route53_excluded_zones').split(',') if a] + + # Include RDS instances? + self.rds_enabled = config.getboolean('ec2', 'rds') + + # Include RDS cluster instances? + self.include_rds_clusters = config.getboolean('ec2', 'include_rds_clusters') + + # Include ElastiCache instances? + self.elasticache_enabled = config.getboolean('ec2', 'elasticache') + + # Return all EC2 instances? + self.all_instances = config.getboolean('ec2', 'all_instances') + + # Instance states to be gathered in inventory. Default is 'running'. + # Setting 'all_instances' to 'yes' overrides this option. + ec2_valid_instance_states = [ + 'pending', + 'running', + 'shutting-down', + 'terminated', + 'stopping', + 'stopped' + ] + self.ec2_instance_states = [] + if self.all_instances: + self.ec2_instance_states = ec2_valid_instance_states + elif config.has_option('ec2', 'instance_states'): + for instance_state in config.get('ec2', 'instance_states').split(','): + instance_state = instance_state.strip() + if instance_state not in ec2_valid_instance_states: + continue + self.ec2_instance_states.append(instance_state) + else: + self.ec2_instance_states = ['running'] + + # Return all RDS instances? (if RDS is enabled) + self.all_rds_instances = config.getboolean('ec2', 'all_rds_instances') + + # Return all ElastiCache replication groups? (if ElastiCache is enabled) + self.all_elasticache_replication_groups = config.getboolean('ec2', 'all_elasticache_replication_groups') + + # Return all ElastiCache clusters? (if ElastiCache is enabled) + self.all_elasticache_clusters = config.getboolean('ec2', 'all_elasticache_clusters') + + # Return all ElastiCache nodes? (if ElastiCache is enabled) + self.all_elasticache_nodes = config.getboolean('ec2', 'all_elasticache_nodes') + + # boto configuration profile (prefer CLI argument then environment variables then config file) + self.boto_profile = self.args.boto_profile or \ + os.environ.get('AWS_PROFILE') or \ + config.get('ec2', 'boto_profile') + + # AWS credentials (prefer environment variables) + if not (self.boto_profile or os.environ.get('AWS_ACCESS_KEY_ID') or + os.environ.get('AWS_PROFILE')): + + aws_access_key_id = config.get('credentials', 'aws_access_key_id') + aws_secret_access_key = config.get('credentials', 'aws_secret_access_key') + aws_security_token = config.get('credentials', 'aws_security_token') + + if aws_access_key_id: + self.credentials = { + 'aws_access_key_id': aws_access_key_id, + 'aws_secret_access_key': aws_secret_access_key + } + if aws_security_token: + self.credentials['security_token'] = aws_security_token + + # Cache related + cache_dir = os.path.expanduser(config.get('ec2', 'cache_path')) + if self.boto_profile: + cache_dir = os.path.join(cache_dir, 'profile_' + self.boto_profile) + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + + cache_name = 'ansible-ec2' + cache_id = self.boto_profile or os.environ.get('AWS_ACCESS_KEY_ID', self.credentials.get('aws_access_key_id')) + if cache_id: + cache_name = '%s-%s' % (cache_name, cache_id) + cache_name += '-' + str(abs(hash(__file__)))[1:7] + self.cache_path_cache = os.path.join(cache_dir, "%s.cache" % cache_name) + self.cache_path_index = os.path.join(cache_dir, "%s.index" % cache_name) + self.cache_max_age = config.getint('ec2', 'cache_max_age') + + self.expand_csv_tags = config.getboolean('ec2', 'expand_csv_tags') + + # Configure nested groups instead of flat namespace. + self.nested_groups = config.getboolean('ec2', 'nested_groups') + + # Replace dash or not in group names + self.replace_dash_in_groups = config.getboolean('ec2', 'replace_dash_in_groups') + + # IAM role to assume for connection + self.iam_role = config.get('ec2', 'iam_role') + + # Configure which groups should be created. + + group_by_options = [a for a in DEFAULTS if a.startswith('group_by')] + for option in group_by_options: + setattr(self, option, config.getboolean('ec2', option)) + + # Do we need to just include hosts that match a pattern? + self.pattern_include = config.get('ec2', 'pattern_include') + if self.pattern_include: + self.pattern_include = re.compile(self.pattern_include) + + # Do we need to exclude hosts that match a pattern? + self.pattern_exclude = config.get('ec2', 'pattern_exclude') + if self.pattern_exclude: + self.pattern_exclude = re.compile(self.pattern_exclude) + + # Do we want to stack multiple filters? + self.stack_filters = config.getboolean('ec2', 'stack_filters') + + # Instance filters (see boto and EC2 API docs). Ignore invalid filters. + self.ec2_instance_filters = [] + + if config.has_option('ec2', 'instance_filters') or 'EC2_INSTANCE_FILTERS' in os.environ: + filters = os.getenv('EC2_INSTANCE_FILTERS', config.get('ec2', 'instance_filters') if config.has_option('ec2', 'instance_filters') else '') + + if self.stack_filters and '&' in filters: + self.fail_with_error("AND filters along with stack_filter enabled is not supported.\n") + + filter_sets = [f for f in filters.split(',') if f] + + for filter_set in filter_sets: + filters = {} + filter_set = filter_set.strip() + for instance_filter in filter_set.split("&"): + instance_filter = instance_filter.strip() + if not instance_filter or '=' not in instance_filter: + continue + filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)] + if not filter_key: + continue + filters[filter_key] = filter_value + self.ec2_instance_filters.append(filters.copy()) + + def parse_cli_args(self): + ''' Command line argument processing ''' + + parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on EC2') + parser.add_argument('--list', action='store_true', default=True, + help='List instances (default: True)') + parser.add_argument('--host', action='store', + help='Get all the variables about a specific instance') + parser.add_argument('--refresh-cache', action='store_true', default=False, + help='Force refresh of cache by making API requests to EC2 (default: False - use cache files)') + parser.add_argument('--profile', '--boto-profile', action='store', dest='boto_profile', + help='Use boto profile for connections to EC2') + self.args = parser.parse_args() + + def do_api_calls_update_cache(self): + ''' Do API calls to each region, and save data in cache files ''' + + if self.route53_enabled: + self.get_route53_records() + + for region in self.regions: + self.get_instances_by_region(region) + if self.rds_enabled: + self.get_rds_instances_by_region(region) + if self.elasticache_enabled: + self.get_elasticache_clusters_by_region(region) + self.get_elasticache_replication_groups_by_region(region) + if self.include_rds_clusters: + self.include_rds_clusters_by_region(region) + + self.write_to_cache(self.inventory, self.cache_path_cache) + self.write_to_cache(self.index, self.cache_path_index) + + def connect(self, region): + ''' create connection to api server''' + if self.eucalyptus: + conn = boto.connect_euca(host=self.eucalyptus_host, **self.credentials) + conn.APIVersion = '2010-08-31' + else: + conn = self.connect_to_aws(ec2, region) + return conn + + def boto_fix_security_token_in_profile(self, connect_args): + ''' monkey patch for boto issue boto/boto#2100 ''' + profile = 'profile ' + self.boto_profile + if boto.config.has_option(profile, 'aws_security_token'): + connect_args['security_token'] = boto.config.get(profile, 'aws_security_token') + return connect_args + + def connect_to_aws(self, module, region): + connect_args = self.credentials + + # only pass the profile name if it's set (as it is not supported by older boto versions) + if self.boto_profile: + connect_args['profile_name'] = self.boto_profile + self.boto_fix_security_token_in_profile(connect_args) + + if self.iam_role: + sts_conn = sts.connect_to_region(region, **connect_args) + role = sts_conn.assume_role(self.iam_role, 'ansible_dynamic_inventory') + connect_args['aws_access_key_id'] = role.credentials.access_key + connect_args['aws_secret_access_key'] = role.credentials.secret_key + connect_args['security_token'] = role.credentials.session_token + + conn = module.connect_to_region(region, **connect_args) + # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported + if conn is None: + self.fail_with_error("region name: %s likely not supported, or AWS is down. connection to region failed." % region) + return conn + + def get_instances_by_region(self, region): + ''' Makes an AWS EC2 API call to the list of instances in a particular + region ''' + + try: + conn = self.connect(region) + reservations = [] + if self.ec2_instance_filters: + if self.stack_filters: + filters_dict = {} + for filters in self.ec2_instance_filters: + filters_dict.update(filters) + reservations.extend(conn.get_all_instances(filters=filters_dict)) + else: + for filters in self.ec2_instance_filters: + reservations.extend(conn.get_all_instances(filters=filters)) + else: + reservations = conn.get_all_instances() + + # Pull the tags back in a second step + # AWS are on record as saying that the tags fetched in the first `get_all_instances` request are not + # reliable and may be missing, and the only way to guarantee they are there is by calling `get_all_tags` + instance_ids = [] + for reservation in reservations: + instance_ids.extend([instance.id for instance in reservation.instances]) + + max_filter_value = 199 + tags = [] + for i in range(0, len(instance_ids), max_filter_value): + tags.extend(conn.get_all_tags(filters={'resource-type': 'instance', 'resource-id': instance_ids[i:i + max_filter_value]})) + + tags_by_instance_id = defaultdict(dict) + for tag in tags: + tags_by_instance_id[tag.res_id][tag.name] = tag.value + + if (not self.aws_account_id) and reservations: + self.aws_account_id = reservations[0].owner_id + + for reservation in reservations: + for instance in reservation.instances: + instance.tags = tags_by_instance_id[instance.id] + self.add_instance(instance, region) + + except boto.exception.BotoServerError as e: + if e.error_code == 'AuthFailure': + error = self.get_auth_error_message() + else: + backend = 'Eucalyptus' if self.eucalyptus else 'AWS' + error = "Error connecting to %s backend.\n%s" % (backend, e.message) + self.fail_with_error(error, 'getting EC2 instances') + + def tags_match_filters(self, tags): + ''' return True if given tags match configured filters ''' + if not self.ec2_instance_filters: + return True + + for filters in self.ec2_instance_filters: + for filter_name, filter_value in filters.items(): + if filter_name[:4] != 'tag:': + continue + filter_name = filter_name[4:] + if filter_name not in tags: + if self.stack_filters: + return False + continue + if isinstance(filter_value, list): + if self.stack_filters and tags[filter_name] not in filter_value: + return False + if not self.stack_filters and tags[filter_name] in filter_value: + return True + if isinstance(filter_value, six.string_types): + if self.stack_filters and tags[filter_name] != filter_value: + return False + if not self.stack_filters and tags[filter_name] == filter_value: + return True + + return self.stack_filters + + def get_rds_instances_by_region(self, region): + ''' Makes an AWS API call to the list of RDS instances in a particular + region ''' + + if not HAS_BOTO3: + self.fail_with_error("Working with RDS instances requires boto3 - please install boto3 and try again", + "getting RDS instances") + + client = ec2_utils.boto3_inventory_conn('client', 'rds', region, **self.credentials) + db_instances = client.describe_db_instances() + + try: + conn = self.connect_to_aws(rds, region) + if conn: + marker = None + while True: + instances = conn.get_all_dbinstances(marker=marker) + marker = instances.marker + for index, instance in enumerate(instances): + # Add tags to instances. + instance.arn = db_instances['DBInstances'][index]['DBInstanceArn'] + tags = client.list_tags_for_resource(ResourceName=instance.arn)['TagList'] + instance.tags = {} + for tag in tags: + instance.tags[tag['Key']] = tag['Value'] + if self.tags_match_filters(instance.tags): + self.add_rds_instance(instance, region) + if not marker: + break + except boto.exception.BotoServerError as e: + error = e.reason + + if e.error_code == 'AuthFailure': + error = self.get_auth_error_message() + elif e.error_code == "OptInRequired": + error = "RDS hasn't been enabled for this account yet. " \ + "You must either log in to the RDS service through the AWS console to enable it, " \ + "or set 'rds = False' in ec2.ini" + elif not e.reason == "Forbidden": + error = "Looks like AWS RDS is down:\n%s" % e.message + self.fail_with_error(error, 'getting RDS instances') + + def include_rds_clusters_by_region(self, region): + if not HAS_BOTO3: + self.fail_with_error("Working with RDS clusters requires boto3 - please install boto3 and try again", + "getting RDS clusters") + + client = ec2_utils.boto3_inventory_conn('client', 'rds', region, **self.credentials) + + marker, clusters = '', [] + while marker is not None: + resp = client.describe_db_clusters(Marker=marker) + clusters.extend(resp["DBClusters"]) + marker = resp.get('Marker', None) + + account_id = boto.connect_iam().get_user().arn.split(':')[4] + c_dict = {} + for c in clusters: + # remove these datetime objects as there is no serialisation to json + # currently in place and we don't need the data yet + if 'EarliestRestorableTime' in c: + del c['EarliestRestorableTime'] + if 'LatestRestorableTime' in c: + del c['LatestRestorableTime'] + + if not self.ec2_instance_filters: + matches_filter = True + else: + matches_filter = False + + try: + # arn:aws:rds:::: + tags = client.list_tags_for_resource( + ResourceName='arn:aws:rds:' + region + ':' + account_id + ':cluster:' + c['DBClusterIdentifier']) + c['Tags'] = tags['TagList'] + + if self.ec2_instance_filters: + for filters in self.ec2_instance_filters: + for filter_key, filter_values in filters.items(): + # get AWS tag key e.g. tag:env will be 'env' + tag_name = filter_key.split(":", 1)[1] + # Filter values is a list (if you put multiple values for the same tag name) + matches_filter = any(d['Key'] == tag_name and d['Value'] in filter_values for d in c['Tags']) + + if matches_filter: + # it matches a filter, so stop looking for further matches + break + + if matches_filter: + break + + except Exception as e: + if e.message.find('DBInstanceNotFound') >= 0: + # AWS RDS bug (2016-01-06) means deletion does not fully complete and leave an 'empty' cluster. + # Ignore errors when trying to find tags for these + pass + + # ignore empty clusters caused by AWS bug + if len(c['DBClusterMembers']) == 0: + continue + elif matches_filter: + c_dict[c['DBClusterIdentifier']] = c + + self.inventory['db_clusters'] = c_dict + + def get_elasticache_clusters_by_region(self, region): + ''' Makes an AWS API call to the list of ElastiCache clusters (with + nodes' info) in a particular region.''' + + # ElastiCache boto module doesn't provide a get_all_instances method, + # that's why we need to call describe directly (it would be called by + # the shorthand method anyway...) + try: + conn = self.connect_to_aws(elasticache, region) + if conn: + # show_cache_node_info = True + # because we also want nodes' information + response = conn.describe_cache_clusters(None, None, None, True) + + except boto.exception.BotoServerError as e: + error = e.reason + + if e.error_code == 'AuthFailure': + error = self.get_auth_error_message() + elif e.error_code == "OptInRequired": + error = "ElastiCache hasn't been enabled for this account yet. " \ + "You must either log in to the ElastiCache service through the AWS console to enable it, " \ + "or set 'elasticache = False' in ec2.ini" + elif not e.reason == "Forbidden": + error = "Looks like AWS ElastiCache is down:\n%s" % e.message + self.fail_with_error(error, 'getting ElastiCache clusters') + + try: + # Boto also doesn't provide wrapper classes to CacheClusters or + # CacheNodes. Because of that we can't make use of the get_list + # method in the AWSQueryConnection. Let's do the work manually + clusters = response['DescribeCacheClustersResponse']['DescribeCacheClustersResult']['CacheClusters'] + + except KeyError as e: + error = "ElastiCache query to AWS failed (unexpected format)." + self.fail_with_error(error, 'getting ElastiCache clusters') + + for cluster in clusters: + self.add_elasticache_cluster(cluster, region) + + def get_elasticache_replication_groups_by_region(self, region): + ''' Makes an AWS API call to the list of ElastiCache replication groups + in a particular region.''' + + # ElastiCache boto module doesn't provide a get_all_instances method, + # that's why we need to call describe directly (it would be called by + # the shorthand method anyway...) + try: + conn = self.connect_to_aws(elasticache, region) + if conn: + response = conn.describe_replication_groups() + + except boto.exception.BotoServerError as e: + error = e.reason + + if e.error_code == 'AuthFailure': + error = self.get_auth_error_message() + if not e.reason == "Forbidden": + error = "Looks like AWS ElastiCache [Replication Groups] is down:\n%s" % e.message + self.fail_with_error(error, 'getting ElastiCache clusters') + + try: + # Boto also doesn't provide wrapper classes to ReplicationGroups + # Because of that we can't make use of the get_list method in the + # AWSQueryConnection. Let's do the work manually + replication_groups = response['DescribeReplicationGroupsResponse']['DescribeReplicationGroupsResult']['ReplicationGroups'] + + except KeyError as e: + error = "ElastiCache [Replication Groups] query to AWS failed (unexpected format)." + self.fail_with_error(error, 'getting ElastiCache clusters') + + for replication_group in replication_groups: + self.add_elasticache_replication_group(replication_group, region) + + def get_auth_error_message(self): + ''' create an informative error message if there is an issue authenticating''' + errors = ["Authentication error retrieving ec2 inventory."] + if None in [os.environ.get('AWS_ACCESS_KEY_ID'), os.environ.get('AWS_SECRET_ACCESS_KEY')]: + errors.append(' - No AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY environment vars found') + else: + errors.append(' - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment vars found but may not be correct') + + boto_paths = ['/etc/boto.cfg', '~/.boto', '~/.aws/credentials'] + boto_config_found = [p for p in boto_paths if os.path.isfile(os.path.expanduser(p))] + if len(boto_config_found) > 0: + errors.append(" - Boto configs found at '%s', but the credentials contained may not be correct" % ', '.join(boto_config_found)) + else: + errors.append(" - No Boto config found at any expected location '%s'" % ', '.join(boto_paths)) + + return '\n'.join(errors) + + def fail_with_error(self, err_msg, err_operation=None): + '''log an error to std err for ansible-playbook to consume and exit''' + if err_operation: + err_msg = 'ERROR: "{err_msg}", while: {err_operation}'.format( + err_msg=err_msg, err_operation=err_operation) + sys.stderr.write(err_msg) + sys.exit(1) + + def get_instance(self, region, instance_id): + conn = self.connect(region) + + reservations = conn.get_all_instances([instance_id]) + for reservation in reservations: + for instance in reservation.instances: + return instance + + def add_instance(self, instance, region): + ''' Adds an instance to the inventory and index, as long as it is + addressable ''' + + # Only return instances with desired instance states + if instance.state not in self.ec2_instance_states: + return + + # Select the best destination address + # When destination_format and destination_format_tags are specified + # the following code will attempt to find the instance tags first, + # then the instance attributes next, and finally if neither are found + # assign nil for the desired destination format attribute. + if self.destination_format and self.destination_format_tags: + dest_vars = [] + inst_tags = getattr(instance, 'tags') + for tag in self.destination_format_tags: + if tag in inst_tags: + dest_vars.append(inst_tags[tag]) + elif hasattr(instance, tag): + dest_vars.append(getattr(instance, tag)) + else: + dest_vars.append('nil') + + dest = self.destination_format.format(*dest_vars) + elif instance.subnet_id: + dest = getattr(instance, self.vpc_destination_variable, None) + if dest is None: + dest = getattr(instance, 'tags').get(self.vpc_destination_variable, None) + else: + dest = getattr(instance, self.destination_variable, None) + if dest is None: + dest = getattr(instance, 'tags').get(self.destination_variable, None) + + if not dest: + # Skip instances we cannot address (e.g. private VPC subnet) + return + + # Set the inventory name + hostname = None + if self.hostname_variable: + if self.hostname_variable.startswith('tag_'): + hostname = instance.tags.get(self.hostname_variable[4:], None) + else: + hostname = getattr(instance, self.hostname_variable) + + # set the hostname from route53 + if self.route53_enabled and self.route53_hostnames: + route53_names = self.get_instance_route53_names(instance) + for name in route53_names: + if name.endswith(self.route53_hostnames): + hostname = name + + # If we can't get a nice hostname, use the destination address + if not hostname: + hostname = dest + # to_safe strips hostname characters like dots, so don't strip route53 hostnames + elif self.route53_enabled and self.route53_hostnames and hostname.endswith(self.route53_hostnames): + hostname = hostname.lower() + else: + hostname = self.to_safe(hostname).lower() + + # if we only want to include hosts that match a pattern, skip those that don't + if self.pattern_include and not self.pattern_include.match(hostname): + return + + # if we need to exclude hosts that match a pattern, skip those + if self.pattern_exclude and self.pattern_exclude.match(hostname): + return + + # Add to index + self.index[hostname] = [region, instance.id] + + # Inventory: Group by instance ID (always a group of 1) + if self.group_by_instance_id: + self.inventory[instance.id] = [hostname] + if self.nested_groups: + self.push_group(self.inventory, 'instances', instance.id) + + # Inventory: Group by region + if self.group_by_region: + self.push(self.inventory, region, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'regions', region) + + # Inventory: Group by availability zone + if self.group_by_availability_zone: + self.push(self.inventory, instance.placement, hostname) + if self.nested_groups: + if self.group_by_region: + self.push_group(self.inventory, region, instance.placement) + self.push_group(self.inventory, 'zones', instance.placement) + + # Inventory: Group by Amazon Machine Image (AMI) ID + if self.group_by_ami_id: + ami_id = self.to_safe(instance.image_id) + self.push(self.inventory, ami_id, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'images', ami_id) + + # Inventory: Group by instance type + if self.group_by_instance_type: + type_name = self.to_safe('type_' + instance.instance_type) + self.push(self.inventory, type_name, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'types', type_name) + + # Inventory: Group by instance state + if self.group_by_instance_state: + state_name = self.to_safe('instance_state_' + instance.state) + self.push(self.inventory, state_name, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'instance_states', state_name) + + # Inventory: Group by platform + if self.group_by_platform: + if instance.platform: + platform = self.to_safe('platform_' + instance.platform) + else: + platform = self.to_safe('platform_undefined') + self.push(self.inventory, platform, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'platforms', platform) + + # Inventory: Group by key pair + if self.group_by_key_pair and instance.key_name: + key_name = self.to_safe('key_' + instance.key_name) + self.push(self.inventory, key_name, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'keys', key_name) + + # Inventory: Group by VPC + if self.group_by_vpc_id and instance.vpc_id: + vpc_id_name = self.to_safe('vpc_id_' + instance.vpc_id) + self.push(self.inventory, vpc_id_name, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'vpcs', vpc_id_name) + + # Inventory: Group by security group + if self.group_by_security_group: + try: + for group in instance.groups: + key = self.to_safe("security_group_" + group.name) + self.push(self.inventory, key, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'security_groups', key) + except AttributeError: + self.fail_with_error('\n'.join(['Package boto seems a bit older.', + 'Please upgrade boto >= 2.3.0.'])) + + # Inventory: Group by AWS account ID + if self.group_by_aws_account: + self.push(self.inventory, self.aws_account_id, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'accounts', self.aws_account_id) + + # Inventory: Group by tag keys + if self.group_by_tag_keys: + for k, v in instance.tags.items(): + if self.expand_csv_tags and v and ',' in v: + values = map(lambda x: x.strip(), v.split(',')) + else: + values = [v] + + for v in values: + if v: + key = self.to_safe("tag_" + k + "=" + v) + else: + key = self.to_safe("tag_" + k) + self.push(self.inventory, key, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k)) + if v: + self.push_group(self.inventory, self.to_safe("tag_" + k), key) + + # Inventory: Group by Route53 domain names if enabled + if self.route53_enabled and self.group_by_route53_names: + route53_names = self.get_instance_route53_names(instance) + for name in route53_names: + self.push(self.inventory, name, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'route53', name) + + # Global Tag: instances without tags + if self.group_by_tag_none and len(instance.tags) == 0: + self.push(self.inventory, 'tag_none', hostname) + if self.nested_groups: + self.push_group(self.inventory, 'tags', 'tag_none') + + # Global Tag: tag all EC2 instances + self.push(self.inventory, 'ec2', hostname) + + self.inventory["_meta"]["hostvars"][hostname] = self.get_host_info_dict_from_instance(instance) + self.inventory["_meta"]["hostvars"][hostname]['ansible_host'] = dest + + def add_rds_instance(self, instance, region): + ''' Adds an RDS instance to the inventory and index, as long as it is + addressable ''' + + # Only want available instances unless all_rds_instances is True + if not self.all_rds_instances and instance.status != 'available': + return + + # Select the best destination address + dest = instance.endpoint[0] + + if not dest: + # Skip instances we cannot address (e.g. private VPC subnet) + return + + # Set the inventory name + hostname = None + if self.hostname_variable: + if self.hostname_variable.startswith('tag_'): + hostname = instance.tags.get(self.hostname_variable[4:], None) + else: + hostname = getattr(instance, self.hostname_variable) + + # If we can't get a nice hostname, use the destination address + if not hostname: + hostname = dest + + hostname = self.to_safe(hostname).lower() + + # Add to index + self.index[hostname] = [region, instance.id] + + # Inventory: Group by instance ID (always a group of 1) + if self.group_by_instance_id: + self.inventory[instance.id] = [hostname] + if self.nested_groups: + self.push_group(self.inventory, 'instances', instance.id) + + # Inventory: Group by region + if self.group_by_region: + self.push(self.inventory, region, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'regions', region) + + # Inventory: Group by availability zone + if self.group_by_availability_zone: + self.push(self.inventory, instance.availability_zone, hostname) + if self.nested_groups: + if self.group_by_region: + self.push_group(self.inventory, region, instance.availability_zone) + self.push_group(self.inventory, 'zones', instance.availability_zone) + + # Inventory: Group by instance type + if self.group_by_instance_type: + type_name = self.to_safe('type_' + instance.instance_class) + self.push(self.inventory, type_name, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'types', type_name) + + # Inventory: Group by VPC + if self.group_by_vpc_id and instance.subnet_group and instance.subnet_group.vpc_id: + vpc_id_name = self.to_safe('vpc_id_' + instance.subnet_group.vpc_id) + self.push(self.inventory, vpc_id_name, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'vpcs', vpc_id_name) + + # Inventory: Group by security group + if self.group_by_security_group: + try: + if instance.security_group: + key = self.to_safe("security_group_" + instance.security_group.name) + self.push(self.inventory, key, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'security_groups', key) + + except AttributeError: + self.fail_with_error('\n'.join(['Package boto seems a bit older.', + 'Please upgrade boto >= 2.3.0.'])) + # Inventory: Group by tag keys + if self.group_by_tag_keys: + for k, v in instance.tags.items(): + if self.expand_csv_tags and v and ',' in v: + values = map(lambda x: x.strip(), v.split(',')) + else: + values = [v] + + for v in values: + if v: + key = self.to_safe("tag_" + k + "=" + v) + else: + key = self.to_safe("tag_" + k) + self.push(self.inventory, key, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k)) + if v: + self.push_group(self.inventory, self.to_safe("tag_" + k), key) + + # Inventory: Group by engine + if self.group_by_rds_engine: + self.push(self.inventory, self.to_safe("rds_" + instance.engine), hostname) + if self.nested_groups: + self.push_group(self.inventory, 'rds_engines', self.to_safe("rds_" + instance.engine)) + + # Inventory: Group by parameter group + if self.group_by_rds_parameter_group: + self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), hostname) + if self.nested_groups: + self.push_group(self.inventory, 'rds_parameter_groups', self.to_safe("rds_parameter_group_" + instance.parameter_group.name)) + + # Global Tag: instances without tags + if self.group_by_tag_none and len(instance.tags) == 0: + self.push(self.inventory, 'tag_none', hostname) + if self.nested_groups: + self.push_group(self.inventory, 'tags', 'tag_none') + + # Global Tag: all RDS instances + self.push(self.inventory, 'rds', hostname) + + self.inventory["_meta"]["hostvars"][hostname] = self.get_host_info_dict_from_instance(instance) + self.inventory["_meta"]["hostvars"][hostname]['ansible_host'] = dest + + def add_elasticache_cluster(self, cluster, region): + ''' Adds an ElastiCache cluster to the inventory and index, as long as + it's nodes are addressable ''' + + # Only want available clusters unless all_elasticache_clusters is True + if not self.all_elasticache_clusters and cluster['CacheClusterStatus'] != 'available': + return + + # Select the best destination address + if 'ConfigurationEndpoint' in cluster and cluster['ConfigurationEndpoint']: + # Memcached cluster + dest = cluster['ConfigurationEndpoint']['Address'] + is_redis = False + else: + # Redis sigle node cluster + # Because all Redis clusters are single nodes, we'll merge the + # info from the cluster with info about the node + dest = cluster['CacheNodes'][0]['Endpoint']['Address'] + is_redis = True + + if not dest: + # Skip clusters we cannot address (e.g. private VPC subnet) + return + + # Add to index + self.index[dest] = [region, cluster['CacheClusterId']] + + # Inventory: Group by instance ID (always a group of 1) + if self.group_by_instance_id: + self.inventory[cluster['CacheClusterId']] = [dest] + if self.nested_groups: + self.push_group(self.inventory, 'instances', cluster['CacheClusterId']) + + # Inventory: Group by region + if self.group_by_region and not is_redis: + self.push(self.inventory, region, dest) + if self.nested_groups: + self.push_group(self.inventory, 'regions', region) + + # Inventory: Group by availability zone + if self.group_by_availability_zone and not is_redis: + self.push(self.inventory, cluster['PreferredAvailabilityZone'], dest) + if self.nested_groups: + if self.group_by_region: + self.push_group(self.inventory, region, cluster['PreferredAvailabilityZone']) + self.push_group(self.inventory, 'zones', cluster['PreferredAvailabilityZone']) + + # Inventory: Group by node type + if self.group_by_instance_type and not is_redis: + type_name = self.to_safe('type_' + cluster['CacheNodeType']) + self.push(self.inventory, type_name, dest) + if self.nested_groups: + self.push_group(self.inventory, 'types', type_name) + + # Inventory: Group by VPC (information not available in the current + # AWS API version for ElastiCache) + + # Inventory: Group by security group + if self.group_by_security_group and not is_redis: + + # Check for the existence of the 'SecurityGroups' key and also if + # this key has some value. When the cluster is not placed in a SG + # the query can return None here and cause an error. + if 'SecurityGroups' in cluster and cluster['SecurityGroups'] is not None: + for security_group in cluster['SecurityGroups']: + key = self.to_safe("security_group_" + security_group['SecurityGroupId']) + self.push(self.inventory, key, dest) + if self.nested_groups: + self.push_group(self.inventory, 'security_groups', key) + + # Inventory: Group by engine + if self.group_by_elasticache_engine and not is_redis: + self.push(self.inventory, self.to_safe("elasticache_" + cluster['Engine']), dest) + if self.nested_groups: + self.push_group(self.inventory, 'elasticache_engines', self.to_safe(cluster['Engine'])) + + # Inventory: Group by parameter group + if self.group_by_elasticache_parameter_group: + self.push(self.inventory, self.to_safe("elasticache_parameter_group_" + cluster['CacheParameterGroup']['CacheParameterGroupName']), dest) + if self.nested_groups: + self.push_group(self.inventory, 'elasticache_parameter_groups', self.to_safe(cluster['CacheParameterGroup']['CacheParameterGroupName'])) + + # Inventory: Group by replication group + if self.group_by_elasticache_replication_group and 'ReplicationGroupId' in cluster and cluster['ReplicationGroupId']: + self.push(self.inventory, self.to_safe("elasticache_replication_group_" + cluster['ReplicationGroupId']), dest) + if self.nested_groups: + self.push_group(self.inventory, 'elasticache_replication_groups', self.to_safe(cluster['ReplicationGroupId'])) + + # Global Tag: all ElastiCache clusters + self.push(self.inventory, 'elasticache_clusters', cluster['CacheClusterId']) + + host_info = self.get_host_info_dict_from_describe_dict(cluster) + + self.inventory["_meta"]["hostvars"][dest] = host_info + + # Add the nodes + for node in cluster['CacheNodes']: + self.add_elasticache_node(node, cluster, region) + + def add_elasticache_node(self, node, cluster, region): + ''' Adds an ElastiCache node to the inventory and index, as long as + it is addressable ''' + + # Only want available nodes unless all_elasticache_nodes is True + if not self.all_elasticache_nodes and node['CacheNodeStatus'] != 'available': + return + + # Select the best destination address + dest = node['Endpoint']['Address'] + + if not dest: + # Skip nodes we cannot address (e.g. private VPC subnet) + return + + node_id = self.to_safe(cluster['CacheClusterId'] + '_' + node['CacheNodeId']) + + # Add to index + self.index[dest] = [region, node_id] + + # Inventory: Group by node ID (always a group of 1) + if self.group_by_instance_id: + self.inventory[node_id] = [dest] + if self.nested_groups: + self.push_group(self.inventory, 'instances', node_id) + + # Inventory: Group by region + if self.group_by_region: + self.push(self.inventory, region, dest) + if self.nested_groups: + self.push_group(self.inventory, 'regions', region) + + # Inventory: Group by availability zone + if self.group_by_availability_zone: + self.push(self.inventory, cluster['PreferredAvailabilityZone'], dest) + if self.nested_groups: + if self.group_by_region: + self.push_group(self.inventory, region, cluster['PreferredAvailabilityZone']) + self.push_group(self.inventory, 'zones', cluster['PreferredAvailabilityZone']) + + # Inventory: Group by node type + if self.group_by_instance_type: + type_name = self.to_safe('type_' + cluster['CacheNodeType']) + self.push(self.inventory, type_name, dest) + if self.nested_groups: + self.push_group(self.inventory, 'types', type_name) + + # Inventory: Group by VPC (information not available in the current + # AWS API version for ElastiCache) + + # Inventory: Group by security group + if self.group_by_security_group: + + # Check for the existence of the 'SecurityGroups' key and also if + # this key has some value. When the cluster is not placed in a SG + # the query can return None here and cause an error. + if 'SecurityGroups' in cluster and cluster['SecurityGroups'] is not None: + for security_group in cluster['SecurityGroups']: + key = self.to_safe("security_group_" + security_group['SecurityGroupId']) + self.push(self.inventory, key, dest) + if self.nested_groups: + self.push_group(self.inventory, 'security_groups', key) + + # Inventory: Group by engine + if self.group_by_elasticache_engine: + self.push(self.inventory, self.to_safe("elasticache_" + cluster['Engine']), dest) + if self.nested_groups: + self.push_group(self.inventory, 'elasticache_engines', self.to_safe("elasticache_" + cluster['Engine'])) + + # Inventory: Group by parameter group (done at cluster level) + + # Inventory: Group by replication group (done at cluster level) + + # Inventory: Group by ElastiCache Cluster + if self.group_by_elasticache_cluster: + self.push(self.inventory, self.to_safe("elasticache_cluster_" + cluster['CacheClusterId']), dest) + + # Global Tag: all ElastiCache nodes + self.push(self.inventory, 'elasticache_nodes', dest) + + host_info = self.get_host_info_dict_from_describe_dict(node) + + if dest in self.inventory["_meta"]["hostvars"]: + self.inventory["_meta"]["hostvars"][dest].update(host_info) + else: + self.inventory["_meta"]["hostvars"][dest] = host_info + + def add_elasticache_replication_group(self, replication_group, region): + ''' Adds an ElastiCache replication group to the inventory and index ''' + + # Only want available clusters unless all_elasticache_replication_groups is True + if not self.all_elasticache_replication_groups and replication_group['Status'] != 'available': + return + + # Skip clusters we cannot address (e.g. private VPC subnet or clustered redis) + if replication_group['NodeGroups'][0]['PrimaryEndpoint'] is None or \ + replication_group['NodeGroups'][0]['PrimaryEndpoint']['Address'] is None: + return + + # Select the best destination address (PrimaryEndpoint) + dest = replication_group['NodeGroups'][0]['PrimaryEndpoint']['Address'] + + # Add to index + self.index[dest] = [region, replication_group['ReplicationGroupId']] + + # Inventory: Group by ID (always a group of 1) + if self.group_by_instance_id: + self.inventory[replication_group['ReplicationGroupId']] = [dest] + if self.nested_groups: + self.push_group(self.inventory, 'instances', replication_group['ReplicationGroupId']) + + # Inventory: Group by region + if self.group_by_region: + self.push(self.inventory, region, dest) + if self.nested_groups: + self.push_group(self.inventory, 'regions', region) + + # Inventory: Group by availability zone (doesn't apply to replication groups) + + # Inventory: Group by node type (doesn't apply to replication groups) + + # Inventory: Group by VPC (information not available in the current + # AWS API version for replication groups + + # Inventory: Group by security group (doesn't apply to replication groups) + # Check this value in cluster level + + # Inventory: Group by engine (replication groups are always Redis) + if self.group_by_elasticache_engine: + self.push(self.inventory, 'elasticache_redis', dest) + if self.nested_groups: + self.push_group(self.inventory, 'elasticache_engines', 'redis') + + # Global Tag: all ElastiCache clusters + self.push(self.inventory, 'elasticache_replication_groups', replication_group['ReplicationGroupId']) + + host_info = self.get_host_info_dict_from_describe_dict(replication_group) + + self.inventory["_meta"]["hostvars"][dest] = host_info + + def get_route53_records(self): + ''' Get and store the map of resource records to domain names that + point to them. ''' + + if self.boto_profile: + r53_conn = route53.Route53Connection(profile_name=self.boto_profile) + else: + r53_conn = route53.Route53Connection() + all_zones = r53_conn.get_zones() + + route53_zones = [zone for zone in all_zones if zone.name[:-1] not in self.route53_excluded_zones] + + self.route53_records = {} + + for zone in route53_zones: + rrsets = r53_conn.get_all_rrsets(zone.id) + + for record_set in rrsets: + record_name = record_set.name + + if record_name.endswith('.'): + record_name = record_name[:-1] + + for resource in record_set.resource_records: + self.route53_records.setdefault(resource, set()) + self.route53_records[resource].add(record_name) + + def get_instance_route53_names(self, instance): + ''' Check if an instance is referenced in the records we have from + Route53. If it is, return the list of domain names pointing to said + instance. If nothing points to it, return an empty list. ''' + + instance_attributes = ['public_dns_name', 'private_dns_name', + 'ip_address', 'private_ip_address'] + + name_list = set() + + for attrib in instance_attributes: + try: + value = getattr(instance, attrib) + except AttributeError: + continue + + if value in self.route53_records: + name_list.update(self.route53_records[value]) + + return list(name_list) + + def get_host_info_dict_from_instance(self, instance): + instance_vars = {} + for key in vars(instance): + value = getattr(instance, key) + key = self.to_safe('ec2_' + key) + + # Handle complex types + # state/previous_state changed to properties in boto in https://github.com/boto/boto/commit/a23c379837f698212252720d2af8dec0325c9518 + if key == 'ec2__state': + instance_vars['ec2_state'] = instance.state or '' + instance_vars['ec2_state_code'] = instance.state_code + elif key == 'ec2__previous_state': + instance_vars['ec2_previous_state'] = instance.previous_state or '' + instance_vars['ec2_previous_state_code'] = instance.previous_state_code + elif isinstance(value, (int, bool)): + instance_vars[key] = value + elif isinstance(value, six.string_types): + instance_vars[key] = value.strip() + elif value is None: + instance_vars[key] = '' + elif key == 'ec2_region': + instance_vars[key] = value.name + elif key == 'ec2__placement': + instance_vars['ec2_placement'] = value.zone + elif key == 'ec2_tags': + for k, v in value.items(): + if self.expand_csv_tags and ',' in v: + v = list(map(lambda x: x.strip(), v.split(','))) + key = self.to_safe('ec2_tag_' + k) + instance_vars[key] = v + elif key == 'ec2_groups': + group_ids = [] + group_names = [] + for group in value: + group_ids.append(group.id) + group_names.append(group.name) + instance_vars["ec2_security_group_ids"] = ','.join([str(i) for i in group_ids]) + instance_vars["ec2_security_group_names"] = ','.join([str(i) for i in group_names]) + elif key == 'ec2_block_device_mapping': + instance_vars["ec2_block_devices"] = {} + for k, v in value.items(): + instance_vars["ec2_block_devices"][os.path.basename(k)] = v.volume_id + else: + pass + # TODO Product codes if someone finds them useful + # print key + # print type(value) + # print value + + instance_vars[self.to_safe('ec2_account_id')] = self.aws_account_id + + return instance_vars + + def get_host_info_dict_from_describe_dict(self, describe_dict): + ''' Parses the dictionary returned by the API call into a flat list + of parameters. This method should be used only when 'describe' is + used directly because Boto doesn't provide specific classes. ''' + + # I really don't agree with prefixing everything with 'ec2' + # because EC2, RDS and ElastiCache are different services. + # I'm just following the pattern used until now to not break any + # compatibility. + + host_info = {} + for key in describe_dict: + value = describe_dict[key] + key = self.to_safe('ec2_' + self.uncammelize(key)) + + # Handle complex types + + # Target: Memcached Cache Clusters + if key == 'ec2_configuration_endpoint' and value: + host_info['ec2_configuration_endpoint_address'] = value['Address'] + host_info['ec2_configuration_endpoint_port'] = value['Port'] + + # Target: Cache Nodes and Redis Cache Clusters (single node) + if key == 'ec2_endpoint' and value: + host_info['ec2_endpoint_address'] = value['Address'] + host_info['ec2_endpoint_port'] = value['Port'] + + # Target: Redis Replication Groups + if key == 'ec2_node_groups' and value: + host_info['ec2_endpoint_address'] = value[0]['PrimaryEndpoint']['Address'] + host_info['ec2_endpoint_port'] = value[0]['PrimaryEndpoint']['Port'] + replica_count = 0 + for node in value[0]['NodeGroupMembers']: + if node['CurrentRole'] == 'primary': + host_info['ec2_primary_cluster_address'] = node['ReadEndpoint']['Address'] + host_info['ec2_primary_cluster_port'] = node['ReadEndpoint']['Port'] + host_info['ec2_primary_cluster_id'] = node['CacheClusterId'] + elif node['CurrentRole'] == 'replica': + host_info['ec2_replica_cluster_address_' + str(replica_count)] = node['ReadEndpoint']['Address'] + host_info['ec2_replica_cluster_port_' + str(replica_count)] = node['ReadEndpoint']['Port'] + host_info['ec2_replica_cluster_id_' + str(replica_count)] = node['CacheClusterId'] + replica_count += 1 + + # Target: Redis Replication Groups + if key == 'ec2_member_clusters' and value: + host_info['ec2_member_clusters'] = ','.join([str(i) for i in value]) + + # Target: All Cache Clusters + elif key == 'ec2_cache_parameter_group': + host_info["ec2_cache_node_ids_to_reboot"] = ','.join([str(i) for i in value['CacheNodeIdsToReboot']]) + host_info['ec2_cache_parameter_group_name'] = value['CacheParameterGroupName'] + host_info['ec2_cache_parameter_apply_status'] = value['ParameterApplyStatus'] + + # Target: Almost everything + elif key == 'ec2_security_groups': + + # Skip if SecurityGroups is None + # (it is possible to have the key defined but no value in it). + if value is not None: + sg_ids = [] + for sg in value: + sg_ids.append(sg['SecurityGroupId']) + host_info["ec2_security_group_ids"] = ','.join([str(i) for i in sg_ids]) + + # Target: Everything + # Preserve booleans and integers + elif isinstance(value, (int, bool)): + host_info[key] = value + + # Target: Everything + # Sanitize string values + elif isinstance(value, six.string_types): + host_info[key] = value.strip() + + # Target: Everything + # Replace None by an empty string + elif value is None: + host_info[key] = '' + + else: + # Remove non-processed complex types + pass + + return host_info + + def get_host_info(self): + ''' Get variables about a specific host ''' + + if len(self.index) == 0: + # Need to load index from cache + self.load_index_from_cache() + + if self.args.host not in self.index: + # try updating the cache + self.do_api_calls_update_cache() + if self.args.host not in self.index: + # host might not exist anymore + return self.json_format_dict({}, True) + + (region, instance_id) = self.index[self.args.host] + + instance = self.get_instance(region, instance_id) + return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True) + + def push(self, my_dict, key, element): + ''' Push an element onto an array that may not have been defined in + the dict ''' + group_info = my_dict.setdefault(key, []) + if isinstance(group_info, dict): + host_list = group_info.setdefault('hosts', []) + host_list.append(element) + else: + group_info.append(element) + + def push_group(self, my_dict, key, element): + ''' Push a group as a child of another group. ''' + parent_group = my_dict.setdefault(key, {}) + if not isinstance(parent_group, dict): + parent_group = my_dict[key] = {'hosts': parent_group} + child_groups = parent_group.setdefault('children', []) + if element not in child_groups: + child_groups.append(element) + + def get_inventory_from_cache(self): + ''' Reads the inventory from the cache file and returns it as a JSON + object ''' + + with open(self.cache_path_cache, 'r') as f: + json_inventory = f.read() + return json_inventory + + def load_index_from_cache(self): + ''' Reads the index from the cache file sets self.index ''' + + with open(self.cache_path_index, 'rb') as f: + self.index = json.load(f) + + def write_to_cache(self, data, filename): + ''' Writes data in JSON format to a file ''' + + json_data = self.json_format_dict(data, True) + with open(filename, 'w') as f: + f.write(json_data) + + def uncammelize(self, key): + temp = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', key) + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', temp).lower() + + def to_safe(self, word): + ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups ''' + regex = r"[^A-Za-z0-9\_" + if not self.replace_dash_in_groups: + regex += r"\-" + return re.sub(regex + "]", "_", word) + + def json_format_dict(self, data, pretty=False): + ''' Converts a dict to a JSON object and dumps it as a formatted + string ''' + + if pretty: + return json.dumps(data, sort_keys=True, indent=2) + else: + return json.dumps(data) + + +if __name__ == '__main__': + # Run the script + Ec2Inventory() \ No newline at end of file diff --git a/provisioning/group_vars/production/vars.yml b/provisioning/group_vars/production/vars.yml new file mode 100644 index 00000000..e4de58e7 --- /dev/null +++ b/provisioning/group_vars/production/vars.yml @@ -0,0 +1,60 @@ +--- +### Variables for Regluit Production Server ### +### Sensitive vars are references to actual values in vault.yml ### +### Use ansible-vault view vault.yml to see the secret values ### + +project_path: "/opt/regluit" +django_settings_module: "regluit.settings.prod" +virtualenv_name: "venv" +user_name: "ubuntu" +server_name: "m.unglue.it" + +wsgi_home: "/opt/regluit/venv" +wsgi_python_path: "/opt/regluit/venv/bin/python" + +# Branch to checkout +repo_version: "newfoundation" + +### Variables in settings.prod.py ### +mysql_db_name: "regluit" +mysql_db_user: "regluit" +mysql_db_pass: "password123" +mysql_db_host: "localhost" +mysql_db_port: 3306 +email_host: "{{ vault_email_host }}" +email_port: "{{ vault_email_port }}" +default_from_email: "notices@gluejar.com" +broker_transport: "redis" +broker_host: "localhost" +broker_port: 6379 +broker_vhost: "0" + +### Variables in common.py ### +common_keys: + booxtream_api_key: "{{ vault_booxtream_api_key }}" + booxtream_api_user: "{{ vault_booxtream_api_user }}" + dropbox_key: "{{ vault_dropbox_key }}" + github_public_token: "{{ vault_github_public_token }}" + mailchimp_api_key: "{{ vault_mailchimp_api_key }}" + mailchimp_news_id: "{{ vault_mailchimp_news_id }}" + mobigen_url: "{{ vault_mobigen_url }}" + mobigen_user_id: "{{ vault_mobigen_user_id }}" + mobigen_password: "{{ vault_mobigen_password }}" + +### Variables in host.py ### +host_keys: + secret_key: '{{ vault_secret_key }}' + google_books_api_key: "{{ vault_google_books_api_key }}" + goodreads_api_key: "{{ vault_goodreads_api_key }}" + goodreads_api_secret: "{{ vault_goodreads_api_secret }}" + email_host_user: '{{ vault_email_host_user }}' + email_host_password: '{{ vault_email_host_password }}' + social_auth_twitter_key: '{{ vault_social_auth_twitter_key }}' + social_auth_twitter_secret: '{{ vault_social_auth_twitter_secret }}' + social_auth_facebook_key: '{{ vault_social_auth_facebook_key }}' + social_auth_facebook_secret: '{{ vault_social_auth_facebook_secret }}' + social_auth_google_oauth2_key: '{{ vault_social_auth_google_oauth2_key }}' + social_auth_google_oauth2_secret: '{{ vault_social_auth_google_oauth2_secret }}' + aws_access_key_id: '{{ vault_aws_access_key_id }}' + aws_secret_access_key: '{{ vault_aws_secret_access_key }}' + aws_storage_bucket_name: '{{ vault_aws_storage_bucket_name }}' \ No newline at end of file diff --git a/provisioning/group_vars/production/vault.yml b/provisioning/group_vars/production/vault.yml new file mode 100644 index 00000000..8be63f37 --- /dev/null +++ b/provisioning/group_vars/production/vault.yml @@ -0,0 +1,92 @@ +$ANSIBLE_VAULT;1.1;AES256 +36653766626632633437383061323964393266313432373930623833333534336239323666336565 +3565323337323164646362303335353830646330666130300a366661323362356164343366613265 +35383637336431323965373730653539356530643839373866663330373066313337616330653966 +3939363433663434620a323761336338393463303230363931366130393866653462383561343234 +65383838626238366239326665306237373632666635653533333330643738326235643335663932 +33643238613365636233383437663262663466333463353938316166626130373663323864393665 +39646639393832383133373166373539343563343363633261646362373530643161646563396336 +35336362653339656231376166356463666361336437636431636663633530623036656131663035 +61656631626637313837646434356363373931336130626336363964656434366636616432303031 +63366532343463663732353330613264633734366162326631356631616161303238323062613035 +31373532646437316130333461636538333166346237323561323633333363373038323564316262 +30666666663966313332313561373236666636363366313631343730376362393833663739343630 +61313133393631323534623962633830343766316566613039323263643436626665393462626531 +34366232656535316533363234383466663335346465386333383563626131383032346434653639 +35656663613163383262643661306635633733336261613739316232636334373965353261663333 +61333338623039343166333339613464656663643765643236353561626331643134363661393631 +33343134376161333230326261656531386635643165653966393631636361663631353639353036 +64393336643364333239333830333166626638643031666237383466623563653336613763336434 +37383137643264616162373138313431393534376633333938383161663736633236303631666365 +30383430326430613131353839376162363839636364363865376231336336386232353333656135 +31336231366236386532633563663432383334336563323862393330343133346135336332303832 +38386232303036656164363963623363393063353437363031623638386336373363623765393636 +31303230666136366164623835366463316436363633303563386533623431313035323564626164 +38353166333038633634396534363137626536623862376231643332633762383065613161343538 +62646565396530623931653031626331383163333332373637393661353236396662363133326637 +35623531366239363336313164623035353339623265323734393437336137366330396362623664 +33366639666161373666663035373537656364313534663137316165363862623630643933353437 +31303131353063656139366466376564393230613235396333343230313639353734343731376333 +38636235306535316361663433333462383637633435306432623833396435356263663235373262 +64316462383064653733396633376564646665343432616231646332393032383463656263376338 +64303836366236613865353137366333643334313764383538633832363532346334663734633130 +65333664666163363838613063663139306435613038653133396363353532343865376462393835 +61366539383731613430636132306130626466306334343035646436336230376434383633326436 +36343338653961613566636230326665653733666139303537623531363439613931316331633131 +63663234663664343031326462326464393532363364383332626263616132386434333665303132 +36336332633030313061666637643532393833353465656531663939636538623864316561353266 +36366162343962383964396163653865633164363262373833383931373537623865373466626666 +66613466623734633131626161356136363864656366346437393638306431623531306133643861 +66303066633566653733386466326635303463383262626664643162323839323239303661316330 +37393634346166633730393038636234326338386565613461656636343837393934303666633836 +37663061633761636361316562383733623265373863363066383634356633383132623366643237 +37623430643933393663353130613933396331363336626530383364373432613436333633646537 +37643231353364333962616138386239366634356430303637323466346662633965346365653935 +32303763363631653037666634366165363739373436356562396565363438386538353730366634 +61613262633134323735353165363738336363626136306665613530306532386565336537303137 +30653364626233663165316263633466326638313166383539306437396139343863656337663161 +66303036326264353637646330323230616264343034303630323334363632353963303739383262 +33396331306631323065363334323633623461613934633932303962333965653564313164376530 +35613539376434373265663563326436653439323866326235313562643239303237373931656363 +64306533396261626562323838613236623031353932383638636565323035623566383063313663 +38656665663065346136666433373361373562626233333863633964346535366234643430336135 +61613439303030663339363436343462373531356165366433303437613961376137326236333139 +36336238376261336538663933306539313737333033633039626230336336383165346664323565 +64626334656663396462396134313634333362363537663566663137363863636434366361306364 +64316563616135616264316431396365353037316433313364653461616466303563626461353633 +63346238383964333533386264613637663865373833656334303938663661613035356335663036 +62326630646330396135313931366631323362653031623034623239386162383663376135653537 +38336165376365666166623937663930653636666230323565653966303361393436373637623833 +35663333613531623837336663373863616431343434373337626230646261366333666232626263 +36393130303739363834643737383532626536623662356539373264363730396639643864633931 +37313038343732653731383262353637386464396435356561656137613835656465306464626436 +66383837376236363965666237396534376566623536393437633031336461393666336466613962 +62656337356331366233316537626665633264666637633433303130653736343033313830326131 +31396531356139653466376661663961626637316130316661653963653938646362343862653136 +62303664623938313863636238636532376566303863633861663035333834303966636263636532 +64386539643230366139303961326230633366336534656536323463313366626137353261666539 +30333265653361643230643337616131663339373630663464656438373434363431356661343131 +37386435633137316538623964643731643936643562656565386164653032356530353166316437 +34346637336430373566366232323166386132383962616165633239366237646238366236343862 +34633733396635333762616436656365346232393637366630386634376636366531363535633032 +35303232636239663734396632363163366133396237646661346466303464323937323164663838 +35376661643662343932316635356631653865633765663035396366396666353965646365396236 +66633162323830653162306339383437363463663130633531336662303162663861323337396133 +36646363323438313337313031623330323230313063373634366162653533646365353864336233 +65653335333436316363393866303763353365386333326662353166333230316336353761343833 +32376435376237303966366230396232303236626361366637323666303732303934643231373630 +62313563633865336532616438396562363337306332643236333861616162353761666162636262 +63393464353166633666366466616465323531396366303337306262653339373435633135613436 +65323062623361326465313636663732346665323261383730313131636130626566363433623733 +62656637353564336237333139303338396262306462643337316163356638356365343838643030 +62613538363765613638643439333034653830653464663364613464346339643136353239393263 +33333766363137306365616230343963646338353263663964653739663239646232666233306339 +65336366306139363038333034313364633637366133303931613030313733343966323161353563 +62306134626166633339336139393630336166666637373364643037343232333035353830346136 +38623139326536386130363034333831393961316261643531313335316261396266666561356633 +31366636343733616230306563666239643134666363343831363731663439396234656365363535 +66363935356633303166316635663239313338333335623931663234616334373536616634363739 +62343739353733336663336535376161376132326462306166353963303030633764343663646261 +36376131663764313339316330316638653331376139616635363461623036386434373564323334 +63336465623166343261363262636665363838383866393234343631663136303265313430386432 +616437326539303165323962636534633063 diff --git a/provisioning/hosts b/provisioning/hosts new file mode 100644 index 00000000..58f71868 --- /dev/null +++ b/provisioning/hosts @@ -0,0 +1,2 @@ +[production] +regluit-prod ansible_host=ec2-18-206-73-169.compute-1.amazonaws.com ansible_user=ubuntu \ No newline at end of file diff --git a/provisioning/roles/regluit_common/defaults/main.yml b/provisioning/roles/regluit_common/defaults/main.yml new file mode 100644 index 00000000..a1f9926f --- /dev/null +++ b/provisioning/roles/regluit_common/defaults/main.yml @@ -0,0 +1,21 @@ +project_path: "/opt/regluit" +django_settings_module: "regluit.settings.me" +virtualenv_name: "venv" + +# MySQL +mysql_db_name: "regluit" +mysql_db_user: "regluit" +mysql_db_pass: "password123" +mysql_db_host: "localhost" +mysql_db_port: 3306 + +# Task Broker +broker_transport: "redis" +broker_host: "localhost" +broker_port: 6379 +broker_vhost: "0" + +# Common.py defaults +boxstream_api_key: "012345678901234567890123456789" +boxstream_api_user: "user" +dropbox_key: "012345678901234" diff --git a/provisioning/roles/regluit_common/files/celerybeat b/provisioning/roles/regluit_common/files/celerybeat new file mode 100644 index 00000000..34b9ad6a --- /dev/null +++ b/provisioning/roles/regluit_common/files/celerybeat @@ -0,0 +1,154 @@ +#!/bin/bash +# ========================================================= +# celerybeat - Starts the Celery periodic task scheduler. +# ========================================================= +# +# :Usage: /etc/init.d/celerybeat {start|stop|force-reload|restart|try-restart|status} +# :Configuration file: /etc/default/celerybeat or /etc/default/celeryd +# +# See http://docs.celeryq.org/en/latest/cookbook/daemonizing.html#init-script-celerybeat +# This file is copied from https://github.com/ask/celery/blob/2.4/contrib/generic-init.d/celerybeat + +### BEGIN INIT INFO +# Provides: celerybeat +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery periodic task scheduler +### END INIT INFO + +# Cannot use set -e/bash -e since the kill -0 command will abort +# abnormally in the absence of a valid process ID. +#set -e + +DEFAULT_PID_FILE="/var/run/celerybeat.pid" +DEFAULT_LOG_FILE="/var/log/celerybeat.log" +DEFAULT_LOG_LEVEL="INFO" +DEFAULT_CELERYBEAT="celerybeat" + +# /etc/init.d/ssh: start and stop the celery task worker daemon. + +if test -f /etc/default/celeryd; then + . /etc/default/celeryd +fi + +if test -f /etc/default/celerybeat; then + . /etc/default/celerybeat +fi + +CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT} +CELERYBEAT_PID_FILE=${CELERYBEAT_PID_FILE:-${CELERYBEAT_PIDFILE:-$DEFAULT_PID_FILE}} +CELERYBEAT_LOG_FILE=${CELERYBEAT_LOG_FILE:-${CELERYBEAT_LOGFILE:-$DEFAULT_LOG_FILE}} +CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} + +export CELERY_LOADER + +CELERYBEAT_OPTS="$CELERYBEAT_OPTS -f $CELERYBEAT_LOG_FILE -l $CELERYBEAT_LOG_LEVEL" + +if [ -n "$2" ]; then + CELERYBEAT_OPTS="$CELERYBEAT_OPTS $2" +fi + +CELERYBEAT_LOG_DIR=`dirname $CELERYBEAT_LOG_FILE` +CELERYBEAT_PID_DIR=`dirname $CELERYBEAT_PID_FILE` +if [ ! -d "$CELERYBEAT_LOG_DIR" ]; then + mkdir -p $CELERYBEAT_LOG_DIR +fi +if [ ! -d "$CELERYBEAT_PID_DIR" ]; then + mkdir -p $CELERYBEAT_PID_DIR +fi + +# Extra start-stop-daemon options, like user/group. +if [ -n "$CELERYBEAT_USER" ]; then + DAEMON_OPTS="$DAEMON_OPTS --uid $CELERYBEAT_USER" + chown "$CELERYBEAT_USER" $CELERYBEAT_LOG_DIR $CELERYBEAT_PID_DIR +fi +if [ -n "$CELERYBEAT_GROUP" ]; then + DAEMON_OPTS="$DAEMON_OPTS --gid $CELERYBEAT_GROUP" + chgrp "$CELERYBEAT_GROUP" $CELERYBEAT_LOG_DIR $CELERYBEAT_PID_DIR +fi + +CELERYBEAT_CHDIR=${CELERYBEAT_CHDIR:-$CELERYD_CHDIR} +if [ -n "$CELERYBEAT_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir $CELERYBEAT_CHDIR" +fi + + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 1 + fi +} + +wait_pid () { + pid=$1 + forever=1 + i=0 + while [ $forever -gt 0 ]; do + kill -0 $pid 1>/dev/null 2>&1 + if [ $? -eq 1 ]; then + echo "OK" + forever=0 + else + kill -TERM "$pid" + i=$((i + 1)) + if [ $i -gt 60 ]; then + echo "ERROR" + echo "Timed out while stopping (30s)" + forever=0 + else + sleep 0.5 + fi + fi + done +} + + +stop_beat () { + echo -n "Stopping celerybeat... " + if [ -f "$CELERYBEAT_PID_FILE" ]; then + wait_pid $(cat "$CELERYBEAT_PID_FILE") + else + echo "NOT RUNNING" + fi +} + +start_beat () { + echo "Starting celerybeat..." + if [ -n "$VIRTUALENV" ]; then + source $VIRTUALENV/bin/activate + fi + $CELERYBEAT $CELERYBEAT_OPTS $DAEMON_OPTS --detach \ + --pidfile="$CELERYBEAT_PID_FILE" +} + + + +case "$1" in + start) + check_dev_null + start_beat + ;; + stop) + stop_beat + ;; + reload|force-reload) + echo "Use start+stop" + ;; + restart) + echo "Restarting celery periodic task scheduler" + stop_beat + check_dev_null + start_beat + ;; + + *) + echo "Usage: /etc/init.d/celerybeat {start|stop|restart}" + exit 1 +esac + +exit 0 \ No newline at end of file diff --git a/provisioning/roles/regluit_common/files/celeryd b/provisioning/roles/regluit_common/files/celeryd new file mode 100644 index 00000000..12ff8445 --- /dev/null +++ b/provisioning/roles/regluit_common/files/celeryd @@ -0,0 +1,217 @@ +#!/bin/bash +# ============================================ +# celeryd - Starts the Celery worker daemon. +# ============================================ +# +# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status} +# +# :Configuration file: /etc/default/celeryd +# +# To configure celeryd you probably need to tell it where to chdir. +# +# EXAMPLE CONFIGURATION +# ===================== +# +# this is an example configuration for a Python project: +# +# /etc/default/celeryd: +# +# # List of nodes to start +# CELERYD_NODES="worker1 worker2 worker3"k +# # ... can also be a number of workers +# CELERYD_NODES=3 +# +# # Where to chdir at start. +# CELERYD_CHDIR="/opt/Myproject/" +# +# # Extra arguments to celeryd +# CELERYD_OPTS="--time-limit=300" +# +# # Name of the celery config module.# +# CELERY_CONFIG_MODULE="celeryconfig" +# +# EXAMPLE DJANGO CONFIGURATION +# ============================ +# +# # Where the Django project is. +# CELERYD_CHDIR="/opt/Project/" +# +# # Name of the projects settings module. +# export DJANGO_SETTINGS_MODULE="settings" +# +# # Path to celeryd +# CELERYD="/opt/Project/manage.py celeryd" +# +# AVAILABLE OPTIONS +# ================= +# +# * CELERYD_NODES +# +# A space separated list of nodes, or a number describing the number of +# nodes, to start +# +# * CELERYD_OPTS +# Additional arguments to celeryd-multi, see `celeryd-multi --help` +# and `celeryd --help` for help. +# +# * CELERYD_CHDIR +# Path to chdir at start. Default is to stay in the current directory. +# +# * CELERYD_PIDFILE +# Full path to the pidfile. Default is /var/run/celeryd.pid. +# +# * CELERYD_LOGFILE +# Full path to the celeryd logfile. Default is /var/log/celeryd.log +# +# * CELERYD_LOG_LEVEL +# Log level to use for celeryd. Default is INFO. +# +# * CELERYD +# Path to the celeryd program. Default is `celeryd`. +# You can point this to an virtualenv, or even use manage.py for django. +# +# * CELERYD_USER +# User to run celeryd as. Default is current user. +# +# * CELERYD_GROUP +# Group to run celeryd as. Default is current user. + +# VARIABLE EXPANSION +# ================== +# +# The following abbreviations will be expanded +# +# * %n -> node name +# * %h -> host name + + +### BEGIN INIT INFO +# Provides: celeryd +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery task worker daemon +### END INIT INFO + +#set -e + +DEFAULT_PID_FILE="/var/run/celeryd@%n.pid" +DEFAULT_LOG_FILE="/var/log/celeryd@%n.log" +DEFAULT_LOG_LEVEL="INFO" +DEFAULT_NODES="celery" +DEFAULT_CELERYD="-m celery.bin.celeryd_detach" + +# /etc/init.d/celeryd: start and stop the celery task worker daemon. + +CELERY_DEFAULTS=${CELERY_DEFAULTS:-"/etc/default/celeryd"} + +test -f "$CELERY_DEFAULTS" && . "$CELERY_DEFAULTS" +if [ -f "/etc/default/celeryd" ]; then + . /etc/default/celeryd +fi + +if [ -f $VIRTUALENV_ACTIVATE ]; then + echo "activating virtualenv $VIRTUALENV_ACTIVATE" + source "$VIRTUALENV_ACTIVATE" +fi + +CELERYD_PID_FILE=${CELERYD_PID_FILE:-${CELERYD_PIDFILE:-$DEFAULT_PID_FILE}} +CELERYD_LOG_FILE=${CELERYD_LOG_FILE:-${CELERYD_LOGFILE:-$DEFAULT_LOG_FILE}} +CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} +CELERYD_MULTI=${CELERYD_MULTI:-"celeryd-multi"} +CELERYD=${CELERYD:-$DEFAULT_CELERYD} +CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES} + +export CELERY_LOADER + +if [ -n "$2" ]; then + CELERYD_OPTS="$CELERYD_OPTS $2" +fi + +# Extra start-stop-daemon options, like user/group. +if [ -n "$CELERYD_USER" ]; then + DAEMON_OPTS="$DAEMON_OPTS --uid=$CELERYD_USER" +fi +if [ -n "$CELERYD_GROUP" ]; then + DAEMON_OPTS="$DAEMON_OPTS --gid=$CELERYD_GROUP" +fi + +if [ -n "$CELERYD_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir=\"$CELERYD_CHDIR\"" +fi + + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 1 + fi +} + + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + + +stop_workers () { + $CELERYD_MULTI stop $CELERYD_NODES --pidfile="$CELERYD_PID_FILE" +} + + +start_workers () { + $CELERYD_MULTI start $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + --cmd="$CELERYD" \ + $CELERYD_OPTS +} + + +restart_workers () { + $CELERYD_MULTI restart $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + --cmd="$CELERYD" \ + $CELERYD_OPTS +} + + + +case "$1" in + start) + check_dev_null + start_workers + ;; + + stop) + check_dev_null + stop_workers + ;; + + reload|force-reload) + echo "Use restart" + ;; + + status) + celeryctl status + ;; + + restart) + check_dev_null + restart_workers + ;; + + try-restart) + check_dev_null + restart_workers + ;; + + *) + echo "Usage: /etc/init.d/celeryd {start|stop|restart|try-restart|kill}" + exit 1 + ;; +esac + +exit 0 diff --git a/provisioning/roles/regluit_common/tasks/celery.yml b/provisioning/roles/regluit_common/tasks/celery.yml new file mode 100644 index 00000000..c2f52036 --- /dev/null +++ b/provisioning/roles/regluit_common/tasks/celery.yml @@ -0,0 +1,37 @@ +--- +# Tasks for Celeryd and Celerybeat processes +- name: Create /var/log/celery + become: true + file: + path: "/var/log/celery" + state: directory + #owner: celery + #group: celery + mode: 0775 + +- name: Create /var/run/celery + become: true + file: + path: "/var/run/celery" + state: directory + #owner: celery + #group: celery + mode: 0775 + +- name: Copy celery init.d scripts + become: true + copy: + src: "{{ item }}" + dest: "/etc/init.d/{{ item }}" + with_items: + - 'celeryd' + - 'celerybeat' + +- name: Copy celery config files + become: true + template: + src: "celery/{{ item }}.j2" + dest: "/etc/default/{{ item }}" + with_items: + - 'celeryd' + - 'celerybeat' diff --git a/provisioning/roles/regluit_common/tasks/main.yml b/provisioning/roles/regluit_common/tasks/main.yml new file mode 100644 index 00000000..84350481 --- /dev/null +++ b/provisioning/roles/regluit_common/tasks/main.yml @@ -0,0 +1,85 @@ +--- +# Need to install python2.7 and pip first so Ansible will function +# This is due to Ubuntu 16 shipping with Python3 by default +- name: Install python2.7 and pip + become: true + raw: bash -c "apt -qqy update && apt install -qqy python2.7-dev python-pip" + register: output + changed_when: output.stdout != "" + +- name: Gathering Facts + setup: + +- name: Install base regluit dependencies + become: true + apt: + name: "{{ item }}" + update_cache: true + state: present + with_items: + - 'git' + - 'python-setuptools' + - 'python-lxml' + - 'build-essential' + - 'libssl-dev' + - 'libffi-dev' + - 'libxml2-dev' + - 'libxslt-dev' + - 'mysql-server' + - 'mysql-client' + - 'libmysqlclient-dev' + - 'python-mysqldb' + +- name: Install virtualenv + pip: + name: "virtualenv" + state: present + +- name: Install python packages to virtualenv + pip: + requirements: "{{ project_path }}/requirements_versioned.pip" + state: present + virtualenv: "{{ project_path }}/venv" + +- name: Add project to PYTHONPATH of virtualenv + template: + src: "{{ item }}.j2" + dest: "{{ project_path }}/venv/lib/python2.7/site-packages/{{ item }}" + with_items: + - 'regluit.pth' + - 'opt.pth' + +- name: Create keys directory + file: + path: "{{ project_path}}/settings/keys" + state: directory + +- name: Copy keys files + copy: + src: "{{ project_path }}/settings/dummy/__init__.py" + dest: "{{ project_path }}/settings/keys/__init__.py" + remote_src: yes + +- name: Copy django settings template + template: + src: me.py.j2 + dest: "{{ project_path }}/settings/me.py" + +- name: Copy key templates to keys directory + template: + src: "{{ item }}.j2" + dest: "{{ project_path }}/settings/keys/{{ item }}" + with_items: + - 'common.py' + - 'host.py' + +- name: MySQL setup + become: true + import_tasks: mysql.yml + +- name: Redis setup + become: true + import_tasks: redis.yml + +# - name: Celery setup +# import_tasks: celery.yml diff --git a/provisioning/roles/regluit_common/tasks/mysql.yml b/provisioning/roles/regluit_common/tasks/mysql.yml new file mode 100644 index 00000000..ca5f6171 --- /dev/null +++ b/provisioning/roles/regluit_common/tasks/mysql.yml @@ -0,0 +1,12 @@ +--- +- name: Create MySQL database + mysql_db: + name: "{{ mysql_db_name }}" + state: present + +- name: Create MySQL user + mysql_user: + name: "{{ mysql_db_user }}" + password: "{{ mysql_db_pass }}" + priv: '*.*:ALL' + state: present diff --git a/provisioning/roles/regluit_common/tasks/redis.yml b/provisioning/roles/regluit_common/tasks/redis.yml new file mode 100644 index 00000000..ca646847 --- /dev/null +++ b/provisioning/roles/regluit_common/tasks/redis.yml @@ -0,0 +1,13 @@ +--- +- name: Install Redis server + become: yes + apt: + name: "redis-server" + state: present + +- name: Ensure Redis is started + become: yes + service: + name: "redis-server" + state: started + enabled: yes diff --git a/provisioning/roles/regluit_common/templates/celery/celerybeat.j2 b/provisioning/roles/regluit_common/templates/celery/celerybeat.j2 new file mode 100644 index 00000000..d459ad8b --- /dev/null +++ b/provisioning/roles/regluit_common/templates/celery/celerybeat.j2 @@ -0,0 +1,35 @@ +# http://docs.celeryproject.org/en/latest/cookbook/daemonizing.html#generic-initd-celerybeat-example +# to be placed at /etc/defaults/celerybeat + +# Where to chdir at start. +CELERYBEAT_CHDIR="{{ project_path }}t/" + +# Extra arguments to celerybeat +#CELERYBEAT_OPTS="--schedule=/var/run/celerybeat-schedule" + +# Name of the celery config module.# +CELERY_CONFIG_MODULE="celeryconfig" + +# Name of the projects settings module. +export DJANGO_SETTINGS_MODULE="{{ django_settings_module }}" + +# Path to celerybeat +CELERYBEAT="{{ project_path }}/{{ virtualenv_name }}/bin/django-admin.py celerybeat" + +# virtualenv to use +VIRTUALENV="{{ project_path }}/{{ virtualenv_name }}" + +#Full path to the PID file. Default is /var/run/celeryd.pid +CELERYBEAT_PIDFILE="/var/log/celerybeat/celerybeat.pid" + +#Full path to the celeryd log file. Default is /var/log/celeryd.log +CELERYBEAT_LOGFILE="/var/log/celerybeat/celerybeat.log" + +#Log level to use for celeryd. Default is INFO. +CELERYBEAT_LOG_LEVEL="INFO" + +#User to run celeryd as. Default is current user. +#CELERYBEAT_USER + +#Group to run celeryd as. Default is current user. +#CELERYBEAT_GROUP diff --git a/provisioning/roles/regluit_common/templates/celery/celeryd.j2 b/provisioning/roles/regluit_common/templates/celery/celeryd.j2 new file mode 100644 index 00000000..c918efb4 --- /dev/null +++ b/provisioning/roles/regluit_common/templates/celery/celeryd.j2 @@ -0,0 +1,9 @@ +CELERYD_NODES="w1" +CELERYD_CHDIR="{{ project_path }}/" +CELERYD_LOG_FILE="/var/log/celery/%n.log" +CELERYD_PID_FILE="/var/log/celery/%n.pid" +CELERYD="{{ project_path }}/{{ virtualenv_name }}/bin/django-admin.py celeryd" +CELERYD_MULTI="{{ project_path }}/{{ virtualenv_name }}/bin/django-admin.py celeryd_multi" + +VIRTUALENV_ACTIVATE="{{ project_path }}/{{ virtualenv_name }}/bin/activate" +export DJANGO_SETTINGS_MODULE="{{ django_settings_module }}" diff --git a/provisioning/roles/regluit_common/templates/common.py.j2 b/provisioning/roles/regluit_common/templates/common.py.j2 new file mode 100644 index 00000000..d835f1c8 --- /dev/null +++ b/provisioning/roles/regluit_common/templates/common.py.j2 @@ -0,0 +1,13 @@ +import os + +# all the COMMON_KEYS +# copy this file to settings/keys/ and replace the dummy values with real ones +BOOXTREAM_API_KEY = os.environ.get('BOOXTREAM_API_KEY', '{{ boxstream_api_key }}') +BOOXTREAM_API_USER = os.environ.get('BOOXTREAM_API_USER', '{{ boxstream_api_user }}') +DROPBOX_KEY = os.environ.get('DROPBOX_KEY', '{{ dropbox_key }}') +GITHUB_PUBLIC_TOKEN = os.environ.get('GITHUB_PUBLIC_TOKEN', None) # 40 chars; null has lower limit +MAILCHIMP_API_KEY = os.environ.get('MAILCHIMP_API_KEY', '-us2') # [32chars]-xx# +MAILCHIMP_NEWS_ID = os.environ.get('MAILCHIMP_NEWS_ID', '0123456789') +MOBIGEN_PASSWORD = os.environ.get('MOBIGEN_PASSWORD', '012345678901234') +MOBIGEN_URL = os.environ.get('MOBIGEN_URL', '') # https://host/mobigen +MOBIGEN_USER_ID = os.environ.get('MOBIGEN_USER_ID', 'user') diff --git a/provisioning/roles/regluit_common/templates/host.py.j2 b/provisioning/roles/regluit_common/templates/host.py.j2 new file mode 100644 index 00000000..9819dc7f --- /dev/null +++ b/provisioning/roles/regluit_common/templates/host.py.j2 @@ -0,0 +1,47 @@ +# host.py +# copy this file to settings/keys/ and replace the dummy values with real ones +# or generate it from the ansible vault +import os + +# you can use this to generate a key: http://www.miniwebtool.com/django-secret-key-generator/ +SECRET_KEY = os.environ.get("SECRET_KEY", '01234567890123456789012345678901234567890123456789') + +# you'll need to register a GoogleBooks API key +# https://code.google.com/apis/console +GOOGLE_BOOKS_API_KEY = os.environ.get("GOOGLE_BOOKS_API_KEY", "012345678901234567890123456789012345678") + +# +GOODREADS_API_KEY = os.environ.get("GOODREADS_API_KEY", "01234567890123456789") +GOODREADS_API_SECRET = os.environ.get("GOODREADS_API_SECRET", None) #43 chars + +# Amazon SES +# create with https://console.aws.amazon.com/ses/home?region=us-east-1#smtp-settings: +EMAIL_HOST_USER = os.environ.get("EMAIL_HOST_USER", '01234567890123456789') +EMAIL_HOST_PASSWORD = os.environ.get("EMAIL_HOST_PASSWORD", '01234567890123456789012345678901234567890123') + +# twitter auth +# you'll need to create a new Twitter application to fill in these blanks +# https://dev.twitter.com/apps/new +SOCIAL_AUTH_TWITTER_KEY = os.environ.get("SOCIAL_AUTH_TWITTER_KEY", '0123456789012345678901234') +SOCIAL_AUTH_TWITTER_SECRET = os.environ.get("SOCIAL_AUTH_TWITTER_SECRET", '01234567890123456789012345678901234567890123456789') + +# support@icontact.nl +BOOXTREAM_API_KEY = os.environ.get("BOOXTREAM_API_KEY", None) # 30 chars +BOOXTREAM_API_USER = os.environ.get("BOOXTREAM_API_USER", 'user') + +# you'll need to create a new Facebook application to fill in these blanks +# https://developers.facebook.com/apps/ +SOCIAL_AUTH_FACEBOOK_KEY = os.environ.get("SOCIAL_AUTH_FACEBOOK_KEY", '012345678901234') +SOCIAL_AUTH_FACEBOOK_SECRET = os.environ.get("SOCIAL_AUTH_FACEBOOK_SECRET", '01234567890123456789012345678901') + +# https://console.developers.google.com/apis/credentials/oauthclient/ +# unglue.it (prod) SOCIAL_AUTH_GOOGLE_OAUTH2_KEY #2 +SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = os.environ.get("_KEY", '012345678901-01234567890123456789012345678901.apps.googleusercontent.com') +SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = os.environ.get("_SECRET", '012345678901234567890123') + +AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID", '01234567890123456789') +AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", '') # 40 chars + +DATABASE_USER = os.environ.get("DATABASE_USER", 'root') +DATABASE_PASSWORD = os.environ.get("DATABASE_PASSWORD", '') +DATABASE_HOST = os.environ.get("DATABASE_HOST", '') diff --git a/provisioning/roles/regluit_common/templates/me.py.j2 b/provisioning/roles/regluit_common/templates/me.py.j2 new file mode 100644 index 00000000..b82ede1b --- /dev/null +++ b/provisioning/roles/regluit_common/templates/me.py.j2 @@ -0,0 +1,90 @@ +# coding=utf-8 +from .common import * +try: + from .keys.host import * +except ImportError: + from .dummy.host import * + +DEBUG = True +TEMPLATES[0]['OPTIONS']['debug'] = DEBUG + +# if you're doing development work, you'll want this to be zero +IS_PREVIEW = False + +# SITE_ID for your particular site -- must be configured in /core/fixtures/initial_data.json +SITE_ID = 3 + +ADMINS = ( + ('Raymond Yee', 'rdhyee+ungluebugs@gluejar.com'), + ('Eric Hellman', 'eric@gluejar.com'), +) + +MANAGERS = ADMINS + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': '{{ mysql_db_name }}', + 'USER': '{{ mysql_db_user }}', + 'PASSWORD': '{{ mysql_db_pass }}', + 'HOST': '{{ mysql_db_host }}', + 'PORT': '{{ mysql_db_port }} ', + 'TEST_CHARSET': 'utf8', + } +} + +STATIC_ROOT = '/var/www/static' +CKEDITOR_UPLOAD_PATH = '/var/www/static/media/' + + +TIME_ZONE = 'America/New_York' + +# settings for outbout email +# if you have a gmail account you can use your email address and password + +EMAIL_USE_TLS = True +EMAIL_HOST = 'smtp.gmail.com' +# EMAIL_HOST_USER is in keys/host +# EMAIL_HOST_PASSWORD is in keys/host +EMAIL_PORT = 587 +DEFAULT_FROM_EMAIL = 'info@ebookfoundation.org' + +# for use with test google account only +GOOGLE_DISPLAY_NAME = 'Unglue.It' +REDIRECT_IS_HTTPS = False + + +#BASE_URL = 'http://0.0.0.0' +BASE_URL_SECURE = 'https://0.0.0.0' + +# use redis as queuing service +BROKER_TRANSPORT = "{{ broker_transport }}" +BROKER_HOST = "{{ broker_host }}" +BROKER_PORT = {{ broker_port }} +BROKER_VHOST = "{{ broker_vhost }}" + +# send celery log to Python logging +CELERYD_HIJACK_ROOT_LOGGER = False + +# a debug_toolbar setting +INTERNAL_IPS = ('127.0.0.1',) + +CELERYD_LOG_LEVEL = "INFO" + +# decide which of the period tasks to add to the schedule +#CELERYBEAT_SCHEDULE['send_test_email'] = SEND_TEST_EMAIL_JOB +#CELERYBEAT_SCHEDULE['refresh_acqs'] = REFRESH_ACQS_JOB + +# if you're doing development work, you'll want this to be zero +IS_PREVIEW = False + +# username, password to pass to LIVE_SERVER_TEST_URL + +UNGLUEIT_TEST_USER = None +UNGLUEIT_TEST_PASSWORD = None + +# local settings for maintenance mode +MAINTENANCE_MODE = False + +# assume that CSS will get generated on dev +SASS_OUTPUT_STYLE = 'compressed' diff --git a/provisioning/roles/regluit_common/templates/opt.pth.j2 b/provisioning/roles/regluit_common/templates/opt.pth.j2 new file mode 100644 index 00000000..27d47a41 --- /dev/null +++ b/provisioning/roles/regluit_common/templates/opt.pth.j2 @@ -0,0 +1 @@ +/opt/ diff --git a/provisioning/roles/regluit_common/templates/regluit.pth.j2 b/provisioning/roles/regluit_common/templates/regluit.pth.j2 new file mode 100644 index 00000000..8964ac81 --- /dev/null +++ b/provisioning/roles/regluit_common/templates/regluit.pth.j2 @@ -0,0 +1 @@ +{{ project_path }}/ diff --git a/provisioning/roles/regluit_dev/defaults/main.yml b/provisioning/roles/regluit_dev/defaults/main.yml new file mode 100644 index 00000000..3194c5f1 --- /dev/null +++ b/provisioning/roles/regluit_dev/defaults/main.yml @@ -0,0 +1,5 @@ +django_settings_module: "regluit.settings.me" +project_path: "/opt/regluit" +virtualenv_name: "venv" +django_server_ip: "0.0.0.0" +django_server_port: 8000 diff --git a/provisioning/roles/regluit_dev/tasks/main.yml b/provisioning/roles/regluit_dev/tasks/main.yml new file mode 100644 index 00000000..a702d772 --- /dev/null +++ b/provisioning/roles/regluit_dev/tasks/main.yml @@ -0,0 +1,69 @@ +--- + +- name: Install dev dependencies + become: true + apt: + name: "{{ item }}" + update_cache: true + state: present + with_items: + - 'git' + - 'python-setuptools' + - 'python-lxml' + - 'build-essential' + - 'libssl-dev' + - 'libffi-dev' + - 'libxml2-dev' + - 'libxslt-dev' + - 'mysql-server' + - 'mysql-client' + - 'libmysqlclient-dev' + - 'python-mysqldb' + +- name: Migrate databse + django_manage: + app_path: "{{ project_path }}" + command: "migrate --noinput" + virtualenv: "{{ project_path }}/venv" + settings: "{{ django_settings_module }}" + +- name: Import fixtures + django_manage: + app_path: "{{ project_path }}" + command: "loaddata" + virtualenv: "{{ project_path }}/venv" + settings: "{{ django_settings_module }}" + fixtures: "core/fixtures/initial_data.json core/fixtures/bookloader.json" + +- name: Start Celery Worker + django_manage: + app_path: "{{ project_path }}" + command: "celery worker --detach --loglevel=INFO" + virtualenv: "{{ project_path }}/venv" + settings: "{{ django_settings_module }}" + +- name: Start Celery Beat + django_manage: + app_path: "{{ project_path }}" + command: "celery beat --detach --loglevel=INFO" + virtualenv: "{{ project_path }}/venv" + settings: "{{ django_settings_module }}" + +- name: Copy activation script + template: + src: "activate_venv.sh.j2" + dest: "/home/{{ ansible_user }}/activate_venv.sh" + owner: "{{ ansible_user }}" + mode: "u=rx,g=rx,o=rwx" + +- name: Source activation script in bash profile + blockinfile: + path: "/home/{{ ansible_user }}/.profile" + block: | + if [ -f ~/activate_venv.sh ]; then + source ~/activate_venv.sh + fi + marker: "# {mark} SOURCE REGLUIT ACTIVATION SCRIPT ON LOGIN" + +- debug: + msg: "Successfully provisioned regluit development environment." diff --git a/provisioning/roles/regluit_dev/templates/activate_venv.sh.j2 b/provisioning/roles/regluit_dev/templates/activate_venv.sh.j2 new file mode 100644 index 00000000..75fb3609 --- /dev/null +++ b/provisioning/roles/regluit_dev/templates/activate_venv.sh.j2 @@ -0,0 +1,7 @@ +#!/bin/bash +cd {{ project_path }} +source {{ virtualenv_name }}/bin/activate +echo Local setup of Regluit complete! +echo To start the django development server, run: +echo ./manage.py runserver {{ django_server_ip }}:{{ django_server_port }} +echo Then leave this session running and access the site on your host machine at http://127.0.0.1:{{ django_server_port }} diff --git a/provisioning/roles/regluit_prod/files/celerybeat b/provisioning/roles/regluit_prod/files/celerybeat new file mode 100644 index 00000000..34b9ad6a --- /dev/null +++ b/provisioning/roles/regluit_prod/files/celerybeat @@ -0,0 +1,154 @@ +#!/bin/bash +# ========================================================= +# celerybeat - Starts the Celery periodic task scheduler. +# ========================================================= +# +# :Usage: /etc/init.d/celerybeat {start|stop|force-reload|restart|try-restart|status} +# :Configuration file: /etc/default/celerybeat or /etc/default/celeryd +# +# See http://docs.celeryq.org/en/latest/cookbook/daemonizing.html#init-script-celerybeat +# This file is copied from https://github.com/ask/celery/blob/2.4/contrib/generic-init.d/celerybeat + +### BEGIN INIT INFO +# Provides: celerybeat +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery periodic task scheduler +### END INIT INFO + +# Cannot use set -e/bash -e since the kill -0 command will abort +# abnormally in the absence of a valid process ID. +#set -e + +DEFAULT_PID_FILE="/var/run/celerybeat.pid" +DEFAULT_LOG_FILE="/var/log/celerybeat.log" +DEFAULT_LOG_LEVEL="INFO" +DEFAULT_CELERYBEAT="celerybeat" + +# /etc/init.d/ssh: start and stop the celery task worker daemon. + +if test -f /etc/default/celeryd; then + . /etc/default/celeryd +fi + +if test -f /etc/default/celerybeat; then + . /etc/default/celerybeat +fi + +CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT} +CELERYBEAT_PID_FILE=${CELERYBEAT_PID_FILE:-${CELERYBEAT_PIDFILE:-$DEFAULT_PID_FILE}} +CELERYBEAT_LOG_FILE=${CELERYBEAT_LOG_FILE:-${CELERYBEAT_LOGFILE:-$DEFAULT_LOG_FILE}} +CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} + +export CELERY_LOADER + +CELERYBEAT_OPTS="$CELERYBEAT_OPTS -f $CELERYBEAT_LOG_FILE -l $CELERYBEAT_LOG_LEVEL" + +if [ -n "$2" ]; then + CELERYBEAT_OPTS="$CELERYBEAT_OPTS $2" +fi + +CELERYBEAT_LOG_DIR=`dirname $CELERYBEAT_LOG_FILE` +CELERYBEAT_PID_DIR=`dirname $CELERYBEAT_PID_FILE` +if [ ! -d "$CELERYBEAT_LOG_DIR" ]; then + mkdir -p $CELERYBEAT_LOG_DIR +fi +if [ ! -d "$CELERYBEAT_PID_DIR" ]; then + mkdir -p $CELERYBEAT_PID_DIR +fi + +# Extra start-stop-daemon options, like user/group. +if [ -n "$CELERYBEAT_USER" ]; then + DAEMON_OPTS="$DAEMON_OPTS --uid $CELERYBEAT_USER" + chown "$CELERYBEAT_USER" $CELERYBEAT_LOG_DIR $CELERYBEAT_PID_DIR +fi +if [ -n "$CELERYBEAT_GROUP" ]; then + DAEMON_OPTS="$DAEMON_OPTS --gid $CELERYBEAT_GROUP" + chgrp "$CELERYBEAT_GROUP" $CELERYBEAT_LOG_DIR $CELERYBEAT_PID_DIR +fi + +CELERYBEAT_CHDIR=${CELERYBEAT_CHDIR:-$CELERYD_CHDIR} +if [ -n "$CELERYBEAT_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir $CELERYBEAT_CHDIR" +fi + + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 1 + fi +} + +wait_pid () { + pid=$1 + forever=1 + i=0 + while [ $forever -gt 0 ]; do + kill -0 $pid 1>/dev/null 2>&1 + if [ $? -eq 1 ]; then + echo "OK" + forever=0 + else + kill -TERM "$pid" + i=$((i + 1)) + if [ $i -gt 60 ]; then + echo "ERROR" + echo "Timed out while stopping (30s)" + forever=0 + else + sleep 0.5 + fi + fi + done +} + + +stop_beat () { + echo -n "Stopping celerybeat... " + if [ -f "$CELERYBEAT_PID_FILE" ]; then + wait_pid $(cat "$CELERYBEAT_PID_FILE") + else + echo "NOT RUNNING" + fi +} + +start_beat () { + echo "Starting celerybeat..." + if [ -n "$VIRTUALENV" ]; then + source $VIRTUALENV/bin/activate + fi + $CELERYBEAT $CELERYBEAT_OPTS $DAEMON_OPTS --detach \ + --pidfile="$CELERYBEAT_PID_FILE" +} + + + +case "$1" in + start) + check_dev_null + start_beat + ;; + stop) + stop_beat + ;; + reload|force-reload) + echo "Use start+stop" + ;; + restart) + echo "Restarting celery periodic task scheduler" + stop_beat + check_dev_null + start_beat + ;; + + *) + echo "Usage: /etc/init.d/celerybeat {start|stop|restart}" + exit 1 +esac + +exit 0 \ No newline at end of file diff --git a/provisioning/roles/regluit_prod/files/celeryd b/provisioning/roles/regluit_prod/files/celeryd new file mode 100644 index 00000000..12ff8445 --- /dev/null +++ b/provisioning/roles/regluit_prod/files/celeryd @@ -0,0 +1,217 @@ +#!/bin/bash +# ============================================ +# celeryd - Starts the Celery worker daemon. +# ============================================ +# +# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status} +# +# :Configuration file: /etc/default/celeryd +# +# To configure celeryd you probably need to tell it where to chdir. +# +# EXAMPLE CONFIGURATION +# ===================== +# +# this is an example configuration for a Python project: +# +# /etc/default/celeryd: +# +# # List of nodes to start +# CELERYD_NODES="worker1 worker2 worker3"k +# # ... can also be a number of workers +# CELERYD_NODES=3 +# +# # Where to chdir at start. +# CELERYD_CHDIR="/opt/Myproject/" +# +# # Extra arguments to celeryd +# CELERYD_OPTS="--time-limit=300" +# +# # Name of the celery config module.# +# CELERY_CONFIG_MODULE="celeryconfig" +# +# EXAMPLE DJANGO CONFIGURATION +# ============================ +# +# # Where the Django project is. +# CELERYD_CHDIR="/opt/Project/" +# +# # Name of the projects settings module. +# export DJANGO_SETTINGS_MODULE="settings" +# +# # Path to celeryd +# CELERYD="/opt/Project/manage.py celeryd" +# +# AVAILABLE OPTIONS +# ================= +# +# * CELERYD_NODES +# +# A space separated list of nodes, or a number describing the number of +# nodes, to start +# +# * CELERYD_OPTS +# Additional arguments to celeryd-multi, see `celeryd-multi --help` +# and `celeryd --help` for help. +# +# * CELERYD_CHDIR +# Path to chdir at start. Default is to stay in the current directory. +# +# * CELERYD_PIDFILE +# Full path to the pidfile. Default is /var/run/celeryd.pid. +# +# * CELERYD_LOGFILE +# Full path to the celeryd logfile. Default is /var/log/celeryd.log +# +# * CELERYD_LOG_LEVEL +# Log level to use for celeryd. Default is INFO. +# +# * CELERYD +# Path to the celeryd program. Default is `celeryd`. +# You can point this to an virtualenv, or even use manage.py for django. +# +# * CELERYD_USER +# User to run celeryd as. Default is current user. +# +# * CELERYD_GROUP +# Group to run celeryd as. Default is current user. + +# VARIABLE EXPANSION +# ================== +# +# The following abbreviations will be expanded +# +# * %n -> node name +# * %h -> host name + + +### BEGIN INIT INFO +# Provides: celeryd +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery task worker daemon +### END INIT INFO + +#set -e + +DEFAULT_PID_FILE="/var/run/celeryd@%n.pid" +DEFAULT_LOG_FILE="/var/log/celeryd@%n.log" +DEFAULT_LOG_LEVEL="INFO" +DEFAULT_NODES="celery" +DEFAULT_CELERYD="-m celery.bin.celeryd_detach" + +# /etc/init.d/celeryd: start and stop the celery task worker daemon. + +CELERY_DEFAULTS=${CELERY_DEFAULTS:-"/etc/default/celeryd"} + +test -f "$CELERY_DEFAULTS" && . "$CELERY_DEFAULTS" +if [ -f "/etc/default/celeryd" ]; then + . /etc/default/celeryd +fi + +if [ -f $VIRTUALENV_ACTIVATE ]; then + echo "activating virtualenv $VIRTUALENV_ACTIVATE" + source "$VIRTUALENV_ACTIVATE" +fi + +CELERYD_PID_FILE=${CELERYD_PID_FILE:-${CELERYD_PIDFILE:-$DEFAULT_PID_FILE}} +CELERYD_LOG_FILE=${CELERYD_LOG_FILE:-${CELERYD_LOGFILE:-$DEFAULT_LOG_FILE}} +CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} +CELERYD_MULTI=${CELERYD_MULTI:-"celeryd-multi"} +CELERYD=${CELERYD:-$DEFAULT_CELERYD} +CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES} + +export CELERY_LOADER + +if [ -n "$2" ]; then + CELERYD_OPTS="$CELERYD_OPTS $2" +fi + +# Extra start-stop-daemon options, like user/group. +if [ -n "$CELERYD_USER" ]; then + DAEMON_OPTS="$DAEMON_OPTS --uid=$CELERYD_USER" +fi +if [ -n "$CELERYD_GROUP" ]; then + DAEMON_OPTS="$DAEMON_OPTS --gid=$CELERYD_GROUP" +fi + +if [ -n "$CELERYD_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir=\"$CELERYD_CHDIR\"" +fi + + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 1 + fi +} + + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + + +stop_workers () { + $CELERYD_MULTI stop $CELERYD_NODES --pidfile="$CELERYD_PID_FILE" +} + + +start_workers () { + $CELERYD_MULTI start $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + --cmd="$CELERYD" \ + $CELERYD_OPTS +} + + +restart_workers () { + $CELERYD_MULTI restart $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + --cmd="$CELERYD" \ + $CELERYD_OPTS +} + + + +case "$1" in + start) + check_dev_null + start_workers + ;; + + stop) + check_dev_null + stop_workers + ;; + + reload|force-reload) + echo "Use restart" + ;; + + status) + celeryctl status + ;; + + restart) + check_dev_null + restart_workers + ;; + + try-restart) + check_dev_null + restart_workers + ;; + + *) + echo "Usage: /etc/init.d/celeryd {start|stop|restart|try-restart|kill}" + exit 1 + ;; +esac + +exit 0 diff --git a/provisioning/roles/regluit_prod/files/certs/STAR_unglue_it.ca-bundle b/provisioning/roles/regluit_prod/files/certs/STAR_unglue_it.ca-bundle new file mode 100644 index 00000000..9258132a --- /dev/null +++ b/provisioning/roles/regluit_prod/files/certs/STAR_unglue_it.ca-bundle @@ -0,0 +1,210 @@ +$ANSIBLE_VAULT;1.1;AES256 +30636262636630653738383536613136363733643931316362326266656433376333386239373962 +6635343138326335313430623566656130663136393531310a343562653666623235336539316561 +31386163663632383436343735376266356134386335363637383536613531323231626535313236 +6163326630306530360a663636393463303062316331623630353235383137333831333239393134 +64353830366463376130633831633835313034643930636166396436373063623039316465613964 +61333538656264353735613034356533353331313735393965363164616331306538383231303932 +31373838363862343931643034643863313738636238376634363430613365356136323237613530 +34323339323337383539363462303863663163373465373262636161346662333738663532316363 +65313464333363643461353539306662353364643663373032346531633362373230653061646436 +34393462326137336163316361363937323738663632386662643338626433626433633235323838 +65653036643961626131383264393138316263366534656663386264376638393639636164323661 +61646166376637396131356339313162343762636234643236316363643063326638366536656534 +31616164313164373336666634356639653831623966333739356462656137363161663635353136 +30653665353334646165663333643863643539373331613263386561353030363862666264353933 +64643439336236386631393934623839626163373463653134656539376262316233626130363663 +65636534303431616631633534643030346534633635393563366266356138383938636264383632 +33616365333863393733646565643236613064616462633661303630613963666236663265636361 +61376336346262346632313833633735356364636239366439393265613866316439643732326330 +38396332363666656636646164646262303162656331636661393738626263383563646261323036 +63363634616362323965616639373961333966303366363937333233356631393938383339613262 +39393461316563383131666539333938313437336462376233353464303739356463323265366665 +64316161333431353264336236373466613230363237316633386661303538666536366462323666 +32373164626330313665653133356563383436663765666437636138343163663533646331346631 +31643638373537653533353036653766663935663636663561343239383666333661343830353133 +34363363333466393761643863393436386231626134663236326335336438633338313963646363 +30623262386437313733353038353231353163326133643139633963303864633739383935363664 +61336265656666643037316139306534323135333763383230633261336139636362396539643539 +34303937656634653862383665356530643064383832363163343331383538623236366535393061 +38373738393064376539343837356335326132363135323935333964383635383965656462383636 +63663661626632396663383838353633663839303936646537363266346661373033663064326435 +62646130376163623363346232346530376638613934623831303636366365313331613436626337 +61346262333933363366653166326330663638643230373364653161333263323035366263623330 +66636532616234636263323366626432363337326564663531323663356166303930653661313030 +32363133313162313331623138356432653630663263623935303739663363323130646133613735 +66333933393266663265333036616465353338656664313230643761303439363035663566393930 +31303633396665316330623962643439313132633531656362333630653537393061316663356566 +39376230643830333035316439346336313464623238373739623636313236316162356337313339 +62376533663138613962306331313066383439373039636631633431316232643266326361666134 +64383437636561323964346134663032663466333831663836393039353562663065643337303330 +62356666353662386439353563643735633730613463666263666539323333613130633765393834 +32343439306636306131386537373164363636386330623534373535323631396464626661323361 +38353935623566373834643661616631373333303735393835643737626234643337653130396365 +65666332323665356330396238633866653761346531303439666236653264323661626135353238 +36643562623266636333623831303439336366646133653332313130336436386537653234666435 +31343863376233623335366333363232653966316230363836613231613130613166643864373565 +35356264383335623539643630323534623265313631383130643739306533663330396434653236 +64376166373639363731663437663331346338373539363637353933386239373738373565353030 +64353261366232643061356165323961383537626239623732393936623461393961353837633733 +34356166393938346232396532353166373534623530646335633338633636336636646630356561 +32363061356632313661386535306339636135623834333231303433343037353633613365326333 +37666235323239383661323536353830383833326565346564663631336465313664346533323337 +38613961313233623033386166336235383362393463393832646538373139333533313266303937 +62346532356632316136633533373665373839623336383435353163386531366466646664326565 +30303639393933373361623333326432366332396137396238336464376131323631343637306531 +37343933373139333333623233386334636436356432303764653235353133366566383031633136 +34616133336663623664616432336265323534373163316338386561623864623861343835633062 +34343738616134366164356337663566326462303933386139643734376332313066613230326363 +66623837323238356366333836386662366435626236333835346662623631373636663033623733 +38316639343638313937353636616638396261333561313032383130393538663366636630376539 +63353566613835343964356635303939343764396631393738646432313765666564316363626431 +65346637656137363266346266616666316361383231343666643063353131386163356638346636 +30383932633264373031373935616230323933373863663564363233313232363066326662303165 +63346532313837396365353365636663313334626433303732326164363330303438636333636163 +35656537623165323666633435386465303063633534656630306664343637336164626433373361 +65376530333361303337623634353434373164626133626263613561613436303161626432626239 +37343536313531626233343530633634613166633735613233393962323165623166366161356336 +35386635366631326266343432386161663033323962313763343163663137643631393166333031 +39346564656665343835616464663231333562616661643234326663633563303964386131656363 +63666363356562383830646633396238623537336439326162376135633665653165326536353037 +37313362666531333837633439643165356534383265643637363933366463343839333961376235 +63333063643634623735363233646334633235373132646139313365303336646264633061643131 +32616465633131356130396465643235353530326564373136313133613064313562306333313965 +38396237653430353533353462373130303262663564653361313465353934353864656235653135 +34613639613633633234386639633230653537393965323363623338656465326531346432333637 +64343436373436356535333865643865373539393036626137626538626635653637666362356162 +30313632396539623962346562646563316264613232343730643565386135346664646363333330 +66656661636235633730393063343634636138383264336535326533356231626530613565336161 +35626232343436616535373230653039303065343432343738616363626635383364373266323333 +33376364666638313564366237356635653731623566626661373961663736666638343061646134 +35313438356531383433636565326365313063383834613837666330396331353738323237363333 +63363963666465313137386537356462663636323737306362326362363266343062626130616230 +64363433343738343632636633373330386464396364383031343531663432363563653032336332 +65366364333133386139636165303964366165326336316534323636633462356335633066643165 +34613030323234383965396233373333393336653237613062323530353762633931343333323865 +30663862373132633834353039653032373737313564333638333762303335636265646162313931 +66393662653933363832623061633634626661356437663331373937396631356131333462656666 +32303166396533643261646130306161326561326533626531663038643565376439633339333432 +64333437366566666366303230616539663564333463666631646163356235636333363633663639 +37666332346161376239656137306430623463653466626663636539666532373565373731613165 +62663532383165656432313762376235333331343738646532373265353066363262333261626137 +31396332383963353066336666366133336138653363613863653833653964663164313831333065 +35333738383232656634356564643634346633313137333863363766643433363631333630643731 +65363161303235323830376638303135613630663362653566363965313436323033643366306630 +39303935636639333961666135633962653362666536663333623430356164376262346662643635 +33323663333832316633373035386334623134326536366236636561373635353366386237323933 +62393435363235396262653065656562383063663236323134313833363833396131643533666235 +37386366343933376338336533316239383430343566626434303537653431626261363438336334 +35363961656332336230386535386531363761303766313563333832633033646461393635376339 +39633630336366373663666239623030326331346361323132363565366366616236346230636632 +38376534363933383965386338313835356135663035326536343532643930383466633538316538 +31353038656533336335376265626139333364633537633662653362393666346163386431326261 +35393261353230373034633666663638306134643066643533323761666264326162343630376661 +65613733343866306364333665353639646263323863663562333965636335613966323564373535 +39363939383031393265613035376635663962386530663639356331393830383130626439346666 +31393532316431393666616535363430636637366134366636643730636431343034336539616135 +38633063333063383766386265336363386362383136653536616663376463333935356162343930 +36383831666537313936393033363738666335653739646431363039373939626538343037633631 +39383439373336336561663431626265666434646565356438336530643863656464323034353565 +36366563323836336162373465313938646239373637613066333238323533346338353834636631 +30333262376638336537623466643736653562336336663436333963366464666131303862313335 +61353862656435653939343362393934303633303333366231646431656234656638613033383133 +31353262353231663633633063663537353536383138366132646364353661363135336333323133 +64613231626662666364313232333633386265353935363065333933666165633862393965326637 +63623962616164626237303466623565623032666431383633323233633861326132346263386631 +64386364653633363965356664636434393031383663393137343733343732363239633436663636 +64346237653966363538303632363461663030396237623661356438653562396232313935626164 +64326162363264323831663965383639393937613931663239316663323164306537326363376339 +33333063373464363065336132613230643831303937373163386263646562313361306133306639 +36636264373638326663613236353439383532323032393431373333626330313235623362643137 +30636462643536363638613936383931363663343964363038373834343164633062343265666137 +30373734396232343936326330623362316636616535643136313765356439323934386364643631 +34316630366332366432373739363637623437363132646330326361363366313633343634383132 +34626633343163613136343665373931363065306232313336356461373366623934306563383935 +35373930633334343864306165653034386133393636386662303737303136613432333531313461 +62323065343037386231656335326364363761396532346661343664636637633334383131613833 +34306465643133383239306534383035303363353766613631643464383264656536323866326464 +38666665613166653638633039666261326135363833393832373564333765353134636138343266 +64373536313666353332613535633061363835363839323430333430643863343461643236333937 +62346262396331613536343539653766323561343639643766653535353335383736333464613331 +61336337613737356236636333393236653263663065353431633537666161373638663861363863 +36393364616566343561393364313531356134626431646261346363343939616332313631636331 +31393435373265313836653533323237613336333061306330636665626233613537383932316235 +35646633653664313834376466306335653465613866393939323265316436333062646662656462 +36313861306533326163306266353538396436386365393464613335646432366461326331623664 +35616665666337663163663631393539663731376333393339373065643363326535643563383532 +63656466623939376632636433316661613730386363363930613830386638653837323938373030 +34653565386564333866336337316164663931396630333634336130383465643061633239656638 +66666166303835326365353433636130653038643938653365333431646561306536623037666231 +30333830376637653932613961353931363537336462643030376537386539623636303038363862 +34373036393931303465626563623664663532666166336334653234346561303538313036343565 +61363734333737393131643661326331383036333363653333373536373134306436646139636561 +61386462323832653061313034616532643265343636313366346537356164346461313436373637 +34316663356135636231613330323431316566323361363636633436653864633564366366383339 +61633266343336643430623138326436383361636161326264346365643536343838333138646563 +65383230373361663461383236313832366334303632313935313263646266393030313238613334 +66626631323033383334636561313666653165356135366435613438323662353135633234346662 +33333336646166643862396361383636306431353439333562633533303265303266613662306162 +34383062333166393739656365303738373361373436343064323636336634353232383232356334 +35633561383735316663383333303334633631346339323562353634396533343636653963653462 +61393039303366626563393737306336376437666138393562373933333335633435396632333232 +39666435383832663462626563646162666565393530393164373963386161666436343263316639 +30343064663133616135613164653937366636353062303139366261313165373362643365316663 +33653630633732623466343236646136663730663938333234613163663835393036363161386332 +62383833396164313132633535323530633937626437383462663866346163623234663564306533 +65326362636433333930623663653031313438363930373462663639366434633665656138303866 +35626333336532346663366530303061343835376239643466326361613030613730326130633165 +39333465393466336231333335616662386238346537633164666636333133306266613064343165 +39653963346133333234353266323065656237303233623264376233616334623665323433623632 +63316262306337386535653233336635616438356565343433393662653036663736663663666435 +33333433343566623331333961613662306236386439386135653138333638366566313534383934 +62626261303530306237653637363933393634663330336462393932353230313730336362376134 +35343530363431323366613361643636663961366238636537616237633965666639633265376437 +63363930353064346664323435313136316132373237356332383537623032633831353563326230 +65626166336166363164613130393165356139353439353436666537346362326635386136313464 +34383633336636376137643733313637373535643966376435353339363535653139313739313362 +65343434343364666639306663663665356533323635316534316561313263663662623764326138 +38386434316535653262616530643164663335323436386365313361656430616535346463643765 +64613231623333363835386337643334333039363835343731316165346438363234666134343263 +63663036663566373734656661613432623761393964313166393532396664326230363230303161 +36373364663137373234383134363966633935666365303430343539663461623561383236626137 +66623964333766323534316439643666323666336539393566316331363862666561656634326166 +64383236633638663163666236633531346330643066353566336337646265623361326261373862 +32626538316631303465396436643236376161613232666339373537383438633630316365363065 +31653433326333613431303936333137323632383163323561313636396331376664656365326536 +39323632623861333466333530643864616166623261346237356431326335343431373164633366 +34363932656234613431306135323434353164323461346262656337336137616430386533313933 +65306566373634363933616233316562643930666161643234323864336631343032326462623365 +65363036633061643463343637313933396436636333663861393333353732383736633534303864 +62376339343938383639626237373961623931616264353264353337333133396564623831643338 +39653962623766643239393061643938643066386564613932656565306564303066316165336230 +62363330653137616666343333323636663462643264656364396138626265623964663730636561 +31356339323934366637623165623039323439333434653336623962643734653436376462386232 +64393863353164373734313262383638393961623466653034626666353361393230623933373336 +35323661333330393835363064396564613766626665623866346334643463663865323264386661 +33663561616131616134333564343334343765373061393131376231653536353163643435613564 +32376466363230363930393734663433313133646238396366356361663636343362313237303766 +63613538383836646333366664313637333537303333373537646130393631656636623461383533 +31333661376236323736666136393062643566313131383165663965656637303634366330666539 +61336265356662336236626432643738343038396263393665396337633837303761613332623436 +33313235323938333333666437613731376562376233353664313362356433383938376436373863 +32633131386462333838323536333439323733393266376632386530366363633337323963353437 +37653334626434616433643930363765366439613236373731343339613237633962356531303433 +38333261613062323963366131633738613463363364653531663639626234383263633963336635 +65343330636330353363356134363138333031393838623635343335393462343036316638623861 +30323638373032653331616361383338333761386264623435656561623337303863323536633932 +61613964363538313435363231613834323333336364353962346265646233613935313632343738 +65303732376561643763336162333962353136663136613466666631323063646432323130616266 +65663431626562646537366661393638346538336538666466343261396431383232356238353062 +37653031653766333931393763346366356563393261363438363535303163303962306630306665 +62346633356463653465363965396164323437346132396138316562316237303033396233636336 +30623039326132393534633937623363656666663366643130633161646566386135303939383762 +33356334636565643761323638346162373636323433306436303331336432333830623739643237 +64646432366466326139306663356438613433366534616462383938646239643262373836363032 +61623735666238626435316530363833366135633563663530643631623539333832343561323138 +32383138303464393265373665333736303861646464373861613730653861353463323837383737 +30356437653833376631303636393739636663663936656663393130616637303332393636363431 +38663837393231326536643033376663316138363234306637613135396365616636656236646335 +33633962626135343564633337663936393031316133646334303635653130626131646531396134 +38343631613562393435326139366330373064393839636138323466646533303538 diff --git a/provisioning/roles/regluit_prod/files/certs/STAR_unglue_it.crt b/provisioning/roles/regluit_prod/files/certs/STAR_unglue_it.crt new file mode 100644 index 00000000..3dd020d3 --- /dev/null +++ b/provisioning/roles/regluit_prod/files/certs/STAR_unglue_it.crt @@ -0,0 +1,100 @@ +$ANSIBLE_VAULT;1.1;AES256 +66366138323934613133623237633539616236326462373461303832393739313466373236323765 +3461353265343631356335643139363335356262346238360a383136303237393662303762393766 +61346362386338663037396631633932373834303265383662356539323766393466656564633465 +3835623862356266390a646463633764653431353537643265363764653934656430646363373335 +32343135303266623365656532633061373564623664616564343638636138623433363964303366 +63323939396365363539366562626135626662336236313963613233633637313731313465363636 +39386237393034383263346164623036646535623463306330663034383632373836333661666435 +61656662353762653266363036373430343333653835646365613835303935396230363032636164 +64613730663339316261313664633636613763313631653839313465336562633663306563633561 +38613661333766366633333463376162306365323330356339613266326331353866316638363237 +33646165313865313431666163613234366133616263396630303362333336323638373131396334 +39656534616331643530646530316335633730323166373830353262366465306631636339316266 +32303162313438373531616439356563303030383136613531353561316234353632646232396433 +38643463333836326435303733373239626562363264653532323334356262346133633361373963 +64396564313630393164323231313937643435613234376436386563623435633666616331643530 +39366536616165303562663638313739353763656134316132616162303161623130616263306366 +65613264623935333134383733353637653336636532383165646338303633353330623231383239 +39343834643038656539623162353561646364393162323839643533333363616437393239313034 +61356134643031356536653262663833336361653632626364336565616163376238326334663661 +35623338393730636237616161383032353762383965633962343330353235643363346633313462 +36653365346263323062653239373132323734626363623337393635373738663931656363383136 +62613836356562363866623436393131323130306636356235343035333534326331343337383431 +38613565383365666632326238363165313631373262336234666434313065363363346636653339 +36363432306639303266366665643934346562663934666665343030396233666534633438396332 +31323662363130616338333233373961316639633436313737353530646135373533613433353034 +62346432623261346334623738663835666639616564373961643439336432316365666665393135 +38623936616361313634333339353133633165663936616332323938396533393235636435376439 +62633537633136386366313934373263383730633334343636373035316638343334356530613530 +63396438343139383439666539383531386437313865303864316437363563663065643266353138 +66323133646463653066323466653662653736306162636565326663386362366332303761653564 +31383133396131663563343339623563363239363763643530383833353263646264363565656535 +64343737663261653530623836356430666462633964353832663964396462363430623336646336 +65356566653938376132663230333033376261376538306266643565613561373636383532616139 +30396564613564333964303262656162313839663435666539393734376639653562643735393037 +34613864353764303661653561383466663730333932663139616164333239633961326632346230 +30316138626562346434643033303333333234396533626633333437316636323062643035623664 +38626434366261326663393839343765363133623339373738313563653736373565336164356435 +37623837343437623833623137373662653934363133633366636436663831653737376631376431 +34363461323535323337343632653430333961343165633864346132343938313361373864363565 +32663734626631323930613638323133396135313562643536343038366233373136653330626239 +65313933636136303365353466646533353236343934356330356161343433643139383764393134 +32356331653831633832376262356238336634353362373837646563653835613634356463653466 +30383361356631616538333565646435656361313839346165376231643633643634313863663137 +61333236313436383464663439366637643663356535613861343831663737623364303339643733 +36373461323130333639613235373961313736303436666361613134323265386165616237393164 +34366530363034376437393866333861646636396434636631343033633565396164663833623331 +32666161636163333266393361383838653333326437303235326565306663356366613430303237 +65343937653439353334343834303234643136656565366461393838653739336233613234616437 +62643065356462306565363964663332663564313734336239633833306135613662633535666662 +36616639323332353864383733393561666464313466393535373961613831313463306461663266 +37653230663732393061616365656638623830346536373932636461663532343532366566343436 +63376365643061623839393139333661643439376363396564363237346461393264663131616162 +32396464363231613231363561373334646362636536343732396364653132663664366434313538 +39396262333232383934396335653461346338316564303563626363646136633266393937343434 +34396234363362336232653136666639613466663833303062356264633461643932613162303031 +33663539366332643538353632336433393564366565323034393531626236353432623531383033 +64613435623761623562363563323364396166353265363039663932626431666339376632353539 +63656330626438653366396635353236663762666439613237623638663731306331313564333239 +37363663366134303037376433323735366266613831356635663932626162343639306366386439 +61643831336139303133336366366165303934646362303665643135386462353565303932383930 +33393334353235306534366339613066633434393038636331366265336163306334323638393264 +64313538316632373430656532623532313063613337626165353365303832373566316131666336 +62353062343565633836306235343134323433303633346130363362633263343564616535396263 +65666666646231336636616364646565303538343361653932356135393161316335343333653439 +61373266633033653931623631643430613137393633393063353833306463663630333434616462 +33653230613639616531666335336131373636323065616239313733323664623138313964303930 +35366665626362303161396562306639613435396331366363616138613735383030366362396337 +66356666643131393433343237346533383335376462383566643035616438396366633737393133 +66323630656539363237646663363764363536373266383036636166333834366663613863663533 +62393235666666653433613665376431306439616138633533656362323436613231303264303764 +30336665336661393866343935313465626264303139353632356137663439373232326339393136 +64306365353639626565653965666133346364613538333135653831663032613135653263353965 +63373532323062653630646363323063383065646562313539373637623463313333313535323866 +32363631613035653637663935313535626333333433623735663439383239373231613037303736 +39633436383361356633373037333362363861346263313038373131653938663930666538306432 +34386163316163623064376132653062333535396631393363393265303964356431663439636234 +64636138333031656364656565346163663533353333353666373466373734633263626439316230 +65313031396564613163333364376637303539343563613133613133376239623066303139393866 +32623135396134396439653061386561666433613536336566613530656530333433343537623866 +31333364636631336264636531343365633264633433643661353164653831343836323763343033 +37386131306563626437343034333330623932303639646631613239303331646364323465616562 +62353030633765393237653161636165363465336136333264373832616537356531313066643263 +38346331383061373133353132646562313630643335376565396662653830656538356165356137 +31396232326666613236313961636334383962373533613566333930313866373334613064376561 +65313565303630326531613131316463386636623830346131656532363632613032303466623334 +33353633663730306638623566326533363065663363373537353130393938313936383763663966 +34336535663565633162313732376237336463343833303939353965653665356563326534643033 +34343638666463363462363235343731346461336235306264643866366235663961363461373930 +63326334323363656263616531373861633936636339623835323936373661373364336233316631 +66366462643663316162383438376162663065633333353138363836353331343162396636623130 +62383461626534386664633166323764303631373731366133633930643232303934666535393237 +35373462313561353561643866623761313839316662373134363431373562373062313430613462 +37333635313037656433656363313864653037313732656232323639383838326263323564643263 +38373337353635353836366136326334343661393064316139373431313330653966323763306337 +62333066323961623535336232303234636138306630313732373364613363663434633734346231 +64663930633162313937343130343961356437353366306338663636366631326366663938363531 +63353863306137653237656330656237356563346337343262363664313339323863616438373537 +63643235306137363264383238316434313331336337356364366261313835656562643566666130 +633536646634326262663866316566376638 diff --git a/provisioning/roles/regluit_prod/files/certs/server.key b/provisioning/roles/regluit_prod/files/certs/server.key new file mode 100644 index 00000000..2e3e98f8 --- /dev/null +++ b/provisioning/roles/regluit_prod/files/certs/server.key @@ -0,0 +1,89 @@ +$ANSIBLE_VAULT;1.1;AES256 +32373237336239343336393066383464393861636235623430343533636365306430323738643234 +3538336532623734333038333832393735343363363566340a643638636634653735656635336338 +39396461646661353061373636313032333065373562373036336232346133313334333862353034 +6337653431323235650a663536386561366531623363303963353539326339393034313961393362 +30306235623461326365646365373835333235303965336635363361366336303236333237383539 +61383438316464336338633163366532323435616563343463623266346632393930353332626137 +34396538653134373433333461373337333639393530383738373764353731363264373835626364 +35393163333462343562303136313264653764333038343838656562653133353934336666643534 +63366264393462643065323865303033636133663164366334313564333262633736626263363338 +39376164343330666332333534313634633137653432343232656464363066356336623166303030 +34653432366563663432343164636665373139346435396135303736653030653930393963663538 +36333636623832633630386235623365363065343936363031613165383534353837353231373164 +37373038363361633330653633396562323738663739316634666137306161313233633561383064 +30323236643635663766393862343363653866303431346535626565356162353433623132363031 +62656332383835386136656534636135346234326561623037396239303034353961396134396334 +35303463366439363336363666623064623564323032646337366332666536373064353962373461 +63306166616535356231396566316433633062376164303434323639376339643731393461393536 +65633363616235343532386435323736313138666661666434643935623266323133336463326532 +33326364346364626165666262396133343736656335353732666362613234343163343465326361 +33383136636364383466363666316131383366623665323561636130343337613539383134646535 +31383131623237636433313133336162633537653533383435336461616439343035336164333363 +31623638333465306538313637636436626132656534313064383663393230653132333938343537 +65343339333037613332316534336133373864666130303166393038373531396266303832383632 +65343531616661303363303966303230353630356437336164383038366563363166363763396464 +37386330653563613063323438363737313565363130353264613236346565323837326631363935 +31346365653938653566646437623333326533646235343966383031656132396539616661393461 +65333235306432666539623964393132616436653430383236353033333366336363323539333636 +64616563376464646534343838616437306232393262313065353936356235373231613561323866 +61383536636361363039656138306131383732326464613931363837376234326433376533643861 +32616434383536373532383238663263643862373738333338343332303735663863346331663037 +35643637356531323639623533313865656132353139323436363661643263323633363161653235 +37636635623865303536353264313263373938366165623364643337313831653161623231343764 +33386162623531396234623937633465653930636637623837653138373365346132323731346130 +64613338386662653833653034313066636566333839306539666161333333616261653537313363 +64656632333462633232396236373666366464623464653434383832333365376532626531313735 +30313335353664643566313437393834393262333665666366326463663761626135326434363931 +36306439376138313838613532343663383938366333316432303930326239646232623633373564 +62316632383131626432333461323330366165366331303735333330363335366265316330663563 +64393231393136626162386335626433336337373765623933346237643532366662616434366163 +37373832303664303836373434343032313731323362353031346438643335323131366538653334 +33643432646339383865386238666134383861616437663238366365663737623663323137613865 +63633965323933366332363131313834653564636336303966656663366361353731316336633233 +32623339343838373639343534326365346135393137303736346634303863333664376332636634 +30303938353536636365353338393932353435383635326435666133353430323464333531356563 +31353733323331656538373932643332316332386437373938373635633832383662316537346132 +35316339343538343462363237666332376461363262623438623738623732383337373738346235 +64333335396336333337643235616436373736623339366236313938653433343661366332646265 +34636530666636363764366533643739653561626134393936616632616264663230616164376132 +30656336633230323535653766636361626562306262326265626166343036643034303730306164 +63356131333739333338353835363430363864333063363238303861323033643534366264623433 +30653639633862376232393065626636396137343635343030316630313863323365363764616630 +38346336663932626430613437656437313462633039393934653532386535323836366465383761 +39323732666339386565653836323461623238316564323333363461396161643338633137646335 +66373461633233333337663332323064373662636535376433613232653137353834373630663366 +61343066303366666362353837343337386631653461646230656533663965626135383131346635 +31623131653264353861353630646662303835643738323565383233336334356264353939373933 +31303964346563636437633532383762383234303130366331313864386330653130653437356466 +38373135333066336139336332643666373932666565303062636465326236623963323337633634 +66646633366532646232326435663065393035386438343030323038663033356639356264663532 +61396130643135666266366664323234353065303031343531366532633532363534646536646235 +62383531393832643137633332353364333230383361663264623564366539323132623639623939 +61323263373866343933653734323661383634636139383833323837303236326364666466386336 +34316465346261366131316535303563623238346530643264326539653432633237656133613532 +30663738313139333461346163626130343535306538623763343134623832346132656666393761 +35316164646635613032326565353163656134616366386364306436303134386339346462653262 +61383961656334653564623262366265316364366538353362313233343239396265383235303534 +33666366323466303534333165653334343437316435613566326338666336326664616133313062 +66333339346335626663323335373939303163623331303133643937653362666430323930623462 +30393239633763666463643066393161353235386461633066306235376261346139363330663235 +31643539616335396331656364383265346333323661333066353732653336616330303931376164 +37643931313337646337333236323939613333303834356662373836633631393736616462363338 +31623064303936306330343361313763336239316362633732326564653566313265616566336135 +61616538653365313666613366313064623232653563383465386535636530643831633735323433 +30353239333166343366643738363834613230316463636339666434633961386335623238353030 +34356536376164613338376534353438383039383930316439653732643339653531316530393534 +35326165666439396537396464656339333366333535366530353064396436663966333465616630 +33666233313537363165333738393362656631386564376365616266393137633931643833343232 +62393663663836303563386235383363623966316635366133353165346635373063383666373833 +35633264666139616463333339653733376431363761653433653138356364383865633937363433 +33313361653536633066656164616331343033373339336334666635353630636532323632323261 +39303937626633613434653835376538356164396631326163346465663337386230656139396237 +36613762346462626135323233393537616539646234663866353433396530383966613333376137 +37373461333561313966303239633834663837656230613830303433353639643431323930633238 +63383161313938663737396539663163303161633732393130363737303732313166623534303339 +65323162386430393639303435653436656239303032663761626462633565626161656632626162 +62386631316434383239303631343536363034626663626430633635353933313533316337613532 +33383335326637626535336664626663376332316236633735663339373131373630643665356133 +31393664663737326638 diff --git a/provisioning/roles/regluit_prod/files/certs/unglue.it.wildcard.csr b/provisioning/roles/regluit_prod/files/certs/unglue.it.wildcard.csr new file mode 100644 index 00000000..7328e869 --- /dev/null +++ b/provisioning/roles/regluit_prod/files/certs/unglue.it.wildcard.csr @@ -0,0 +1,58 @@ +$ANSIBLE_VAULT;1.1;AES256 +65656166313438643163366632316530386139623538636439633638636566383566646239306131 +3938333565313734613639376539323233353932373635640a313831613637383734636539666463 +64306331656564626233383831633862623861346364366632313733623263306464636264363965 +3933326435396164390a646439366537323232633232373339343538616431343137373338326166 +65373365613465616631623463313364396332366539386462333737623737343330393662646533 +62353663306531663935623765353838333138346630313333373066303835323461383630306662 +34316363316661613865616163393133626139663162393336356665373465353766656661633731 +37373563396437366661373134623664393430616532353530303861336238666539306562343662 +32613431393331306234643362323961343261376238353635366132623638663463343730343730 +37323733313739636238376538323264333238643532306439366463626233613237313933383333 +31373064343036616239616562373264653163396265613935656638613662663364633866393638 +62303563363763356433376634626234663234323636366232616431646534383730346233333963 +31616263353462653932343537343863326337643037326265666433623965646264303066316139 +31653864383331626231313736623563316330393563376562303933313237383939396438623231 +65383666336565393236326433646561346635356661356136306165323432383531646264323064 +31353738663861306135636131333261333964663338636637306665333663313164393537396663 +36666663616634323137333835626133383737666632316336313661656434376239616437346636 +62643035376637373031636330646638356562643962313336653964616337643064346562366531 +33613935656630633736386632616162663535366466663637313339323939646136636165656166 +38346562346331323738393934396362316361643538653163333935656139356535363432313532 +38633336646332386661643464666263646432373134306639306561663134373164326632613531 +64313162303263333534666436333634663262623632366631633865336433303564363138353566 +36303862656531343765643965666531613231343035636537373030613932303039623131363830 +37346263666637373639643930303438316430663462333430613335613131396566303533343864 +62373439663236636232353230343433336137613930623133303936323562313061336637306566 +34383336646336663263373334323131316262613136353239616630303264633238383266323464 +37623864646331363735306132393936306162393463313736366166656439343039626434353739 +37613532393830323431393964333437306366313335393434323463346133343237303732343962 +33373164363135363335323435636666386231363463373864663933373430656239366539643164 +30323038323739373965616630336166356362653833303132336230323661373737666232346630 +38393933303736306537383364356162656339613461333262656137636662303532663737636237 +36333661663734643363636661353337616162396666356663393432633361313534333537393161 +66666165363635346162623965333061386563663166643736653134303638613332313637333064 +65333438333661343664303361316631643763323164386330636139353932646635393963656336 +36343362323332336130633533663234343236393639323062663962633336626435653137306539 +63633866616438623764616631663533363233346362336132303861653039386133323965643863 +30633932646132633862666266323761656438343935636332326263653437333366306338636137 +65303538613661373538313437646466393536313665623666396235353962306634386261396337 +61376439636531383631383234313230303731653435393964393835393236653739316634623363 +61363730333632323039326438366633336462656135366164663434383561356337393636353661 +64616435386238316265386565633938623935663332633062316163356337303765366262313534 +61396361326333616136366435666662346637663864326561653634313030303033303132366132 +64353635383932633733303333636137323237393764396136313661353463306137316563303232 +32363063626630343230333065376638626236653532383766363161313862616336396334303764 +38393262613531343134653130313237343036383464393739353631636235343930653537373935 +63306431643132643362656433313061303838306164643236316166303331313636376431326239 +66386635333631653232383664343864383739663364363131373062623835363635393930326639 +33303733353761336638363935316232336533366339396166383539363963373536346232663865 +39613463386138363364316365373161313130316638616635313235363235396166323935353664 +39646163306161323536366532333231373530636234643764366134306164333431336636333533 +32333966393634393135333031356533666531613262313432346366373262373565323962323939 +36343764303866653638336134653263616561396332643030356434383531343166356466646633 +37336564613131633534313064303933633134393131656536383439393633366164376636353634 +34663236663163633338303838303733643262363834633064313532626631646239306564313361 +62616331306330636366306633366430353964376466313032356434346532366362356136376536 +33613962316539376138323033623736366531643261386535653733303465333331383131666530 +38336561353633303639616635656566323130616561656235393135616631356334 diff --git a/provisioning/roles/regluit_prod/handlers/main.yml b/provisioning/roles/regluit_prod/handlers/main.yml new file mode 100644 index 00000000..0c679221 --- /dev/null +++ b/provisioning/roles/regluit_prod/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: restart apache + become: yes + service: + name: apache2 + state: restarted \ No newline at end of file diff --git a/provisioning/roles/regluit_prod/tasks/apache.yml b/provisioning/roles/regluit_prod/tasks/apache.yml new file mode 100644 index 00000000..3063add3 --- /dev/null +++ b/provisioning/roles/regluit_prod/tasks/apache.yml @@ -0,0 +1,79 @@ +--- +- name: Install apache + become: yes + apt: + name: "{{ item }}" + state: present + with_items: + - 'apache2' + - 'libapache2-mod-wsgi' + +- name: Ensure apache is running and enabled + become: yes + service: + name: apache2 + state: started + enabled: yes + +- name: Create apache config + become: yes + template: + src: apache.conf.j2 + dest: "/etc/apache2/sites-available/prod.conf" + owner: "{{ user_name }}" + group: "{{ user_name }}" + mode: 0664 + notify: + - restart apache + +- name: Create static directory + become: yes + file: + path: "/var/www/static" + state: directory + owner: "{{ user_name }}" + group: "{{ user_name }}" + mode: 0755 + +- name: Create WSGI Script + template: + src: prod.wsgi.j2 + dest: "{{ project_path }}/deploy/prod.wsgi" + owner: "{{ user_name }}" + group: "{{ user_name }}" + mode: 0664 + +- name: Remove apache2 logrotate file + become: yes + file: + path: /etc/logrotate.d/apache2 + state: absent + notify: + - restart apache + +- name: Disable default site + become: yes + command: a2dissite 000-default + notify: + - restart apache + +- name: Enable prod site + become: yes + command: a2ensite prod + notify: + - restart apache + +- name: Enable SSL rewrite headers + become: yes + command: a2enmod ssl rewrite headers + notify: + - restart apache + +- name: Generate static files + django_manage: + app_path: "{{ project_path }}" + command: "collectstatic" + virtualenv: "{{ project_path }}/venv" + settings: "{{ django_settings_module }}" + notify: + - restart apache \ No newline at end of file diff --git a/provisioning/roles/regluit_prod/tasks/celery.yml b/provisioning/roles/regluit_prod/tasks/celery.yml new file mode 100644 index 00000000..9d6d1d59 --- /dev/null +++ b/provisioning/roles/regluit_prod/tasks/celery.yml @@ -0,0 +1,67 @@ +--- +- name: Create celery user + become: yes + user: + create_home: no + name: "celery" + tags: + - celery + +- name: Add current user to celery and www-data groups + become: yes + user: + name: "{{ user_name }}" + groups: + - celery + - www-data + append: yes + tags: + - celery + +- name: Create directories for celery + become: yes + file: + path: "{{ item }}" + state: directory + owner: celery + group: celery + mode: 0775 + with_items: + - '/var/log/celery' + - '/var/run/celery' + tags: + - celery + +- name: Copy celery init.d scripts + become: yes + copy: + src: "{{ item }}" + dest: "/etc/init.d/{{ item }}" + mode: 0755 + with_items: + - 'celeryd' + - 'celerybeat' + tags: + - celery + +- name: Copy celery config files + become: yes + template: + src: "celery/{{ item }}.j2" + dest: "/etc/default/{{ item }}" + mode: 0644 + with_items: + - 'celeryd' + - 'celerybeat' + tags: + - celery + +- name: Start celeryd + django_manage: + app_path: "{{ project_path }}" + command: "celeryd_multi restart w1" + virtualenv: "{{ project_path }}/venv" + settings: "{{ django_settings_module }}" + +- name: Start celerybeat + command: /etc/init.d/celerybeat start \ No newline at end of file diff --git a/provisioning/roles/regluit_prod/tasks/certs.yml b/provisioning/roles/regluit_prod/tasks/certs.yml new file mode 100644 index 00000000..0804aceb --- /dev/null +++ b/provisioning/roles/regluit_prod/tasks/certs.yml @@ -0,0 +1,39 @@ +--- +- name: Copy server key + become: yes + copy: + src: certs/server.key + dest: /etc/ssl/private/server.key + owner: "{{ user_name }}" + group: "{{ user_name }}" + mode: 0600 + notify: + - restart apache + tags: + - certs + +- name: Copy STAR_unglue_it.crt + become: yes + copy: + src: certs/STAR_unglue_it.crt + dest: /etc/ssl/certs/server.crt + owner: "{{ user_name }}" + group: "{{ user_name }}" + mode: 0644 + notify: + - restart apache + tags: + - certs + +- name: Copy STAR_unglue_it.ca-bundle + become: yes + copy: + src: certs/STAR_unglue_it.ca-bundle + dest: /etc/ssl/certs/STAR_unglue_it.ca-bundle + owner: "{{ user_name }}" + group: "{{ user_name }}" + mode: 0600 + notify: + - restart apache + tags: + - certs \ No newline at end of file diff --git a/provisioning/roles/regluit_prod/tasks/main.yml b/provisioning/roles/regluit_prod/tasks/main.yml new file mode 100644 index 00000000..1a8bf265 --- /dev/null +++ b/provisioning/roles/regluit_prod/tasks/main.yml @@ -0,0 +1,120 @@ +--- +- name: Install prod dependencies + become: true + apt: + name: "{{ item }}" + update_cache: true + state: present + with_items: + - 'git' + - 'python-setuptools' + - 'python-lxml' + - 'python-dev' + - 'python-virtualenv' + - 'build-essential' + - 'libssl-dev' + - 'libffi-dev' + - 'libxml2-dev' + - 'libxslt-dev' + - 'mysql-client' + - 'libmysqlclient-dev' + - 'python-mysqldb' + - 'postfix' + - 'libjpeg-dev' + +- name: Create project directory + become: true + file: + path: "{{ project_path }}" + state: directory + owner: "{{ user_name }}" + mode: 0755 + +- name: Checkout regluit repo + git: + accept_hostkey: yes + force: yes + repo: "https://github.com/EbookFoundation/regluit.git" + dest: "{{ project_path }}" + version: "{{ repo_version }}" + +- name: Install python packages to virtualenv + pip: + requirements: "{{ project_path }}/requirements_versioned.pip" + state: present + virtualenv: "{{ project_path }}/venv" + +- name: Add project to PYTHONPATH of virtualenv + template: + src: "{{ item }}.j2" + dest: "{{ project_path }}/venv/lib/python2.7/site-packages/{{ item }}" + with_items: + - 'regluit.pth' + - 'opt.pth' + +- name: Create keys directory + file: + path: "{{ project_path}}/settings/keys" + state: directory + owner: "{{ user_name }}" + mode: 0755 + +- name: Copy keys files + copy: + src: "{{ project_path }}/settings/dummy/__init__.py" + dest: "{{ project_path }}/settings/keys/__init__.py" + remote_src: yes + +- name: Copy django settings template + template: + src: prod.py.j2 + dest: "{{ project_path }}/settings/prod.py" + +- name: Copy key templates to keys directory + template: + src: "{{ item }}.j2" + dest: "{{ project_path }}/settings/keys/{{ item }}" + with_items: + - 'common.py' + - 'host.py' + +- name: Create django log directory + become: yes + file: + path: "/var/log/regluit" + state: directory + owner: "{{ user_name }}" + group: "www-data" + mode: 2775 + +- name: Open ports on firewall + become: yes + ufw: + rule: allow + port: "{{ item }}" + proto: tcp + with_items: + - 22 + - 80 + - 443 + +- name: Run redis tasks + import_tasks: redis.yml + +- name: Run mysql tasks + import_tasks: mysql.yml + +- name: Run cert tasks + import_tasks: certs.yml + +- name: Run apache tasks + import_tasks: apache.yml + +- name: Run celery tasks + import_tasks: celery.yml + + + + + + diff --git a/provisioning/roles/regluit_prod/tasks/mysql.yml b/provisioning/roles/regluit_prod/tasks/mysql.yml new file mode 100644 index 00000000..14733069 --- /dev/null +++ b/provisioning/roles/regluit_prod/tasks/mysql.yml @@ -0,0 +1,48 @@ +--- +- name: Install mysql-server + become: yes + apt: + name: "{{ item }}" + state: present + with_items: + - 'mysql-server' + +# - name: Ensure mysql-server is started and enabled +# become: yes +# service: +# name: mysql-server +# state: started +# enabled: yes + +- name: Create MySQL database + become: yes + mysql_db: + name: "{{ mysql_db_name }}" + state: present + +- name: Create MySQL user + become: yes + mysql_user: + name: "{{ mysql_db_user }}" + password: "{{ mysql_db_pass }}" + priv: '*.*:ALL' + state: present + +- name: Migrate databse + django_manage: + app_path: "{{ project_path }}" + command: "migrate --noinput" + virtualenv: "{{ project_path }}/venv" + settings: "{{ django_settings_module }}" + notify: + - restart apache + +- name: Import fixtures + django_manage: + app_path: "{{ project_path }}" + command: "loaddata" + virtualenv: "{{ project_path }}/venv" + settings: "{{ django_settings_module }}" + fixtures: "core/fixtures/initial_data.json core/fixtures/bookloader.json" + notify: + - restart apache \ No newline at end of file diff --git a/provisioning/roles/regluit_prod/tasks/redis.yml b/provisioning/roles/regluit_prod/tasks/redis.yml new file mode 100644 index 00000000..ca646847 --- /dev/null +++ b/provisioning/roles/regluit_prod/tasks/redis.yml @@ -0,0 +1,13 @@ +--- +- name: Install Redis server + become: yes + apt: + name: "redis-server" + state: present + +- name: Ensure Redis is started + become: yes + service: + name: "redis-server" + state: started + enabled: yes diff --git a/provisioning/roles/regluit_prod/templates/apache.conf.j2 b/provisioning/roles/regluit_prod/templates/apache.conf.j2 new file mode 100644 index 00000000..874f8dba --- /dev/null +++ b/provisioning/roles/regluit_prod/templates/apache.conf.j2 @@ -0,0 +1,71 @@ +WSGIPythonHome {{ wsgi_home }} +WSGIPythonPath {{ wsgi_python_path }} +WSGISocketPrefix {{ project_path }} + + + + +ServerName {{ server_name }} + +ServerAdmin info@ebookfoundation.org + +Redirect permanent / https://{{ server_name }} + + + + + + +ServerName {{ server_name }}:443 + +ServerAdmin info@ebookfoundation.org + +SSLEngine on +SSLProtocol All -SSLv2 -SSLv3 + +SSLCertificateFile /etc/ssl/certs/server.crt +SSLCertificateKeyFile /etc/ssl/private/server.key +SSLCertificateChainFile /etc/ssl/certs/STAR_unglue_it.ca-bundle + +#SSLCertificateChainFile /etc/ssl/certs/gd_bundle.crt + +WSGIDaemonProcess regluit processes=4 threads=4 python-eggs=/tmp/regluit-python-eggs +WSGIScriptAlias / /opt/regluit/deploy/prod.wsgi + +# generated using https://mozilla.github.io/server-side-tls/ssl-config-generator/ +# intermediate mode +# 2015.03.04 (with Apache v 2.2.22 and OpenSSL 1.0.1 and HSTS enabled) + +SSLProtocol all -SSLv2 -SSLv3 +SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA +SSLHonorCipherOrder on + +# HSTS (mod_headers is required) (15768000 seconds = 6 months) +Header always add Strict-Transport-Security "max-age=15768000" + + + + Require all granted + + + + + Options Indexes FollowSymLinks + AllowOverride None + + Require all granted + + +Alias /static /var/www/static + +BrowserMatch "MSIE [2-6]" \ + nokeepalive ssl-unclean-shutdown \ + downgrade-1.0 force-response-1.0 +# MSIE 7 and newer should be able to use keepalive +BrowserMatch "MSIE [17-9]" ssl-unclean-shutdown + +ErrorLog "|/usr/bin/cronolog /var/log/apache2/%Y%m%d_error.log" +LogLevel warn +CustomLog "|/usr/bin/cronolog /var/log/apache2/%Y%m%d_access.log" combined + + \ No newline at end of file diff --git a/provisioning/roles/regluit_prod/templates/celery/celerybeat.j2 b/provisioning/roles/regluit_prod/templates/celery/celerybeat.j2 new file mode 100644 index 00000000..d459ad8b --- /dev/null +++ b/provisioning/roles/regluit_prod/templates/celery/celerybeat.j2 @@ -0,0 +1,35 @@ +# http://docs.celeryproject.org/en/latest/cookbook/daemonizing.html#generic-initd-celerybeat-example +# to be placed at /etc/defaults/celerybeat + +# Where to chdir at start. +CELERYBEAT_CHDIR="{{ project_path }}t/" + +# Extra arguments to celerybeat +#CELERYBEAT_OPTS="--schedule=/var/run/celerybeat-schedule" + +# Name of the celery config module.# +CELERY_CONFIG_MODULE="celeryconfig" + +# Name of the projects settings module. +export DJANGO_SETTINGS_MODULE="{{ django_settings_module }}" + +# Path to celerybeat +CELERYBEAT="{{ project_path }}/{{ virtualenv_name }}/bin/django-admin.py celerybeat" + +# virtualenv to use +VIRTUALENV="{{ project_path }}/{{ virtualenv_name }}" + +#Full path to the PID file. Default is /var/run/celeryd.pid +CELERYBEAT_PIDFILE="/var/log/celerybeat/celerybeat.pid" + +#Full path to the celeryd log file. Default is /var/log/celeryd.log +CELERYBEAT_LOGFILE="/var/log/celerybeat/celerybeat.log" + +#Log level to use for celeryd. Default is INFO. +CELERYBEAT_LOG_LEVEL="INFO" + +#User to run celeryd as. Default is current user. +#CELERYBEAT_USER + +#Group to run celeryd as. Default is current user. +#CELERYBEAT_GROUP diff --git a/provisioning/roles/regluit_prod/templates/celery/celeryd.j2 b/provisioning/roles/regluit_prod/templates/celery/celeryd.j2 new file mode 100644 index 00000000..c918efb4 --- /dev/null +++ b/provisioning/roles/regluit_prod/templates/celery/celeryd.j2 @@ -0,0 +1,9 @@ +CELERYD_NODES="w1" +CELERYD_CHDIR="{{ project_path }}/" +CELERYD_LOG_FILE="/var/log/celery/%n.log" +CELERYD_PID_FILE="/var/log/celery/%n.pid" +CELERYD="{{ project_path }}/{{ virtualenv_name }}/bin/django-admin.py celeryd" +CELERYD_MULTI="{{ project_path }}/{{ virtualenv_name }}/bin/django-admin.py celeryd_multi" + +VIRTUALENV_ACTIVATE="{{ project_path }}/{{ virtualenv_name }}/bin/activate" +export DJANGO_SETTINGS_MODULE="{{ django_settings_module }}" diff --git a/provisioning/roles/regluit_prod/templates/common-old.py.j2 b/provisioning/roles/regluit_prod/templates/common-old.py.j2 new file mode 100644 index 00000000..57e97fe5 --- /dev/null +++ b/provisioning/roles/regluit_prod/templates/common-old.py.j2 @@ -0,0 +1,13 @@ +import os + +# all the COMMON_KEYS +# copy this file to settings/keys/ and replace the dummy values with real ones +BOOXTREAM_API_KEY = os.environ.get('BOOXTREAM_API_KEY', '{{ booxtream_api_key }}') +BOOXTREAM_API_USER = os.environ.get('BOOXTREAM_API_USER', '{{ booxtream_api_user }}') +DROPBOX_KEY = os.environ.get('DROPBOX_KEY', '{{ dropbox_key }}') +GITHUB_PUBLIC_TOKEN = os.environ.get('GITHUB_PUBLIC_TOKEN', '{{ github_public_token }}') # 40 chars; null has lower limit +MAILCHIMP_API_KEY = os.environ.get('MAILCHIMP_API_KEY', '{{ mailchimp_api_key }}') # [32chars]-xx# +MAILCHIMP_NEWS_ID = os.environ.get('MAILCHIMP_NEWS_ID', '{{ mailchimp_news_id }}') +MOBIGEN_PASSWORD = os.environ.get('MOBIGEN_PASSWORD', '{{ mobigen_password }}') +MOBIGEN_URL = os.environ.get('MOBIGEN_URL', '{{ mobigen_url }}') # https://host/mobigen +MOBIGEN_USER_ID = os.environ.get('MOBIGEN_USER_ID', '{{ mobigen_user_id }}') diff --git a/provisioning/roles/regluit_prod/templates/common.py.j2 b/provisioning/roles/regluit_prod/templates/common.py.j2 new file mode 100644 index 00000000..6e2e938e --- /dev/null +++ b/provisioning/roles/regluit_prod/templates/common.py.j2 @@ -0,0 +1,5 @@ +import os + +{% for key in common_keys %} +{{ key|upper }} = os.environ.get('{{ key|upper }}', '{{ common_keys[key] }}') +{% endfor %} diff --git a/provisioning/roles/regluit_prod/templates/host-old.py.j2 b/provisioning/roles/regluit_prod/templates/host-old.py.j2 new file mode 100644 index 00000000..b036d9c7 --- /dev/null +++ b/provisioning/roles/regluit_prod/templates/host-old.py.j2 @@ -0,0 +1,47 @@ +# host.py +# copy this file to settings/keys/ and replace the dummy values with real ones +# or generate it from the ansible vault +import os + +# you can use this to generate a key: http://www.miniwebtool.com/django-secret-key-generator/ +SECRET_KEY = os.environ.get("SECRET_KEY", '{{ secret_key }}') + +# you'll need to register a GoogleBooks API key +# https://code.google.com/apis/console +GOOGLE_BOOKS_API_KEY = os.environ.get("GOOGLE_BOOKS_API_KEY", "{{ google_books_api_key }}") + +# +GOODREADS_API_KEY = os.environ.get("GOODREADS_API_KEY", "{{ goodreads_api_key }}") +GOODREADS_API_SECRET = os.environ.get("GOODREADS_API_SECRET", "{{ goodreads_api_secret }}") #43 chars + +# Amazon SES +# create with https://console.aws.amazon.com/ses/home?region=us-east-1#smtp-settings: +EMAIL_HOST_USER = os.environ.get("EMAIL_HOST_USER", '{{ email_host_user }}') +EMAIL_HOST_PASSWORD = os.environ.get("EMAIL_HOST_PASSWORD", '{{ email_host_password }}') + +# twitter auth +# you'll need to create a new Twitter application to fill in these blanks +# https://dev.twitter.com/apps/new +SOCIAL_AUTH_TWITTER_KEY = os.environ.get("SOCIAL_AUTH_TWITTER_KEY", '{{ social_auth_twitter_key }}') +SOCIAL_AUTH_TWITTER_SECRET = os.environ.get("SOCIAL_AUTH_TWITTER_SECRET", '{{ social_auth_twitter_secret }}') + +# support@icontact.nl +BOOXTREAM_API_KEY = os.environ.get("BOOXTREAM_API_KEY", "{{ booxtream_api_key }}") # 30 chars +BOOXTREAM_API_USER = os.environ.get("BOOXTREAM_API_USER", '{{ booxtream_api_user }}') + +# you'll need to create a new Facebook application to fill in these blanks +# https://developers.facebook.com/apps/ +SOCIAL_AUTH_FACEBOOK_KEY = os.environ.get("SOCIAL_AUTH_FACEBOOK_KEY", '{{ social_auth_facebook_key }}') +SOCIAL_AUTH_FACEBOOK_SECRET = os.environ.get("SOCIAL_AUTH_FACEBOOK_SECRET", '{{ social_auth_facebook_secret }}') + +# https://console.developers.google.com/apis/credentials/oauthclient/ +# unglue.it (prod) SOCIAL_AUTH_GOOGLE_OAUTH2_KEY #2 +SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = os.environ.get("_KEY", '{{ social_auth_google_oauth2_key }}') +SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = os.environ.get("_SECRET", '{{ social_auth_google_oauth2_secret }}') + +AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID", '{{ aws_access_key_id }}') +AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", '{{ aws_secret_access_key }}') # 40 chars + +DATABASE_USER = os.environ.get("DATABASE_USER", '{{ mysql_db_user }}') +DATABASE_PASSWORD = os.environ.get("DATABASE_PASSWORD", '{{ mysql_db_pass }}') +DATABASE_HOST = os.environ.get("DATABASE_HOST", '{{ mysql_db_host }}') diff --git a/provisioning/roles/regluit_prod/templates/host.py.j2 b/provisioning/roles/regluit_prod/templates/host.py.j2 new file mode 100644 index 00000000..845c962f --- /dev/null +++ b/provisioning/roles/regluit_prod/templates/host.py.j2 @@ -0,0 +1,5 @@ +import os + +{% for key in host_keys %} +{{ key|upper }} = os.environ.get('{{ key|upper }}', '{{ host_keys[key] }}') +{% endfor %} diff --git a/provisioning/roles/regluit_prod/templates/opt.pth.j2 b/provisioning/roles/regluit_prod/templates/opt.pth.j2 new file mode 100644 index 00000000..27d47a41 --- /dev/null +++ b/provisioning/roles/regluit_prod/templates/opt.pth.j2 @@ -0,0 +1 @@ +/opt/ diff --git a/provisioning/roles/regluit_prod/templates/prod.py.j2 b/provisioning/roles/regluit_prod/templates/prod.py.j2 new file mode 100644 index 00000000..a46a9177 --- /dev/null +++ b/provisioning/roles/regluit_prod/templates/prod.py.j2 @@ -0,0 +1,135 @@ +from .common import * + +ALLOWED_HOSTS = ['.unglue.it'] +DEBUG = False +TEMPLATES[0]['OPTIONS']['debug'] = DEBUG +# we are launched! +IS_PREVIEW = False + +SITE_ID = 1 + +ADMINS = ( + ('Raymond Yee', 'rdhyee+ungluebugs@gluejar.com'), + ('Eric Hellman', 'eric@gluejar.com'), +) + +MANAGERS = ADMINS + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': '{{ mysql_db_name }}', + 'USER': '{{ mysql_db_user }}', + 'PASSWORD': '{{ mysql_db_pass }}', + 'HOST': '{{ mysql_db_host }}', + 'PORT': '{{ mysql_db_port }}', + 'TEST_CHARSET': 'utf8', + } +} + +TIME_ZONE = 'America/New_York' + +# settings for outbout email +# if you have a gmail account you can use your email address and password + + +# Amazon SES + +EMAIL_BACKEND = 'django_smtp_ssl.SSLEmailBackend' +MAIL_USE_TLS = True +EMAIL_HOST = '{{ email_host }}' +EMAIL_PORT = '{{ email_port }}' +DEFAULT_FROM_EMAIL = '{{ default_from_email }}' + +# send celery log to Python logging +CELERYD_HIJACK_ROOT_LOGGER = False + +# Next step to try https +#BASE_URL = 'http://{{ server_name }}' +BASE_URL_SECURE = 'https://{{ server_name }}' +IPN_SECURE_URL = False + +# use redis for production queue +BROKER_TRANSPORT = '{{ broker_transport }}' +BROKER_HOST = '{{ broker_host }}' +BROKER_PORT = '{{ broker_port }}' +BROKER_VHOST = '{{ broker_vhost }}' + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'brief': { + 'format': '%(asctime)s %(levelname)s %(name)s[%(funcName)s]: %(message)s', + }, + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'class': 'django.utils.log.AdminEmailHandler' + }, + 'null': { + 'level': 'DEBUG', + 'class': 'logging.NullHandler', + }, + 'file': { + 'level': 'INFO', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': join('/var/log/regluit', 'unglue.it.log'), + 'maxBytes': 1024*1024*5, # 5 MB + 'backupCount': 5, + 'formatter': 'brief', + }, + }, + 'loggers': { + 'django.request': { + 'handlers': ['mail_admins'], + 'level': 'ERROR', + 'propagate': True, + }, + 'django.security.DisallowedHost': { + 'handlers': ['null'], + 'propagate': False, + }, + '': { + 'handlers': ['file'], + 'level': 'WARNING', + 'propagate': False, + }, + } +} + +STATIC_ROOT = '/var/www/static' +#CKEDITOR_UPLOAD_PATH = '/var/www/static/media/' +#CKEDITOR_UPLOAD_PREFIX = 'https://unglue.it/static/media/' + +# decide which of the period tasks to add to the schedule +CELERYBEAT_SCHEDULE['send_test_email'] = SEND_TEST_EMAIL_JOB +# update the statuses of campaigns +CELERYBEAT_SCHEDULE['update_active_campaign_statuses'] = UPDATE_ACTIVE_CAMPAIGN_STATUSES +CELERYBEAT_SCHEDULE['report_new_ebooks'] = EBOOK_NOTIFICATIONS_JOB +CELERYBEAT_SCHEDULE['notify_ending_soon'] = NOTIFY_ENDING_SOON_JOB +CELERYBEAT_SCHEDULE['update_account_statuses'] = UPDATE_ACCOUNT_STATUSES +CELERYBEAT_SCHEDULE['notify_expiring_accounts'] = NOTIFY_EXPIRING_ACCOUNTS +CELERYBEAT_SCHEDULE['refresh_acqs'] = REFRESH_ACQS_JOB +CELERYBEAT_SCHEDULE['refresh_acqs'] = NOTIFY_UNCLAIMED_GIFTS + +# set -- sandbox or production Amazon FPS? +#AMAZON_FPS_HOST = "fps.sandbox.amazonaws.com" +AMAZON_FPS_HOST = "fps.amazonaws.com" + +# local settings for maintenance mode +MAINTENANCE_MODE = False + +# Amazon keys to permit S3 access +# https://console.aws.amazon.com/iam/home?region=us-east-1#/users/s3user?section=security_credentials +DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' + +# we should suppress Google Analytics outside of production +SHOW_GOOGLE_ANALYTICS = True + +# if settings/local.py exists, import those settings -- allows for dynamic generation of parameters such as DATABASES +try: + from regluit.settings.local import * +except ImportError: + pass diff --git a/provisioning/roles/regluit_prod/templates/prod.wsgi.j2 b/provisioning/roles/regluit_prod/templates/prod.wsgi.j2 new file mode 100644 index 00000000..096c5f4e --- /dev/null +++ b/provisioning/roles/regluit_prod/templates/prod.wsgi.j2 @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +import os + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "regluit.settings.prod") +os.environ['CELERY_LOADER'] = 'django' + +{% for key in host_keys %} +os.environ['{{ key|upper }}'] = '{{ host_keys[key] }}' +{% endfor %} + +from django.core.wsgi import get_wsgi_application +application = get_wsgi_application() \ No newline at end of file diff --git a/provisioning/roles/regluit_prod/templates/regluit.pth.j2 b/provisioning/roles/regluit_prod/templates/regluit.pth.j2 new file mode 100644 index 00000000..8964ac81 --- /dev/null +++ b/provisioning/roles/regluit_prod/templates/regluit.pth.j2 @@ -0,0 +1 @@ +{{ project_path }}/ diff --git a/provisioning/setup-prod.retry b/provisioning/setup-prod.retry new file mode 100644 index 00000000..d3b06b7c --- /dev/null +++ b/provisioning/setup-prod.retry @@ -0,0 +1 @@ +regluit-prod diff --git a/provisioning/setup-prod.yml b/provisioning/setup-prod.yml new file mode 100644 index 00000000..a792de6b --- /dev/null +++ b/provisioning/setup-prod.yml @@ -0,0 +1,18 @@ +- hosts: regluit-prod + gather_facts: false + tasks: + # Need to install python2.7 and pip first so Ansible will function + # This is due to Ubuntu 16 shipping with Python3 by default + - name: Install python2.7 and pip + become: true + raw: bash -c "apt -qqy update && apt install -qqy python2.7-dev python-pip" + register: output + changed_when: output.stdout != "" + + - name: Gathering Facts + setup: + + - include_role: + name: regluit_prod + + \ No newline at end of file diff --git a/provisioning/setup-regluit.yml b/provisioning/setup-regluit.yml new file mode 100644 index 00000000..8bb3776b --- /dev/null +++ b/provisioning/setup-regluit.yml @@ -0,0 +1,6 @@ +--- +- hosts: regluit-local + gather_facts: false + roles: + - regluit_common + - regluit_dev diff --git a/utils/crypto.py b/utils/encryption.py similarity index 100% rename from utils/crypto.py rename to utils/encryption.py diff --git a/vagrant/group_vars/all/misc.yml b/vagrant/group_vars/all/misc.yml index c08529f2..2d4a280a 100644 --- a/vagrant/group_vars/all/misc.yml +++ b/vagrant/group_vars/all/misc.yml @@ -1,11 +1,11 @@ $ANSIBLE_VAULT;1.1;AES256 -63613831616639376633626266303462633762313365343031313561363336333536303436623033 -6431323039346335366131353366616466663932363365320a343964663464313835663930386236 -64313434633233613764393834646431666632373033636561373833623931363033396235653031 -6237326633343230330a303432623634363363343036363534396162303532326335633532663065 -39336337653362346134396266353639643530643430363963663633623163373336633966343038 -66633563356332663136636434323635633164313232343337346335623337633337316533313531 -38306332303562383663303033653237386563653339653230313939623435663166653364653862 -65373863306531633963653639303033343733653339656563656264633362663238663136643733 -39373565353639383830333462393935353165633236346238613334643863336637326161636166 -6162303863363232643039313835313036363865663531363230 +34396634353236366232316262326532383330346466656632396266363962363330323331656538 +6463386133646665623738313130636139363361323839350a653766666239353962633534353261 +37343761653238333833316131373035663266336336383564613230366639663861656631646362 +3462383738373136630a303131383066613030653564316335656332373534396336323538393933 +39383232613833393763663564633738376436626665623333663163306630393866343236613230 +66376334653631663365323432363364303431326336623266376535393338393733643964363462 +63333031376134623632383865653136393766346637346563313231393766363331656336663639 +35363461646561616136616334306232376165363364323931313539373438613564633630376237 +66323936356136646233373662323030376130356337326263353239323338653330623830663432 +6434393837656430346632383766636664613436373339396430 diff --git a/vagrant/group_vars/all/secrets.yml b/vagrant/group_vars/all/secrets.yml index d4a96e78..47820687 100644 --- a/vagrant/group_vars/all/secrets.yml +++ b/vagrant/group_vars/all/secrets.yml @@ -1,51 +1,51 @@ $ANSIBLE_VAULT;1.1;AES256 -63376563643062623466613335633730323031653736643437643030303533383931666434323939 -3266633436313635643463333266356238316638326131610a653336303661326334323561623239 -65623834633736663861343464333764303865663934303533333031343934333635616333643631 -3933383131353062630a633133353430386437626130656632393230636664376438613862303466 -66343337303961303361353030313936626363383465316166316663643830313164646661363366 -66656630323437643031363237633439366434666537396464336364373134383061623364636635 -31393435326337373162333365386465633730323135383364383433366664353938303962313461 -61633835393963646666376562633731626639313438326235306335653639666635386566343033 -36383064653634613065303538346638363666326336643631616636623632376662323037633165 -32633662636531643366343963663439313439636335336435616666633863323433643930666536 -30623234313838616461383534663230616364336131383763316235356265363964636539316637 -61376433306366346362303439303665333961393239373263373365306363336334623536353031 -61636339353561303532383833336164346538396139373835633162323339333732373431613035 -62333739373832623762356364383035343937656337666439323830663235666637326134346531 -66386135323064333732663136396431616563653731343134363262373639333836623033656536 -38653931346337666566336331363631666238666638353237663065653164343936636239343631 -62636339396332663637643239376335343731643764343666356361623239383462313936366363 -32616638653666666638316563656366666361353538303133656433376637663532313666343631 -63633436336137653339653364616532333434623036396439343433613361626231613331373931 -64343232313839383332353036633633383762316531383936393539316531316336636537666439 -33613166613733346336643932363263663833366533623965623039313036373230396135343663 -66653336313437336561333630393365653133636263333031356562623332623733616464333262 -35333535396337616461353361616337613138363239656162323232346436336233633335376666 -63626661373639306665613431393735343937616539336233666235343936653932366337326663 -33353930653965623661383265643837306637363231656435663030393562623438623361353666 -36626635323837363733353365373738646230303436343834636666373165653937333130343637 -31666566643434653663626531383730666238306438656133303862626538613339626437633432 -64393432333834623339393631363932353862646332313062636565326135636138333931333431 -64633937356630326666373939343635323263373934306330393032316630646333363239313137 -65383666663063356430663965336463393537363266636431633362343263666234323463653532 -30376262656335653662656366636336336135626130303835326432366438393962616266373435 -39613234326136623263356131376239313136653131326632343739313137653833316434656265 -30323863383639363462613034353062393964333130656161306633306566646233356537313162 -65376237303961333239303630313831303232643831383036643166343164363934323138353234 -33623130383339343162636239343234393731626530326164643438633239343738306236623433 -37636662353263373866656564316334303736336566336636653937663566343261333335333437 -62313833353235393131326430323263383861366230663639363661383362363438373432346663 -62643166363633633134303836383239386236386361376533353330333930633530333139666462 -63623361356134333337643634393461343635363232313435656137303731326461653330613831 -66613536666635313539643130373132623965333863343930363761613031306664333732393564 -34333937326234383833306233386638323263323430613863346436643030353732356261643834 -62343230633034363839373965646133353238346261373538313833323264616531313965376537 -32646166326135306134356262353461626337663137323335613339646532636663326462393936 -30316265636335336138383930313138306366383766653866613532626366373463613436303638 -31333738306164613031626435373966323361626564666537396539313832613038363865366534 -62656565316466393366303137646431303231626534613031316561336233353137353265623032 -32616134366439363630376666343937633462613731613961343862303639373135373466363663 -34663235333431323764613330303539323164313332313331613761306439623232396436323138 -35623265613335636430323335313230373865633331303633613632306431323339396165626135 -3662393265643861393433333230353233366330646435343361 +38353435396232353338653362653465343063393836633332633964663162363636316433396365 +3336316136626134653262626630663738333938666336330a626562646335323264626363316233 +62383131353536323166353364633437383333663434653037353964646662633865323466306237 +3336306364653366320a323464383061353862663932623461346230316364396530653564353762 +33363864663730643265396565363430376564386164366163323266393735633830363534353665 +31376233393136306365383366333866383962313864376561333964623639623834373931396133 +31646665313165376266336132663833663436633165393536316262373635353837306365363964 +38656539313935366663336539376334313863633439343664353530393336383064623533336264 +34346630643737643938373366383432303139653566306163333463366465623465303862616361 +61363139613265636432616533656564386231323066623863663463313336626435323433343337 +31626530626132646262366334643261626431643261633334393262636137643935376461643739 +38633631626366303934393139356166633536333634396163333164306532373765656138393464 +39623636353034343363396361383837663863313465636134616333666563666534653930626331 +65633536383230653665356233323330666439643665396335303566366230646366356534663732 +30313835323633306130343865626131383032666236663362633733383235643239383765663332 +32643332313361353736313035653137323239616635653662386435373939386636653138393036 +38396231623635316633333366646339336662313164656634363536376163333631353031373232 +37643636303934313664633765353562343666353961643237316461383131343566326466663936 +32613933303430633165363339373433363637343733633833356339373965336131666234373737 +33653763326461343164313663633361303261333737326266663030646136643234313263376432 +33636332376634313364383463643030333462323266383263643535393061313665636637323266 +62626334303936626535656431366339393561356461393862633464336534386665623735383862 +37336232336662633331386661643131343663356133643838636231396336303364393562336139 +30623030316365303937333037386362613065636161336530613637353737323535636138366263 +30303533353531396135663139646566646631643533306165383364643764656330666134366633 +64353333333065663238353666303637366162346535656534666465336630323764303639616564 +63646638646566663033373263653062343466363364333537623661643331666664343065656633 +32356239316638613264303939363830373030353263613863356465663538666638333462336234 +65653563306562663530383334643763353464643363356462343863653965623634663338333136 +31366531383837396564353134333232376630613063613134393038366631616365366439616263 +36363231396665626263373830383330393432633934626363393866353538663433343563323631 +65346430666666333237366162636363356462313165383233313438353431636539663131386134 +39626136613935393265363534336265653435656333386363303735613336633062663131396336 +65616339376561383633636362323530323061356134303365303234383063336261643330363363 +39396332666532353563363966326266333632313164643965343538633763376466353864383435 +34366266666162306334613262663861646538346139343631343139353234333661386234666431 +39353038373434363130633435633161623565333062343734626239653434383035653361393865 +64646135353236313261303633346161393636323531326334393036663137626365303934346337 +33336136366534343238623262313064356634333661356339383462303037383630656631643762 +38346439353338616165323562633436623131623532313663323131303737353734336234386164 +37343465636332663363396536393266613235333762363538373261316263653135663931333962 +31613630303266363265623363393364636437316465646532623264326239346531363165346566 +34333831666236643636663738383363616231333166336334663035336639666165303965633064 +33306434363730633830346535326565643563306338386235313633363631353666393161313934 +63346531653062336666336366356330646566393563356464346436656333363962383562363463 +64313466633064316331646130366333326438633662323034383037396363643861353736336333 +66663531373230386330393065663762653932646561373962633034343635333265313964343965 +33663239316334623636646530363565636235666536646132303533663230396363313732373063 +37663333646639383037336333343438643933643238396236653935396663303163363962633237 +3038323032383666326138306366633063636366363534393333 diff --git a/vagrant/host_vars/please/secrets.yml b/vagrant/host_vars/please/secrets.yml index 1c3921a0..ae1a221d 100644 --- a/vagrant/host_vars/please/secrets.yml +++ b/vagrant/host_vars/please/secrets.yml @@ -1,93 +1,93 @@ $ANSIBLE_VAULT;1.1;AES256 -38386461333437313133643631373538633664303964383463643764363236343164383636653035 -3966626566393633313332623565383763643434383930360a323663363833646139343039323139 -34333837383866646565346561313032393334363966383564666662326366633532623132626435 -6263306636633738300a613362393561363931356634333430366164616539353737646438363962 -31356266643864393332343962653364363836363235343932623839363562313966353337366463 -32333832663536633731393666383966663365646662663664366330326330343564316238346462 -64316231373230393530306563323136626631633137323636383432366235306437623930663666 -39613830326339353335653537356262313332383765313438623236326131316466326631366336 -31653338646666313533646365303537636439313438623266383231376231613534303630383338 -37363461373262623732613536323362343036373238313434393666313033353032633830663532 -62353365343937333133306138626337626536386465633536613331343635343931356431303964 -63636136643938646332633833623733343835366133323832393236343330373632626663656231 -37653537663739336432313563643865663238303535636463623339313161626138313961346239 -36316631353935613265616137363962333039663138366439346461373365633464333265393435 -30633436633533323038386664666439313939373033333134396437386335366633306164363831 -31363561333633343464646566643361366365306264613536363334623663626236306562393664 -37383163373538306632616330326433393461323961353565373561393361656661333962663136 -64343761613134363637316466393161326264396263346264656263613931373465633030306265 -61353961356230316361316566643537333031343635663837323238356231356234313839306331 -61383863323766656461653737363533336333333635386662343563333233386131623734316139 -61666661323962303631646365363735363135343733653265646532613166313962663230386638 -30343563313237373039623933313161313465343337366331303761626664306163353632626465 -66643964323638356462316635323264376235323734633732393538643032343461646262663235 -35313934303362386335306264346231353431333437383038643233346336396465336333326364 -63656135363764616538383666366131323464623337343939356665376366363161626330393131 -34336463336566646138336562633931353531366635326632306335303865363232373964663664 -31363965386431363961343962666538633837663933623362653137363365623037623631626537 -65366534643533346630393837383130613961653636613835643665346337633363613732383234 -36626164626461356238646135353338393463333839323964643966643463353264393362616466 -32336338373936343733393436333536383562613365323538633163656634333536383930643532 -64383834396166343931346166646234343231643665623836346131663238663365613063623161 -34343131323063366434613164383761366265623061646665363964346230313038306664363561 -33316664633861313632343161613236343934386166626334666632356266353966663163303731 -30373166363637663131613564303634373337626538626665303139656362383935383535663366 -66663437323039343262333636623363366231313366626434383039336231323938666633303663 -63326264373065613539323433646163653334613733323536633862343535306165633731343361 -64366136663535306433636338653763653030643837633830343034363361363132666132386633 -63393331396165366239313637303836313737326531646635613036373436346239383333336336 -31623434333336643863303334313539336533636439393738353766313338313161336239386363 -64353032633934613533646532613961346531353664333230613036393737333966373437363863 -36626139333030326134386164653931393432323731373562343735303737376532613534396463 -32346432313361643235653165376630316631616463336238316234626231396364303364643136 -35393165333439373336323630636138333730316266336261316637613738383639343661373239 -64336535306465333161346637346636643034316236353664653464323061323834366633653561 -37663730353531373630613034643464333933663331393337303335313631383134353330323238 -63363030386236323738383661633366316265663566613331386166633436666564303738356263 -34356166656465656230336166636533643866666565646336626338643135633236353033626230 -66646230303262626635653665336333383831313232353565313638396462353035376137666230 -36313931626665316234346335343838363533623838353263313732633735653861633936633239 -37363139666137623665376131666636306233363861363931333034383063656335663333656134 -37323461373261343563663239623533313532386539646562333864633337393966613532336265 -62646161633830303630356636356138383962633935386335336432663461613039333333666263 -31303431336266616463393238663662643961656536353238613965646138316630663934313235 -32353261346337346438623732383036393961356332363838316266313566623437336263393836 -37353939626536326538643962363731373833356532366232323034393137373463383630646165 -35633439613330663065356630353631326136363633666233343837633433626136623332303263 -33306435643734633865316565363832376137616231666562306335643637663132316435383765 -34666638343031323762653363333062363034626134626463636331383331353939613865366633 -37333639626136373964366234393939633664656535306539646362656437636539643036363364 -32343939323763303966343230616136643132623132333230613663363931363832326661356364 -39363231643036633464643539346362376566343933393861643335626338656335306434393464 -63633537656166653638333532333633626439306436333736316465646538343137326430613934 -63313538373435353535323734616562323631393763313065336461363237393331616330393435 -62366238373532313132396132323561643866396365373335326462303163613339666536396630 -63646564373431333734633832316633316531623730356462663232636662353131646265376435 -65353562613066353830623239333265346362653861316432636361613134613366633735666138 -31346265383164623062363431306664383436653265343434326366653066306630323635633136 -64366163383161303735613935653231356635316338333132363836313165333932653635643062 -63633530636631643565333765666438343231326439303239303530306436323365646639346433 -66373161643164613861323661303730653635306662366234323132346533643164303338353136 -37353664636239663066323265313330303631396562646636623136663836643236333139656434 -66313531373161333038393161383663313062356439323839663139393736633637386332633938 -63313737393166316361623762303237623835336534353764373139666539343766333461373437 -39356438313039306333396262373330323561343065653765646566653831336363646638333637 -61326662353765326566663266646365336533636437383862326561663032366230613764313136 -64303232396534303136613931376633323937376464373133646636336263643462646365333164 -32623833363264376233316136633666366435303635386330636238343139316136306231396533 -35316438623539393362366363353635363732323839373063636531363462383334623365656436 -36383335333561326333626532653132663766363231353966396633303333343165653838663262 -66376666626135376564393235616130393436613064653130616362613031383532313633313936 -38333532376638383461643332353837363061656632353430376163393431623032336530643234 -62313331323561343166653636366461613834326534396231326163303462303235663162626462 -63333766363330633066386466346235366335383266386561643336306536363034396231373137 -31376261663937393439386462613531616366613362623863326235326531663337316531623831 -64666636313035323235323236393238346261343765613639346163363430386130326365313331 -38613365623438383339613664643037316431636632393066363531316631336264623930346532 -31636333616139636236326464323333376236393862646333656464653063656263303433623164 -35636161623463353363636334383266356236656163633064303231663430343430306131333662 -63633131653262346636633632646638356263306238616135393938393238303461303965313361 -39656138373639373335316362336266313838396661343933633162376237616635363764623630 -36336666333531353237323263356664303932616136373736326361623733336331343136303637 -31336630613031373564 +64346437633164303231353763353632353638376565623935303562383939306134356238643662 +3863623565353533333032343339616464393235393735640a616438353833353930663639393262 +61303932623163343666383238386137343631353032333339376339613464626263346530376166 +3333663433643530300a366165343831343435396134343162656432313035626333396465373066 +37623635396630363335653438643037303662643733386262356134633563326536303064623164 +35356339643466613931636430376561616438616535653936316238626430333637313966333330 +65653033386636623538316162663539393064383061653163343766363237626438353662373661 +38343966326661323466393634343561666463623766663331346361643538393234336163386462 +38343664323166616332353637343064396332333663353165313661366134333239316166313761 +33346161316130353432386132373633356430306237393937326432396630333239306339643239 +66306133386131376366353836326164613030323133373561363334623964346265363861343833 +31663963313336323463643931663364663437616335653534663130623137613638336333663333 +36353331616466386439633363383861323232626562386362376262356338663935386434623663 +65653833623163376231323232316239353432616663346637616434313931386265633933333339 +30343662316365346533316333636131303434633039346139373032373533313164346232623734 +39653262633432383063313231383465633039653237646235663466386230663135376330333432 +35643931666135353234396439303664376236646238363335303762653335316266376363633130 +61396665333432313461653666336637623039383439316332333034346530383763623166626530 +61383036393530633636383764653937323463656563306131666130636161366531313838633438 +66313261343165333430343233393861646138373931363831336636663239363535306533623034 +34306232613932323065316630363262373437626230323563626339306630633034663765366161 +39396535303262653264633135656264386539363239636630626330343864663635616530323062 +31333365393538346432313335616138316538663237306364656232333137666165313861376361 +39383830336637326466316432303232323165303633343866623963343234636439623434643435 +38633361356238313263663235323237643733646637313035316462393935396562376366386538 +38396665303831393838663435353435663631383062326533353236396439383531623464636537 +34356239663635373439393137346430386135306166313931313665353631613930663663313862 +66616164616463663963383263626333383962373262643837636530366230303562383938333961 +64323565393830646633383262313466363736646637396336343236333439396638643931636162 +63313338366332623131386539353435653234336632663631383939356161346664656265643563 +33633661393566303835643530616465316530613161363663313631613436633461323861313138 +61346665363130376639356235633031646633303838393963333030353962333466626633616131 +32643762623630613237313535636337306636356530336339663932333361313733333861306436 +66346631656665626439353566393034303833326630616532656434396639373335353162343936 +64633665613139623435303164363536326630373838613261616534326534666631613962653962 +35333264396235343931633066653763356666393065653933353032623231366564643939363437 +64646631613339326432336363626538393162663161646661333532383462663733383237636230 +32613661653931346231613137353731633833303630356532623635313538386364663366303261 +38383761336530326663613231353164343535613962633835643462366262336137633162333762 +37353964346562626133616235363661343166613062353934636132363631306266323031646637 +31356633653766376137663139373362373631653338613263306462343130613461333230646461 +33646132663135626231613937353065373034383131353733333131313930616235376134643663 +65366638636366646363386130663838306331623035626334373639313539666661663364316532 +65313334333965336531353233336135303439653538633266393065656235643166373432346362 +63653864376634353336353338353964643931353763646665363034316532623031663762666663 +32353934643961393536303864373761356534333462343866333161396236326531383563613435 +33333965313062366436666337323831653361383530343130643065613232356263393334386234 +63386162343838363465383037326163643165323038343936303061653837373934343662656337 +38353530666366303765336439383562666666623164336233333532306530393437626461373935 +61363830376266666438383766303832393630623033666565363432613631343239326563366136 +64666539666134373430333537343963623366633039336339626437636638323262373431333936 +63633932346539376463336232623565306262656239366433313264376538646365653065316534 +34623635623336636438373232383063393166653932353331386162643332326265383837666432 +63623262353434333437333863393930336337616337626337623531626462356538663531366432 +34363030343363316461393866323034353432656636326164333532626534363434313635663362 +37626435303134353435366437393435633565656166646165633531323965623234623035346236 +35346239333133313663643231356233303530303034313230333039613963366237373961623036 +31313438356239336437653035633133626166633066643663333935653038356638393461316261 +34343861633664656263656133326333653065393731346537613064393565303861613433386230 +35646165363662386538653139656264616565313562363534376637396432313233343133633038 +64343631613563623939303234313631333533623033376633376330643666623563376564326333 +65326663386664613465363161353431653465663333633436343031323464383533313532316465 +64363835663835363933666230353362636635333063663731643139633939626234656336333836 +34376338343237636433313939623036346635643735373735626530333761376436396238393266 +38646163653265303732623962303733346161646437366135633565373432616261353964633730 +33313935333532333336623631366466623231396538373134616137643333616437396534373837 +61633065613864653163333738626664303036353733306464353263366233633731393664383065 +66363533643363363365653662323038383864383664636137613163323932646433353066663331 +32386562633532366435393833623537626664396164303830653539373362376330323463316436 +32303830323332333131366164316164303836383238643636373838666462366233626661336461 +31623235323165303339303737343363396136656161666531653866316461363439303737363665 +34373235356632326562343563633735373834626665396362353732636232316139326665383063 +66653336303036363864663533633262363937373566306135626430653065393638656662633233 +32643938663030363436633463646562653961383332306462353933396335333765303936366230 +35396266653135346334323632633235626663626330343063666233643136343435383366393233 +33333761333937373164613861353039393633616361613339366633643439386365316162393264 +38313035613033626330323535666531323939313437383666323730343065366138303930643362 +36663863623534633930346336343761383632633737386332623533363632333262616330613063 +33333765633335636132646663306431313138356538366464643062663032343830653131633932 +65343265373939303433646333336133313132346535363064343261353463666137393362303862 +37363633336430666236646161643863633932343564353937386463366561613239656638666536 +34303961306333303938373666316637393964386231363736663731306435353833373032636135 +33303065383235363639383031323431363764306636633435323638616666313135633135303065 +64303531323030323430663762396132373037303363366333366639303164643335393861343035 +39356363623466633638633663393734363534313531356361663564376330343263393066353064 +31323965386231393736393631626438616635666130333630343461303339356537333165306338 +62643765656638626162356236383163363863653863613437633634323461313063623230323063 +38346131303737643233313435303361616165663138326431363361613536366366643831333362 +61383236613438373862373739656661323537616463386466333035646166383065343263343339 +36353233363266633435366630303463393430316134313965353262626631396230396462353030 +66653234343439353061623738663830663937363932623530653962633138323365623766386365 +31303933636438653532 diff --git a/vagrant/host_vars/prod/secrets.yml b/vagrant/host_vars/prod/secrets.yml index a71118d3..e30172a9 100644 --- a/vagrant/host_vars/prod/secrets.yml +++ b/vagrant/host_vars/prod/secrets.yml @@ -1,159 +1,159 @@ $ANSIBLE_VAULT;1.1;AES256 -38346539303366323035383961646532326632643533366661623764633665326530326262396532 -3239366336626230653262336661396465393364646234610a643261613233653431393763653035 -66343232326133356264363862353439353961363637616462376334616437343037643831646339 -3135633930303338620a613133343161373033666134653762373431656331633330303638346366 -65366466376138646535373438356335643530656564393234633731323336653339616161663961 -36363363643863313731343464373239336364333434353232636238383463353366303534396664 -37653061393863383839636132343130363837613835333934623131343562383362386339393235 -30336438316535633438393235663961616531386631666561643438616532363733356463343030 -66336561336239373530356336303935363137666436346563613065316533363339303863626430 -62373738386531386336323561643139303931353432366137396463313335323163326362343737 -61643466623838656136356137623535666363353864353832616633633964303765653838636336 -61306232353566353265353131383031306665373365336561666365373835346539383634336266 -66356232316364643036316463396564323436393463633839633538366234373563353465643934 -32383334383930623937346538346534366335663430623731366231313764643437363739363734 -32323939323836396137313365326534343834623536333432626238313232383233653661303464 -64316163653436383630323739636566663234336331666430333638666261623861323334613737 -39383866666132313963373338313864643866396261373664663134623862616665346435393030 -34356130353564303139393835333332393662626338303461376338396465326437636436383038 -62336362373138326633383866333738303262316334663335623530366330333665613039353466 -39303239346436326239386461636266313364656335643331623738663439393234383939306662 -63663231303161323066663333336137323436313337323938353536303736323639636138316361 -64396138363232393861653464366661666361646263623430376361353463643634623739643939 -30623934346135616261646662393339316536303836343130383365656339373231653633383730 -33333930653261306333666631626633313162626639353935343934663336336433613165303863 -64376364323062656161616538643865613535623166363863396133343434643663633765373535 -64316533356438373534366330646262643361363865333733333730303633663639626231633866 -62323835346264316263323035343437356430626437343564396164663961613236383637353236 -39323861343231646564653834386637653436323561336661663334636361653861343863643164 -61646666623436633661383530326563356534323166333465616236343930613839346637326662 -38643235656133366566663364653732343132386337383063613364636462643463393536356532 -66323738356635333161353535386234343631313433336237316665376639613861663864646438 -36633233333632643231663765346235376162363962346334653330663036626436646366353263 -34373562616334383366306531376134643666646164663964663634613731313139643839366630 -31623339376464323437653165666234373561306234643731356665613636363639323866306662 -38366533353962386664393634386266393163323466336633656132616431306631643463626532 -34663263303434613064373532326232636266336266626661663963616335353433306336653237 -34326633623333633961343361316665346235333637383239326333353038306566346139306361 -31323030393632363765646534306266633164383435383666313464613863353662663964386635 -32613830386261333635376538656138636639633031313233323839353961646432303439636437 -63326166336535393565343734616137626437303136613432653337656164323164383730653066 -34633831643266343736616437633236343061386162656432393035323863366566303365646262 -64383561626164373166376131663763393135336166316539373230313461316133633238343361 -31323366613730363766373662376636386339383435393033346232663632636131343134333065 -38656335386462366162306433633261623263383036316365343861666137633162623232333133 -34386665373930636337663063376439616131363961343833616437333539616566323363313134 -35383834613065313231363133633937306638316231343765306436316433616563653862376335 -36623933633061376132616263383632656231303465656132393533646261303766383532633538 -32333139323934623133666662643263646438613332613664323463653164353366373166356135 -63323131303836613638336435333937613564623138376234393635396538623536623762653234 -63646337636638653964663131626534313437333461623933373334396133353734316663356365 -38643066316261313366353938333765313731383039353731383137626531633633343031623732 -36313232326136393761626236643366323031393634323237623632343163386565313363613738 -61616439656637636562333232363832363636663935636435336537313133666633656134643135 -64336634313966343262393930626637623537366532363563366633306134356438613839386632 -34623537646235363930396137313363646266376465653539316566623365373237323263376366 -64356139373330313937373861353739373738613164396639623336306262386463373462316663 -66643961383764653337393031363262656139383536353862356530326231653739653730656330 -61653830663932666533323461663262383066633135343439346261653663663038623335666435 -34373531336663316437633236666237333036323830653932666563353239623130643061393363 -62393135313938326265343334646333643739343662613432386239333365313963646233383764 -33613865343561336635333237373861323637323639386431353766633061616566643763653365 -63346530623263376665303635306635376138376530373933343263666339373637653337656439 -32326137383364343332323730626461666431333164343130633437383730653138323632336330 -62636431633734643732336532633139396535306463373431376262313533346632643465646332 -33303037343231643737643763613164636538396137356538373962386564656364636535656633 -35623765393633653063343938376331373336653464326335623331306666656366306139326465 -66333833653439363830636538616462363632383831626532626539363134333531346161343166 -32323833656333626633363664396463376665383239353362353531323730633761363332623034 -33386133663936623938643661333936653463313864323864343834383134336339656334353432 -35623631376532666534336432646563323037363637333735653961653261613963643335353061 -38663161383432653333333261393866656366386334333933383965636161616633393835646530 -33323963306365646138623063333039386666636263366536316232353633643739303631663338 -35333961343963663639633865343138373438363939356662373932373930313931373164383865 -38653364306165656439643866343863383735386265353131396261336536646439313339613635 -30643436616539643632366133336330363761336164396662656236613331643931633434653638 -37646239613833333836656362323338306136626337303935623461366664663532333961666638 -66636432666336663363376133326637646566326239373934336562636235346133353734313964 -38643538373530646461346265643663333263396262383764396466393664663637336535396333 -38663237326265333162363734636630653739313464383364316432303761363437353964363066 -64376463306639623266346437623637363234616131646230366438356633383662366130313533 -37643132303239643731663635303761373931646537633533613162326536346362356636373364 -31636461623435396438373235613066376565626566353766656437623631653536663934636365 -63316235323134326432343038373839326264363161653730613265366666633534353262643865 -34316130626230373733656631376137346665366336666635346664633334373134363930386538 -33353835383261343334366133393263326631326537353432666434323531326332303937393438 -38623234396263343435303063303161646237353937353464306231373435643239663764373264 -63366662636434363031366664666431663238346666323262383463616138643265336363343232 -30353235366437643865633964323962343330303539613963633131363333396435346166636434 -66383439636561653265646131613537313733656135623034383931346564666135336434393965 -34353364636665323361323636343766303265393664343033616531653535353333663939663838 -64373939343934396235666532343431646434313237326234353362356230653165633864336537 -64663039343066343364313531353536323338623562373261343362643063313139623262323935 -38616631393339343265386466393161653862336637303638383438383563646161363530626435 -34313132333436356136396130336136323235386261666264656564303733653039393465666665 -38626661666131343837346436616437663738353733663661366462323461333636636539393639 -35383934636666646335366563363462376433616161376162333139323030396131623663356630 -38663563613431646532303634373739366266613439383436373465346338366334646633663963 -32623234666565316564633362323061626638646264656237633266626661653464353866393964 -34646461366639316431653136363761663837373333393164323861636136653661383931616464 -37363931336234613734336362363865663336326163386236366237303434326566663935633838 -32383264373435353762363030353236333130616636656539373438633134306162356536396338 -37623038633330346134333438613864636333613363313266656263653334313430346265633235 -35333936346262316535653234373430373431663365623865613061343939383837373064393766 -62366465343238386332613465613933303537383137623137366336303462386236333237663435 -64613839316431363734303438616561356634623237626133326633316263303665626430356661 -39306432613237376666306163613063323431646631623261353035323761656631613131633466 -30383162323266623162653739666264363462353063663837313866653232613732616634383563 -65656433616461393839626335653361386466626266623037636231623230376561396236646234 -30636633633561326562363365326536333530313134313535323032636361353637323465313334 -33663436383466363065393237306533643739366133313961656636613836343162316466333233 -31336533363162623535333363376262383038323065386563613539383565336635656635343932 -65666130383336313830313538666138633861306232643437393030646461636163373261666337 -62626633303432376531653333353764383533383465316135313362373433613631336632626232 -65396337306665396238353266306134396466623338393365313634373064313334653235336133 -31643963623338303135363537323137623936666565316236653430306232623830353266303330 -62323466316631316433356139643333383461336134333734303637316463666330613030356134 -35623263663066653963653933346433303534343630313933613439653830613131393162643961 -65643133373664633061316236623962326131303437663836383531396131356633383833636563 -61373738386438626139303561373235633363326563303737613430663866663936343736376137 -62353634613337666539333265336630306565643037363533663966393134346661643030353534 -36313938633732323265633563313339393564356132346664373163313662333732613736633130 -33636439643937333536363039396561326333376535363134326461333239393133393563326532 -33393062306539336163316631306539363939656663663136666561333465313462383632313330 -31323331316336373164393962646665346438653637306366366563323331366439663566653734 -62336562613738623564663632393163626632643865616162643566326565326430373661633837 -32336435356663373738383734613937363334333430353630646130666131656664653463386563 -33343564666163316532663737333961333963396162663561316135313539393736316665613334 -30373466613633636465363533366262626261626137323937656165323535383539383461363264 -31356161313332613234343734626565313731303730623063646664343364366531613935333365 -31386262633538386132333333303937643039363738343835663837363135303637386439386439 -32336535313436666132646133653866623731383236646534303765616439313665303137636430 -32333038343132373433313339656662653361316639336166643333333530653365353036343363 -33613131373963313630653363313636636339376264383730663862346233613434656638333935 -32333433626461396137636165393432636533303961386432343666333266623933363562653164 -39613237366437643134363433386339623839363531363763343735636238333464373033343064 -30626533356632613732303835623764363664333631623665643634336463363934393339393061 -33653433643732616633343564656634316632343838636530396331333163613265626334373437 -34653730626563633864363065383431333466653362643630353839333035393961643739393034 -65616261303864343332333163633735326131313762613335646365343135616166646238353036 -61336536613661383837313633613065313937393939643238346538626633623139356661376335 -30386533376631313732646533326562383332656336613537303432363331653764396331653438 -62323035346230356562363032393032336632663363393331333130316430663830303934666636 -64326162343836326330393130343737343633613161653731306535313637343932306339663539 -35616530326565656161313764663866366163383539643631663939363836303538303966616361 -30643235356537326132646432396665323331383961343162393964373364646634623935656564 -37383831303661316665373461306361666536626163633238366134336662663539313966376230 -32656433346631316638656362313535633630303938323332346135323162633831313736616536 -62613733383763613235383932623239663666316337316164333333626637616433373561366332 -32616261626139393830623162373639356533393439653366363266336538366166623330346361 -37653865356235356637643832396334303631633633366332353035313239333664646431363264 -38356437623431396665663061316630663662376436646634333930306238323533373538353466 -31323565313434353635333334333536356438623231393237303032356462303030396166666330 -34376463356437396635343062353034666438383733383066383333326235636436396633393532 -32653662343732396635303936343531383838323832646236333239653036633835316635343638 -61306138386364633137303237323265633737396537306435343161646139326238623938316436 -30613638323563396232323464633430393331646130336164386463303337616366663262623833 -61623833633338333438373839643361616165633762343333333864386235363537323732356536 -3931643361643034346432333064646435323030363763353431 +37303235636630316435383233653430366438623337383033636361383766666232616330303263 +3138326465636535373638353765616237306265656639620a343362663063326666313237653263 +65383565353338303137316563346636356535616365343762633762643136653736646131626264 +3132633030623562390a663936353138313262653431336238656662366462373961393739373639 +35616533346132346462663633356636363134363737663834316466393435313366666138376435 +32323862663864616463346239663432646430643765386633643432356462333263386665326438 +31346632636166323163323730343937336563313235396265393261326339636263316438396461 +38316238633932326664386235313365613430313761646162383265613030356434633631336365 +62323830346631656635623438646162323136633038643233663963376163306661393332376130 +65346234343566613836306461613261623561313765663237353962663834396635393161626138 +65663731663162623062633436303238626631326632613339343335353036313231626331373538 +32393062363665666131646337613231383934323132326332396338396361323832353263623938 +66393635323836656365663866616637613335646163353065626538386639393365373366363163 +65643266363537313237373765366665643863363239383330646233366333363462396233393733 +31376464623436623165316365613665643236316136633337663762323363386163396364636332 +30343065623238313139343839653633316664313634636535666435366230333961343163613839 +61646666313136613734663238306563633837613134376362633966353233636433363166383363 +31363962636633616433646236626132633534663430653631356538336139613430653366353061 +33383634316137323132396336383062633866616639366633616363393161656361363834326435 +33373561396466383638613665376137653738616461333236323463313264643334313662363337 +61663434303464633163353366323965643865623931383565623165343565393665396239386533 +34353431313561336630663963363933666639636237323033343566626132386138363433346435 +32353761313765616230383639633061386161343363333938646163363732346337393930363761 +61396237633065366135386263613463663633303734313438383965613964316232633265636631 +65623236323930353665313165653764393263613339666634353364303165346432313739646537 +62333632383139666436313135653130643562643936323336626361336364313764346263316231 +30613730356133333762356631383032366538393139643337303963646630643934333431303234 +63636536633763616434623432646137376639616535313535373835363933613265363236346438 +62393331653435333539623461343239393230653462333463633538393063343139343564626234 +64323337336366643362613563333835353839386333613361383937633166333633333763646133 +35353031303734313161396430646137653636363631646465663739316334663339643832373836 +36316131356366633834393130393834643138663361366562313139313862393464306462336165 +34623335303831323336366265393938313738303837653435633735613139383738386561386639 +66633333373466313063643063623332383566656366336531326632396162333835653865663934 +33306139633462613661323861363563393334653662376565616437636436653263316536303532 +33663339636561646135383962666237666630663130623665653864383436356462613466393034 +64623033653966666533323465346230313662613237343035303033616263383165633865633535 +35366334376335313831303030336637636462656534353830623431363536643361663366656361 +31623162373161383866623465306162346534336435393639623536316662346531386434326537 +64383931663034656633636563386561396164636232363037316230393132303237333132383339 +66303665353839626462396132366666356339666238626132663531323236346532363866623530 +64386561646661336162656264346539343866343763616439306436636635663264613237366462 +66343337383265343832393139633562323738313232333762313131373636306234303266316435 +62663433323166386532643337666262656430663530643564303334393732653666656138303232 +38363835333137643537666561653030626633666263336661326437636464363261343938373335 +65346237323231653338366235393831653237393634633064353133323539306263336430373061 +66663134363931663263303031383736353135613839333638643435386234663334376366663139 +36663432653036666430313333343639316430613036393338326338373862653931383338376339 +62656236613764326636623238663030613864386633656337373064626632323733333164383238 +62363462336366396632626237663033383533313832353161393238336131353566323566396339 +62376566326535376532306632623531643138313334343165356665353631363033303730613935 +30633032343134353937653638373130343237613632303238333065663135633661383865336632 +31393138353535376538326466656132373666326332336364356530613862653562303765333832 +31386565393538656236626263356436393437353133663032613139333261373239356465353432 +33643635346334393136346236396636343861316364616465366339356637623061383830313863 +33333230643435356334376634363931653933666262383464643861373539326439396438333132 +62363166346664326264633163363134636331643130336232313565666664363062653863323165 +35396333396561316233353839643332633639303939663033386563313831366238666136393464 +61623662373638656336623962336461626161363261613935643965333938616265396635323433 +34323431333837323163313730323537653135663633333366376238643930663863636262303838 +38663034653339366237303830336537353137333866393033383463323931373135633538303234 +39353664623839386633636430636565333631633531383362396131336534376664326139326535 +38613434373737636464663737313430323536393737393262653866656463336561386566646133 +61393239636633616332643336383438396536383932633266303361393066393830663237393137 +66316437653865393561633862633061643635306334613432633839633332356638356461383961 +31376531373866303231313463653661656636623235613639666439323661316230313937333038 +36333362353638383332343438363330333263666566356263336639626332313363346539393362 +61333136396136313737633234353466383933376537346365393665633739636439313430323037 +62666331613034346137333334376165613162623432346461646561613233393235363631656431 +37666632643837353238306235613766613131333239316434656237663965616431383761653966 +36346130633765306336393033326135323336326530396635333264306535353965353031306662 +34663766373935653464353561636430626538653464363030653335613231643733303831646437 +65396136306330663432613932623932383566346636643932393036613030636139323730376639 +30663132323431396339663963666566383765393935616335393030366332313665383035646631 +37313166396636363464333763316235613562363164643133303930616632366434656531633536 +65376430313631373934316330316234666336613263643133323632653761623035343966356165 +32323231646537366565623963373630353735343463663361373837306261373337656363633166 +64393930326431613433316232343461363530663761666166376665656130643665633238396264 +32643933353666383365323262623733353033653766623734353837333430333664306662636434 +34623635353536333539373032323835383638393033316661366661316565396161666334323830 +31353631313165393230353438623132323339303664373264666464656336633966313539636564 +37616636646334663133376130663731653461373664343966306637346632386262656133663432 +37353062633434643063623136356364623833336336666235366366656666366232623136383739 +37626436643562646430633564396439323730623563616366303137653765666533306331643331 +66633934386430383631623166386539313035326632386639646261643236366333663539643330 +30613535666238623137626236313038656335353863623031366466303934616438336135656433 +36653966333236326636663763393766643265366164303930326361313265326430333766366139 +64356138666230303263383061376138613938353039313163646662336435313966316239313662 +31353334303532646164313634316666663562613037373564646566646136633638326333613834 +64383131356433653837353132623263313330376338613836633365353965383965666239313066 +62623462376566613039343235653635353064393861326134653535373932623962376339613637 +30646461646230333330623832356166653866633731306465663566653733363630653037643638 +64326462663830653335663932616665313064333333626433616637656662663834653935393531 +31313062326639393231303064636262386361643635353737383261333035646638383030343963 +62393963346464383966663766363262396636656163633638306361373166633762613963666339 +37353130666463653634333233633334666364623834313532363237396233613834653237653365 +62356432633539363530393131323865366464383465623065333637613733313935663661393665 +33313364643833336131353962373361363162373863356463316132356135316561313564383636 +65316364636666333063666232363164656139623636636661646433613964656631346438366163 +30343931633234633137616235303563643039663035623334316631376533323132623334376135 +33353137313534323335333464306336626139373532356536323630393066303633373836326636 +63636263343061366233323134356131316232616162313361656639306332313438316436353236 +65643239333730666633313962386236393663393537613435333935336331353664386538346139 +39616532653239656135393535306261656563386331313734633633333161326333353331613637 +66656662393830343435303462393865386337353539346663303535633738336639396337336466 +65383661626462653037326636356537313039313066613862386462643634326331326236366631 +31303238613531393766363165306534383037623534323232313463343766336661383637616538 +38343864386532303736316638663038636332633264353464393662336532353635643165633533 +63373632653066613133653962356439663966326333623332333737323934346139346330343634 +63613130376533656633356532613963386666306435366237643135613937633032656431353833 +31393033656531643761313431623638383038356233643531343466613637376661343463633437 +63336233623034303038343636333566306331343966393865643234613233316539313762396161 +34316638383163366435643535633834613930393432616464336434363230353964646433613433 +31623064393733393065653662613334313239366333323438396365666335666666306363396133 +35356131666132336134386237666633653231636361633930663065346666656137376661613137 +66333564653232303266386435333734316535333162323338323265363439373935666138666438 +30623161353134353964373334383530613663643061323864363662393035336136666135646330 +63363737643538643935613962643866663936323264333864393764376130386233663334366431 +38356438623238313966623130626139313664623139656166366334326238333031363539363037 +36313930633037353835326635376164633239643737326161646634653933333239313637626662 +61313762346539383036336330313665623337356639326237376330613465653736396138316466 +36326538313161663935306166653864386637613432316266633534396365336338303939346536 +31623664353935333930366236666634363133316438633439336637376366366331373339653835 +31343637643838656465343037613561643063323563653263636532386633303966316364343461 +32656464383661666335323037656436373132306566366262353037353438396337316364313666 +66363030306465616164343035376337636431636339663531333239663864316465373332373534 +65313462376535333661343931623966336435633730313035353631383436333162623030373061 +31383239333164363538356430373934356161633634613432623834396562616134643665373064 +39656364303962363035333835616463636331666336616335383035626538663266633330363231 +34636630373534613033616165383935343436386661626161633336366265333738376663333233 +38623438376434306133323365636635643234616533333063323565313036313739626464643535 +38333662646635373337396666376638346337346536656263656663333266653932323936336633 +64643366386161323037653335663533613066313862333630653530323765646233643861653837 +61656231333039633463653762343463366334623137663032613861646539343732376264336139 +38366138383762383531386336333463663839333064623765306363356536323362346433343665 +64616261653933376561393632316139313534376233313036313963613534623539353232376534 +66646362316364373334323533326433383230326332343765623566653035336266383634353238 +66353164623565306230353834396434633233396335333930353237633663353862346437343436 +37383666666263646538616665646264643535333532376662396538393062393238376263326235 +30323437623261356565636466313032343135346166663934656537663837383639323638323937 +38323562323565306336636131313430656231376339666535633035363861373639633535343533 +39313563353863663439616132316136393231353266396638643761653363653564376335646365 +34393338323462356363613661313536333233653965323434336361633730346666393034613966 +30353437323366653738376461346531316137376164616164363766386131313332323535656236 +39633038303636393536363938643638383037383130663263313832653064396330316134613736 +32316663393832373133306435616364656633646565326661633535396634643861323831633661 +65303737663731653066343531336330633732376563303163653065326663366264303537646238 +34663833363538646236646263383634366133376365336638633962343864393131326565363333 +39666636626461346632643361643862323233613963323266623434303338383366656436653832 +35633865663062386437336665323566616538623265323365616666373062306331643835316564 +61643962373131366432616562323463383534376534636132653761663232613765353132313964 +33356634636335373837396530663830613331383235323965356165356235623036303631376139 +32313532636433653433646662633664383666633566313335613530346564353161376536646466 +62353533366434366534353261383363633762653532333431383930323637623736613637653930 +33623037373439623736356264346562383533323330323164633638343364383035376463633831 +38626634393937356534653237303835336666623338646361313333303462653661623133356236 +37636133363863633765396561363163343365613030633638393630653165323861333337313231 +6536633566373865643733383466646439383065636532336431