First pass at a Vagrant setup for please.unglue.it

vagrant/README.md

@@ -0,0 +1,19 @@
+So far, the vagrant/ansible setup is meant to build please.unglue.it on AWS.
+
+For this script to work, you need to have AWS_ACCESS_KEY and AWS_SECRET_KEY environment variables set.
+Run:
+
+```
+vagrant up --provider=aws
+```
+
+followed by
+
+```
+a-play-aws.sh unglueit.yml
+```
+
+I might not have removed all the dependencies on my own laptop yet for this setup.
+
+Need to fill out host_vars/unglueit.template and copy it to hosts_vars/unglueit
+

vagrant/Vagrantfile

@@ -0,0 +1,72 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  # All Vagrant configuration is done here. The most common configuration
+  # options are documented and commented below. For a complete reference,
+  # please see the online documentation at vagrantup.com.
+
+  # Every Vagrant virtual environment requires a box to build off of.
+  config.vm.box = "precise64"
+  config.vm.box = "dummy"
+
+
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+  config.vm.network "private_network", ip: "192.168.33.10"
+  config.ssh.forward_agent = true
+  #config.vm.network "private_network", type: "dhcp"
+
+  #config.vm.synced_folder "data", "vagrant_data"
+
+  #you can replace box001 with another name
+
+  config.vm.define 'unglueit' do |node|
+    node.vm.provision 'ansible' do |ansible|
+      # http://bertvv.github.io/vagrant-presentation/#/vagrant-configuration
+      # "define directive is important to make automatic inventory work"
+      # I wish I could comment out the following line to allow for
+      #  .vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory
+      # to be calculated w/o running any ansible playbook -- but no luck
+      ansible.playbook = 'hello.yml'
+    end
+  end
+
+ # 512MB not enough for compiling lxml: http://stackoverflow.com/a/25916353/7782
+ # http://stackoverflow.com/a/26468913/7782 --> for how to get to this setting
+  config.vm.provider "virtualbox" do |v|
+      v.memory = 1024
+      v.cpus = 2
+  end
+
+  config.vm.provider :aws do |aws, override|
+    aws.access_key_id =  ENV['AWS_ACCESS_KEY_ID']
+    aws.secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+    
+    #aws.region = "us-east-1"
+    #aws.region_config "us-east-1", :ami => "ami-7747d01e"
+    
+    #aws.session_token = ""
+    aws.keypair_name = "ry-laptop"
+
+    # Ubuntu 12.04 LTS Precise
+    # PV EBS-SSD boot
+    # alestic 2015.05.05
+    
+    aws.instance_type="t1.micro"
+    
+    aws.region = "us-east-1"
+    aws.availability_zone = "us-east-1c"
+    aws.ami = "ami-d8132bb0"
+    
+ 
+    override.vm.box = "dummy"
+    override.ssh.username = "ubuntu"
+    override.ssh.private_key_path = "/Users/raymondyee/.ssh/id_rsa"
+  end
+
+
+
+end

vagrant/a-cmd.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+ansible -i .vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory --private-key=~/.vagrant.d/insecure_private_key -u vagrant unglueit  -a $1

vagrant/a-play-aws.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+ansible-playbook -vvvv -i .vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory \
+          --private-key=/Users/raymondyee/.ssh/id_rsa \
+          -e aws_access_key=$AWS_ACCESS_KEY \
+          -e aws_secret_key=$AWS_SECRET_ACCESS_KEY \
+          -u ubuntu $1

vagrant/a-play.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+ansible-playbook -vvvv  -i .vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory --private-key=.vagrant/machines/unglueit/virtualbox/private_key -u vagrant $1

vagrant/ansible.cfg

@@ -0,0 +1,5 @@
+[defaults]
+library = ./library
+inventory = .vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory
+private-key = ~/.vagrant.d/insecure_private_key
+remote_user = vagrant

vagrant/hello.yml

@@ -0,0 +1,7 @@
+- name: Hello Ansible
+  hosts: unglueit
+  sudo: no
+
+  tasks:
+    - name: Hello server
+      shell: date > now.txt

vagrant/hello2.yml

@@ -0,0 +1,21 @@
+- name: Hello Ansible
+  hosts: unglueit
+  vars:
+    user: "{{ ansible_ssh_user }}"
+  sudo: yes
+
+  tasks:
+    - name: Hello server
+      shell: date > now.txt
+            
+    - name: copy deploy/celeryd_please.conf
+      command: cp /opt/regluit/deploy/celeryd_please.conf /etc/default/celeryd
+      
+    - name: set mode on /etc/default/celeryd
+      file: path=/etc/default/celeryd mode=0644
+      
+    - name: copy deploy/celerybeat_please.conf
+      command: cp /opt/regluit/deploy/celerybeat_please.conf /etc/default/celerybeat
+      
+    - name: set mode on /etc/default/celerybeat
+      file: path=/etc/default/celerybeat mode=0755

vagrant/host_vars/unglueit.template

@@ -0,0 +1,4 @@
+github_auth_key: 
+mysql_root_pw: unglueit_pw_123
+mysql_regluit_pw: regluit
+

vagrant/library/ansible_module_template.py

@@ -0,0 +1,82 @@
+#!/Users/raymondyee/anaconda/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2015, Raymond Yee <raymond.yee@gmail.com>
+
+
+import json
+import base64
+
+DOCUMENTATION = '''
+---
+module: MODULE_NAME
+short_description: MODULE_SHORT_DESCRIPTION
+description:
+     - Adds service hooks and removes service hooks that have an error status.
+version_added: "1.4"
+options:
+  user:
+    description:
+      - Github username.
+    required: true
+  oauthkey:
+    description:
+      - The oauth key provided by github. It can be found/generated on github under "Edit Your Profile" >> "Applications" >> "Personal Access Tokens"
+    required: true
+    
+   
+
+
+author: Raymond Yee, raymond.yee@gmail.com
+'''
+
+EXAMPLES = '''
+# Example creating a new service hook. It ignores duplicates.
+- github_hooks: action=create hookurl=http://11.111.111.111:2222 user={{ gituser }} oauthkey={{ oauthkey }} repo=https://api.github.com/repos/pcgentry/Github-Auto-Deploy
+
+# Cleaning all hooks for this repo that had an error on the last update. Since this works for all hooks in a repo it is probably best that this would be called from a handler.
+- local_action: github_hooks action=cleanall user={{ gituser }} oauthkey={{ oauthkey }} repo={{ repo }}
+'''
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(
+        oauthkey=dict(required=True),
+        repo=dict(required=True),
+        user=dict(required=True),
+        validate_certs=dict(default='yes', type='bool'),
+        content_type=dict(default='json', choices=['json', 'form']),
+        )
+    )
+
+    action = module.params['action']
+    hookurl = module.params['hookurl']
+    oauthkey = module.params['oauthkey']
+    repo = module.params['repo']
+    user = module.params['user']
+    content_type = module.params['content_type']
+
+    if action == "list":
+        (rc, out) = _list(module, hookurl, oauthkey, repo, user)
+
+    if action == "clean504":
+        (rc, out) = _clean504(module, hookurl, oauthkey, repo, user)
+
+    if action == "cleanall":
+        (rc, out) = _cleanall(module, hookurl, oauthkey, repo, user)
+
+    if action == "create":
+        (rc, out) = _create(module, hookurl, oauthkey, repo, user, content_type)
+
+    if rc != 0:
+        module.fail_json(msg="failed", result=out)
+
+    module.exit_json(msg="success", result=out)
+
+
+# import module snippets
+from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
+main()

vagrant/library/filetest.py

@@ -0,0 +1,36 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2015, Raymond Yee <raymond.yee@gmail.com>
+
+import os
+import json
+
+# import module snippets
+from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
+
+DOCUMENTATION = '''
+---
+module: filetest
+short_description: tests for existence of a path
+description:
+     - Checks on the existence of a given path
+#options:
+#  path:
+#    description:
+#      - path to test.
+#    required: true
+   
+
+author: Raymond Yee, raymond.yee@gmail.com
+'''
+
+def main():
+    cwd = str(os.getcwd())
+    print ( json.dumps({
+    "cwd" : cwd
+}))
+
+main()

vagrant/library/github_deploy_key.py

@@ -0,0 +1,92 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2015, Raymond Yee <raymond.yee@gmail.com>
+
+import os
+import json
+import base64
+from StringIO import StringIO
+   
+from github import Github
+
+DOCUMENTATION = '''
+---
+module: github_deploy_key
+short_description: create a deploy key to a github repository
+description:
+     - create a deploy key to a github repository
+options:
+  repo_name:
+    description:
+      - the repository to write key to
+    required: true
+  github_auth_key:
+    description:
+      - The oauth key provided by github. It can be found/generated on github under "Edit Your Profile" >> "Applications" >> "Personal Access Tokens"
+    required: true
+  key_path:
+    description:
+      - location of the key to upload
+  key_name:
+    description:
+      - name for the key
+
+
+author: Raymond Yee, raymond.yee@gmail.com
+
+dependency: pygithub
+'''
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            github_auth_key=dict(required=True),
+            repo_name=dict(required=True),
+            key_path=dict(required=True),
+            key_name=dict(required=True),
+        )
+    )
+
+    github_auth_key = module.params['github_auth_key']
+    repo_name = module.params['repo_name']
+    key_path = module.params['key_path']
+    key_name = module.params['key_name']
+
+    failed = True
+    
+    try:
+        g = Github(github_auth_key)
+        s = open(key_path).read()
+        repo = g.get_repo(repo_name)
+        key = repo.create_key(key_name, s)
+        
+        failed = False
+    except Exception, e:
+        failed = False
+    
+    # error handling and what to return with success?
+    
+    if not failed:
+        msg = "None:success"
+    else:
+        msg = str(e) + " " + str(e.message) + " " + str(e.get(args))
+
+        
+    module.exit_json(
+      changed = True,
+      github_auth_key = github_auth_key,
+      repo_name =  repo_name,
+      key_path = key_path,
+      key_name = key_name,
+      msg = msg,
+      failed = failed
+    )
+ 
+
+# import module snippets
+from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
+main()

vagrant/load_data_please.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+cd /opt/regluit
+source ENV/bin/activate
+export DJANGO_SETTINGS_MODULE=regluit.settings.please
+
+sysadmin/drop_tables.sh | django-admin.py dbshell
+cat test/campaign_starter.sql | django-admin.py dbshell
+
+django-admin.py migrate default 0001 --fake
+django-admin.py syncdb --migrate --noinput

vagrant/unglueit.yml

@@ -0,0 +1,351 @@
+- name: unglueit setup
+  hosts: unglueit
+  vars:
+    user: "{{ ansible_ssh_user }}"
+  sudo: yes
+
+  pre_tasks:
+  - name: check apt last update
+    stat: path=/var/cache/apt
+    register: apt_cache_stat
+  - name: update apt if needed
+    apt: update_cache=yes
+    when: ansible_date_time.epoch|float - apt_cache_stat.stat.mtime > 60*60*12
+
+  tasks:
+    
+    - name: set DNS record for please.unglue.it
+      route53:
+        command: create
+        zone: unglue.it
+        record: please.unglue.it
+        type: A
+        ttl: 60
+        value: "{{ansible_ssh_host}}"
+        overwrite: yes         
+        aws_access_key: "{{aws_access_key}}"
+        aws_secret_key: "{{aws_secret_key}}"
+        
+    - name: do apt-get update --fix-missing
+      command: apt-get update --fix-missing
+      
+    - name: installing dependencies
+      apt: pkg={{ item }} update_cache=yes state=present
+      with_items:
+        - git-core
+        - apache2
+        - libapache2-mod-wsgi
+        - mysql-client
+        - python-virtualenv
+        - python-mysqldb
+        - redis-server
+        - python-lxml
+        - python-dev
+        - libmysqlclient-dev
+        - libxml2-dev
+        - libxslt1-dev
+        - python-setuptools
+        - python-dev
+        - postfix
+      tags: install
+      
+    - name: make {{user}} group
+      group: name={{user}}
+      
+    - name: make {{user}} user 
+      user: name={{user}} shell=/bin/bash group={{user}} generate_ssh_key=yes
+      
+    - name: install some python modules to use
+      #pip: name={{item}} virtualenv=/home/{{user}}/venv
+      pip: name={{item}}
+      with_items:
+        - PyGithub
+      
+    - name: create /opt/regluit
+      file: path=/opt/regluit state=directory owner={{user}} group={{user}} mode=0745
+    
+    - name: git config
+      command: "{{item}}"
+      with_items:
+        - git config --global user.name "Raymond Yee"
+        - git config --global user.email "rdhyee@gluejar.com"
+ 
+    - name: ssh-keygen
+      #command: pwd
+      command: ssh-keygen -b 2048 -t rsa -f /home/{{user}}/.ssh/id_rsa -P ""
+      sudo: no
+      args:
+        creates: /home/{{user}}/.ssh/id_rsa
+ 
+    - name: create deploy key for repo
+      action: github_deploy_key
+      sudo: no
+      args:
+        github_auth_key: "{{github_auth_key}}"
+        repo_name: Gluejar/regluit
+        key_name: vagrant_ansible_test
+        key_path: /home/{{user}}/.ssh/id_rsa.pub
+ 
+    - name: postfix install
+      raw: DEBIAN_FRONTEND='noninteractive' apt-get install -y -q --force-yes postfix
+      
+    - name: clone the regluit git repo into /opt/regluit
+      sudo: no
+      git: repo=ssh://git@github.com/Gluejar/regluit.git dest=/opt/regluit accept_hostkey=True force=yes version=sysadmin
+    
+    
+    # installing mysql
+    # https://github.com/bennojoy/mysql --> probably the right way
+    # how do you make use of other people's playbooks in the right way?
+    # http://stackoverflow.com/a/7740571/7782
+    
+    - name: mysql setup
+      raw: debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password password {{mysql_root_pw}}'
+    - raw: debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password_again password {{mysql_root_pw}}'
+    - raw: apt-get -y install mysql-server
+    
+    - name: Create regluit database
+      mysql_db: db=regluit state=present encoding=utf8 collation=utf8_bin login_user=root login_password={{mysql_root_pw}}
+      
+      # GRANT ALL PRIVILEGES ON regluit.* TO 'regluit'@'localhost' WITH GRANT OPTION; (covered?)
+    - name: Create database user
+      mysql_user: >
+          user=regluit
+          password={{mysql_regluit_pw}}
+          host=localhost
+          priv=*.*:ALL
+          state=present
+          login_user=root
+          login_password={{mysql_root_pw}}
+
+ 
+    # running stuff within a virtualenv
+    # http://stackoverflow.com/a/20572360
+    # http://stackoverflow.com/questions/20575084/best-way-to-always-run-ansible-inside-a-virtualenv-on-remote-machines?rq=1
+
+
+    ## hard coding of please
+    #sudo("ln -s /opt/regluit/deploy/please.conf /etc/apache2/sites-available/please")
+
+    - name: link please.conf into sites-available
+      file: src=/opt/regluit/deploy/please.conf  dest=/etc/apache2/sites-available/please state=link
+
+    #run('pip install -r requirements_versioned.pip')
+
+    - name: pip install requests (to see whether in right place)
+      pip: >
+           executable=/opt/regluit/ENV/bin/pip
+           name={{item}}
+           virtualenv=/opt/regluit/ENV
+           virtualenv_command=virtualenv
+      with_items:
+        - requests
+        - census
+      sudo: no
+           
+    
+    - name: pip requirments
+      pip: >
+           executable=/opt/regluit/ENV/bin/pip 
+           requirements=/opt/regluit/requirements_versioned.pip
+           virtualenv=/opt/regluit/ENV
+           virtualenv_command=virtualenv
+           virtualenv_site_packages=yes
+      sudo: no
+           
+           
+    #run('echo "/opt/regluit/" > ENV/lib/python2.7/site-packages/regluit.pth')
+    #run('echo "/opt/" > ENV/lib/python2.7/site-packages/opt.pth')
+
+    - name: establish regluit.pth
+      lineinfile: create=yes dest=/opt/regluit/ENV/lib/python2.7/site-packages/regluit.pth line="/opt/regluit/"
+      
+    - name: establish opt.pth
+      lineinfile: create=yes dest=/opt/regluit/ENV/lib/python2.7/site-packages/regluit.pth line="/opt/"
+ 
+    #sudo('mkdir /var/www/static')
+    #sudo('chown ubuntu:ubuntu /var/www/static')
+
+    - name: create /var/www/static
+      file: path=/var/www/static state=directory owner={{user}} group={{user}} mode=0755
+      
+    #
+    #run('django-admin.py syncdb --migrate --noinput --settings regluit.settings.please')
+
+    #Run syncdb on the application
+
+    - name: django syncdb
+      django_manage: >
+          command=syncdb
+          app_path=/opt/regluit/
+          settings="regluit.settings.please"
+          virtualenv=/opt/regluit/ENV
+      notify:
+        - restart apache2
+
+
+    - name: django migrations
+      django_manage: >
+          command=migrate
+          app_path=/opt/regluit/
+          settings="regluit.settings.please"
+          virtualenv=/opt/regluit/ENV
+      notify:
+        - restart apache2
+      
+    #run('django-admin.py collectstatic  --noinput --settings regluit.settings.please')
+
+    - name: django collectstatic
+      django_manage: >
+          command=collectstatic
+          app_path=/opt/regluit/
+          settings="regluit.settings.please"
+          virtualenv=/opt/regluit/ENV
+      notify:
+        - restart apache2
+          
+    - name: copy STAR_unglue_it.crt 
+      copy: > 
+        src=/Volumes/ryvault1/gluejar/other_keys/unglue_it/STAR_unglue_it.crt 
+        dest=/etc/ssl/certs/server.crt 
+        owner={{user}} 
+        group={{user}} 
+        mode=0644
+      notify:
+        - restart apache2
+
+    - name: copy server.key
+      copy: > 
+        src=/Volumes/ryvault1/gluejar/other_keys/unglue_it/server.key 
+        dest=/etc/ssl/private/server.key
+        owner={{user}} 
+        group={{user}} 
+        mode=0600
+      notify:
+        - restart apache2
+
+    - name: copy STAR_unglue_it.ca-bundle
+      copy: > 
+        src=/Volumes/ryvault1/gluejar/other_keys/unglue_it/STAR_unglue_it.ca-bundle 
+        dest=/etc/ssl/certs/STAR_unglue_it.ca-bundle
+        owner={{user}} 
+        group={{user}} 
+        mode=0600
+      notify:
+        - restart apache2
+
+    - name: a2dissite default
+      command: a2dissite default
+      notify:
+        - restart apache2
+
+    - name: a2ensite please
+      command: a2ensite please
+      notify:
+        - restart apache2
+     
+    - name: a2enmod ssl rewrite
+      command: a2enmod ssl rewrite
+      notify:
+        - restart apache2
+
+    #
+    #sudo ("/etc/init.d/apache2 restart")
+    #
+    
+    - name: turn on ports 22, 80, 443
+      ufw: rule=allow port={{ item }} proto=tcp
+      with_items:
+      - 22
+      - 80
+      - 443    
+
+    - name: enable ufw
+      ufw: state=enabled
+      
+    #with cd("/opt/regluit"):
+    #
+    #    sudo ("yes | adduser --no-create-home celery --disabled-password --disabled-login")
+    
+    - name: make celery group
+      group: name=celery
+      
+    - name: create celery user
+      user: >
+         name=celery
+         createhome=no
+         group=celery
+         generate_ssh_key=no
+
+    #    sudo ("cp deploy/celeryd /etc/init.d/celeryd")
+    #    sudo ("chmod 755 /etc/init.d/celeryd")         
+    
+    - name: copy deploy/celeryd
+      command: cp /opt/regluit/deploy/celeryd /etc/init.d/celeryd
+    
+    - name: set mode on /etc/init.d/celeryd
+      file: path=/etc/init.d/celeryd mode=0755
+    
+    #    sudo ("cp deploy/celeryd.conf /etc/default/celeryd")
+    
+    - name: copy deploy/celeryd_please.conf
+      command: cp /opt/regluit/deploy/celeryd_please.conf /etc/default/celeryd
+    
+    - name: set mode on /etc/default/celeryd
+      file: path=/etc/default/celeryd mode=0644    
+    
+    #    sudo ("mkdir /var/log/celery")
+    - name: make /var/log/celery
+      file: path=/var/log/celery state=directory owner=celery group=celery mode=0755
+      
+    #    sudo ("mkdir /var/run/celery")
+    #    sudo ("chown celery:celery /var/log/celery /var/run/celery")
+    
+    - name: make /var/run/celery
+      file: path=/var/run/celery state=directory owner=celery group=celery mode=0755
+    
+    #    sudo ("/etc/init.d/celeryd start")
+    
+    - name: start celeryd
+      command: /etc/init.d/celeryd start
+    
+    #    sudo ("cp deploy/celerybeat /etc/init.d/celerybeat")   
+    #    sudo ("chmod 755 /etc/init.d/celerybeat")
+    # https://stackoverflow.com/questions/24162996/how-to-move-rename-a-file-using-an-ansible-task-on-a-remote-system
+
+    - name: copy deploy/celerybeat
+      command: cp /opt/regluit/deploy/celerybeat /etc/init.d/celerybeat
+    
+    - name: set mode on /etc/init.d/celerybeat
+      file: path=/etc/init.d/celerybeat mode=0755
+    
+    #    sudo ("cp deploy/celerybeat.conf /etc/default/celerybeat")
+    
+    - name: copy deploy/celerybeat_please.conf
+      command: cp /opt/regluit/deploy/celerybeat_please.conf /etc/default/celerybeat
+    
+    - name: set mode on /etc/default/celerybeat
+      file: path=/etc/default/celerybeat mode=0755
+    
+    #    sudo ("mkdir /var/log/celerybeat")
+    #    sudo ("chown celery:celery /var/log/celerybeat")
+
+    - name: make /var/log/celerybeat
+      file: path=/var/log/celerybeat state=directory owner=celery group=celery mode=0755
+    
+    #    sudo ("/etc/init.d/celerybeat start")
+    - name: start celerybeat
+      command: /etc/init.d/celerybeat start
+      
+    # run data loading script
+    - name: run data loading script
+      script: load_data_please.sh
+      
+    # set up crontab
+    - name: crontab for please
+      command: crontab /opt/regluit/deploy/crontab_please.txt
+
+
+  handlers:
+    - name: restart apache2
+      service: name=apache2 state=restarted