regluit/build_ec2_instances_for_django.ipynb

{
 "metadata": {
  "name": "build_ec2_instances_for_django"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from regluit.sysadmin import aws\n",
      "reload(aws)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "AMI_UBUNTU_12_04_ID = 'ami-79c0ae10'\n",
      "image = aws.ec2.get_all_images(image_ids=[AMI_UBUNTU_12_04_ID])[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "image.id, image.name"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# use default security group for now -- probably want to make a new one\n",
      "\n",
      "INSTANCE_NAME = 'new_test'\n",
      "SECURITY_GROUP_NAME = 'testsg1'\n",
      "\n",
      "(instance, cmd) = aws.launch_instance(ami=AMI_UBUNTU_12_04_ID, \n",
      "                                      instance_type='t1.micro',\n",
      "                                      key_name='rdhyee_public_key',\n",
      "                                      group_name=SECURITY_GROUP_NAME,\n",
      "                                      tag='new_instance',\n",
      "                                      cmd_shell=False) \n",
      "                                      \n",
      "                                      "
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "instance.update()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# add name\n",
      "instance.add_tag('Name', INSTANCE_NAME)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# configure security group testsg1\n",
      "\n",
      "PORTS_TO_OPEN = [80, 443]\n",
      "\n",
      "for port in PORTS_TO_OPEN:\n",
      "    aws.ec2.authorize_security_group(group_name=SECURITY_GROUP_NAME, ip_protocol='tcp', from_port=port, to_port=port,\n",
      "         cidr_ip='0.0.0.0/0')\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "console_output = instance.get_console_output()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# it takes time for the console output to show not be None -- I don't know exactly how long\n",
      "\n",
      "print console_output.output"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# http://ubuntu-smoser.blogspot.com/2010/07/verify-ssh-keys-on-ec2-instances.html\n",
      "\n",
      "[line for line in console_output.output.split(\"\\n\") if line.startswith(\"ec2\")]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "instance_id = instance.id\n",
      "instance_id"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "output = !source ~/gj_aws.sh; ec2-get-console-output $instance_id  | grep -i ec2"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "output"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# copy a command to ssh into the instance\n",
      "\n",
      "cmdstring = \"ssh -oStrictHostKeyChecking=no ubuntu@{0}\".format(instance.dns_name)\n",
      "# works on a mac\n",
      "! echo \"$cmdstring\" | pbcopy\n",
      "cmdstring"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "dynamic execution of fabric tasks"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# http://docs.fabfile.org/en/1.6/usage/execution.html#using-execute-with-dynamically-set-host-lists\n",
      "\n",
      "import fabric\n",
      "from fabric.api import run, local, env, cd, sudo\n",
      "from fabric.operations import get\n",
      "\n",
      "from regluit.sysadmin import aws\n",
      "from StringIO import StringIO\n",
      "\n",
      "import github\n",
      "\n",
      "# uncomment for debugging\n",
      "# github.enable_console_debug_logging()\n",
      "\n",
      "from github import Github\n",
      "\n",
      "from django.conf import settings \n",
      "\n",
      "# allow us to use our ssh config files (e.g., ~/.ssh/config)\n",
      "env.use_ssh_config = True\n",
      "\n",
      "GITHUB_REPO_NAME = \"Gluejar/regluit\"\n",
      "#GITHUB_REPO_NAME = \"rdhyee/working-open-data\"\n",
      "\n",
      "# maybe generate some random pw -- not sure how important it is to generate some complicated PW if we configure \n",
      "# security groups properly\n",
      "MYSQL_ROOT_PW = \"unglueit_pw_123\"\n",
      "\n",
      "\n",
      "# can use 3 different types of authn: https://github.com/jacquev6/PyGithub/issues/15\n",
      "# can be empty, username/pw, or personal API token (https://github.com/blog/1509-personal-api-tokens)\n",
      "g = Github(settings.GITHUB_AUTH_TOKEN)\n",
      "\n",
      "def host_type():\n",
      "    run('uname -s')\n",
      "\n",
      "    \n",
      "def deploy():\n",
      "    sudo(\"aptitude update\")\n",
      "    sudo(\"yes | aptitude upgrade\")\n",
      "    sudo(\"yes | aptitude install git-core apache libapache2-mod-wsgi mysql-client python-virtualenv python-mysqldb redis-server python-lxml\")\n",
      "    sudo(\"yes | aptitude install python-dev\")\n",
      "    # http://www.whatastruggle.com/postfix-non-interactive-install\n",
      "    sudo(\"DEBIAN_FRONTEND='noninteractive' apt-get install -y -q --force-yes postfix\")\n",
      "\n",
      "    sudo (\"mkdir /opt/regluit\")\n",
      "    sudo (\"chown ubuntu:ubuntu /opt/regluit\")\n",
      "\n",
      "    run('git config --global user.name \"Raymond Yee\"')\n",
      "    run('git config --global user.email \"rdhyee@gluejar.com\"')\n",
      "\n",
      "    run('ssh-keygen -b 2048 -t rsa -f /home/ubuntu/.ssh/id_rsa -P \"\"')\n",
      "\n",
      "    # how to get the key and push it to github\n",
      "    s = StringIO()\n",
      "    get('/home/ubuntu/.ssh/id_rsa.pub', s)\n",
      "    repo = g.get_repo(GITHUB_REPO_NAME)\n",
      "    key = repo.create_key('test deploy key', s.getvalue())    \n",
      "    \n",
      "    # http://debuggable.com/posts/disable-strict-host-checking-for-git-clone:49896ff3-0ac0-4263-9703-1eae4834cda3\n",
      "    run('echo -e \"Host github.com\\n\\tStrictHostKeyChecking no\\n\" >> ~/.ssh/config')\n",
      "    \n",
      "    # clone the regluit git repo into /opt/regluit\n",
      "    with cd(\"/opt\"):\n",
      "        run(\"yes | git clone git@github.com:Gluejar/regluit.git\")\n",
      "        \n",
      "    #  for configuring local mysql server (5.5)\n",
      "    #  http://stackoverflow.com/a/7740571/7782\n",
      "    sudo(\"debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password password {0}'\".format(MYSQL_ROOT_PW))\n",
      "    sudo(\"debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password_again password {0}'\".format(MYSQL_ROOT_PW))\n",
      "    sudo(\"apt-get -y install mysql-server\")\n",
      "    \n",
      "    \n",
      "def deploy_next():\n",
      "\n",
      "    pass   \n",
      "    \n",
      "#hosts = ['ubuntu@ec2-75-101-232-46.compute-1.amazonaws.com']\n",
      "hosts = [\"ubuntu@{0}\".format(instance.dns_name)]\n",
      "\n",
      "fabric.tasks.execute(deploy, hosts=hosts)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# security groups\n",
      "\n",
      "security_groups = aws.ec2.get_all_security_groups()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "web_prod_sgroup = [(group.id, group.name, group.description, group.rules) for group in security_groups if group.name=='web-production'][0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "web_prod_sgroup"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# http://boto.readthedocs.org/en/latest/security_groups.html\n",
      "rules = web_prod_sgroup[3]\n",
      "[(rule.ip_protocol, rule.from_port, rule.to_port, rule.grants, rule.groups) for rule in rules]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[(grant.cidr_ip) for grant in rule.grants]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[(grant.owner_id, grant.group_id, grant.name, grant.cidr_ip) for grant in rule.grants]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# let's make a new security group to replicate the web-production sg\n",
      "\n",
      "test8_sg = aws.ec2.create_security_group('test8', 'test8 sg')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# You need to pass in either src_group_name OR ip_protocol, from_port, to_port, and cidr_ip.\n",
      "\n",
      "test8_sg.authorize('tcp', 80, 80, '0.0.0.0/0')\n",
      "test8_sg.authorize('tcp', 22, 22, '0.0.0.0/0')\n",
      "test8_sg.authorize('tcp', 443, 443, '0.0.0.0/0')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test9_sg = aws.ec2.create_security_group('test9', 'test9 sg')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test9_sg.authorize(src_group=test8_sg)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test8_sg.rules"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rules = test9_sg.rules\n",
      "rule = rules[0]\n",
      "grant = rule.grants[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "(rule.ip_protocol, rule.from_port, rule.to_port, rule.grants)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "grant.owner_id, grant.group_id, grant.name, grant.cidr_ip"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test9_sg = [(group.id, group.name, group.description, group.rules) for group in security_groups if group.name=='test9'][0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rules = test9_sg[3]\n",
      "[(rule.ip_protocol, rule.from_port, rule.to_port, [(grant.owner_id, grant.group_id, grant.name, grant.cidr_ip) for grant in rule.grants], rule.groups) for rule in rules]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "aws.ec2.authorize_security_group(group_name='test8', ip_protocol='tcp', from_port=80, to_port=80, cidr_ip='0.0.0.0/0')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Launch new_test\n",
      "\n",
      "inst = aws.instance('new_test')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "inst.start()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "inst.update()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Setting up MySQL"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "* plain old mysql on the server ( https://help.ubuntu.com/12.04/serverguide/mysql.html )\n",
      "* RDS parameters to figure out\n",
      "\n",
      "to run mysql on server:\n",
      "\n",
      "> `sudo apt-get install mysql-server`"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\"ubuntu@{0}\".format(inst.dns_name)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Creating an Image out of Instance"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "instance.id"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "new_image = aws.ec2.create_image(instance.id, \"script_built_after_local_mysql_2013-05-24\", \n",
      "     description=\"next step figure out RDS\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# sometimes it really does take a surprisingly long time to make an image out of an instance\n",
      "\n",
      "# new_image = aws.ec2.get_image(image_id=u'ami-853a51ec')\n",
      "new_image"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "new_image = aws.ec2.get_image(new_image)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "new_image.state"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Fire up an instance"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "(instance, cmd) = aws.launch_instance(ami=u'ami-853a51ec', \n",
      "                                      instance_type='t1.micro',\n",
      "                                      key_name='rdhyee_public_key',\n",
      "                                      group_name=SECURITY_GROUP_NAME,\n",
      "                                      tag='new_instance',\n",
      "                                      cmd_shell=False) \n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "<pre>\n",
      "sudo debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password password unglueit_pw_123'\n",
      "sudo debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password_again password unglueit_pw_123'\n",
      "sudo apt-get -y install mysql-server\n",
      "</pre>\n",
      "\n",
      "<pre>\n",
      "mysql -h 127.0.0.1 --user=root --password=unglueit_pw_123 \n",
      "</pre>"
     ]
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "RDS"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "http://calculator.s3.amazonaws.com/calc5.html can be used to estimate costs\n",
      "\n",
      "A barebones micro rds costs about $20/month\n",
      "\n",
      "References:\n",
      "\n",
      "* [boto rds intro](http://boto.readthedocs.org/en/latest/rds_tut.html)\n",
      "* [boto rds api ref](http://boto.readthedocs.org/en/latest/ref/rds.html)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dbs = aws.all_rds()\n",
      "dbs"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n",
      "db = dbs[1]\n",
      "(db.id, db.allocated_storage, db.instance_class, db.engine, db.master_username, \n",
      " db.parameter_group, db.security_group, db.availability_zone, db.multi_az)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# I forgot I already have a working rds db info displayer\n",
      "aws.db_info(db)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# parameter group\n",
      "# http://boto.readthedocs.org/en/latest/ref/rds.html#module-boto.rds.parametergroup\n",
      "\n",
      "# I think functionality is more primitive"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pg = aws.rds.get_all_dbparameters('production1')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rds = aws.rds\n",
      "\n",
      "def parameter_group_iteritems(group_name):\n",
      "\n",
      "    first_page = True\n",
      "    get_next_page = True\n",
      "    \n",
      "    while get_next_page:\n",
      "        if first_page:\n",
      "            pg = rds.get_all_dbparameters(group_name)\n",
      "            first_page = False\n",
      "        else:\n",
      "            pg = rds.get_all_dbparameters(group_name, marker = pg.Marker)\n",
      "            \n",
      "        for key in pg.keys():\n",
      "            yield (key, pg[key])\n",
      "    \n",
      "        get_next_page = hasattr(pg, 'Marker')\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# try to turn parameter group into a dict to enable reproducibiity of group\n",
      "\n",
      "pg_dict = {}\n",
      "for (key, param) in parameter_group_iteritems('production1'):\n",
      "    try:\n",
      "        key, {'name':param.name, 'type':param.type, 'description':param.description, 'value':param.value}\n",
      "    except Exception as e:\n",
      "        print key, e\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sorted(pg_dict.keys())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# https://github.com/boto/boto/blob/2.8.0/boto/rds/parametergroup.py#L71\n",
      "\n",
      "param = pg_dict.get('tx_isolation')\n",
      "{'name':param.name, 'type':param.type, 'description':param.description, 'value':param.value}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# security group"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# how to create RDS\n",
      "# db = conn.create_dbinstance(\"db-master-1\", 10, 'db.m1.small', 'root', 'hunter2')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Rebooting instance"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "instance"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rebooted_instance = instance.reboot()\n",
      "rebooted_instance"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "instance.update()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# looks like reboot works, but that the instance status remains running throughout time reboot happens...\n",
      "# maybe we wait a specific amount of time and the try to connect "
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}