\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: management.execute_from_command_line()\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: File \"/opt/regluit/ENV/local/lib/python2.7/site-packages/django/core/management/__init__.py\", line 443, in execute_from_command_line\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: utility.execute()\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: File \"/opt/regluit/ENV/local/lib/python2.7/site-packages/django/core/management/__init__.py\", line 382, in execute\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: self.fetch_command(subcommand).run_from_argv(self.argv)\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: File \"/opt/regluit/ENV/local/lib/python2.7/site-packages/djcelery/management/base.py\", line 74, in run_from_argv\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: return super(CeleryCommand, self).run_from_argv(argv)\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: File \"/opt/regluit/ENV/local/lib/python2.7/site-packages/django/core/management/base.py\", line 196, in run_from_argv\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: self.execute(*args, **options.__dict__)\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: File \"/opt/regluit/ENV/local/lib/python2.7/site-packages/djcelery/management/base.py\", line 67, in execute\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: super(CeleryCommand, self).execute(*args, **options)\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: File \"/opt/regluit/ENV/local/lib/python2.7/site-packages/django/core/management/base.py\", line 231, in execute\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: self.validate()\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: File \"/opt/regluit/ENV/local/lib/python2.7/site-packages/django/core/management/base.py\", line 266, in validate\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: num_errors = get_validation_errors(s, app)\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: File \"/opt/regluit/ENV/local/lib/python2.7/site-packages/django/core/management/validation.py\", line 103, in get_validation_errors\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: connection.validation.validate_field(e, opts, f)\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: File \"/opt/regluit/ENV/local/lib/python2.7/site-packages/django/db/backends/mysql/validation.py\", line 14, in validate_field\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: db_version = self.connection.get_server_version()\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: File \"/opt/regluit/ENV/local/lib/python2.7/site-packages/django/db/backends/mysql/base.py\", line 415, in get_server_version\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: self.cursor().close()\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: File \"/opt/regluit/ENV/local/lib/python2.7/site-packages/django/db/backends/__init__.py\", line 319, in cursor\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: cursor = util.CursorWrapper(self._cursor(), self)\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: File \"/opt/regluit/ENV/local/lib/python2.7/site-packages/django/db/backends/mysql/base.py\", line 387, in _cursor\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: self.connection = Database.connect(**kwargs)\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: File \"/opt/regluit/ENV/local/lib/python2.7/site-packages/MySQLdb/__init__.py\", line 81, in Connect\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: return Connection(*args, **kwargs)\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: File \"/opt/regluit/ENV/local/lib/python2.7/site-packages/MySQLdb/connections.py\", line 187, in __init__\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: super(Connection, self).__init__(*args, **kwargs2)\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: _mysql_exceptions.OperationalError: (2003, \"Can't connect to MySQL server on 'production.cboagmr25pjs.us-east-1.rds.amazonaws.com' (110)\")\n",
"[ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com] out: \n"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
"\n"
]
},
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 8,
"text": [
"{'ubuntu@ec2-54-237-136-92.compute-1.amazonaws.com': None}"
]
}
],
"prompt_number": 8
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# the following code is prone to fail \n",
"\n",
"(candidate_rrset, rrsets) = aws.route53_records('unglue.it', 'just', 'A')\n",
"\n",
"\n",
"if len(candidate_rrset) == 1: # assuming we should 0 or 1\n",
" r = candidate_rrset[0]\n",
" r.resource_records[0], r.ttl, r.name, r.type\n",
" change = rrsets.add_change(\"DELETE\",\"just.unglue.it\", r.type, ttl=r.ttl)\n",
" change.add_value(r.resource_records[0])\n",
" rrsets.commit()\n",
"\n",
"\n",
"instance = aws.instance('just_unglue_it')\n",
"\n",
"change = rrsets.add_change(\"CREATE\", 'just.unglue.it.', 'A', 60)\n",
"change.add_value(instance.ip_address)\n",
"rrsets.commit()"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
" # create regluit db an user\n",
" \n",
"# for configuring local mysql server (5.5)\n",
"# http://stackoverflow.com/a/7740571/7782\n",
"sudo(\"debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password password {0}'\".format(MYSQL_ROOT_PW))\n",
"sudo(\"debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password_again password {0}'\".format(MYSQL_ROOT_PW))\n",
"sudo(\"apt-get -y install mysql-server\") \n",
"\n",
" \n",
"# maybe generate some random pw -- not sure how important it is to generate some complicated PW if we configure \n",
"# security groups properly\n",
"MYSQL_ROOT_PW = \"unglueit_pw_123\"\n",
"\n",
"\n",
"command = \"\"\"mysql -h 127.0.0.1 --user=root --password={0} <<'EOF'\n",
"\n",
"CREATE DATABASE regluit CHARACTER SET utf8 COLLATE utf8_bin;\n",
"CREATE USER 'regluit'@'localhost' IDENTIFIED BY 'regluit';\n",
"\n",
"FLUSH PRIVILEGES;\n",
"\n",
"GRANT ALL PRIVILEGES ON regluit.* TO 'regluit'@'localhost' WITH GRANT OPTION; \n",
"EOF\n",
"\"\"\".format(MYSQL_ROOT_PW)\n",
"\n",
"def test_mysql_connection():\n",
" # test connectivity to mysql-server\n",
" command = \"\"\"mysql -h 127.0.0.1 --user=root --password={0} <<'EOF'\n",
"\n",
"SHOW VARIABLES;\n",
"EOF\n",
"\"\"\".format(MYSQL_ROOT_PW)\n",
" run(command) \n",
" "
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "heading",
"level": 1,
"metadata": {},
"source": [
"Debugging route53"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"(candidate_rrset, rrsets) = aws.route53_records('unglue.it', 'please', 'A')\n",
"(candidate_rrset, rrsets)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"len(candidate_rrset)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"instance.ip_address"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"(candidate_rrset, rrsets) = aws.route53_records('unglue.it', 'please', 'A')\n",
"\n",
"#http://hackinghabits.com/2011/08/20/boto-for-amazon-route53/\n",
"# even for deletion, value and TTL have to match record to delete. \n",
"\n",
"if len(candidate_rrset) == 1: # assuming we should 0 or 1\n",
" r = candidate_rrset[0]\n",
" r.resource_records[0], r.ttl, r.name, r.type\n",
" change = rrsets.add_change(\"DELETE\",\"please.unglue.it\", r.type, ttl=r.ttl)\n",
" change.add_value(r.resource_records[0])\n",
" rrsets.commit()\n",
"\n",
" \n",
"instance = aws.instance('please_unglue_it')\n",
"\n",
"change = rrsets.add_change(\"CREATE\", 'please.unglue.it.', 'A', 60)\n",
"change.add_value(instance.ip_address)\n",
"rrsets.commit()"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Commands to add?\n",
"\n",
"By the time we run through a lot of the fabric script, a reboot of the system is required. After installing mysql locally, it seems that the instance needs to be rebooted. Here's some code to do so. Problem remaining is how to reboot, wait for reboot to be completed, and then pick up the next steps.\n",
"\n",
"I could issue a fabric command to apply security upgrade: `sudo unattended-upgrade`\n",
"\n",
"or \n",
"\n",
"I think there is a boto command to restart instance\n",
"\n",
" * `sudo unattended-upgrade`\n"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rebooted_instance = instance.reboot()\n",
"rebooted_instance"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# looks like reboot works, but that the instance status remains running throughout time reboot happens...\n",
"# maybe we wait a specific amount of time and the try to connect "
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "heading",
"level": 1,
"metadata": {},
"source": [
"hand-installing things for expedient job on gluejar.com"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* git repo"
]
},
{
"cell_type": "heading",
"level": 1,
"metadata": {},
"source": [
"EC2 security groups"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* listing key existing security groups\n",
"* how to copy parameters\n",
"\n"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# security groups\n",
"\n",
"\n",
"security_groups = aws.ec2.get_all_security_groups()\n",
"security_groups"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# pull out the security group used for unglue.it\n",
"\n",
"web_prod_sgroup = [(group.id, group.name, group.description, group.rules) for group in security_groups if group.name=='web-production'][0]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"web_prod_sgroup"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# http://boto.readthedocs.org/en/latest/security_groups.html\n",
"rules = web_prod_sgroup[3]\n",
"[(rule.ip_protocol, rule.from_port, rule.to_port, rule.grants, rule.groups) for rule in rules]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"[(grant.cidr_ip) for grant in rule.grants]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"[(grant.owner_id, grant.group_id, grant.name, grant.cidr_ip) for grant in rule.grants]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# let's make a new security group to replicate the web-production sg\n",
"\n",
"test8_sg = aws.ec2.create_security_group('test8', 'test8 sg')"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# You need to pass in either src_group_name OR ip_protocol, from_port, to_port, and cidr_ip.\n",
"\n",
"test8_sg.authorize('tcp', 80, 80, '0.0.0.0/0')\n",
"test8_sg.authorize('tcp', 22, 22, '0.0.0.0/0')\n",
"test8_sg.authorize('tcp', 443, 443, '0.0.0.0/0')"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"test9_sg = aws.ec2.create_security_group('test9', 'test9 sg')"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"test9_sg.authorize(src_group=test8_sg)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"test8_sg.rules"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rules = test9_sg.rules\n",
"rule = rules[0]\n",
"grant = rule.grants[0]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"(rule.ip_protocol, rule.from_port, rule.to_port, rule.grants)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"grant.owner_id, grant.group_id, grant.name, grant.cidr_ip"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"test9_sg = [(group.id, group.name, group.description, group.rules) for group in security_groups if group.name=='test9'][0]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rules = test9_sg[3]\n",
"[(rule.ip_protocol, rule.from_port, rule.to_port, [(grant.owner_id, grant.group_id, grant.name, grant.cidr_ip) for grant in rule.grants], rule.groups) for rule in rules]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"aws.ec2.authorize_security_group(group_name='test8', ip_protocol='tcp', from_port=80, to_port=80, cidr_ip='0.0.0.0/0')"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# let's compute the instances that are tied to the various security groups\n",
"# http://boto.readthedocs.org/en/latest/ref/ec2.html#module-boto.ec2.securitygroup\n",
"# This calculation is useful for reconstructing the relationships among instances and security groups\n",
"\n",
"\n",
"from boto.ec2 import securitygroup\n",
"\n",
"for security_group in aws.ec2.get_all_security_groups():\n",
" sg = securitygroup.SecurityGroup(name=security_group.name, connection=aws.ec2)\n",
" print security_group, [inst.id for inst in sg.instances()]\n"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# with the exception of frontend-lb, let's delete the security groups that have no attached instances \n",
"\n",
"for sg in [sg for sg in aws.ec2.get_all_security_groups() if len(sg.instances()) == 0 and sg.name != 'frontend-lb']:\n",
" print sg.name, sg.id, aws.ec2.delete_security_group(group_id=sg.id)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "heading",
"level": 1,
"metadata": {},
"source": [
"Setting up MySQL"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* plain old mysql on the server ( https://help.ubuntu.com/12.04/serverguide/mysql.html )\n",
"* RDS parameters to figure out\n",
"\n",
"to run mysql on server -- if you didn't have to worry about interactivity:\n",
"\n",
"> `sudo apt-get install mysql-server`"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"\"ubuntu@{0}\".format(inst.dns_name)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# once mysql installed, how to test the basic connectivity?\n",
"\n"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"sudo debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password password unglueit_pw_123'\n",
"sudo debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password_again password unglueit_pw_123'\n",
"sudo apt-get -y install mysql-server\n",
"\n",
"\n",
"\n",
"mysql -h 127.0.0.1 --user=root --password=unglueit_pw_123 \n",
""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
" mysql -h 127.0.0.1 --user=root --password=unglueit_pw_123 <<'EOF'\n",
"\n",
" SHOW DATABASES;\n",
" EOF\n",
"\n"
]
},
{
"cell_type": "heading",
"level": 1,
"metadata": {},
"source": [
"Creating an Image out of Instance"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"instance.id"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"new_image = aws.ec2.create_image(instance.id, \"script_built_after_local_mysql_2013-05-24\", \n",
" description=\"next step figure out RDS\")"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# sometimes it really does take a surprisingly long time to make an image out of an instance\n",
"\n",
"# new_image = aws.ec2.get_image(image_id=u'ami-853a51ec')\n",
"new_image"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"new_image = aws.ec2.get_image(new_image)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"new_image.state"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Fire up an instance"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"(instance, cmd) = aws.launch_instance(ami=u'ami-853a51ec', \n",
" instance_type='t1.micro',\n",
" key_name='rdhyee_public_key',\n",
" group_name=SECURITY_GROUP_NAME,\n",
" tag='new_instance',\n",
" cmd_shell=False) \n"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "heading",
"level": 1,
"metadata": {},
"source": [
"Route53"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"from regluit.sysadmin import aws\n",
"reload(aws)\n",
"\n",
"(candidate_rrset, rrsets) = aws.route53_records('unglue.it', 'please', 'A')\n",
"\n",
"#http://hackinghabits.com/2011/08/20/boto-for-amazon-route53/\n",
"# even for deletion, value and TTL have to match record to delete. \n",
"\n",
"if len(candidate_rrset) == 1: # assuming we should 0 or 1\n",
" r = candidate_rrset[0]\n",
" r.resource_records[0], r.ttl, r.name, r.type\n",
" change = rrsets.add_change(\"DELETE\",\"please.unglue.it\", r.type, ttl=r.ttl)\n",
" change.add_value(r.resource_records[0])\n",
" rrsets.commit()\n",
"\n",
" \n",
"instance = aws.instance('please_unglue_it')\n",
"\n",
"change = rrsets.add_change(\"CREATE\", 'please.unglue.it.', 'A', 60)\n",
"change.add_value(instance.ip_address)\n",
"rrsets.commit()"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "heading",
"level": 1,
"metadata": {},
"source": [
"RDS"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"http://calculator.s3.amazonaws.com/calc5.html can be used to estimate costs\n",
"\n",
"A barebones micro rds costs about $20/month\n",
"\n",
"References:\n",
"\n",
"* [boto rds intro](http://boto.readthedocs.org/en/latest/rds_tut.html)\n",
"* [boto rds api ref](http://boto.readthedocs.org/en/latest/ref/rds.html)"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"dbs = aws.all_rds()\n",
"dbs"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"\n",
"db = dbs[1]\n",
"(db.id, db.allocated_storage, db.instance_class, db.engine, db.master_username, \n",
" db.parameter_group, db.security_group, db.availability_zone, db.multi_az)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# I forgot I already have a working rds db info displayer\n",
"aws.db_info(db)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# parameter group\n",
"# http://boto.readthedocs.org/en/latest/ref/rds.html#module-boto.rds.parametergroup\n",
"\n",
"# I think functionality is more primitive"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"pg = aws.rds.get_all_dbparameters('production1')"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rds = aws.rds\n",
"\n",
"def parameter_group_iteritems(group_name):\n",
"\n",
" first_page = True\n",
" get_next_page = True\n",
" \n",
" while get_next_page:\n",
" if first_page:\n",
" pg = rds.get_all_dbparameters(group_name)\n",
" first_page = False\n",
" else:\n",
" pg = rds.get_all_dbparameters(group_name, marker = pg.Marker)\n",
" \n",
" for key in pg.keys():\n",
" yield (key, pg[key])\n",
" \n",
" get_next_page = hasattr(pg, 'Marker')\n"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# try to turn parameter group into a dict to enable reproducibiity of group\n",
"\n",
"pg_dict = {}\n",
"for (key, param) in parameter_group_iteritems('production1'):\n",
" try:\n",
" key, {'name':param.name, 'type':param.type, 'description':param.description, 'value':param.value}\n",
" pg_dict[key] = {'name':param.name, 'type':param.type, 'description':param.description, 'value':param.value}\n",
" except Exception as e:\n",
" print key, e\n"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"sorted(pg_dict.keys())"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# https://github.com/boto/boto/blob/2.8.0/boto/rds/parametergroup.py#L71\n",
"\n",
"param = pg_dict.get('character_set_database')\n",
"{'name':param[\"name\"], 'type':param[\"type\"], 'description':param[\"description\"], 'value':param[\"value\"]}"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# security group"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# how to create RDS\n",
"# db = conn.create_dbinstance(\"db-master-1\", 10, 'db.m1.small', 'root', 'hunter2')"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "heading",
"level": 1,
"metadata": {},
"source": [
"IAM"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"good to get an automated handle of the IAM groups and users. To use boto to manage IAM, you will need to have AWS keys with sufficient permissions."
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"from regluit.sysadmin import aws\n",
"iam = aws.boto.connect_iam()\n",
"\n",
"\n",
"IAM_POWER_USER_PERMISSION = \"\"\"{\n",
" \"Version\": \"2012-10-17\",\n",
" \"Statement\": [\n",
" {\n",
" \"Effect\": \"Allow\",\n",
" \"NotAction\": \"iam:*\",\n",
" \"Resource\": \"*\"\n",
" }\n",
" ]\n",
"}\"\"\"\n",
"\n",
"\n",
"# get group names\n",
"\n",
"def all_iam_group_names():\n",
" return [g.group_name for g in iam.get_all_groups()['list_groups_response']['list_groups_result']['groups']]\n",
"\n",
"# get user names\n",
"\n",
"def all_iam_user_names():\n",
" return [u.user_name for u in iam.get_all_users()[u'list_users_response'][u'list_users_result']['users']]\n",
"\n",
"# mapping between groups and users\n",
"# list users and their corresponding groups.\n",
"\n",
"def iam_group_names_for_user(user_name):\n",
" return [g.group_name for g in iam.get_groups_for_user(user_name)['list_groups_for_user_response'][u'list_groups_for_user_result']['groups']]\n",
"\n",
"# for given groups, list corresponding users\n",
"\n",
"def iam_user_names_for_group(group_name):\n",
" return [u.user_name for u in iam.get_group(group_name=group_name)[u'get_group_response'][u'get_group_result']['users']]\n",
"\n",
"# find keys associated with user\n",
"\n",
"def access_keys_for_user_name(user_name):\n",
" keys = iam.get_all_access_keys(user_name=user_name)['list_access_keys_response'][u'list_access_keys_result']['access_key_metadata']\n",
" return keys\n",
"\n",
"# can we use IAM to create new IAM user and get the key / secret?\n",
"\n",
"def create_iam_user(user_name, generate_key=True):\n",
" iam_user = iam.create_user(user_name=user_name)\n",
" if generate_key:\n",
" key_output = iam.create_access_key(user_name=user_name)\n",
" access_key = key_output['create_access_key_response']['create_access_key_result']['access_key']\n",
" (key, secret) = (access_key['access_key_id'], access_key['secret_access_key'])\n",
" return (iam_user, key, secret)\n",
" else:\n",
" return (iam_user, key, None, None)\n",
"\n",
"def delete_iam_user(user_name):\n",
" \n",
" # check to see whether there is such a user_name.\n",
" try:\n",
" iam.get_user(user_name)\n",
" except boto.exception.BotoServerError as e:\n",
" return None\n",
"\n",
" # delete associated keys\n",
" \n",
" keys = access_keys_for_user_name(user_name)\n",
" \n",
" for key in keys:\n",
" # print key.access_key_id, key.status\n",
" iam.delete_access_key(access_key_id=key.access_key_id, user_name=user_name)\n",
" #result = iam.update_access_key(access_key_id=key.access_key_id, user_name=user_name, status='Inactive')\n",
" \n",
" # also need to delete associated policies\n",
" \n",
" policy_names = iam_policy_names_for_user(user_name)\n",
" \n",
" for policy_name in policy_names:\n",
" iam.delete_user_policy(user_name=user_name,policy_name=policy_name)\n",
" \n",
" # once the keys associated with the user are deleted, then proceed to delete the user\n",
"\n",
" result = iam.delete_user(user_name)\n",
" return result\n",
" \n",
"# policies\n",
"\n",
"def iam_policy_names_for_group(group_name):\n",
" return iam.get_all_group_policies(group_name=group_name)['list_group_policies_response'][u'list_group_policies_result']['policy_names']\n",
"\n",
"def iam_policy_names_for_user(user_name):\n",
" return iam.get_all_user_policies(user_name=user_name)['list_user_policies_response'][u'list_user_policies_result']['policy_names']\n",
"\n",
"def policy_document(policy_name, user_name=None, group_name=None):\n",
" if group_name is not None:\n",
" document = iam.get_group_policy(group_name=group_name, policy_name=policy_name)[u'get_group_policy_response'][u'get_group_policy_result'][u'policy_document']\n",
" return urlparse.parse_qs(\"policy={0}\".format(document))['policy'][0]\n",
" if user_name is not None:\n",
" document = iam.get_user_policy(user_name=user_name, policy_name=policy_name)[u'get_user_policy_response'][u'get_user_policy_result'][u'policy_document']\n",
" return urlparse.parse_qs(\"policy={0}\".format(document))['policy'][0]\n",
" \n",
"# get general IAM stats\n",
"\n",
"(iam.get_account_summary(), all_iam_group_names(), all_iam_user_names(),\n",
" iam_group_names_for_user('eric'), iam_user_names_for_group('gluejar'),\n",
" access_keys_for_user_name('ry-dev')\n",
" )\n",
"\n"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# test -> grab all groups and list of corresponding users\n",
"\n",
"for g in all_iam_group_names():\n",
" print g, iam_user_names_for_group(g)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# list all keys by looping through users\n",
"\n",
"for u in all_iam_user_names():\n",
" print u, [(k.access_key_id, k.status) for k in access_keys_for_user_name(u)]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# look at permission structures of groups and users\n",
"\n",
"from urllib import urlencode\n",
"import urlparse\n",
"\n",
"policy_names = iam_policy_names_for_group('gluejar')\n",
"\n",
"for p in policy_names:\n",
" print policy_document(group_name='gluejar', policy_name=p)\n",
" \n"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"iam_user, key, secret = create_iam_user('ry-dev-3', True)\n",
"iam.put_user_policy( user_name='ry-dev-3', policy_name='power_user_2013-06-12', policy_json=IAM_POWER_USER_PERMISSION)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# write out a shell script for configuring the environment with the keys for AWS\n",
"\n",
"print \"\"\"#!/bin/bash\n",
"\n",
"\n",
"export AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID}\n",
"export AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY}\n",
"\n",
"# EC2 API tools\n",
"export EC2_ACCESS_KEY=$AWS_ACCESS_KEY_ID\n",
"export EC2_SECRET_KEY=$AWS_SECRET_ACCESS_KEY\"\"\".format(**{'AWS_SECRET_ACCESS_KEY':secret, 'AWS_ACCESS_KEY_ID':key})"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"[policy_document(p, user_name='ry-dev-3') for p in iam_policy_names_for_user('ry-dev-3')]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"delete_iam_user(user_name='ry-dev-3')"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Different ways to pass in AWS keys to boto\n",
"\n",
"http://boto.readthedocs.org/en/latest/boto_config_tut.html\n",
"\n",
"* Credentials passed into Connection class constructor.\n",
"* Credentials specified by environment variables\n",
"* Credentials specified as options in the config file.\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"\n",
"#!/bin/bash\n",
"\n",
"export AWS_ACCESS_KEY={AWS_ACCESS_KEY}\n",
"export AWS_SECRET_KEY={AWS_SECRET_KEY}\n",
"\n",
"# EC2 API tools\n",
"export EC2_ACCESS_KEY=$AWS_ACCESS_KEY\n",
"export EC2_SECRET_KEY=$AWS_SECRET_KEY\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"%%bash\n",
"# something to convert this notebook to Python source\n",
"cd /Users/raymondyee/D/Document/Gluejar/Gluejar.github/regluit; python ~/C/src/nbconvert/nbconvert.py python build_ec2_instances_for_django.ipynb"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"import fabric\n",
"from fabric.api import run, local, env, cd, sudo\n",
"from fabric.operations import get\n",
"\n",
"\n",
"def run_on_ry_dev():\n",
"\n",
" run(\"ls -lt\") \n",
" \n",
"hosts = [\"ubuntu@ry-dev.unglue.it\"]\n",
"\n",
"fabric.tasks.execute(run_on_ry_dev, hosts=hosts)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"instance.state"
],
"language": "python",
"metadata": {},
"outputs": []
}
],
"metadata": {}
}
]
}