regluit/aws_cleanup.ipynb

861 lines
21 KiB
Plaintext

{
"metadata": {
"name": ""
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "code",
"collapsed": false,
"input": [
"from regluit.sysadmin import aws\n",
"reload(aws)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 1,
"text": [
"<module 'regluit.sysadmin.aws' from '/Users/raymondyee/C/src/Gluejar/regluit/sysadmin/aws.pyc'>"
]
}
],
"prompt_number": 1
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Reminding myself the relationship among various resources/entities in AWS\n",
"\n",
"* instances\n",
"* images\n",
"* volumes\n",
"* security groups\n",
"* keypairs"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Pull up all instances"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"instances = aws.all_instances()\n",
"len(instances)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 3,
"text": [
"8"
]
}
],
"prompt_number": 3
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"we can use ec2 utils to get list of instances"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"%%bash\n",
"source ~/gj_aws.sh \n",
"ec2-describe-instances | head -10"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"RESERVATION\tr-ee67d78f\t439256357102\tDjango Stack powered by BitNami-1-4-5-0 on Ubuntu 12-04-AutogenByAWSMP-\n",
"INSTANCE\ti-4f7a8327\tami-7afa9c13\t\t\tterminated\trdhyee_public_key\t0\t\tt1.micro\t2013-05-17T22:54:29+0000\tus-east-1b\taki-88aa75e1\t\t\tmonitoring-disabled\t\t\t\t\tebs\t\t\t\t\tparavirtual\txen\t55a4792d-f3aa-4a08-8787-018da2e5c988\tsg-f539f99e\tdefault\tfalse\t\n",
"TAG\tinstance\ti-4f7a8327\tName\tbitnamitest0\n",
"RESERVATION\tr-0ecd2c6e\t439256357102\tdevelopment\n",
"INSTANCE\ti-512dd634\tami-e358958a\t\t\tstopped\tselenium-dev\t0\t\tm1.small\t2012-05-04T02:04:13+0000\tus-east-1b\taki-407d9529\t\t\tmonitoring-disabled\t\t\t\t\tebs\t\t\t\t\tparavirtual\txen\t\tsg-646a250d\tdefault\tfalse\t\n",
"BLOCKDEVICE\t/dev/sda1\tvol-392c3f54\t2012-01-26T17:48:14.000Z\ttrue\t\t\n",
"TAG\tinstance\ti-512dd634\tName\tselenium-dev\n",
"RESERVATION\tr-37b43b55\t439256357102\try-dev\n",
"INSTANCE\ti-f17cc397\tami-3a6cc853\t\t\tstopped\t\t0\t\tt1.micro\t2013-05-08T21:41:21+0000\tus-east-1c\taki-427d952b\t\t\tmonitoring-disabled\t\t\t\t\tebs\t\t\t\t\tparavirtual\txen\t\tsg-c509bead\tdefault\tfalse\t\n",
"BLOCKDEVICE\t/dev/sda1\tvol-8c0834e3\t2012-05-11T23:34:56.000Z\ttrue\t\t\n"
]
}
],
"prompt_number": 4
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"import subprocess\n",
"\n",
"# don't know how to use boto to modify this flag -- even possible?\n",
"# UPDATE: yes, I think we can use modify_attribute https://github.com/boto/boto/blob/2.9.3/boto/ec2/instance.py#L526\n",
"\n",
"if please_inst.get_attribute('disableApiTermination').get('disableApiTermination'):\n",
" # http://alestic.com/2010/01/ec2-instance-locking\n",
" # ec2-modify-instance-attribute --disable-api-termination false INSTANCEID\n",
" output = subprocess.check_output(\n",
" \"source ~/gj_aws.sh; ec2-modify-instance-attribute --disable-api-termination false {0}\".format(please_inst.id),\n",
" shell=True,\n",
" )\n",
" print output\n",
"\n",
"print please_inst.get_attribute('disableApiTermination').get('disableApiTermination')"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# delete the stopped 'please' instance\n",
"please_inst = aws.instance('please')\n",
"\n",
"# check termination protection?\n",
"if please_inst.state == 'stopped' and not please_inst.get_attribute('disableApiTermination').get('disableApiTermination'):\n",
" please_inst.terminate()"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"from collections import Counter\n",
"Counter([instance.state for instance in instances])"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 5,
"text": [
"Counter({u'running': 4, u'terminated': 2, u'stopped': 2})"
]
}
],
"prompt_number": 5
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# running vs stopped instances\n",
"[(instance.id, instance.dns_name, instance.tags.get('Name'), instance.groups[0].name) for instance in instances if instance.state == 'running']"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 6,
"text": [
"[(u'i-41f44a27', u'ec2-23-23-184-9.compute-1.amazonaws.com', u'just', u'just'),\n",
" (u'i-80857bee',\n",
" u'ec2-184-73-231-81.compute-1.amazonaws.com',\n",
" u'gluejar_dot_com',\n",
" u'gluejar_dot_com_sg'),\n",
" (u'i-5197452c',\n",
" u'ec2-23-20-143-15.compute-1.amazonaws.com',\n",
" u'jenkins_small',\n",
" u'default'),\n",
" (u'i-4ec6fd33',\n",
" u'ec2-50-17-216-246.compute-1.amazonaws.com',\n",
" u'web2_relauch_c_med',\n",
" u'web-production')]"
]
}
],
"prompt_number": 6
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# stopped instances grouped by ami-id\n",
"\n",
"from collections import defaultdict\n",
"\n",
"i_dict = defaultdict(list)\n",
"\n",
"for instance in instances:\n",
" if instance.state == 'stopped':\n",
" i_dict[instance.image_id].append(instance)\n",
" \n",
"# I need to retrieve ami info\n",
"\n",
"image_ids = i_dict.keys()\n",
"images_dict = dict(zip(image_ids, aws.ec2.get_all_images(image_ids=image_ids)))\n",
"\n",
"for image_id in image_ids:\n",
" print images_dict[image_id].name, i_dict[image_id]\n",
" print\n"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"rdhyee-dev_2012_05_11 [Instance:i-f17cc397]\n",
"\n",
"ebs/ubuntu-images/ubuntu-natty-11.04-i386-server-20111003 [Instance:i-512dd634]\n",
"\n"
]
}
],
"prompt_number": 7
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"inst0 = instances[0]\n",
"inst0.id, inst0.image_id, inst0.state, inst0.image_id"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 8,
"text": [
"(u'i-4f7a8327', u'ami-7afa9c13', u'terminated', u'ami-7afa9c13')"
]
}
],
"prompt_number": 8
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"set([instance.image_id for instance in instances])"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 9,
"text": [
"{u'ami-35f9445c',\n",
" u'ami-3a6cc853',\n",
" u'ami-7afa9c13',\n",
" u'ami-a26dc9cb',\n",
" u'ami-e358958a',\n",
" u'ami-e7582d8e',\n",
" u'ami-f8d66891'}"
]
}
],
"prompt_number": 9
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"if an ami is not one of my own, then can I query about it? (e.g., if it's some ubuntu instance....what can I learn about it?)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Images / Volumes / Snapshots"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"len(aws.all_images())"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"images = aws.all_images()"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"img0 = images[0]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"img0.id, img0.description, "
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# which images don't correspond to any of the instances? which do -- and certainly should not be deleted."
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"img_ids_from_instances = set([instance.image_id for instance in instances])\n",
"image_ids = set([img.id for img in images])\n",
"\n",
"img_ids_from_instances - image_ids"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"not_my_images = aws.ec2.get_all_images(image_ids=list(img_ids_from_instances - image_ids))"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"img0 = not_my_images[0]\n",
"img0.id, img0.description, img0.name"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"[(img.id, img.description, img.name) for img in not_my_images]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# which of my images aren't tied to any of the running or stopped instances?\n",
"\n"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# keypairs\n",
"\n",
"rdhyee_public_key = aws.ec2.get_all_key_pairs()[0]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rdhyee_public_key.fingerprint"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# volumes\n",
"aws.ec2.get_all_volumes()"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"aws.ec2.get_all_snapshots(owner=aws.GLUEJAR_ACCOUNT_ID)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"len(_)"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"how to get at Route53?"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"hzones = aws.all_hosted_zones()\n",
"hzones"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"unglue_it_zone = aws.route53.get_hosted_zone_by_name('unglue.it')"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"unglue_it_zone"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"bitnamitest53.to_xml()"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rrsets.commit()"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rrsets = aws.route53.get_all_rrsets('ZNLXEIJIA7XKC')"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# http://stackoverflow.com/questions/9706883/using-route53-to-update-dns-with-boto-should-i-use-a-name-or-cname-and-what-ttl\n",
"# add_change(action, name, type, ttl=600, alias_hosted_zone_id=None, alias_dns_name=None, identifier=None, weight=None, region=None)\n",
"change = rrsets.add_change(\"CREATE\",\"bitnami2.unglue.it\", \"A\", ttl=60)\n",
"change.add_value(bitnamitest0_i.ip_address)\n",
"rrsets.commit()"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rrsets.changes"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"#http://hackinghabits.com/2011/08/20/boto-for-amazon-route53/\n",
"# even for deletion, value and TTL have to match record to delete. \n",
"\n",
"change = rrsets.add_change(\"DELETE\",\"bitnami2.unglue.it\", \"A\", ttl=60)\n",
"change.add_value(\"23.20.67.193\")\n"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rrsets.to_xml()"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rrsets.commit()"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# unglue.it\n",
"[rset for rset in route53.get_all_rrsets('ZNLXEIJIA7XKC')]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"bitnamitest53 = route53.get_all_rrsets('ZNLXEIJIA7XKC')[5]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"bitnamitest53.name, bitnamitest53.type, bitnamitest53.ttl, bitnamitest53.resource_records"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# now try to change IP attached to bitnamitest53\n",
"\n",
"bitnamitest53.\n",
"\n",
"#bitnamitest0_i.ip_address"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"route53."
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"i0 = instances[0]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"[aname for aname in dir(i0) if not aname.startswith(\"__\")]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"[aws.instance_info(i) for i in instances]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"aws.extraneous_snapshot_ids()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 10,
"text": [
"{u'snap-09159e7e',\n",
" u'snap-13f59c6f',\n",
" u'snap-1ada476f',\n",
" u'snap-1db3a040',\n",
" u'snap-23f09e5f',\n",
" u'snap-35f43345',\n",
" u'snap-7d760501',\n",
" u'snap-88c4a8ec',\n",
" u'snap-9b3f7bf9',\n",
" u'snap-9d113ee1',\n",
" u'snap-ad706ff0',\n",
" u'snap-b5d341f4',\n",
" u'snap-ba5e0ec7',\n",
" u'snap-d957d0ae',\n",
" u'snap-e313b99d',\n",
" u'snap-ea5da29e',\n",
" u'snap-f957ee8d'}"
]
}
],
"prompt_number": 10
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"aws.stats_for_instances(aws.all_instances())"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"len(aws.all_images())"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"[s.description for s in aws.all_snapshots()]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"[i.id for i in aws.all_images()]"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"import re\n",
"\n",
"rexp = re.compile(r'(ami-\\S+)')\n",
"d = u'Created by CreateImage(i-4ba65a2e) for ami-2d865744 from vol-d58f99b8'\n",
"\n",
"images = set([i.id for i in aws.all_images()])\n",
"\n",
"s = re.search(rexp, d)\n",
"s.group(1)\n",
"\n",
"snapshot_amis = set([re.search(rexp,s.description).group(1) for s in aws.all_snapshots()])\n",
"\n",
"# overlap\n",
"\n",
"print \"overlap\", images & snapshot_amis\n",
"\n",
"print \"in snapshot_amis, but not images\", snapshot_amis - images\n",
"print \"in images, but not snapshot_amis\", images - snapshot_amis\n"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Consolidate code to pull up route53 record \n",
"\n",
"def route53_records(domain_name, name, record_type):\n",
" \n",
" zone = route53.get_hosted_zone_by_name(domain_name)\n",
" \n",
" if zone is None:\n",
" return ([], None)\n",
" \n",
" zone_id = zone.GetHostedZoneResponse.Id.replace('/hostedzone/', '')\n",
" rrsets = route53.get_all_rrsets(zone_id)\n",
" \n",
" full_name = \"{0}.{1}.\".format(name, domain_name)\n",
" \n",
" return ([rset for rset in route53.get_all_rrsets(zone_id) if rset.name == full_name and rset.type == record_type ],\n",
" rrsets)\n",
" \n",
" "
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"#http://hackinghabits.com/2011/08/20/boto-for-amazon-route53/\n",
"# even for deletion, value and TTL have to match record to delete. \n",
"\n",
"(rrsets0, rrsets) = route53_records('unglue.it', 'bitnamitest', 'A')\n",
"\n",
"# if an old record exists, the get rid of it\n",
"\n",
"if len(rrsets0) == 1:\n",
" change = rrsets.add_change(\"DELETE\", rrsets0[0].name, rrsets0[0].type, rrsets0[0].ttl)\n",
" change.add_value(rrsets0[0].resource_records[0])\n",
" rrsets.commit()\n",
" rrsets.changes[:] = []\n",
" "
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"bitnamitest0_i = aws.instance('bitnamitest0')\n",
"\n",
"change = rrsets.add_change(\"CREATE\", 'bitnamitest.unglue.it.', 'A', 60)\n",
"change.add_value(bitnamitest0_i.ip_address)\n",
"rrsets.commit()\n"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rrsets.changes[:] = []"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rrsets.changes"
],
"language": "python",
"metadata": {},
"outputs": []
}
],
"metadata": {}
}
]
}