{ "metadata": { "name": "aws_cleanup" }, "nbformat": 3, "nbformat_minor": 0, "worksheets": [ { "cells": [ { "cell_type": "code", "collapsed": false, "input": [ "from regluit.sysadmin import aws\n", "reload(aws)" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "pyout", "prompt_number": 1, "text": [ "" ] } ], "prompt_number": 1 }, { "cell_type": "markdown", "metadata": {}, "source": [ "Reminding myself the relationship among various resources/entities in AWS\n", "\n", "* instances\n", "* images\n", "* volumes\n", "* security groups\n", "* keypairs" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Pull up all instances" ] }, { "cell_type": "code", "collapsed": false, "input": [ "instances = aws.all_instances()\n", "len(instances)" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "pyout", "prompt_number": 2, "text": [ "6" ] } ], "prompt_number": 2 }, { "cell_type": "markdown", "metadata": {}, "source": [ "we can use ec2 utils to get list of instances" ] }, { "cell_type": "code", "collapsed": false, "input": [ "%%bash\n", "source ~/gj_aws.sh \n", "ec2-describe-instances | head -5" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "RESERVATION\tr-0d229c66\t439256357102\tdefault\n", "INSTANCE\ti-7ecb881b\tami-79c0ae10\tec2-107-21-188-164.compute-1.amazonaws.com\tip-10-204-58-177.ec2.internal\trunning\trdhyee_public_key\t0\t\tt1.micro\t2013-05-23T21:29:36+0000\tus-east-1c\taki-88aa75e1\t\t\tmonitoring-disabled\t107.21.188.164\t10.204.58.177\t\t\tebs\t\t\t\t\tparavirtual\txen\t\tsg-cece4ca7\tdefault\tfalse\t\n", "BLOCKDEVICE\t/dev/sda1\tvol-fb7a6ea3\t2013-05-23T21:29:41.000Z\ttrue\t\t\n", "TAG\tinstance\ti-7ecb881b\tName\tnew_test\n", "TAG\tinstance\ti-7ecb881b\tnew_instance\t\n" ] } ], "prompt_number": 3 }, { "cell_type": "code", "collapsed": false, "input": [ "import subprocess\n", "\n", "# don't know how to use boto to modify this flag -- even possible?\n", "# UPDATE: yes, I think we can use modify_attribute https://github.com/boto/boto/blob/2.9.3/boto/ec2/instance.py#L526\n", "\n", "if please_inst.get_attribute('disableApiTermination').get('disableApiTermination'):\n", " # http://alestic.com/2010/01/ec2-instance-locking\n", " # ec2-modify-instance-attribute --disable-api-termination false INSTANCEID\n", " output = subprocess.check_output(\n", " \"source ~/gj_aws.sh; ec2-modify-instance-attribute --disable-api-termination false {0}\".format(please_inst.id),\n", " shell=True,\n", " )\n", " print output\n", "\n", "print please_inst.get_attribute('disableApiTermination').get('disableApiTermination')" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "Have 39 bytes in output\n", "disableApiTermination\ti-1749f371\tfalse\n", "\n", "False" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "\n" ] } ], "prompt_number": 21 }, { "cell_type": "code", "collapsed": false, "input": [ "# delete the stopped 'please' instance\n", "please_inst = aws.instance('please')\n", "\n", "# check termination protection?\n", "if please_inst.state == 'stopped' and not please_inst.get_attribute('disableApiTermination').get('disableApiTermination'):\n", " please_inst.terminate()" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 22 }, { "cell_type": "code", "collapsed": false, "input": [ "from collections import Counter\n", "Counter([instance.state for instance in instances])" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# running vs stopped instances\n", "[(instance.id, instance.dns_name, instance.tags.get('Name'), instance.groups[0].name) for instance in instances if instance.state == 'running']" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# stopped instances grouped by ami-id\n", "\n", "from collections import defaultdict\n", "\n", "i_dict = defaultdict(list)\n", "\n", "for instance in instances:\n", " if instance.state == 'stopped':\n", " i_dict[instance.image_id].append(instance)\n", " \n", "# I need to retrieve ami info\n", "\n", "image_ids = i_dict.keys()\n", "images_dict = dict(zip(image_ids, aws.ec2.get_all_images(image_ids=image_ids)))\n", "\n", "for image_id in image_ids:\n", " print images_dict[image_id].name, i_dict[image_id]\n", " print\n" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "inst0 = instances[0]\n", "inst0.id, inst0.image_id, inst0.state, inst0.image_id" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "set([instance.image_id for instance in instances])" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "if an ami is not one of my own, then can I query about it? (e.g., if it's some ubuntu instance....what can I learn about it?)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Images / Volumes / Snapshots" ] }, { "cell_type": "code", "collapsed": false, "input": [ "len(aws.all_images())" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "images = aws.all_images()" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 28 }, { "cell_type": "code", "collapsed": false, "input": [ "img0 = images[0]" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 29 }, { "cell_type": "code", "collapsed": false, "input": [ "img0.id, img0.description, " ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "pyout", "prompt_number": 30, "text": [ "(u'ami-054fe76c', u'unglue.it template 2012 07 30')" ] } ], "prompt_number": 30 }, { "cell_type": "code", "collapsed": false, "input": [ "# which images don't correspond to any of the instances? which do -- and certainly should not be deleted." ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 31 }, { "cell_type": "code", "collapsed": false, "input": [ "img_ids_from_instances = set([instance.image_id for instance in instances])\n", "image_ids = set([img.id for img in images])\n", "\n", "img_ids_from_instances - image_ids" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "pyout", "prompt_number": 32, "text": [ "set([u'ami-e358958a', u'ami-7afa9c13'])" ] } ], "prompt_number": 32 }, { "cell_type": "code", "collapsed": false, "input": [ "not_my_images = aws.ec2.get_all_images(image_ids=list(img_ids_from_instances - image_ids))" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 33 }, { "cell_type": "code", "collapsed": false, "input": [ "img0 = not_my_images[0]\n", "img0.id, img0.description, img0.name" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "[(img.id, img.description, img.name) for img in not_my_images]" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "pyout", "prompt_number": 34, "text": [ "[(u'ami-7afa9c13',\n", " u'http://bitnami.org',\n", " u'bitnami-djangostack-1.4.5-0-linux-ubuntu-12.04.1-x86_64-ebs-mp-e490fac7-7e6f-4c5b-9f7d-51b555bf7bae-ami-b4d143dd.1'),\n", " (u'ami-e358958a',\n", " None,\n", " u'ebs/ubuntu-images/ubuntu-natty-11.04-i386-server-20111003')]" ] } ], "prompt_number": 34 }, { "cell_type": "code", "collapsed": false, "input": [ "# which of my images aren't tied to any of the running or stopped instances?\n", "\n" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# keypairs\n", "\n", "rdhyee_public_key = aws.ec2.get_all_key_pairs()[0]" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 38 }, { "cell_type": "code", "collapsed": false, "input": [ "rdhyee_public_key.fingerprint" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "pyout", "prompt_number": 39, "text": [ "u'ac:89:8d:0c:df:31:56:ee:6e:78:1d:1e:93:00:52:2d'" ] } ], "prompt_number": 39 }, { "cell_type": "code", "collapsed": false, "input": [ "# volumes\n", "aws.ec2.get_all_volumes()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "aws.ec2.get_all_snapshots(owner=aws.GLUEJAR_ACCOUNT_ID)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "len(_)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "how to get at Route53?" ] }, { "cell_type": "code", "collapsed": false, "input": [ "hzones = aws.all_hosted_zones()\n", "hzones" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "unglue_it_zone = aws.route53.get_hosted_zone_by_name('unglue.it')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "unglue_it_zone" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "bitnamitest53.to_xml()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rrsets.commit()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rrsets = route53.get_all_rrsets('ZNLXEIJIA7XKC')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# http://stackoverflow.com/questions/9706883/using-route53-to-update-dns-with-boto-should-i-use-a-name-or-cname-and-what-ttl\n", "# add_change(action, name, type, ttl=600, alias_hosted_zone_id=None, alias_dns_name=None, identifier=None, weight=None, region=None)\n", "change = rrsets.add_change(\"CREATE\",\"bitnami2.unglue.it\", \"A\", ttl=60)\n", "change.add_value(bitnamitest0_i.ip_address)\n", "rrsets.commit()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rrsets.changes" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "#http://hackinghabits.com/2011/08/20/boto-for-amazon-route53/\n", "# even for deletion, value and TTL have to match record to delete. \n", "\n", "change = rrsets.add_change(\"DELETE\",\"bitnami2.unglue.it\", \"A\", ttl=60)\n", "change.add_value(\"23.20.67.193\")\n" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rrsets.to_xml()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rrsets.commit()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# unglue.it\n", "[rset for rset in route53.get_all_rrsets('ZNLXEIJIA7XKC')]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "bitnamitest53 = route53.get_all_rrsets('ZNLXEIJIA7XKC')[5]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "bitnamitest53.name, bitnamitest53.type, bitnamitest53.ttl, bitnamitest53.resource_records" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# now try to change IP attached to bitnamitest53\n", "\n", "bitnamitest53.\n", "\n", "#bitnamitest0_i.ip_address" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "route53." ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "i0 = instances[0]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "[aname for aname in dir(i0) if not aname.startswith(\"__\")]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "[aws.instance_info(i) for i in instances]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "aws.extraneous_snapshot_ids()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "aws.stats_for_instances(aws.all_instances())" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "len(aws.all_images())" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "[s.description for s in aws.all_snapshots()]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "[i.id for i in aws.all_images()]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "import re\n", "\n", "rexp = re.compile(r'(ami-\\S+)')\n", "d = u'Created by CreateImage(i-4ba65a2e) for ami-2d865744 from vol-d58f99b8'\n", "\n", "images = set([i.id for i in aws.all_images()])\n", "\n", "s = re.search(rexp, d)\n", "s.group(1)\n", "\n", "snapshot_amis = set([re.search(rexp,s.description).group(1) for s in aws.all_snapshots()])\n", "\n", "# overlap\n", "\n", "print \"overlap\", images & snapshot_amis\n", "\n", "print \"in snapshot_amis, but not images\", snapshot_amis - images\n", "print \"in images, but not snapshot_amis\", images - snapshot_amis\n" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# Consolidate code to pull up route53 record \n", "\n", "def route53_records(domain_name, name, record_type):\n", " \n", " zone = route53.get_hosted_zone_by_name(domain_name)\n", " \n", " if zone is None:\n", " return ([], None)\n", " \n", " zone_id = zone.GetHostedZoneResponse.Id.replace('/hostedzone/', '')\n", " rrsets = route53.get_all_rrsets(zone_id)\n", " \n", " full_name = \"{0}.{1}.\".format(name, domain_name)\n", " \n", " return ([rset for rset in route53.get_all_rrsets(zone_id) if rset.name == full_name and rset.type == record_type ],\n", " rrsets)\n", " \n", " " ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "#http://hackinghabits.com/2011/08/20/boto-for-amazon-route53/\n", "# even for deletion, value and TTL have to match record to delete. \n", "\n", "(rrsets0, rrsets) = route53_records('unglue.it', 'bitnamitest', 'A')\n", "\n", "# if an old record exists, the get rid of it\n", "\n", "if len(rrsets0) == 1:\n", " change = rrsets.add_change(\"DELETE\", rrsets0[0].name, rrsets0[0].type, rrsets0[0].ttl)\n", " change.add_value(rrsets0[0].resource_records[0])\n", " rrsets.commit()\n", " rrsets.changes[:] = []\n", " " ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "bitnamitest0_i = aws.instance('bitnamitest0')\n", "\n", "change = rrsets.add_change(\"CREATE\", 'bitnamitest.unglue.it.', 'A', 60)\n", "change.add_value(bitnamitest0_i.ip_address)\n", "rrsets.commit()\n" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rrsets.changes[:] = []" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rrsets.changes" ], "language": "python", "metadata": {}, "outputs": [] } ], "metadata": {} } ] }