regluit/aws_cleanup.ipynb

{
 "metadata": {
  "name": "aws_cleanup"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from regluit.sysadmin import aws\n",
      "reload(aws)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 1,
       "text": [
        "<module 'regluit.sysadmin.aws' from '/Users/raymondyee/C/src/Gluejar/regluit/sysadmin/aws.pyc'>"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Reminding myself the relationship among various resources/entities in AWS\n",
      "\n",
      "* instances\n",
      "* images\n",
      "* volumes\n",
      "* security groups\n",
      "* keypairs"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Pull up all instances"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "instances = aws.all_instances()\n",
      "len(instances)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 2,
       "text": [
        "6"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "we can use ec2 utils to get list of instances"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%bash\n",
      "source ~/gj_aws.sh \n",
      "ec2-describe-instances | head -5"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "RESERVATION\tr-0d229c66\t439256357102\tdefault\n",
        "INSTANCE\ti-7ecb881b\tami-79c0ae10\tec2-107-21-188-164.compute-1.amazonaws.com\tip-10-204-58-177.ec2.internal\trunning\trdhyee_public_key\t0\t\tt1.micro\t2013-05-23T21:29:36+0000\tus-east-1c\taki-88aa75e1\t\t\tmonitoring-disabled\t107.21.188.164\t10.204.58.177\t\t\tebs\t\t\t\t\tparavirtual\txen\t\tsg-cece4ca7\tdefault\tfalse\t\n",
        "BLOCKDEVICE\t/dev/sda1\tvol-fb7a6ea3\t2013-05-23T21:29:41.000Z\ttrue\t\t\n",
        "TAG\tinstance\ti-7ecb881b\tName\tnew_test\n",
        "TAG\tinstance\ti-7ecb881b\tnew_instance\t\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import subprocess\n",
      "\n",
      "# don't know how to use boto to modify this flag -- even possible?\n",
      "# UPDATE:  yes, I think we can use modify_attribute https://github.com/boto/boto/blob/2.9.3/boto/ec2/instance.py#L526\n",
      "\n",
      "if please_inst.get_attribute('disableApiTermination').get('disableApiTermination'):\n",
      "    # http://alestic.com/2010/01/ec2-instance-locking\n",
      "    # ec2-modify-instance-attribute   --disable-api-termination false   INSTANCEID\n",
      "    output = subprocess.check_output(\n",
      "      \"source ~/gj_aws.sh; ec2-modify-instance-attribute  --disable-api-termination false {0}\".format(please_inst.id),\n",
      "    shell=True,\n",
      "    )\n",
      "    print output\n",
      "\n",
      "print please_inst.get_attribute('disableApiTermination').get('disableApiTermination')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Have 39 bytes in output\n",
        "disableApiTermination\ti-1749f371\tfalse\n",
        "\n",
        "False"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# delete the stopped 'please' instance\n",
      "please_inst = aws.instance('please')\n",
      "\n",
      "# check termination protection?\n",
      "if please_inst.state == 'stopped' and not please_inst.get_attribute('disableApiTermination').get('disableApiTermination'):\n",
      "    please_inst.terminate()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from collections import Counter\n",
      "Counter([instance.state for instance in instances])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# running vs stopped instances\n",
      "[(instance.id, instance.dns_name, instance.tags.get('Name'), instance.groups[0].name) for instance in instances if instance.state == 'running']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# stopped instances grouped by ami-id\n",
      "\n",
      "from collections import defaultdict\n",
      "\n",
      "i_dict = defaultdict(list)\n",
      "\n",
      "for instance in instances:\n",
      "    if instance.state == 'stopped':\n",
      "        i_dict[instance.image_id].append(instance)\n",
      "        \n",
      "# I need to retrieve ami info\n",
      "\n",
      "image_ids = i_dict.keys()\n",
      "images_dict = dict(zip(image_ids, aws.ec2.get_all_images(image_ids=image_ids)))\n",
      "\n",
      "for image_id in image_ids:\n",
      "    print images_dict[image_id].name, i_dict[image_id]\n",
      "    print\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "inst0 = instances[0]\n",
      "inst0.id, inst0.image_id, inst0.state, inst0.image_id"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "set([instance.image_id for instance in instances])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "if an ami is not one of my own, then can I query about it?  (e.g., if it's some ubuntu instance....what can I learn about it?)"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Images / Volumes / Snapshots"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "len(aws.all_images())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "images = aws.all_images()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "img0 = images[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "img0.id, img0.description, "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 30,
       "text": [
        "(u'ami-054fe76c', u'unglue.it template 2012 07 30')"
       ]
      }
     ],
     "prompt_number": 30
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# which images don't correspond to any of the instances?  which do -- and certainly should not be deleted."
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "img_ids_from_instances = set([instance.image_id for instance in instances])\n",
      "image_ids = set([img.id for img in images])\n",
      "\n",
      "img_ids_from_instances - image_ids"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 32,
       "text": [
        "set([u'ami-e358958a', u'ami-7afa9c13'])"
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "not_my_images = aws.ec2.get_all_images(image_ids=list(img_ids_from_instances - image_ids))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 33
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "img0 = not_my_images[0]\n",
      "img0.id, img0.description, img0.name"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[(img.id, img.description, img.name) for img in not_my_images]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 34,
       "text": [
        "[(u'ami-7afa9c13',\n",
        "  u'http://bitnami.org',\n",
        "  u'bitnami-djangostack-1.4.5-0-linux-ubuntu-12.04.1-x86_64-ebs-mp-e490fac7-7e6f-4c5b-9f7d-51b555bf7bae-ami-b4d143dd.1'),\n",
        " (u'ami-e358958a',\n",
        "  None,\n",
        "  u'ebs/ubuntu-images/ubuntu-natty-11.04-i386-server-20111003')]"
       ]
      }
     ],
     "prompt_number": 34
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# which of my images aren't tied to any of the running or stopped instances?\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# keypairs\n",
      "\n",
      "rdhyee_public_key = aws.ec2.get_all_key_pairs()[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 38
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rdhyee_public_key.fingerprint"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 39,
       "text": [
        "u'ac:89:8d:0c:df:31:56:ee:6e:78:1d:1e:93:00:52:2d'"
       ]
      }
     ],
     "prompt_number": 39
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# volumes\n",
      "aws.ec2.get_all_volumes()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "aws.ec2.get_all_snapshots(owner=aws.GLUEJAR_ACCOUNT_ID)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "len(_)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "how to get at Route53?"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "hzones = aws.all_hosted_zones()\n",
      "hzones"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "unglue_it_zone = aws.route53.get_hosted_zone_by_name('unglue.it')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "unglue_it_zone"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "bitnamitest53.to_xml()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rrsets.commit()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rrsets = route53.get_all_rrsets('ZNLXEIJIA7XKC')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# http://stackoverflow.com/questions/9706883/using-route53-to-update-dns-with-boto-should-i-use-a-name-or-cname-and-what-ttl\n",
      "# add_change(action, name, type, ttl=600, alias_hosted_zone_id=None, alias_dns_name=None, identifier=None, weight=None, region=None)\n",
      "change = rrsets.add_change(\"CREATE\",\"bitnami2.unglue.it\", \"A\", ttl=60)\n",
      "change.add_value(bitnamitest0_i.ip_address)\n",
      "rrsets.commit()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rrsets.changes"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#http://hackinghabits.com/2011/08/20/boto-for-amazon-route53/\n",
      "# even for deletion, value and TTL have to match record to delete. \n",
      "\n",
      "change = rrsets.add_change(\"DELETE\",\"bitnami2.unglue.it\", \"A\", ttl=60)\n",
      "change.add_value(\"23.20.67.193\")\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rrsets.to_xml()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rrsets.commit()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# unglue.it\n",
      "[rset for rset in route53.get_all_rrsets('ZNLXEIJIA7XKC')]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "bitnamitest53 = route53.get_all_rrsets('ZNLXEIJIA7XKC')[5]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "bitnamitest53.name, bitnamitest53.type, bitnamitest53.ttl, bitnamitest53.resource_records"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# now try to change IP attached to bitnamitest53\n",
      "\n",
      "bitnamitest53.\n",
      "\n",
      "#bitnamitest0_i.ip_address"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "route53."
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "i0 = instances[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[aname for aname in dir(i0) if not aname.startswith(\"__\")]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[aws.instance_info(i) for i in instances]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "aws.extraneous_snapshot_ids()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "aws.stats_for_instances(aws.all_instances())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "len(aws.all_images())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[s.description for s in aws.all_snapshots()]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[i.id for i in aws.all_images()]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import re\n",
      "\n",
      "rexp = re.compile(r'(ami-\\S+)')\n",
      "d = u'Created by CreateImage(i-4ba65a2e) for ami-2d865744 from vol-d58f99b8'\n",
      "\n",
      "images = set([i.id for i in aws.all_images()])\n",
      "\n",
      "s = re.search(rexp, d)\n",
      "s.group(1)\n",
      "\n",
      "snapshot_amis = set([re.search(rexp,s.description).group(1) for s in aws.all_snapshots()])\n",
      "\n",
      "# overlap\n",
      "\n",
      "print \"overlap\", images & snapshot_amis\n",
      "\n",
      "print \"in snapshot_amis, but not images\", snapshot_amis - images\n",
      "print \"in images, but not snapshot_amis\", images - snapshot_amis\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Consolidate code to pull up route53 record \n",
      "\n",
      "def route53_records(domain_name, name, record_type):\n",
      "    \n",
      "    zone = route53.get_hosted_zone_by_name(domain_name)\n",
      "    \n",
      "    if zone is None:\n",
      "        return ([], None)\n",
      "    \n",
      "    zone_id = zone.GetHostedZoneResponse.Id.replace('/hostedzone/', '')\n",
      "    rrsets = route53.get_all_rrsets(zone_id)\n",
      "    \n",
      "    full_name = \"{0}.{1}.\".format(name, domain_name)\n",
      "    \n",
      "    return ([rset for rset in route53.get_all_rrsets(zone_id) if rset.name == full_name and rset.type == record_type ],\n",
      "            rrsets)\n",
      "    \n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#http://hackinghabits.com/2011/08/20/boto-for-amazon-route53/\n",
      "# even for deletion, value and TTL have to match record to delete. \n",
      "\n",
      "(rrsets0, rrsets) = route53_records('unglue.it', 'bitnamitest', 'A')\n",
      "\n",
      "# if an old record exists, the get rid of it\n",
      "\n",
      "if len(rrsets0) == 1:\n",
      "    change = rrsets.add_change(\"DELETE\", rrsets0[0].name, rrsets0[0].type, rrsets0[0].ttl)\n",
      "    change.add_value(rrsets0[0].resource_records[0])\n",
      "    rrsets.commit()\n",
      "    rrsets.changes[:] = []\n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "bitnamitest0_i = aws.instance('bitnamitest0')\n",
      "\n",
      "change = rrsets.add_change(\"CREATE\", 'bitnamitest.unglue.it.', 'A', 60)\n",
      "change.add_value(bitnamitest0_i.ip_address)\n",
      "rrsets.commit()\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rrsets.changes[:] = []"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rrsets.changes"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}