regluit/aws_cleanup.ipynb

{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from regluit.sysadmin import aws\n",
      "reload(aws)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 1,
       "text": [
        "<module 'regluit.sysadmin.aws' from '/Users/raymondyee/C/src/Gluejar/regluit/sysadmin/aws.pyc'>"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Reminding myself the relationship among various resources/entities in AWS\n",
      "\n",
      "* instances\n",
      "* images\n",
      "* volumes\n",
      "* security groups\n",
      "* keypairs"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Pull up all instances"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "instances = aws.all_instances()\n",
      "len(instances)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 3,
       "text": [
        "8"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "we can use ec2 utils to get list of instances"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%bash\n",
      "source ~/gj_aws.sh \n",
      "ec2-describe-instances | head -10"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "RESERVATION\tr-ee67d78f\t439256357102\tDjango Stack powered by BitNami-1-4-5-0 on Ubuntu 12-04-AutogenByAWSMP-\n",
        "INSTANCE\ti-4f7a8327\tami-7afa9c13\t\t\tterminated\trdhyee_public_key\t0\t\tt1.micro\t2013-05-17T22:54:29+0000\tus-east-1b\taki-88aa75e1\t\t\tmonitoring-disabled\t\t\t\t\tebs\t\t\t\t\tparavirtual\txen\t55a4792d-f3aa-4a08-8787-018da2e5c988\tsg-f539f99e\tdefault\tfalse\t\n",
        "TAG\tinstance\ti-4f7a8327\tName\tbitnamitest0\n",
        "RESERVATION\tr-0ecd2c6e\t439256357102\tdevelopment\n",
        "INSTANCE\ti-512dd634\tami-e358958a\t\t\tstopped\tselenium-dev\t0\t\tm1.small\t2012-05-04T02:04:13+0000\tus-east-1b\taki-407d9529\t\t\tmonitoring-disabled\t\t\t\t\tebs\t\t\t\t\tparavirtual\txen\t\tsg-646a250d\tdefault\tfalse\t\n",
        "BLOCKDEVICE\t/dev/sda1\tvol-392c3f54\t2012-01-26T17:48:14.000Z\ttrue\t\t\n",
        "TAG\tinstance\ti-512dd634\tName\tselenium-dev\n",
        "RESERVATION\tr-37b43b55\t439256357102\try-dev\n",
        "INSTANCE\ti-f17cc397\tami-3a6cc853\t\t\tstopped\t\t0\t\tt1.micro\t2013-05-08T21:41:21+0000\tus-east-1c\taki-427d952b\t\t\tmonitoring-disabled\t\t\t\t\tebs\t\t\t\t\tparavirtual\txen\t\tsg-c509bead\tdefault\tfalse\t\n",
        "BLOCKDEVICE\t/dev/sda1\tvol-8c0834e3\t2012-05-11T23:34:56.000Z\ttrue\t\t\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import subprocess\n",
      "\n",
      "# don't know how to use boto to modify this flag -- even possible?\n",
      "# UPDATE:  yes, I think we can use modify_attribute https://github.com/boto/boto/blob/2.9.3/boto/ec2/instance.py#L526\n",
      "\n",
      "if please_inst.get_attribute('disableApiTermination').get('disableApiTermination'):\n",
      "    # http://alestic.com/2010/01/ec2-instance-locking\n",
      "    # ec2-modify-instance-attribute   --disable-api-termination false   INSTANCEID\n",
      "    output = subprocess.check_output(\n",
      "      \"source ~/gj_aws.sh; ec2-modify-instance-attribute  --disable-api-termination false {0}\".format(please_inst.id),\n",
      "    shell=True,\n",
      "    )\n",
      "    print output\n",
      "\n",
      "print please_inst.get_attribute('disableApiTermination').get('disableApiTermination')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# delete the stopped 'please' instance\n",
      "please_inst = aws.instance('please')\n",
      "\n",
      "# check termination protection?\n",
      "if please_inst.state == 'stopped' and not please_inst.get_attribute('disableApiTermination').get('disableApiTermination'):\n",
      "    please_inst.terminate()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from collections import Counter\n",
      "Counter([instance.state for instance in instances])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "Counter({u'running': 4, u'terminated': 2, u'stopped': 2})"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# running vs stopped instances\n",
      "[(instance.id, instance.dns_name, instance.tags.get('Name'), instance.groups[0].name) for instance in instances if instance.state == 'running']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "[(u'i-41f44a27', u'ec2-23-23-184-9.compute-1.amazonaws.com', u'just', u'just'),\n",
        " (u'i-80857bee',\n",
        "  u'ec2-184-73-231-81.compute-1.amazonaws.com',\n",
        "  u'gluejar_dot_com',\n",
        "  u'gluejar_dot_com_sg'),\n",
        " (u'i-5197452c',\n",
        "  u'ec2-23-20-143-15.compute-1.amazonaws.com',\n",
        "  u'jenkins_small',\n",
        "  u'default'),\n",
        " (u'i-4ec6fd33',\n",
        "  u'ec2-50-17-216-246.compute-1.amazonaws.com',\n",
        "  u'web2_relauch_c_med',\n",
        "  u'web-production')]"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# stopped instances grouped by ami-id\n",
      "\n",
      "from collections import defaultdict\n",
      "\n",
      "i_dict = defaultdict(list)\n",
      "\n",
      "for instance in instances:\n",
      "    if instance.state == 'stopped':\n",
      "        i_dict[instance.image_id].append(instance)\n",
      "        \n",
      "# I need to retrieve ami info\n",
      "\n",
      "image_ids = i_dict.keys()\n",
      "images_dict = dict(zip(image_ids, aws.ec2.get_all_images(image_ids=image_ids)))\n",
      "\n",
      "for image_id in image_ids:\n",
      "    print images_dict[image_id].name, i_dict[image_id]\n",
      "    print\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "rdhyee-dev_2012_05_11 [Instance:i-f17cc397]\n",
        "\n",
        "ebs/ubuntu-images/ubuntu-natty-11.04-i386-server-20111003 [Instance:i-512dd634]\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "inst0 = instances[0]\n",
      "inst0.id, inst0.image_id, inst0.state, inst0.image_id"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "(u'i-4f7a8327', u'ami-7afa9c13', u'terminated', u'ami-7afa9c13')"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "set([instance.image_id for instance in instances])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 9,
       "text": [
        "{u'ami-35f9445c',\n",
        " u'ami-3a6cc853',\n",
        " u'ami-7afa9c13',\n",
        " u'ami-a26dc9cb',\n",
        " u'ami-e358958a',\n",
        " u'ami-e7582d8e',\n",
        " u'ami-f8d66891'}"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "if an ami is not one of my own, then can I query about it?  (e.g., if it's some ubuntu instance....what can I learn about it?)"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Images / Volumes / Snapshots"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "len(aws.all_images())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "images = aws.all_images()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "img0 = images[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "img0.id, img0.description, "
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# which images don't correspond to any of the instances?  which do -- and certainly should not be deleted."
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "img_ids_from_instances = set([instance.image_id for instance in instances])\n",
      "image_ids = set([img.id for img in images])\n",
      "\n",
      "img_ids_from_instances - image_ids"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "not_my_images = aws.ec2.get_all_images(image_ids=list(img_ids_from_instances - image_ids))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "img0 = not_my_images[0]\n",
      "img0.id, img0.description, img0.name"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[(img.id, img.description, img.name) for img in not_my_images]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# which of my images aren't tied to any of the running or stopped instances?\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# keypairs\n",
      "\n",
      "rdhyee_public_key = aws.ec2.get_all_key_pairs()[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rdhyee_public_key.fingerprint"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# volumes\n",
      "aws.ec2.get_all_volumes()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "aws.ec2.get_all_snapshots(owner=aws.GLUEJAR_ACCOUNT_ID)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "len(_)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "how to get at Route53?"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "hzones = aws.all_hosted_zones()\n",
      "hzones"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "unglue_it_zone = aws.route53.get_hosted_zone_by_name('unglue.it')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "unglue_it_zone"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "bitnamitest53.to_xml()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rrsets.commit()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rrsets = aws.route53.get_all_rrsets('ZNLXEIJIA7XKC')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# http://stackoverflow.com/questions/9706883/using-route53-to-update-dns-with-boto-should-i-use-a-name-or-cname-and-what-ttl\n",
      "# add_change(action, name, type, ttl=600, alias_hosted_zone_id=None, alias_dns_name=None, identifier=None, weight=None, region=None)\n",
      "change = rrsets.add_change(\"CREATE\",\"bitnami2.unglue.it\", \"A\", ttl=60)\n",
      "change.add_value(bitnamitest0_i.ip_address)\n",
      "rrsets.commit()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rrsets.changes"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#http://hackinghabits.com/2011/08/20/boto-for-amazon-route53/\n",
      "# even for deletion, value and TTL have to match record to delete. \n",
      "\n",
      "change = rrsets.add_change(\"DELETE\",\"bitnami2.unglue.it\", \"A\", ttl=60)\n",
      "change.add_value(\"23.20.67.193\")\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rrsets.to_xml()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rrsets.commit()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# unglue.it\n",
      "[rset for rset in route53.get_all_rrsets('ZNLXEIJIA7XKC')]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "bitnamitest53 = route53.get_all_rrsets('ZNLXEIJIA7XKC')[5]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "bitnamitest53.name, bitnamitest53.type, bitnamitest53.ttl, bitnamitest53.resource_records"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# now try to change IP attached to bitnamitest53\n",
      "\n",
      "bitnamitest53.\n",
      "\n",
      "#bitnamitest0_i.ip_address"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "route53."
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "i0 = instances[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[aname for aname in dir(i0) if not aname.startswith(\"__\")]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[aws.instance_info(i) for i in instances]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "aws.extraneous_snapshot_ids()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 10,
       "text": [
        "{u'snap-09159e7e',\n",
        " u'snap-13f59c6f',\n",
        " u'snap-1ada476f',\n",
        " u'snap-1db3a040',\n",
        " u'snap-23f09e5f',\n",
        " u'snap-35f43345',\n",
        " u'snap-7d760501',\n",
        " u'snap-88c4a8ec',\n",
        " u'snap-9b3f7bf9',\n",
        " u'snap-9d113ee1',\n",
        " u'snap-ad706ff0',\n",
        " u'snap-b5d341f4',\n",
        " u'snap-ba5e0ec7',\n",
        " u'snap-d957d0ae',\n",
        " u'snap-e313b99d',\n",
        " u'snap-ea5da29e',\n",
        " u'snap-f957ee8d'}"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "aws.stats_for_instances(aws.all_instances())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "len(aws.all_images())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[s.description for s in aws.all_snapshots()]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[i.id for i in aws.all_images()]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import re\n",
      "\n",
      "rexp = re.compile(r'(ami-\\S+)')\n",
      "d = u'Created by CreateImage(i-4ba65a2e) for ami-2d865744 from vol-d58f99b8'\n",
      "\n",
      "images = set([i.id for i in aws.all_images()])\n",
      "\n",
      "s = re.search(rexp, d)\n",
      "s.group(1)\n",
      "\n",
      "snapshot_amis = set([re.search(rexp,s.description).group(1) for s in aws.all_snapshots()])\n",
      "\n",
      "# overlap\n",
      "\n",
      "print \"overlap\", images & snapshot_amis\n",
      "\n",
      "print \"in snapshot_amis, but not images\", snapshot_amis - images\n",
      "print \"in images, but not snapshot_amis\", images - snapshot_amis\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Consolidate code to pull up route53 record \n",
      "\n",
      "def route53_records(domain_name, name, record_type):\n",
      "    \n",
      "    zone = route53.get_hosted_zone_by_name(domain_name)\n",
      "    \n",
      "    if zone is None:\n",
      "        return ([], None)\n",
      "    \n",
      "    zone_id = zone.GetHostedZoneResponse.Id.replace('/hostedzone/', '')\n",
      "    rrsets = route53.get_all_rrsets(zone_id)\n",
      "    \n",
      "    full_name = \"{0}.{1}.\".format(name, domain_name)\n",
      "    \n",
      "    return ([rset for rset in route53.get_all_rrsets(zone_id) if rset.name == full_name and rset.type == record_type ],\n",
      "            rrsets)\n",
      "    \n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#http://hackinghabits.com/2011/08/20/boto-for-amazon-route53/\n",
      "# even for deletion, value and TTL have to match record to delete. \n",
      "\n",
      "(rrsets0, rrsets) = route53_records('unglue.it', 'bitnamitest', 'A')\n",
      "\n",
      "# if an old record exists, the get rid of it\n",
      "\n",
      "if len(rrsets0) == 1:\n",
      "    change = rrsets.add_change(\"DELETE\", rrsets0[0].name, rrsets0[0].type, rrsets0[0].ttl)\n",
      "    change.add_value(rrsets0[0].resource_records[0])\n",
      "    rrsets.commit()\n",
      "    rrsets.changes[:] = []\n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "bitnamitest0_i = aws.instance('bitnamitest0')\n",
      "\n",
      "change = rrsets.add_change(\"CREATE\", 'bitnamitest.unglue.it.', 'A', 60)\n",
      "change.add_value(bitnamitest0_i.ip_address)\n",
      "rrsets.commit()\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rrsets.changes[:] = []"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rrsets.changes"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}