{ "metadata": { "name": "" }, "nbformat": 3, "nbformat_minor": 0, "worksheets": [ { "cells": [ { "cell_type": "code", "collapsed": false, "input": [ "from regluit.sysadmin import aws\n", "reload(aws)" ], "language": "python", "metadata": {}, "outputs": [ { "metadata": {}, "output_type": "pyout", "prompt_number": 1, "text": [ "" ] } ], "prompt_number": 1 }, { "cell_type": "markdown", "metadata": {}, "source": [ "Reminding myself the relationship among various resources/entities in AWS\n", "\n", "* instances\n", "* images\n", "* volumes\n", "* security groups\n", "* keypairs" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Pull up all instances" ] }, { "cell_type": "code", "collapsed": false, "input": [ "instances = aws.all_instances()\n", "len(instances)" ], "language": "python", "metadata": {}, "outputs": [ { "metadata": {}, "output_type": "pyout", "prompt_number": 3, "text": [ "8" ] } ], "prompt_number": 3 }, { "cell_type": "markdown", "metadata": {}, "source": [ "we can use ec2 utils to get list of instances" ] }, { "cell_type": "code", "collapsed": false, "input": [ "%%bash\n", "source ~/gj_aws.sh \n", "ec2-describe-instances | head -10" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "RESERVATION\tr-ee67d78f\t439256357102\tDjango Stack powered by BitNami-1-4-5-0 on Ubuntu 12-04-AutogenByAWSMP-\n", "INSTANCE\ti-4f7a8327\tami-7afa9c13\t\t\tterminated\trdhyee_public_key\t0\t\tt1.micro\t2013-05-17T22:54:29+0000\tus-east-1b\taki-88aa75e1\t\t\tmonitoring-disabled\t\t\t\t\tebs\t\t\t\t\tparavirtual\txen\t55a4792d-f3aa-4a08-8787-018da2e5c988\tsg-f539f99e\tdefault\tfalse\t\n", "TAG\tinstance\ti-4f7a8327\tName\tbitnamitest0\n", "RESERVATION\tr-0ecd2c6e\t439256357102\tdevelopment\n", "INSTANCE\ti-512dd634\tami-e358958a\t\t\tstopped\tselenium-dev\t0\t\tm1.small\t2012-05-04T02:04:13+0000\tus-east-1b\taki-407d9529\t\t\tmonitoring-disabled\t\t\t\t\tebs\t\t\t\t\tparavirtual\txen\t\tsg-646a250d\tdefault\tfalse\t\n", "BLOCKDEVICE\t/dev/sda1\tvol-392c3f54\t2012-01-26T17:48:14.000Z\ttrue\t\t\n", "TAG\tinstance\ti-512dd634\tName\tselenium-dev\n", "RESERVATION\tr-37b43b55\t439256357102\try-dev\n", "INSTANCE\ti-f17cc397\tami-3a6cc853\t\t\tstopped\t\t0\t\tt1.micro\t2013-05-08T21:41:21+0000\tus-east-1c\taki-427d952b\t\t\tmonitoring-disabled\t\t\t\t\tebs\t\t\t\t\tparavirtual\txen\t\tsg-c509bead\tdefault\tfalse\t\n", "BLOCKDEVICE\t/dev/sda1\tvol-8c0834e3\t2012-05-11T23:34:56.000Z\ttrue\t\t\n" ] } ], "prompt_number": 4 }, { "cell_type": "code", "collapsed": false, "input": [ "import subprocess\n", "\n", "# don't know how to use boto to modify this flag -- even possible?\n", "# UPDATE: yes, I think we can use modify_attribute https://github.com/boto/boto/blob/2.9.3/boto/ec2/instance.py#L526\n", "\n", "if please_inst.get_attribute('disableApiTermination').get('disableApiTermination'):\n", " # http://alestic.com/2010/01/ec2-instance-locking\n", " # ec2-modify-instance-attribute --disable-api-termination false INSTANCEID\n", " output = subprocess.check_output(\n", " \"source ~/gj_aws.sh; ec2-modify-instance-attribute --disable-api-termination false {0}\".format(please_inst.id),\n", " shell=True,\n", " )\n", " print output\n", "\n", "print please_inst.get_attribute('disableApiTermination').get('disableApiTermination')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# delete the stopped 'please' instance\n", "please_inst = aws.instance('please')\n", "\n", "# check termination protection?\n", "if please_inst.state == 'stopped' and not please_inst.get_attribute('disableApiTermination').get('disableApiTermination'):\n", " please_inst.terminate()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "from collections import Counter\n", "Counter([instance.state for instance in instances])" ], "language": "python", "metadata": {}, "outputs": [ { "metadata": {}, "output_type": "pyout", "prompt_number": 5, "text": [ "Counter({u'running': 4, u'terminated': 2, u'stopped': 2})" ] } ], "prompt_number": 5 }, { "cell_type": "code", "collapsed": false, "input": [ "# running vs stopped instances\n", "[(instance.id, instance.dns_name, instance.tags.get('Name'), instance.groups[0].name) for instance in instances if instance.state == 'running']" ], "language": "python", "metadata": {}, "outputs": [ { "metadata": {}, "output_type": "pyout", "prompt_number": 6, "text": [ "[(u'i-41f44a27', u'ec2-23-23-184-9.compute-1.amazonaws.com', u'just', u'just'),\n", " (u'i-80857bee',\n", " u'ec2-184-73-231-81.compute-1.amazonaws.com',\n", " u'gluejar_dot_com',\n", " u'gluejar_dot_com_sg'),\n", " (u'i-5197452c',\n", " u'ec2-23-20-143-15.compute-1.amazonaws.com',\n", " u'jenkins_small',\n", " u'default'),\n", " (u'i-4ec6fd33',\n", " u'ec2-50-17-216-246.compute-1.amazonaws.com',\n", " u'web2_relauch_c_med',\n", " u'web-production')]" ] } ], "prompt_number": 6 }, { "cell_type": "code", "collapsed": false, "input": [ "# stopped instances grouped by ami-id\n", "\n", "from collections import defaultdict\n", "\n", "i_dict = defaultdict(list)\n", "\n", "for instance in instances:\n", " if instance.state == 'stopped':\n", " i_dict[instance.image_id].append(instance)\n", " \n", "# I need to retrieve ami info\n", "\n", "image_ids = i_dict.keys()\n", "images_dict = dict(zip(image_ids, aws.ec2.get_all_images(image_ids=image_ids)))\n", "\n", "for image_id in image_ids:\n", " print images_dict[image_id].name, i_dict[image_id]\n", " print\n" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "rdhyee-dev_2012_05_11 [Instance:i-f17cc397]\n", "\n", "ebs/ubuntu-images/ubuntu-natty-11.04-i386-server-20111003 [Instance:i-512dd634]\n", "\n" ] } ], "prompt_number": 7 }, { "cell_type": "code", "collapsed": false, "input": [ "inst0 = instances[0]\n", "inst0.id, inst0.image_id, inst0.state, inst0.image_id" ], "language": "python", "metadata": {}, "outputs": [ { "metadata": {}, "output_type": "pyout", "prompt_number": 8, "text": [ "(u'i-4f7a8327', u'ami-7afa9c13', u'terminated', u'ami-7afa9c13')" ] } ], "prompt_number": 8 }, { "cell_type": "code", "collapsed": false, "input": [ "set([instance.image_id for instance in instances])" ], "language": "python", "metadata": {}, "outputs": [ { "metadata": {}, "output_type": "pyout", "prompt_number": 9, "text": [ "{u'ami-35f9445c',\n", " u'ami-3a6cc853',\n", " u'ami-7afa9c13',\n", " u'ami-a26dc9cb',\n", " u'ami-e358958a',\n", " u'ami-e7582d8e',\n", " u'ami-f8d66891'}" ] } ], "prompt_number": 9 }, { "cell_type": "markdown", "metadata": {}, "source": [ "if an ami is not one of my own, then can I query about it? (e.g., if it's some ubuntu instance....what can I learn about it?)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Images / Volumes / Snapshots" ] }, { "cell_type": "code", "collapsed": false, "input": [ "len(aws.all_images())" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "images = aws.all_images()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "img0 = images[0]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "img0.id, img0.description, " ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# which images don't correspond to any of the instances? which do -- and certainly should not be deleted." ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "img_ids_from_instances = set([instance.image_id for instance in instances])\n", "image_ids = set([img.id for img in images])\n", "\n", "img_ids_from_instances - image_ids" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "not_my_images = aws.ec2.get_all_images(image_ids=list(img_ids_from_instances - image_ids))" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "img0 = not_my_images[0]\n", "img0.id, img0.description, img0.name" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "[(img.id, img.description, img.name) for img in not_my_images]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# which of my images aren't tied to any of the running or stopped instances?\n", "\n" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# keypairs\n", "\n", "rdhyee_public_key = aws.ec2.get_all_key_pairs()[0]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rdhyee_public_key.fingerprint" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# volumes\n", "aws.ec2.get_all_volumes()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "aws.ec2.get_all_snapshots(owner=aws.GLUEJAR_ACCOUNT_ID)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "len(_)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "how to get at Route53?" ] }, { "cell_type": "code", "collapsed": false, "input": [ "hzones = aws.all_hosted_zones()\n", "hzones" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "unglue_it_zone = aws.route53.get_hosted_zone_by_name('unglue.it')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "unglue_it_zone" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "bitnamitest53.to_xml()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rrsets.commit()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rrsets = aws.route53.get_all_rrsets('ZNLXEIJIA7XKC')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# http://stackoverflow.com/questions/9706883/using-route53-to-update-dns-with-boto-should-i-use-a-name-or-cname-and-what-ttl\n", "# add_change(action, name, type, ttl=600, alias_hosted_zone_id=None, alias_dns_name=None, identifier=None, weight=None, region=None)\n", "change = rrsets.add_change(\"CREATE\",\"bitnami2.unglue.it\", \"A\", ttl=60)\n", "change.add_value(bitnamitest0_i.ip_address)\n", "rrsets.commit()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rrsets.changes" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "#http://hackinghabits.com/2011/08/20/boto-for-amazon-route53/\n", "# even for deletion, value and TTL have to match record to delete. \n", "\n", "change = rrsets.add_change(\"DELETE\",\"bitnami2.unglue.it\", \"A\", ttl=60)\n", "change.add_value(\"23.20.67.193\")\n" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rrsets.to_xml()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rrsets.commit()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# unglue.it\n", "[rset for rset in route53.get_all_rrsets('ZNLXEIJIA7XKC')]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "bitnamitest53 = route53.get_all_rrsets('ZNLXEIJIA7XKC')[5]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "bitnamitest53.name, bitnamitest53.type, bitnamitest53.ttl, bitnamitest53.resource_records" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# now try to change IP attached to bitnamitest53\n", "\n", "bitnamitest53.\n", "\n", "#bitnamitest0_i.ip_address" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "route53." ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "i0 = instances[0]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "[aname for aname in dir(i0) if not aname.startswith(\"__\")]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "[aws.instance_info(i) for i in instances]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "aws.extraneous_snapshot_ids()" ], "language": "python", "metadata": {}, "outputs": [ { "metadata": {}, "output_type": "pyout", "prompt_number": 10, "text": [ "{u'snap-09159e7e',\n", " u'snap-13f59c6f',\n", " u'snap-1ada476f',\n", " u'snap-1db3a040',\n", " u'snap-23f09e5f',\n", " u'snap-35f43345',\n", " u'snap-7d760501',\n", " u'snap-88c4a8ec',\n", " u'snap-9b3f7bf9',\n", " u'snap-9d113ee1',\n", " u'snap-ad706ff0',\n", " u'snap-b5d341f4',\n", " u'snap-ba5e0ec7',\n", " u'snap-d957d0ae',\n", " u'snap-e313b99d',\n", " u'snap-ea5da29e',\n", " u'snap-f957ee8d'}" ] } ], "prompt_number": 10 }, { "cell_type": "code", "collapsed": false, "input": [ "aws.stats_for_instances(aws.all_instances())" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "len(aws.all_images())" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "[s.description for s in aws.all_snapshots()]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "[i.id for i in aws.all_images()]" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "import re\n", "\n", "rexp = re.compile(r'(ami-\\S+)')\n", "d = u'Created by CreateImage(i-4ba65a2e) for ami-2d865744 from vol-d58f99b8'\n", "\n", "images = set([i.id for i in aws.all_images()])\n", "\n", "s = re.search(rexp, d)\n", "s.group(1)\n", "\n", "snapshot_amis = set([re.search(rexp,s.description).group(1) for s in aws.all_snapshots()])\n", "\n", "# overlap\n", "\n", "print \"overlap\", images & snapshot_amis\n", "\n", "print \"in snapshot_amis, but not images\", snapshot_amis - images\n", "print \"in images, but not snapshot_amis\", images - snapshot_amis\n" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "# Consolidate code to pull up route53 record \n", "\n", "def route53_records(domain_name, name, record_type):\n", " \n", " zone = route53.get_hosted_zone_by_name(domain_name)\n", " \n", " if zone is None:\n", " return ([], None)\n", " \n", " zone_id = zone.GetHostedZoneResponse.Id.replace('/hostedzone/', '')\n", " rrsets = route53.get_all_rrsets(zone_id)\n", " \n", " full_name = \"{0}.{1}.\".format(name, domain_name)\n", " \n", " return ([rset for rset in route53.get_all_rrsets(zone_id) if rset.name == full_name and rset.type == record_type ],\n", " rrsets)\n", " \n", " " ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "#http://hackinghabits.com/2011/08/20/boto-for-amazon-route53/\n", "# even for deletion, value and TTL have to match record to delete. \n", "\n", "(rrsets0, rrsets) = route53_records('unglue.it', 'bitnamitest', 'A')\n", "\n", "# if an old record exists, the get rid of it\n", "\n", "if len(rrsets0) == 1:\n", " change = rrsets.add_change(\"DELETE\", rrsets0[0].name, rrsets0[0].type, rrsets0[0].ttl)\n", " change.add_value(rrsets0[0].resource_records[0])\n", " rrsets.commit()\n", " rrsets.changes[:] = []\n", " " ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "bitnamitest0_i = aws.instance('bitnamitest0')\n", "\n", "change = rrsets.add_change(\"CREATE\", 'bitnamitest.unglue.it.', 'A', 60)\n", "change.add_value(bitnamitest0_i.ip_address)\n", "rrsets.commit()\n" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rrsets.changes[:] = []" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "rrsets.changes" ], "language": "python", "metadata": {}, "outputs": [] } ], "metadata": {} } ] }