diff --git a/build_ec2_instances_for_django.ipynb b/build_ec2_instances_for_django.ipynb
index 9b0ee438..d7a67571 100644
--- a/build_ec2_instances_for_django.ipynb
+++ b/build_ec2_instances_for_django.ipynb
@@ -7,6 +7,27 @@
"worksheets": [
{
"cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Goals of this notebook\n",
+ "\n",
+ "* instantiate a new EC2 instance with which to build a new image\n",
+ "* configure the instance -- what's involved? Essentially turn https://github.com/Gluejar/regluit/blob/master/README.md into a fabric script\n",
+ "* security group\n",
+ "* database\n",
+ "* database security group\n",
+ "* IAM\n",
+ "* elastic IP\n",
+ "\n",
+ "\n",
+ "I'm starting to figure out the pieces using this IPython notebook, but ultimately what am I producing? Something that Eric and Andromeda can run:\n",
+ "\n",
+ "* a set of fabric commands (https://github.com/Gluejar/regluit/blob/master/fabfile.py)?\n",
+ "* some other form?\n"
+ ]
+ },
{
"cell_type": "code",
"collapsed": false,
@@ -22,7 +43,12 @@
"cell_type": "code",
"collapsed": false,
"input": [
- "AMI_UBUNTU_12_04_ID = 'ami-79c0ae10'\n",
+ "# look up Ubuntu EC2 image ids from alestic.com\n",
+ "# us-east-1 Ubuntu 12.04 LTS Precise\n",
+ "# EBS boot\tami-e7582d8e\n",
+ "\n",
+ "#AMI_UBUNTU_12_04_ID = 'ami-79c0ae10' # older one\n",
+ "AMI_UBUNTU_12_04_ID = 'ami-e7582d8e'\n",
"image = aws.ec2.get_all_images(image_ids=[AMI_UBUNTU_12_04_ID])[0]"
],
"language": "python",
@@ -33,6 +59,8 @@
"cell_type": "code",
"collapsed": false,
"input": [
+ "# name of image follows Eric Hammond's convention of dating the images\n",
+ "\n",
"image.id, image.name"
],
"language": "python",
@@ -43,7 +71,7 @@
"cell_type": "code",
"collapsed": false,
"input": [
- "# sometimes we have an instance running or created already --\n",
+ "# sometimes we have an instance running or created already\n",
"# so we just need to get a reference to it (instead of creating a new one)\n",
"\n",
"instance = aws.instance('new_test')\n",
@@ -68,9 +96,7 @@
" key_name='rdhyee_public_key',\n",
" group_name=SECURITY_GROUP_NAME,\n",
" tag='new_instance',\n",
- " cmd_shell=False) \n",
- " \n",
- " "
+ " cmd_shell=False)"
],
"language": "python",
"metadata": {},
@@ -80,7 +106,6 @@
"cell_type": "code",
"collapsed": false,
"input": [
- "\n",
"instance.update()"
],
"language": "python",
@@ -200,7 +225,7 @@
"level": 1,
"metadata": {},
"source": [
- "dynamic execution of fabric tasks"
+ "dynamic execution of fabric tasks to setup the instance"
]
},
{
@@ -298,16 +323,21 @@
"source": [
"## Commands to add?\n",
"\n",
- "Apply security upgrade: `sudo unattended-upgrade`\n"
+ "By the time we run through a lot of the fabric script, a reboot of the system is required. After installing mysql locally, it seems that the instance needs to be rebooted. Here's some code to do so. Problem remaining is how to reboot, wait for reboot to be completed, and then pick up the next steps.\n",
+ "\n",
+ "I could issue a fabric command to apply security upgrade: `sudo unattended-upgrade`\n",
+ "\n",
+ "or \n",
+ "\n",
+ "I think there is a boto command to restart instance\n"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
- "# security groups\n",
- "\n",
- "security_groups = aws.ec2.get_all_security_groups()"
+ "rebooted_instance = instance.reboot()\n",
+ "rebooted_instance"
],
"language": "python",
"metadata": {},
@@ -317,6 +347,50 @@
"cell_type": "code",
"collapsed": false,
"input": [
+ "# looks like reboot works, but that the instance status remains running throughout time reboot happens...\n",
+ "# maybe we wait a specific amount of time and the try to connect "
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": []
+ },
+ {
+ "cell_type": "heading",
+ "level": 1,
+ "metadata": {},
+ "source": [
+ "EC2 security groups"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "* listing key existing security groups\n",
+ "* how to copy parameters\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "# security groups\n",
+ "\n",
+ "\n",
+ "security_groups = aws.ec2.get_all_security_groups()\n",
+ "security_groups"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "# pull out the security group used for unglue.it\n",
+ "\n",
"web_prod_sgroup = [(group.id, group.name, group.description, group.rules) for group in security_groups if group.name=='web-production'][0]"
],
"language": "python",
@@ -484,6 +558,38 @@
"metadata": {},
"outputs": []
},
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "# let's compute the instances that are tied to the various security groups\n",
+ "# http://boto.readthedocs.org/en/latest/ref/ec2.html#module-boto.ec2.securitygroup\n",
+ "# This calculation is useful for reconstructing the relationships among instances and security groups\n",
+ "\n",
+ "\n",
+ "from boto.ec2 import securitygroup\n",
+ "\n",
+ "for security_group in aws.ec2.get_all_security_groups():\n",
+ " sg = securitygroup.SecurityGroup(name=security_group.name, connection=aws.ec2)\n",
+ " print security_group, [inst.id for inst in sg.instances()]\n"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "# with the exception of frontend-lb, let's delete the security groups that have no attached instances \n",
+ "\n",
+ "for sg in [sg for sg in aws.ec2.get_all_security_groups() if len(sg.instances()) == 0 and sg.name != 'frontend-lb']:\n",
+ " print sg.name, sg.id, aws.ec2.delete_security_group(group_id=sg.id)"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": []
+ },
{
"cell_type": "heading",
"level": 1,
@@ -499,7 +605,7 @@
"* plain old mysql on the server ( https://help.ubuntu.com/12.04/serverguide/mysql.html )\n",
"* RDS parameters to figure out\n",
"\n",
- "to run mysql on server:\n",
+ "to run mysql on server -- if you didn't have to worry about interactivity:\n",
"\n",
"> `sudo apt-get install mysql-server`"
]
@@ -508,12 +614,49 @@
"cell_type": "code",
"collapsed": false,
"input": [
- "\"ubuntu@{0}\".format(inst.dns_name)"
+ "thatcvamp\"ubuntu@{0}\".format(inst.dns_name)"
],
"language": "python",
"metadata": {},
"outputs": []
},
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "# once mysql installed, how to test the basic connectivity?\n",
+ "\n"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "\n",
+ "sudo debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password password unglueit_pw_123'\n",
+ "sudo debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password_again password unglueit_pw_123'\n",
+ "sudo apt-get -y install mysql-server\n",
+ "
\n",
+ "\n",
+ "\n",
+ "mysql -h 127.0.0.1 --user=root --password=unglueit_pw_123 \n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ " mysql -h 127.0.0.1 --user=root --password=unglueit_pw_123 <<'EOF'\n",
+ "\n",
+ " SHOW DATABASES;\n",
+ " EOF\n",
+ "\n"
+ ]
+ },
{
"cell_type": "heading",
"level": 1,
@@ -598,21 +741,6 @@
"metadata": {},
"outputs": []
},
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "\n",
- "sudo debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password password unglueit_pw_123'\n",
- "sudo debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password_again password unglueit_pw_123'\n",
- "sudo apt-get -y install mysql-server\n",
- "
\n",
- "\n",
- "\n",
- "mysql -h 127.0.0.1 --user=root --password=unglueit_pw_123 \n",
- "
"
- ]
- },
{
"cell_type": "heading",
"level": 1,
@@ -781,63 +909,6 @@
"metadata": {},
"outputs": []
},
- {
- "cell_type": "heading",
- "level": 1,
- "metadata": {},
- "source": [
- "Rebooting instance"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "After installing mysql locally, it seems that the instance needs to be rebooted. Here's some code to do so. Problem remaining is how to reboot, wait for reboot to be completed, and then pick up the next steps."
- ]
- },
- {
- "cell_type": "code",
- "collapsed": false,
- "input": [
- "instance"
- ],
- "language": "python",
- "metadata": {},
- "outputs": []
- },
- {
- "cell_type": "code",
- "collapsed": false,
- "input": [
- "rebooted_instance = instance.reboot()\n",
- "rebooted_instance"
- ],
- "language": "python",
- "metadata": {},
- "outputs": []
- },
- {
- "cell_type": "code",
- "collapsed": false,
- "input": [
- "instance.update()"
- ],
- "language": "python",
- "metadata": {},
- "outputs": []
- },
- {
- "cell_type": "code",
- "collapsed": false,
- "input": [
- "# looks like reboot works, but that the instance status remains running throughout time reboot happens...\n",
- "# maybe we wait a specific amount of time and the try to connect "
- ],
- "language": "python",
- "metadata": {},
- "outputs": []
- },
{
"cell_type": "heading",
"level": 1,
@@ -1029,7 +1100,7 @@
"cell_type": "code",
"collapsed": false,
"input": [
- "# write out a \n",
+ "# write out a shell script for configuring the environment with the keys for AWS\n",
"\n",
"print \"\"\"#!/bin/bash\n",
"\n",
@@ -1107,7 +1178,7 @@
"language": "python",
"metadata": {},
"outputs": [],
- "prompt_number": 19
+ "prompt_number": 57
},
{
"cell_type": "code",
@@ -1133,7 +1204,9 @@
{
"cell_type": "code",
"collapsed": false,
- "input": [],
+ "input": [
+ "instance.state"
+ ],
"language": "python",
"metadata": {},
"outputs": []
diff --git a/build_ec2_instances_for_django.py b/build_ec2_instances_for_django.py
index b79eff27..6450bc14 100644
--- a/build_ec2_instances_for_django.py
+++ b/build_ec2_instances_for_django.py
@@ -1,17 +1,41 @@
+# # Goals of this notebook
+#
+# * instantiate a new EC2 instance with which to build a new image
+# * configure the instance -- what's involved? Essentially turn https://github.com/Gluejar/regluit/blob/master/README.md into a fabric script
+# * security group
+# * database
+# * database security group
+# * IAM
+# * elastic IP
+#
+#
+# I'm starting to figure out the pieces using this IPython notebook, but ultimately what am I producing? Something that Eric and Andromeda can run:
+#
+# * a set of fabric commands (https://github.com/Gluejar/regluit/blob/master/fabfile.py)?
+# * some other form?
+#
+
# In[ ]:
from regluit.sysadmin import aws
reload(aws)
# In[ ]:
-AMI_UBUNTU_12_04_ID = 'ami-79c0ae10'
+# look up Ubuntu EC2 image ids from alestic.com
+# us-east-1 Ubuntu 12.04 LTS Precise
+# EBS boot ami-e7582d8e
+
+#AMI_UBUNTU_12_04_ID = 'ami-79c0ae10' # older one
+AMI_UBUNTU_12_04_ID = 'ami-e7582d8e'
image = aws.ec2.get_all_images(image_ids=[AMI_UBUNTU_12_04_ID])[0]
# In[ ]:
+# name of image follows Eric Hammond's convention of dating the images
+
image.id, image.name
# In[ ]:
-# sometimes we have an instance running or created already --
+# sometimes we have an instance running or created already
# so we just need to get a reference to it (instead of creating a new one)
instance = aws.instance('new_test')
@@ -29,12 +53,9 @@ SECURITY_GROUP_NAME = 'testsg1'
key_name='rdhyee_public_key',
group_name=SECURITY_GROUP_NAME,
tag='new_instance',
- cmd_shell=False)
-
-
+ cmd_shell=False)
# In[ ]:
-
instance.update()
# In[ ]:
@@ -83,7 +104,7 @@ cmdstring = "ssh -oStrictHostKeyChecking=no ubuntu@{0}".format(instance.dns_name
! echo "$cmdstring" | pbcopy
cmdstring
-## dynamic execution of fabric tasks
+## dynamic execution of fabric tasks to setup the instance
# In[ ]:
# http://docs.fabfile.org/en/1.6/usage/execution.html#using-execute-with-dynamically-set-host-lists
@@ -169,15 +190,40 @@ fabric.tasks.execute(deploy, hosts=hosts)
# ## Commands to add?
#
-# Apply security upgrade: `sudo unattended-upgrade`
+# By the time we run through a lot of the fabric script, a reboot of the system is required. After installing mysql locally, it seems that the instance needs to be rebooted. Here's some code to do so. Problem remaining is how to reboot, wait for reboot to be completed, and then pick up the next steps.
+#
+# I could issue a fabric command to apply security upgrade: `sudo unattended-upgrade`
+#
+# or
+#
+# I think there is a boto command to restart instance
+#
+
+# In[ ]:
+rebooted_instance = instance.reboot()
+rebooted_instance
+
+# In[ ]:
+# looks like reboot works, but that the instance status remains running throughout time reboot happens...
+# maybe we wait a specific amount of time and the try to connect
+
+## EC2 security groups
+
+# * listing key existing security groups
+# * how to copy parameters
+#
#
# In[ ]:
# security groups
+
security_groups = aws.ec2.get_all_security_groups()
+security_groups
# In[ ]:
+# pull out the security group used for unglue.it
+
web_prod_sgroup = [(group.id, group.name, group.description, group.rules) for group in security_groups if group.name=='web-production'][0]
# In[ ]:
@@ -236,17 +282,58 @@ rules = test9_sg[3]
# In[ ]:
aws.ec2.authorize_security_group(group_name='test8', ip_protocol='tcp', from_port=80, to_port=80, cidr_ip='0.0.0.0/0')
+# In[ ]:
+# let's compute the instances that are tied to the various security groups
+# http://boto.readthedocs.org/en/latest/ref/ec2.html#module-boto.ec2.securitygroup
+# This calculation is useful for reconstructing the relationships among instances and security groups
+
+
+from boto.ec2 import securitygroup
+
+for security_group in aws.ec2.get_all_security_groups():
+ sg = securitygroup.SecurityGroup(name=security_group.name, connection=aws.ec2)
+ print security_group, [inst.id for inst in sg.instances()]
+
+
+# In[ ]:
+# with the exception of frontend-lb, let's delete the security groups that have no attached instances
+
+for sg in [sg for sg in aws.ec2.get_all_security_groups() if len(sg.instances()) == 0 and sg.name != 'frontend-lb']:
+ print sg.name, sg.id, aws.ec2.delete_security_group(group_id=sg.id)
+
## Setting up MySQL
# * plain old mysql on the server ( https://help.ubuntu.com/12.04/serverguide/mysql.html )
# * RDS parameters to figure out
#
-# to run mysql on server:
+# to run mysql on server -- if you didn't have to worry about interactivity:
#
# > `sudo apt-get install mysql-server`
# In[ ]:
-"ubuntu@{0}".format(inst.dns_name)
+thatcvamp"ubuntu@{0}".format(inst.dns_name)
+
+# In[ ]:
+# once mysql installed, how to test the basic connectivity?
+
+
+
+#
+# sudo debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password password unglueit_pw_123'
+# sudo debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password_again password unglueit_pw_123'
+# sudo apt-get -y install mysql-server
+#
+#
+#
+# mysql -h 127.0.0.1 --user=root --password=unglueit_pw_123
+#
+
+# mysql -h 127.0.0.1 --user=root --password=unglueit_pw_123 <<'EOF'
+#
+# SHOW DATABASES;
+# EOF
+#
+#
## Creating an Image out of Instance
@@ -280,16 +367,6 @@ new_image.state
cmd_shell=False)
-#
-# sudo debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password password unglueit_pw_123'
-# sudo debconf-set-selections <<< 'mysql-server-5.5 mysql-server/root_password_again password unglueit_pw_123'
-# sudo apt-get -y install mysql-server
-#
-#
-#
-# mysql -h 127.0.0.1 --user=root --password=unglueit_pw_123
-#
-
## RDS
# http://calculator.s3.amazonaws.com/calc5.html can be used to estimate costs
@@ -372,24 +449,6 @@ param = pg_dict.get('tx_isolation')
# how to create RDS
# db = conn.create_dbinstance("db-master-1", 10, 'db.m1.small', 'root', 'hunter2')
-## Rebooting instance
-
-# After installing mysql locally, it seems that the instance needs to be rebooted. Here's some code to do so. Problem remaining is how to reboot, wait for reboot to be completed, and then pick up the next steps.
-
-# In[ ]:
-instance
-
-# In[ ]:
-rebooted_instance = instance.reboot()
-rebooted_instance
-
-# In[ ]:
-instance.update()
-
-# In[ ]:
-# looks like reboot works, but that the instance status remains running throughout time reboot happens...
-# maybe we wait a specific amount of time and the try to connect
-
## IAM
# good to get an automated handle of the IAM groups and users. To use boto to manage IAM, you will need to have AWS keys with sufficient permissions.
@@ -534,7 +593,7 @@ iam_user, key, secret = create_iam_user('ry-dev-3', True)
iam.put_user_policy( user_name='ry-dev-3', policy_name='power_user_2013-06-12', policy_json=IAM_POWER_USER_PERMISSION)
# In[ ]:
-# write out a
+# write out a shell script for configuring the environment with the keys for AWS
print """#!/bin/bash
@@ -595,4 +654,4 @@ hosts = ["ubuntu@ry-dev.unglue.it"]
fabric.tasks.execute(run_on_ry_dev, hosts=hosts)
# In[ ]:
-
+instance.state