EbookFoundation
/
regluit
mirror of https://github.com/EbookFoundation/regluit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'ry' of github.com:Gluejar/regluit into fps 

Conflicts:
	requirements.pip
pull/1/head
Raymond Yee 2012-05-02 18:11:22 -07:00
parent b15a3bc6eb 910ad607ba
commit 99332f238b
5 changed files with 306 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
fabfile.py vendored
View File

@ -1,4 +1,5 @@
from fabric.api import run, local, env, cd
from regluit.sysadmin import aws
# allow us to use our ssh config files (e.g., ~/.ssh/config)
env.use_ssh_config = True
@ -18,8 +19,8 @@ def update_prod():
with cd("/opt/regluit"):
run("./deploy/update-prod")
def backup_db():
run("""TS=`date +"%Y-%m-%dT%H:%M:%S"`; /home/ubuntu/dump.sh | gzip > unglue.it.${TS}.sql.gz; scp ./unglue.it.${TS}.sql.gz b235656@hanjin.dreamhost.com: ; rm -f unglue.it.${TS}.sql.gz""")
def backup_db(name='unglue.it'):
run("""TS=`date +"%Y-%m-%dT%H:%M:%S"`; /home/ubuntu/dump.sh | gzip > {0}.${{TS}}.sql.gz; scp ./{0}.${{TS}}.sql.gz b235656@hanjin.dreamhost.com: ; rm -f {0}.${{TS}}.sql.gz""".format(name))
def get_dump():
"""Dump the current db on remote server and scp it over to local machine.
@ -29,6 +30,39 @@ def get_dump():
run("gzip -f unglue.it.sql")
local("scp web1:/home/ubuntu/unglue.it.sql.gz .")
local("gunzip -f unglue.it.sql.gz")
def copy_dump_to_ry_dev():
"""Dump the current db on remote server and scp it over to ry-dev.
Note: web1 has been hardcoded here to represent the name of the unglue.it server
"""
run("./dump.sh > unglue.it.sql ")
run("scp unglue.it.sql ry-dev.dyndns.org:")
def build_prod_instance(ami_id='ami-a29943cb'):
"""Build a new instance to serve as server instance for unglue.it"""
# http://my.safaribooksonline.com/book/-/9781449308100/2dot-ec2-recipes/id2529379
# default ami-a29943cb' is Ubuntu 12.04 Precise EBS boot
def ecdsa():
"""Calculate the host ECSDA host fingerprint http://bridge.grumpy-troll.org/2011/01/openssh.html """
run("""ssh-keygen -f /etc/ssh/ssh_host_ecdsa_key.pub -l""")
def ssh_fingerprint():
"""display ssh fingerprint of /home/ubuntu/.ssh/id_rsa.pub on remote machine"""
run ("""ssh-keygen -l -f /home/ubuntu/.ssh/id_rsa.pub""")
def ssh_fingerprint2():
# http://stackoverflow.com/a/6682934/7782
import base64,hashlib
def lineToFingerprint(line):
key = base64.b64decode(line.strip().partition('ssh-rsa ')[2])
fp_plain = hashlib.md5(key).hexdigest()
return ':'.join(a+b for a,b in zip(fp_plain[::2], fp_plain[1::2]))
def public_key_from_private_key():
# ssh-keygen -y -f ~/.ssh/id_rsa
pass
def email_addresses():
"""list email addresses in unglue.it"""

2
requirements.pip
View File

@ -23,3 +23,5 @@ django-notification
boto
fabric
git+git://github.com/agiliq/merchant.git#egg=django-merchant
paramiko
pyasn1

2
requirements_versioned.pip
View File

@ -36,6 +36,8 @@ mimeparse==0.1.3
MySQL-python==1.2.3
nose==1.1.2
oauth2==1.5.211
paramiko==1.7.7.1
pyasn1==0.1.3
pycrypto==2.5
pyparsing==1.5.6
python-dateutil==1.5

266
sysadmin/aws.py Normal file
View File

@ -0,0 +1,266 @@
from pprint import pprint
import os
import time
import boto
import boto.manage.cmdshell
# connect up parts of the Amazon infrastructure
# notes: to delete snapshots I have made of instances, one has to deregister the AMI first and then delete the snapshot
GLUEJAR_ACCOUNT_ID = 439256357102
ec2 = boto.connect_ec2()
cw = boto.connect_cloudwatch()
rds = boto.connect_rds()
def all_instances():
reservations = ec2.get_all_instances()
instances = [i for r in reservations for i in r.instances]
return instances
def all_zones():
print ec2.get_all_zones()
def all_rds():
return rds.get_all_dbinstances()
def all_rds_parameter_groups():
return rds.get_all_dbparameter_groups()
def modify_rds_parameter(group_name, parameter, value, apply_immediate=False):
"""change parameter in RDS parameter group_name to value
http://stackoverflow.com/a/9085381/7782
Remember to make sure that the parameter group is actually associated with the db.
You will likely need to reboot db too.
"""
pg = rds.get_all_dbparameters(group_name)
while not pg.has_key(parameter) and hasattr(pg, 'Marker'):
pg = rds.get_all_dbparameters(group_name, marker = pg.Marker)
if pg.has_key(parameter):
pg[parameter].value = value
pg[parameter].apply(immediate=apply_immediate)
return True
else:
return False
# how I associated the param_group to a db and then rebooted db
# rds.modify_dbinstance(id='production', param_group='production1', apply_immediately=True)
# rds.reboot_dbinstance(id='production')
def all_snapshots(owner=GLUEJAR_ACCOUNT_ID):
"""by default, return only snapshots owned by Gluejar -- None returns all snapshots available to us"""
return ec2.get_all_snapshots(owner=owner)
def instance(tag_name):
try:
return ec2.get_all_instances(filters={'tag:Name' : tag_name})[0].instances[0]
except Exception, e:
return None
def all_images(owners=(GLUEJAR_ACCOUNT_ID, )):
return ec2.get_all_images(owners=owners)
def stop_instances(instances):
return ec2.stop_instances(instance_ids=[instance.id for instance in instances])
def console_output(instance):
"""returnn console output of instance"""
try:
return instance.get_console_output().output
except Exception, e:
return None
def instance_metrics(instance):
"""metrics that apply to given instance"""
metrics = cw.list_metrics()
my_metrics = []
for metric in metrics:
if 'InstanceId' in metric.dimensions:
if instance.id in metric.dimensions['InstanceId']:
my_metrics.append(metric)
return my_metrics
# how to get average CPU utilization of web1?
# filter(lambda x: x.name == 'CPUUtilization', m)
# metric.query(start_time, end_time,'Average', period=6000)
def instance_metric(instance, metric_name):
m = instance_metrics(instance)
return filter(lambda x: x.name == metric_name, m)
def launch_time(instance):
return boto.utils.parse_ts(instance.launch_time)
def max_cpu(instance):
pass
def stats_for_instances(instances=None):
"""return basic stats for input instances"""
if instances is None:
instances = all_instances()
stats = []
for instance in instances:
instance.update() # to get latest update
stats.append((instance.id, instance.key_name, instance.state, instance.ip_address, instance.dns_name))
return stats
# http://my.safaribooksonline.com/book/-/9781449308100/2dot-ec2-recipes/id2529379
def launch_instance(ami='ami-a29943cb',
instance_type='t1.micro',
key_name='rdhyee_public_key',
key_extension='.pem',
key_dir='~/.ssh',
group_name='default',
ssh_port=22,
cidr='0.0.0.0/0',
tag='paws',
user_data=None,
cmd_shell=True,
login_user='ubuntu',
ssh_pwd=None):
"""
Launch an instance and wait for it to start running.
Returns a tuple consisting of the Instance object and the CmdShell
object, if request, or None.
ami The ID of the Amazon Machine Image that this instance will
be based on. Default is a 64-bit Amazon Linux EBS image.
instance_type The type of the instance.
key_name The name of the SSH Key used for logging into the instance.
It will be created if it does not exist.
key_extension The file extension for SSH private key files.
key_dir The path to the directory containing SSH private keys.
This is usually ~/.ssh.
group_name The name of the security group used to control access
to the instance. It will be created if it does not exist.
ssh_port The port number you want to use for SSH access (default 22).
cidr The CIDR block used to limit access to your instance.
tag A name that will be used to tag the instance so we can
easily find it later.
user_data Data that will be passed to the newly started
instance at launch and will be accessible via
the metadata service running at http://169.254.169.254.
cmd_shell If true, a boto CmdShell object will be created and returned.
This allows programmatic SSH access to the new instance.
login_user The user name used when SSH'ing into new instance. The
default is 'ubuntu'
ssh_pwd The password for your SSH key if it is encrypted with a
passphrase.
"""
cmd = None
# Create a connection to EC2 service.
# You can pass credentials in to the connect_ec2 method explicitly
# or you can use the default credentials in your ~/.boto config file
# as we are doing here.
ec2 = boto.connect_ec2()
# Check to see if specified keypair already exists.
# If we get an InvalidKeyPair.NotFound error back from EC2,
# it means that it doesn't exist and we need to create it.
try:
key = ec2.get_all_key_pairs(keynames=[key_name])[0]
except ec2.ResponseError, e:
if e.code == 'InvalidKeyPair.NotFound':
print 'Creating keypair: %s' % key_name
# Create an SSH key to use when logging into instances.
key = ec2.create_key_pair(key_name)
# AWS will store the public key but the private key is
# generated and returned and needs to be stored locally.
# The save method will also chmod the file to protect
# your private key.
key.save(key_dir)
else:
raise
# Check to see if specified security group already exists.
# If we get an InvalidGroup.NotFound error back from EC2,
# it means that it doesn't exist and we need to create it.
try:
group = ec2.get_all_security_groups(groupnames=[group_name])[0]
except ec2.ResponseError, e:
if e.code == 'InvalidGroup.NotFound':
print 'Creating Security Group: %s' % group_name
# Create a security group to control access to instance via SSH.
group = ec2.create_security_group(group_name,
'A group that allows SSH access')
else:
raise
# Add a rule to the security group to authorize SSH traffic
# on the specified port.
try:
group.authorize('tcp', ssh_port, ssh_port, cidr)
except ec2.ResponseError, e:
if e.code == 'InvalidPermission.Duplicate':
print 'Security Group: %s already authorized' % group_name
else:
raise
# Now start up the instance. The run_instances method
# has many, many parameters but these are all we need
# for now.
reservation = ec2.run_instances(ami,
key_name=key_name,
security_groups=[group_name],
instance_type=instance_type,
user_data=user_data)
# Find the actual Instance object inside the Reservation object
# returned by EC2.
instance = reservation.instances[0]
# The instance has been launched but it's not yet up and
# running. Let's wait for its state to change to 'running'.
print 'waiting for instance'
while instance.state != 'running':
print '.'
time.sleep(5)
instance.update()
print 'done'
# Let's tag the instance with the specified label so we can
# identify it later.
instance.add_tag(tag)
# The instance is now running, let's try to programmatically
# SSH to the instance using Paramiko via boto CmdShell.
if cmd_shell:
key_path = os.path.join(os.path.expanduser(key_dir),
key_name+key_extension)
cmd = boto.manage.cmdshell.sshclient_from_instance(instance,
key_path,
user_name=login_user,
ssh_pwd=ssh_pwd)
return (instance, cmd)
if __name__ == '__main__':
pprint (stats_for_instances(all_instances()))
web1 = instance('web1')
print instance_metrics(web1)

103
sysadmin/cloudwatch.py
View File

@ -1,103 +0,0 @@
from pprint import pprint
import boto
# connect up parts of the Amazon infrastructure
# notes: to delete snapshots I have made of instances, one has to deregister the AMI first and then delete the snapshot
GLUEJAR_ACCOUNT_ID = 439256357102
ec2 = boto.connect_ec2()
cw = boto.connect_cloudwatch()
rds = boto.connect_rds()
def all_instances():
reservations = ec2.get_all_instances()
instances = [i for r in reservations for i in r.instances]
return instances
def all_zones():
print ec2.get_all_zones()
def all_rds():
return rds.get_all_dbinstances()
def all_rds_parameter_groups():
return rds.get_all_dbparameter_groups()
def modify_please1_pg_group():
"""kinda ugly
http://stackoverflow.com/a/9085381/7782
After doing this, I changed please db to talk to this parameter group and rebooted db
"""
pg = conn.get_all_dbparameters('mygroup')
pg2 = rds.get_all_dbparameters('mygroup', marker = pg.Marker)
pg2['tx_isolation'].value = True
pg2['tx_isolation'].apply(True)
def all_snapshots(owner=GLUEJAR_ACCOUNT_ID):
"""by default, return only snapshots owned by Gluejar -- None returns all snapshots available to us"""
return ec2.get_all_snapshots(owner=owner)
def instance(tag_name):
try:
return ec2.get_all_instances(filters={'tag:Name' : tag_name})[0].instances[0]
except Exception, e:
return None
def all_images(owners=(GLUEJAR_ACCOUNT_ID, )):
return ec2.get_all_images(owners=owners)
def stop_instances(instances):
return ec2.stop_instances(instance_ids=[instance.id for instance in instances])
def console_output(instance):
"""returnn console output of instance"""
try:
return instance.get_console_output().output
except Exception, e:
return None
def instance_metrics(instance):
"""metrics that apply to given instance"""
metrics = cw.list_metrics()
my_metrics = []
for metric in metrics:
if 'InstanceId' in metric.dimensions:
if instance.id in metric.dimensions['InstanceId']:
my_metrics.append(metric)
return my_metrics
# how to get average CPU utilization of web1?
# filter(lambda x: x.name == 'CPUUtilization', m)
# metric.query(start_time, end_time,'Average', period=6000)
def instance_metric(instance, metric_name):
m = instance_metrics(instance)
return filter(lambda x: x.name == metric_name, m)
def launch_time(instance):
return boto.utils.parse_ts(instance.launch_time)
def max_cpu(instance):
pass
def stats_for_instances(instances=None):
"""return basic stats for input instances"""
if instances is None:
instances = all_instances()
stats = []
for instance in instances:
instance.update() # to get latest update
stats.append((instance.id, instance.key_name, instance.state, instance.ip_address, instance.dns_name))
return stats
if __name__ == '__main__':
pprint (stats_for_instances(all_instances()))
web1 = instance('web1')
print instance_metrics(web1)