Merge branch 'master' into mobigen_integrate

api/opds.py

@@ -7,6 +7,9 @@ from django.utils.http import urlquote
 
 import pytz
 
+import logging
+logger = logging.getLogger(__name__)
+
 from regluit.core import models, facets
 import regluit.core.cc as cc
 
@@ -137,9 +140,21 @@ def work_node(work):
     for subject in work.subjects.all():
         if subject.is_visible:
             category_node = etree.Element("category")
-            category_node.attrib["term"] = subject.name 
-            node.append(category_node)
-            append_navlink(node, 'related', 'kw.'+ subject.name , 0, 'popular', title=subject.name)
+            try:
+                category_node.attrib["term"] =  subject.name 
+                node.append(category_node)
+                try:
+                    subject.works.filter(is_free=True)[1]
+                    # only show feed if there's another work in it
+                    append_navlink(node, 'related', 'kw.'+ subject.name , 0, 'popular', title=subject.name)
+                except:
+                    pass
+            except ValueError:
+                # caused by control chars in subject.name
+                logger.warning('Deleting subject: %s' % subject.name)
+                subject.delete()
+                
+                
             
     return node
 

core/management/commands/delete_subjects_with_nonxml.py

@@ -0,0 +1,28 @@
+from django.core.management.base import BaseCommand
+
+from regluit.core.models import Subject
+
+#http://stackoverflow.com/questions/8733233/filtering-out-certain-bytes-in-python
+
+def valid_xml_char_ordinal(c):
+    codepoint = ord(c)
+    # conditions ordered by presumed frequency
+    return (
+        0x20 <= codepoint <= 0xD7FF or
+        codepoint in (0x9, 0xA, 0xD) or
+        0xE000 <= codepoint <= 0xFFFD or
+        0x10000 <= codepoint <= 0x10FFFF
+        )
+
+def clean_string(input_string):
+    return ''.join(c for c in input_string if valid_xml_char_ordinal(c))
+
+class Command(BaseCommand):
+    help = "delete subjects whose names are not expressible in XML"
+    
+    def handle(self, **options):
+       bad_subjects = [subject for subject in Subject.objects.all() if clean_string(subject.name) != subject.name]
+       print ("number of bad subjects:", len(bad_subjects))
+       for bad_subject in bad_subjects:
+            print (bad_subject.name.encode('ascii', 'ignore'), bad_subject.works.count())
+            bad_subject.delete()

sysadmin/playbooks/README.md

@@ -0,0 +1,38 @@
+This directory is for [ansible](http://www.ansible.com/home) playbooks for sysadmin tasks around unglue.it.
+
+We start with automating tasks such as [applying security upgrades](unattended-upgrade.yml) to the servers.
+
+Goal is to build playbooks for the configuration of servers.
+
+# Current status of playbooks
+
+* [host](hosts) is currently written by hand -- the inventory might be dynamically generated at some point.
+
+
+# Installing ansible
+
+[installation instructions](http://docs.ansible.com/intro_installation.html).
+
+On the mac, easiest approach is probably to use [homebrew](http://brew.sh/):
+
+    brew install ansible
+    
+You can also use `pip`:
+
+    pip install ansible
+    
+Note: ansible depends on one having ssh access to the servers.
+
+
+# Running unattended-upgrade
+
+in `regluit/sysadmin/playbooks`, run
+
+    ansible-playbook -i hosts -e "target=instances" unattended-upgrade.yml
+    
+to run on all the servers.
+
+    ansible-playbook -i hosts unattended-upgrade.yml
+    
+will run only just.
+

sysadmin/playbooks/hosts

@@ -0,0 +1,6 @@
+[instances]
+just ansible_ssh_port=22 ansible_ssh_host=just.unglue.it ansible_ssh_user=ubuntu
+web1 ansible_ssh_port=22 ansible_ssh_host=unglue.it ansible_ssh_user=ubuntu
+gluejar ansible_ssh_port=22 ansible_ssh_host=gluejar.com ansible_ssh_user=ubuntu
+jenkins ansible_ssh_port=22 ansible_ssh_host=jenkins.unglueit.com ansible_ssh_user=ubuntu
+

sysadmin/playbooks/roles/common/README.md

@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

sysadmin/playbooks/roles/common/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+# defaults file for common

sysadmin/playbooks/roles/common/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for common

sysadmin/playbooks/roles/common/meta/main.yml

@@ -0,0 +1,128 @@
+---
+galaxy_info:
+  author: your name
+  description: 
+  company: your company (optional)
+  # Some suggested licenses:
+  # - BSD (default)
+  # - MIT
+  # - GPLv2
+  # - GPLv3
+  # - Apache
+  # - CC-BY
+  license: license (GPLv2, CC-BY, etc)
+  min_ansible_version: 1.2
+  #
+  # Below are all platforms currently available. Just uncomment
+  # the ones that apply to your role. If you don't see your 
+  # platform on this list, let us know and we'll get it added!
+  #
+  #platforms:
+  #- name: EL
+  #  versions:
+  #  - all
+  #  - 5
+  #  - 6
+  #  - 7
+  #- name: GenericUNIX
+  #  versions:
+  #  - all
+  #  - any
+  #- name: Fedora
+  #  versions:
+  #  - all
+  #  - 16
+  #  - 17
+  #  - 18
+  #  - 19
+  #  - 20
+  #- name: SmartOS
+  #  versions:
+  #  - all
+  #  - any
+  #- name: opensuse
+  #  versions:
+  #  - all
+  #  - 12.1
+  #  - 12.2
+  #  - 12.3
+  #  - 13.1
+  #  - 13.2
+  #- name: Amazon
+  #  versions:
+  #  - all
+  #  - 2013.03
+  #  - 2013.09
+  #- name: GenericBSD
+  #  versions:
+  #  - all
+  #  - any
+  #- name: FreeBSD
+  #  versions:
+  #  - all
+  #  - 8.0
+  #  - 8.1
+  #  - 8.2
+  #  - 8.3
+  #  - 8.4
+  #  - 9.0
+  #  - 9.1
+  #  - 9.1
+  #  - 9.2
+  #- name: Ubuntu
+  #  versions:
+  #  - all
+  #  - lucid
+  #  - maverick
+  #  - natty
+  #  - oneiric
+  #  - precise
+  #  - quantal
+  #  - raring
+  #  - saucy
+  #  - trusty
+  #- name: SLES
+  #  versions:
+  #  - all
+  #  - 10SP3
+  #  - 10SP4
+  #  - 11
+  #  - 11SP1
+  #  - 11SP2
+  #  - 11SP3
+  #- name: GenericLinux
+  #  versions:
+  #  - all
+  #  - any
+  #- name: Debian
+  #  versions:
+  #  - all
+  #  - etch
+  #  - lenny
+  #  - squeeze
+  #  - wheezy
+  #
+  # Below are all categories currently available. Just as with
+  # the platforms above, uncomment those that apply to your role.
+  #
+  #categories:
+  #- cloud
+  #- cloud:ec2
+  #- cloud:gce
+  #- cloud:rax
+  #- clustering
+  #- database
+  #- database:nosql
+  #- database:sql
+  #- development
+  #- monitoring
+  #- networking
+  #- packaging
+  #- system
+  #- web
+dependencies: []
+  # List your role dependencies here, one per line. Only
+  # dependencies available via galaxy should be listed here.
+  # Be sure to remove the '[]' above if you add dependencies
+  # to this list.
+  

sysadmin/playbooks/roles/common/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+# tasks file for common

sysadmin/playbooks/roles/common/vars/main.yml

@@ -0,0 +1,2 @@
+---
+# vars file for common

sysadmin/playbooks/unattended-upgrade.yml

@@ -0,0 +1,15 @@
+- name: unattended-upgrade
+  # by default, run only just
+  hosts: '{{target | default("just")}}'
+
+  tasks:
+
+    - name: apply upgrade
+      command: sudo unattended-upgrade
+   
+    - name: check whether reboot needed 
+      stat: path=/var/run/reboot-required
+      register: reboot_required 
+
+    - debug: var=reboot_required.stat.exists 
+