Merge branch 'master' into tastypie2

Conflicts:
	core/models.py

README.md

@@ -29,6 +29,7 @@ to install python-setuptools in step 1:
 1. `echo 'export DJANGO_SETTINGS_MODULE=regluit.settings.me' >> ~/.virtualenvs/regluit/bin/postactivate`
 1. `deactivate ; workon regluit`
 1. `django-admin.py syncdb --migrate --noinput`
+1. `django-admin.py celeryd --loglevel INFO` start the celery daemon to perform asynchronous tasks like adding related editions, and display logging information in the foreground.`
 1. `django-admin.py runserver 0.0.0.0:8000` (you can change the port number from the default value of 8000)
 1. point your browser at http://localhost:8000/
 
@@ -40,7 +41,7 @@ Below are the steps for getting regluit running on EC2 with Apache and mod_wsgi,
 1. create an ubuntu natty ec2 instance using ami-1aad5273
 1. `sudo aptitude update`
 1. `sudo aptitude upgrade`
-1. `sudo aptitude install git apache libapache2-mod-wsgi mysql-client python-virtualenv python-mysqldb`
+1. `sudo aptitude install git apache libapache2-mod-wsgi mysql-client python-virtualenv python-mysqldb redis-server`
 1. `sudo mkdir /opt/regluit`
 1. `sudo chown ubuntu:ubuntu /opt/regluit`
 1. `cd /opt`
@@ -64,6 +65,15 @@ Below are the steps for getting regluit running on EC2 with Apache and mod_wsgi,
 1. `sudo ln -s /opt/regluit/deploy/regluit.conf /etc/apache2/sites-available/regluit`
 1. `sudo a2ensite regluit`
 1. `sudo /etc/init.d/apache2 restart`
+1. `sudo adduser --no-create-home celery --disabled-password --disabled-login`
+1. `sudo cp celeryd /etc/init.d/celeryd`
+1. `sudo chmod 755 /etc/init.d/celeryd`
+1. `sudo cp celeryd.conf /etc/default/celeryd`
+1. `sudo mkdir /var/log/celery`
+1. `sudo chown celery:celery /var/log/celery`
+1. `sudo mkdir /var/run/celery`
+1. `sudo chown celery:celery /var/run/celery`
+1. `sudo /etc/init.d/celeryd start`
 
 
 OS X Develper Notes

core/bookloader.py

@@ -18,6 +18,7 @@ def add_by_isbn(isbn, work=None, add_related=True):
     is optional, and if not supplied the edition will be associated with
     a stub work.
     """
+    logger.info("adding book for %s", isbn)
     # save a lookup to google if we already have this isbn
     has_isbn = Q(isbn_10=isbn) | Q(isbn_13=isbn)
     for edition in models.Edition.objects.filter(has_isbn):
@@ -49,6 +50,7 @@ def add_by_googlebooks_id(googlebooks_id, work=None):
     if not created:
         return e
 
+    logger.info("loading metadata from google for %s", googlebooks_id)
     url = "https://www.googleapis.com/books/v1/volumes/%s" % googlebooks_id
     d = _get_json(url)['volumeInfo']
 
@@ -89,21 +91,28 @@ def add_related(isbn):
     The initial seed ISBN will be added if it's not already there.
     """
     # make sure the seed edition is there
+    logger.info("adding related editions for %s", isbn)
     edition = add_by_isbn(isbn)
 
     # this is the work everything will hang off
     work = edition.work
 
+    new_editions = []
     for other_isbn in thingisbn(isbn):
         related_edition = add_by_isbn(other_isbn, work)
         if related_edition and related_edition.work != edition.work:
             merge_works(edition.work, related_edition.work)
+        if related_edition:
+            new_editions.append(related_edition)
+
+    return new_editions
 
 
 def thingisbn(isbn):
     """given an ISBN return a list of related edition ISBNs, according to 
     Library Thing.
     """
+    logger.info("looking up %s at ThingISBN" % isbn)
     url = "http://www.librarything.com/api/thingISBN/%s" % isbn
     xml = requests.get(url, headers={"User-Agent": settings.USER_AGENT}).content
     doc = ElementTree.fromstring(xml)
@@ -113,6 +122,7 @@ def thingisbn(isbn):
 def merge_works(w1, w2):
     """will merge the second work (w2) into the first (w1)
     """
+    logger.info("merging work %s into %s", w1, w2)
     for edition in w2.editions.all():
         edition.work = w1
         edition.save()

core/management/commands/add_related.py

@@ -0,0 +1,13 @@
+from django.core.management.base import BaseCommand
+
+from regluit.core import tasks, models
+
+class Command(BaseCommand):
+    help = "queues add related books for works that have only one edition"
+
+    def handle(self, **options):
+        for work in models.Work.objects.all():
+            if work.editions.all().count() == 1:
+                sole_edition = work.editions.all()[0]
+                tasks.add_related.delay(sole_edition.isbn_10)
+

core/models.py

@@ -102,10 +102,8 @@ class Work(models.Model):
     openlibrary_id = models.CharField(max_length=50, null=True)
 
     def cover_image_small(self):
-        server_id = random.randint(0, 9)
+        return self.editions.all()[0].cover_image_small()
-        gb_id = self.editions.all()[0].googlebooks_id
+
-        return "http://bks%i.books.google.com/books?id=%s&printsec=frontcover&img=1&zoom=5" % (server_id, gb_id)                
-    
     def __unicode__(self):
         return self.title
 
@@ -143,6 +141,14 @@ class Edition(models.Model):
     def __unicode__(self):
         return "%s (%s)" % (self.title, self.isbn_13)
 
+    def cover_image_small(self):
+        server_id = random.randint(0, 9)
+        return "http://bks%i.books.google.com/books?id=%s&printsec=frontcover&img=1&zoom=5" % (server_id, self.googlebooks_id)
+
+    def cover_image_thumbnail(self):
+        server_id = random.randint(0, 9)
+        return "http://bks%s.books.google.com/books?id=%s&printsec=frontcover&img=1&zoom=1" % (server_id, self.googlebooks_id)
+
     @classmethod
     def get_by_isbn(klass, isbn):
         for e in Edition.objects.filter(Q(isbn_10=isbn) | Q(isbn_13=isbn)):

core/tasks.py

@@ -0,0 +1,11 @@
+from celery.decorators import task
+
+from regluit.core import bookloader
+
+@task
+def add_related(isbn):
+    return bookloader.add_related(isbn)
+
+@task
+def add_by_isbn(isbn):
+    return bookloader.add_by_isbn(isbn)

deploy/celeryd

@@ -0,0 +1,217 @@
+#!/bin/bash
+# ============================================
+#  celeryd - Starts the Celery worker daemon.
+# ============================================
+#
+# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status}
+#
+# :Configuration file: /etc/default/celeryd
+#
+# To configure celeryd you probably need to tell it where to chdir.
+#
+# EXAMPLE CONFIGURATION
+# =====================
+#
+# this is an example configuration for a Python project:
+#
+# /etc/default/celeryd:
+#
+#   # List of nodes to start
+#   CELERYD_NODES="worker1 worker2 worker3"k
+#   # ... can also be a number of workers
+#   CELERYD_NODES=3
+#
+#   # Where to chdir at start.
+#   CELERYD_CHDIR="/opt/Myproject/"
+#
+#   # Extra arguments to celeryd
+#   CELERYD_OPTS="--time-limit=300"
+#
+#   # Name of the celery config module.#
+#   CELERY_CONFIG_MODULE="celeryconfig"
+#
+# EXAMPLE DJANGO CONFIGURATION
+# ============================
+#
+#   # Where the Django project is.
+#   CELERYD_CHDIR="/opt/Project/"
+#
+#   # Name of the projects settings module.
+#   export DJANGO_SETTINGS_MODULE="settings"
+#
+#   # Path to celeryd
+#   CELERYD="/opt/Project/manage.py celeryd"
+#
+# AVAILABLE OPTIONS
+# =================
+#
+#   * CELERYD_NODES
+#
+#       A space separated list of nodes, or a number describing the number of
+#       nodes, to start
+#
+#   * CELERYD_OPTS
+#       Additional arguments to celeryd-multi, see `celeryd-multi --help`
+#       and `celeryd --help` for help.
+#
+#   * CELERYD_CHDIR
+#       Path to chdir at start. Default is to stay in the current directory.
+#
+#   * CELERYD_PIDFILE
+#       Full path to the pidfile. Default is /var/run/celeryd.pid.
+#
+#   * CELERYD_LOGFILE
+#       Full path to the celeryd logfile. Default is /var/log/celeryd.log
+#
+#   * CELERYD_LOG_LEVEL
+#       Log level to use for celeryd. Default is INFO.
+#
+#   * CELERYD
+#       Path to the celeryd program. Default is `celeryd`.
+#       You can point this to an virtualenv, or even use manage.py for django.
+#
+#   * CELERYD_USER
+#       User to run celeryd as. Default is current user.
+#
+#   * CELERYD_GROUP
+#       Group to run celeryd as. Default is current user.
+
+# VARIABLE EXPANSION
+# ==================
+#
+# The following abbreviations will be expanded
+#
+# * %n -> node name
+# * %h -> host name
+
+
+### BEGIN INIT INFO
+# Provides:          celeryd
+# Required-Start:    $network $local_fs $remote_fs
+# Required-Stop:     $network $local_fs $remote_fs
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: celery task worker daemon
+### END INIT INFO
+
+#set -e
+
+DEFAULT_PID_FILE="/var/run/celeryd@%n.pid"
+DEFAULT_LOG_FILE="/var/log/celeryd@%n.log"
+DEFAULT_LOG_LEVEL="INFO"
+DEFAULT_NODES="celery"
+DEFAULT_CELERYD="-m celery.bin.celeryd_detach"
+
+# /etc/init.d/celeryd: start and stop the celery task worker daemon.
+
+CELERY_DEFAULTS=${CELERY_DEFAULTS:-"/etc/default/celeryd"}
+
+test -f "$CELERY_DEFAULTS" && . "$CELERY_DEFAULTS"
+if [ -f "/etc/default/celeryd" ]; then
+    . /etc/default/celeryd
+fi
+
+if [ -f $VIRTUALENV_ACTIVATE ]; then
+    echo "activating virtualenv $VIRTUALENV_ACTIVATE"
+    source "$VIRTUALENV_ACTIVATE"
+fi
+
+CELERYD_PID_FILE=${CELERYD_PID_FILE:-${CELERYD_PIDFILE:-$DEFAULT_PID_FILE}}
+CELERYD_LOG_FILE=${CELERYD_LOG_FILE:-${CELERYD_LOGFILE:-$DEFAULT_LOG_FILE}}
+CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}}
+CELERYD_MULTI=${CELERYD_MULTI:-"celeryd-multi"}
+CELERYD=${CELERYD:-$DEFAULT_CELERYD}
+CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES}
+
+export CELERY_LOADER
+
+if [ -n "$2" ]; then
+    CELERYD_OPTS="$CELERYD_OPTS $2"
+fi
+
+# Extra start-stop-daemon options, like user/group.
+if [ -n "$CELERYD_USER" ]; then
+    DAEMON_OPTS="$DAEMON_OPTS --uid=$CELERYD_USER"
+fi
+if [ -n "$CELERYD_GROUP" ]; then
+    DAEMON_OPTS="$DAEMON_OPTS --gid=$CELERYD_GROUP"
+fi
+
+if [ -n "$CELERYD_CHDIR" ]; then
+    DAEMON_OPTS="$DAEMON_OPTS --workdir=\"$CELERYD_CHDIR\""
+fi
+
+
+check_dev_null() {
+    if [ ! -c /dev/null ]; then
+        echo "/dev/null is not a character device!"
+        exit 1
+    fi
+}
+
+
+export PATH="${PATH:+$PATH:}/usr/sbin:/sbin"
+
+
+stop_workers () {
+    $CELERYD_MULTI stop $CELERYD_NODES --pidfile="$CELERYD_PID_FILE"
+}
+
+
+start_workers () {
+    $CELERYD_MULTI start $CELERYD_NODES $DAEMON_OPTS        \
+                         --pidfile="$CELERYD_PID_FILE"      \
+                         --logfile="$CELERYD_LOG_FILE"      \
+                         --loglevel="$CELERYD_LOG_LEVEL"    \
+                         --cmd="$CELERYD"                   \
+                         $CELERYD_OPTS
+}
+
+
+restart_workers () {
+    $CELERYD_MULTI restart $CELERYD_NODES $DAEMON_OPTS      \
+                           --pidfile="$CELERYD_PID_FILE"    \
+                           --logfile="$CELERYD_LOG_FILE"    \
+                           --loglevel="$CELERYD_LOG_LEVEL"  \
+                           --cmd="$CELERYD"                 \
+                           $CELERYD_OPTS
+}
+
+
+
+case "$1" in
+    start)
+        check_dev_null
+        start_workers
+    ;;
+
+    stop)
+        check_dev_null
+        stop_workers
+    ;;
+
+    reload|force-reload)
+        echo "Use restart"
+    ;;
+
+    status)
+        celeryctl status
+    ;;
+
+    restart)
+        check_dev_null
+        restart_workers
+    ;;
+
+    try-restart)
+        check_dev_null
+        restart_workers
+    ;;
+
+    *)
+        echo "Usage: /etc/init.d/celeryd {start|stop|restart|try-restart|kill}"
+        exit 1
+    ;;
+esac
+
+exit 0

deploy/celeryd.conf

@@ -0,0 +1,12 @@
+
+CELERYD_NODES="w1"
+CELERYD_CHDIR="/opt/regluit/"
+CELERYD_LOG_FILE="/var/log/celery/%n.log"
+CELERYD_PID_FILE="/var/run/celery/%n.pid"
+CELERYD_USER="celery"
+CELERYD_GROUP="celery"
+CELERYD="/opt/regluit/ENV/bin/django-admin.py celeryd"
+CELERYD_MULTI="/opt/regluit/ENV/bin/django-admin.py celeryd_multi"
+
+VIRTUALENV_ACTIVATE="/opt/regluit/ENV/bin/activate"
+export DJANGO_SETTINGS_MODULE="regluit.settings.prod"

deploy/regluit.wsgi

@@ -4,5 +4,6 @@ import os
 
 import django.core.handlers.wsgi
 
+os.environ['CELERY_LOADER'] = 'django'
 os.environ['DJANGO_SETTINGS_MODULE'] = 'regluit.settings.prod'
 application = django.core.handlers.wsgi.WSGIHandler()

frontend/templates/supporter.html

@@ -233,7 +233,7 @@ how do I integrate the your wishlist thing with the tabs thing?
                         </div>
                         <div class="book-name">
                         <span>
-                            {{ work.title }}
+                            <a href="{% url work work.id %}">{{ work.title }}</a>
                         </span>
                         </div>
                         {% ifequal supporter request.user %}

frontend/templates/work.html

@@ -0,0 +1,40 @@
+{% extends "base.html" %}
+
+{% block content %}
+
+<div id="main-container">
+<div class="js-main">
+{% include "explore.html" %}
+<div id="js-maincol-fr">
+<div class="js-maincol-inner">
+<div class="content-block">
+<h1>{{ work.title }}</h1>
+
+<h2>{{ editions.all.count }} Editions:</h2>
+
+<ul class="edition-list">
+    {% for edition in editions %}
+    <li class="edition">
+    <div style="float: left; margin-right: 10px;">
+        <img class="cover" src="{{ edition.cover_image_small }}" />
+    </div>
+    <div>
+    <span class="publisher">{{ edition.publisher }}</span>,
+    <span class="publication-date">{{ edition.publication_date }}</span><br>
+    <span class="language">{{ edition.language }}</span><br>
+    ISBN-10: 
+    <span class="isbn_10">{{ edition.isbn_10 }}</span><br>
+    ISBN-13:
+    <span class="isbn_13">{{ edition.isbn_13 }}</span>
+    </div>
+    </li>
+    {% endfor %}
+</ul>
+
+</div>
+</div>
+</div>
+</div>
+</div>
+
+{% endblock %}

frontend/urls.py

@@ -9,6 +9,7 @@ from regluit.frontend.views import CampaignFormView
 urlpatterns = patterns(
     "regluit.frontend.views",
     url(r"^$", "home", name="home"),
+    url(r"work/(?P<work_id>.+)/$", "work", name="work"),
     url(r"^supporter/(?P<supporter_username>.+)/$", "supporter", name="supporter"),
     url(r"^search/$", "search", name="search"),
     url(r"^privacy/$", TemplateView.as_view(template_name="privacy.html"),

frontend/views.py

@@ -1,31 +1,25 @@
-from django.template import RequestContext
+import logging
+from decimal import Decimal as D
+
+from django.conf import settings
 from django.contrib.auth.models import User
-# from django.contrib.auth.forms import UserChangeForm
 from django.core.urlresolvers import reverse
 from django.http import HttpResponseRedirect
 from django.http import HttpResponse
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_POST
-from django.views.generic import ListView, DetailView
-from django.views.generic.base import TemplateView
 from django.views.generic.edit import FormView
 from django.contrib.auth.decorators import login_required
 from django.shortcuts import render, render_to_response, get_object_or_404
 
-from django.conf import settings
+from regluit.core import tasks
-
 from regluit.core import models, bookloader
 from regluit.core.search import gluejar_search
-
+from regluit.frontend.forms import UserData, ProfileForm
-from regluit.frontend.forms import UserData,ProfileForm
 from regluit.frontend.forms import CampaignPledgeForm
-
 from regluit.payment.manager import PaymentManager
 from regluit.payment.parameters import TARGET_TYPE_CAMPAIGN
 
-from decimal import Decimal as D
-
-import logging
 logger = logging.getLogger(__name__)
 
 from regluit.payment.models import Transaction
@@ -36,6 +30,11 @@ def home(request):
             args=[request.user.username]))
     return render(request, 'home.html', {'suppress_search_box': True})
 
+def work(request, work_id):
+    work = get_object_or_404(models.Work, id=work_id)
+    editions = work.editions.all().order_by('-publication_date')
+    return render(request, 'work.html', {'work': work, 'editions': editions})
+
 def supporter(request, supporter_username):
     supporter = get_object_or_404(User, username=supporter_username)
     wishlist = supporter.wishlist
@@ -137,6 +136,8 @@ def wishlist(request):
     remove_work_id = request.POST.get('remove_work_id', None)
     if googlebooks_id:
         edition = bookloader.add_by_googlebooks_id(googlebooks_id)
+        # add related editions asynchronously
+        tasks.add_related.delay(edition.isbn_10)
         request.user.wishlist.works.add(edition.work)
         # TODO: redirect to work page, when it exists
         return HttpResponseRedirect('/')
@@ -158,6 +159,7 @@ class CampaignFormView(FormView):
            'campaign': campaign
         })
         return context
+
     def form_valid(self,form):
         pk = self.kwargs["pk"]
         pledge_amount = form.cleaned_data["pledge_amount"]

requirements.pip

@@ -9,3 +9,6 @@ selenium
 django-nose-selenium
 nose
 django-profiles
+django-kombu
+django-celery
+redis

settings/common.py

@@ -107,6 +107,7 @@ INSTALLED_APPS = (
     'registration',
     'social_auth',
     'tastypie',
+    'djcelery',
 )
 
 # A sample logging configuration. The only tangible logging
@@ -182,3 +183,6 @@ USER_AGENT = "unglue.it.bot v0.0.1 <http://unglue.it>"
 SOUTH_TESTS_MIGRATE = True
 
 AUTH_PROFILE_MODULE = "core.userprofile"
+
+import djcelery
+djcelery.setup_loader()

settings/dev.py

@@ -79,3 +79,7 @@ PAYPAL_TEST_RH_EMAIL = "rh1_1317336251_biz@gluejar.com"
 PAYPAL_TEST_NONPROFIT_PARTNER_EMAIL = "nppart_1318957063_per@gluejar.com"
 
 BASE_URL = 'http://0.0.0.0/'
+
+# use database as queuing service in development
+BROKER_TRANSPORT = "djkombu.transport.DatabaseTransport"
+INSTALLED_APPS += ("djkombu",)

settings/jenkins.py

@@ -73,3 +73,7 @@ PAYPAL_BUYER_LOGIN =''
 PAYPAL_BUYER_PASSWORD = ''
 
 BASE_URL = 'http://0.0.0.0/'
+
+# use database as queuing service in development
+BROKER_TRANSPORT = "djkombu.transport.DatabaseTransport"
+INSTALLED_APPS += ("djkombu",)

settings/prod.py

@@ -47,3 +47,31 @@ FACEBOOK_API_SECRET = '5eae483a0e92113d884c427b578ef23a'
 GOOGLE_OAUTH2_CLIENT_ID = '989608723367.apps.googleusercontent.com'
 GOOGLE_OAUTH2_CLIENT_SECRET = '3UqalKyNynnaaarumUIWh8vS'
 GOOGLE_DISPLAY_NAME = 'unglue it!'
+
+PAYPAL_USERNAME = ''
+PAYPAL_PASSWORD =  ''
+PAYPAL_SIGNATURE = ''
+PAYPAL_APPID = ''
+
+PAYPAL_ENDPOINT = 'svcs.sandbox.paypal.com' # sandbox
+PAYPAL_PAYMENT_HOST = 'http://www.sandbox.paypal.com' # sandbox
+
+PAYPAL_SANDBOX_LOGIN = ''
+PAYPAL_SANDBOX_PASSWORD = ''
+
+PAYPAL_BUYER_LOGIN =''
+PAYPAL_BUYER_PASSWORD = ''
+
+PAYPAL_GLUEJAR_EMAIL = ""
+
+# for test purposes have a single RH paypal email
+PAYPAL_TEST_RH_EMAIL = "rh1_1317336251_biz@gluejar.com"
+PAYPAL_TEST_NONPROFIT_PARTNER_EMAIL = ""
+
+BASE_URL = 'http://0.0.0.0/'
+
+# use redis for production queue
+BROKER_TRANSPORT = "redis"
+BROKER_HOST = "localhost"
+BROKER_PORT = 6379
+BROKER_VHOST = "0"

static/css/layout.css

@@ -697,3 +697,21 @@ span.bounce-search {
     border-bottom: 3px solid;
     cursor: pointer;
 }
+
+/* work page */
+
+ul.edition-list {
+    list-style: none;
+}
+ul.edition-list li {
+    clear: both;
+    padding: 10px;
+    height: 80px;
+    border-bottom: thin gray solid;
+}
+li.edition img.cover {
+    vertical-align: top;
+}
+li.edition span.publisher {
+    font-weight: bold;
+}