Merge branch 'master' into tastypie2

Conflicts:
	core/models.py
pull/1/head
Raymond Yee 2011-10-20 10:00:45 -07:00
commit b1526ea284
18 changed files with 402 additions and 18 deletions

View File

@ -29,6 +29,7 @@ to install python-setuptools in step 1:
1. `echo 'export DJANGO_SETTINGS_MODULE=regluit.settings.me' >> ~/.virtualenvs/regluit/bin/postactivate`
1. `deactivate ; workon regluit`
1. `django-admin.py syncdb --migrate --noinput`
1. `django-admin.py celeryd --loglevel INFO` start the celery daemon to perform asynchronous tasks like adding related editions, and display logging information in the foreground.`
1. `django-admin.py runserver 0.0.0.0:8000` (you can change the port number from the default value of 8000)
1. point your browser at http://localhost:8000/
@ -40,7 +41,7 @@ Below are the steps for getting regluit running on EC2 with Apache and mod_wsgi,
1. create an ubuntu natty ec2 instance using ami-1aad5273
1. `sudo aptitude update`
1. `sudo aptitude upgrade`
1. `sudo aptitude install git apache libapache2-mod-wsgi mysql-client python-virtualenv python-mysqldb`
1. `sudo aptitude install git apache libapache2-mod-wsgi mysql-client python-virtualenv python-mysqldb redis-server`
1. `sudo mkdir /opt/regluit`
1. `sudo chown ubuntu:ubuntu /opt/regluit`
1. `cd /opt`
@ -64,6 +65,15 @@ Below are the steps for getting regluit running on EC2 with Apache and mod_wsgi,
1. `sudo ln -s /opt/regluit/deploy/regluit.conf /etc/apache2/sites-available/regluit`
1. `sudo a2ensite regluit`
1. `sudo /etc/init.d/apache2 restart`
1. `sudo adduser --no-create-home celery --disabled-password --disabled-login`
1. `sudo cp celeryd /etc/init.d/celeryd`
1. `sudo chmod 755 /etc/init.d/celeryd`
1. `sudo cp celeryd.conf /etc/default/celeryd`
1. `sudo mkdir /var/log/celery`
1. `sudo chown celery:celery /var/log/celery`
1. `sudo mkdir /var/run/celery`
1. `sudo chown celery:celery /var/run/celery`
1. `sudo /etc/init.d/celeryd start`
OS X Develper Notes

View File

@ -18,6 +18,7 @@ def add_by_isbn(isbn, work=None, add_related=True):
is optional, and if not supplied the edition will be associated with
a stub work.
"""
logger.info("adding book for %s", isbn)
# save a lookup to google if we already have this isbn
has_isbn = Q(isbn_10=isbn) | Q(isbn_13=isbn)
for edition in models.Edition.objects.filter(has_isbn):
@ -49,6 +50,7 @@ def add_by_googlebooks_id(googlebooks_id, work=None):
if not created:
return e
logger.info("loading metadata from google for %s", googlebooks_id)
url = "https://www.googleapis.com/books/v1/volumes/%s" % googlebooks_id
d = _get_json(url)['volumeInfo']
@ -89,21 +91,28 @@ def add_related(isbn):
The initial seed ISBN will be added if it's not already there.
"""
# make sure the seed edition is there
logger.info("adding related editions for %s", isbn)
edition = add_by_isbn(isbn)
# this is the work everything will hang off
work = edition.work
new_editions = []
for other_isbn in thingisbn(isbn):
related_edition = add_by_isbn(other_isbn, work)
if related_edition and related_edition.work != edition.work:
merge_works(edition.work, related_edition.work)
if related_edition:
new_editions.append(related_edition)
return new_editions
def thingisbn(isbn):
"""given an ISBN return a list of related edition ISBNs, according to
Library Thing.
"""
logger.info("looking up %s at ThingISBN" % isbn)
url = "http://www.librarything.com/api/thingISBN/%s" % isbn
xml = requests.get(url, headers={"User-Agent": settings.USER_AGENT}).content
doc = ElementTree.fromstring(xml)
@ -113,6 +122,7 @@ def thingisbn(isbn):
def merge_works(w1, w2):
"""will merge the second work (w2) into the first (w1)
"""
logger.info("merging work %s into %s", w1, w2)
for edition in w2.editions.all():
edition.work = w1
edition.save()

View File

@ -0,0 +1,13 @@
from django.core.management.base import BaseCommand
from regluit.core import tasks, models
class Command(BaseCommand):
help = "queues add related books for works that have only one edition"
def handle(self, **options):
for work in models.Work.objects.all():
if work.editions.all().count() == 1:
sole_edition = work.editions.all()[0]
tasks.add_related.delay(sole_edition.isbn_10)

View File

@ -102,10 +102,8 @@ class Work(models.Model):
openlibrary_id = models.CharField(max_length=50, null=True)
def cover_image_small(self):
server_id = random.randint(0, 9)
gb_id = self.editions.all()[0].googlebooks_id
return "http://bks%i.books.google.com/books?id=%s&printsec=frontcover&img=1&zoom=5" % (server_id, gb_id)
return self.editions.all()[0].cover_image_small()
def __unicode__(self):
return self.title
@ -143,6 +141,14 @@ class Edition(models.Model):
def __unicode__(self):
return "%s (%s)" % (self.title, self.isbn_13)
def cover_image_small(self):
server_id = random.randint(0, 9)
return "http://bks%i.books.google.com/books?id=%s&printsec=frontcover&img=1&zoom=5" % (server_id, self.googlebooks_id)
def cover_image_thumbnail(self):
server_id = random.randint(0, 9)
return "http://bks%s.books.google.com/books?id=%s&printsec=frontcover&img=1&zoom=1" % (server_id, self.googlebooks_id)
@classmethod
def get_by_isbn(klass, isbn):
for e in Edition.objects.filter(Q(isbn_10=isbn) | Q(isbn_13=isbn)):

11
core/tasks.py Normal file
View File

@ -0,0 +1,11 @@
from celery.decorators import task
from regluit.core import bookloader
@task
def add_related(isbn):
return bookloader.add_related(isbn)
@task
def add_by_isbn(isbn):
return bookloader.add_by_isbn(isbn)

217
deploy/celeryd Normal file
View File

@ -0,0 +1,217 @@
#!/bin/bash
# ============================================
# celeryd - Starts the Celery worker daemon.
# ============================================
#
# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status}
#
# :Configuration file: /etc/default/celeryd
#
# To configure celeryd you probably need to tell it where to chdir.
#
# EXAMPLE CONFIGURATION
# =====================
#
# this is an example configuration for a Python project:
#
# /etc/default/celeryd:
#
# # List of nodes to start
# CELERYD_NODES="worker1 worker2 worker3"k
# # ... can also be a number of workers
# CELERYD_NODES=3
#
# # Where to chdir at start.
# CELERYD_CHDIR="/opt/Myproject/"
#
# # Extra arguments to celeryd
# CELERYD_OPTS="--time-limit=300"
#
# # Name of the celery config module.#
# CELERY_CONFIG_MODULE="celeryconfig"
#
# EXAMPLE DJANGO CONFIGURATION
# ============================
#
# # Where the Django project is.
# CELERYD_CHDIR="/opt/Project/"
#
# # Name of the projects settings module.
# export DJANGO_SETTINGS_MODULE="settings"
#
# # Path to celeryd
# CELERYD="/opt/Project/manage.py celeryd"
#
# AVAILABLE OPTIONS
# =================
#
# * CELERYD_NODES
#
# A space separated list of nodes, or a number describing the number of
# nodes, to start
#
# * CELERYD_OPTS
# Additional arguments to celeryd-multi, see `celeryd-multi --help`
# and `celeryd --help` for help.
#
# * CELERYD_CHDIR
# Path to chdir at start. Default is to stay in the current directory.
#
# * CELERYD_PIDFILE
# Full path to the pidfile. Default is /var/run/celeryd.pid.
#
# * CELERYD_LOGFILE
# Full path to the celeryd logfile. Default is /var/log/celeryd.log
#
# * CELERYD_LOG_LEVEL
# Log level to use for celeryd. Default is INFO.
#
# * CELERYD
# Path to the celeryd program. Default is `celeryd`.
# You can point this to an virtualenv, or even use manage.py for django.
#
# * CELERYD_USER
# User to run celeryd as. Default is current user.
#
# * CELERYD_GROUP
# Group to run celeryd as. Default is current user.
# VARIABLE EXPANSION
# ==================
#
# The following abbreviations will be expanded
#
# * %n -> node name
# * %h -> host name
### BEGIN INIT INFO
# Provides: celeryd
# Required-Start: $network $local_fs $remote_fs
# Required-Stop: $network $local_fs $remote_fs
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: celery task worker daemon
### END INIT INFO
#set -e
DEFAULT_PID_FILE="/var/run/celeryd@%n.pid"
DEFAULT_LOG_FILE="/var/log/celeryd@%n.log"
DEFAULT_LOG_LEVEL="INFO"
DEFAULT_NODES="celery"
DEFAULT_CELERYD="-m celery.bin.celeryd_detach"
# /etc/init.d/celeryd: start and stop the celery task worker daemon.
CELERY_DEFAULTS=${CELERY_DEFAULTS:-"/etc/default/celeryd"}
test -f "$CELERY_DEFAULTS" && . "$CELERY_DEFAULTS"
if [ -f "/etc/default/celeryd" ]; then
. /etc/default/celeryd
fi
if [ -f $VIRTUALENV_ACTIVATE ]; then
echo "activating virtualenv $VIRTUALENV_ACTIVATE"
source "$VIRTUALENV_ACTIVATE"
fi
CELERYD_PID_FILE=${CELERYD_PID_FILE:-${CELERYD_PIDFILE:-$DEFAULT_PID_FILE}}
CELERYD_LOG_FILE=${CELERYD_LOG_FILE:-${CELERYD_LOGFILE:-$DEFAULT_LOG_FILE}}
CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}}
CELERYD_MULTI=${CELERYD_MULTI:-"celeryd-multi"}
CELERYD=${CELERYD:-$DEFAULT_CELERYD}
CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES}
export CELERY_LOADER
if [ -n "$2" ]; then
CELERYD_OPTS="$CELERYD_OPTS $2"
fi
# Extra start-stop-daemon options, like user/group.
if [ -n "$CELERYD_USER" ]; then
DAEMON_OPTS="$DAEMON_OPTS --uid=$CELERYD_USER"
fi
if [ -n "$CELERYD_GROUP" ]; then
DAEMON_OPTS="$DAEMON_OPTS --gid=$CELERYD_GROUP"
fi
if [ -n "$CELERYD_CHDIR" ]; then
DAEMON_OPTS="$DAEMON_OPTS --workdir=\"$CELERYD_CHDIR\""
fi
check_dev_null() {
if [ ! -c /dev/null ]; then
echo "/dev/null is not a character device!"
exit 1
fi
}
export PATH="${PATH:+$PATH:}/usr/sbin:/sbin"
stop_workers () {
$CELERYD_MULTI stop $CELERYD_NODES --pidfile="$CELERYD_PID_FILE"
}
start_workers () {
$CELERYD_MULTI start $CELERYD_NODES $DAEMON_OPTS \
--pidfile="$CELERYD_PID_FILE" \
--logfile="$CELERYD_LOG_FILE" \
--loglevel="$CELERYD_LOG_LEVEL" \
--cmd="$CELERYD" \
$CELERYD_OPTS
}
restart_workers () {
$CELERYD_MULTI restart $CELERYD_NODES $DAEMON_OPTS \
--pidfile="$CELERYD_PID_FILE" \
--logfile="$CELERYD_LOG_FILE" \
--loglevel="$CELERYD_LOG_LEVEL" \
--cmd="$CELERYD" \
$CELERYD_OPTS
}
case "$1" in
start)
check_dev_null
start_workers
;;
stop)
check_dev_null
stop_workers
;;
reload|force-reload)
echo "Use restart"
;;
status)
celeryctl status
;;
restart)
check_dev_null
restart_workers
;;
try-restart)
check_dev_null
restart_workers
;;
*)
echo "Usage: /etc/init.d/celeryd {start|stop|restart|try-restart|kill}"
exit 1
;;
esac
exit 0

12
deploy/celeryd.conf Normal file
View File

@ -0,0 +1,12 @@
CELERYD_NODES="w1"
CELERYD_CHDIR="/opt/regluit/"
CELERYD_LOG_FILE="/var/log/celery/%n.log"
CELERYD_PID_FILE="/var/run/celery/%n.pid"
CELERYD_USER="celery"
CELERYD_GROUP="celery"
CELERYD="/opt/regluit/ENV/bin/django-admin.py celeryd"
CELERYD_MULTI="/opt/regluit/ENV/bin/django-admin.py celeryd_multi"
VIRTUALENV_ACTIVATE="/opt/regluit/ENV/bin/activate"
export DJANGO_SETTINGS_MODULE="regluit.settings.prod"

View File

@ -4,5 +4,6 @@ import os
import django.core.handlers.wsgi
os.environ['CELERY_LOADER'] = 'django'
os.environ['DJANGO_SETTINGS_MODULE'] = 'regluit.settings.prod'
application = django.core.handlers.wsgi.WSGIHandler()

View File

@ -233,7 +233,7 @@ how do I integrate the your wishlist thing with the tabs thing?
</div>
<div class="book-name">
<span>
{{ work.title }}
<a href="{% url work work.id %}">{{ work.title }}</a>
</span>
</div>
{% ifequal supporter request.user %}

View File

@ -0,0 +1,40 @@
{% extends "base.html" %}
{% block content %}
<div id="main-container">
<div class="js-main">
{% include "explore.html" %}
<div id="js-maincol-fr">
<div class="js-maincol-inner">
<div class="content-block">
<h1>{{ work.title }}</h1>
<h2>{{ editions.all.count }} Editions:</h2>
<ul class="edition-list">
{% for edition in editions %}
<li class="edition">
<div style="float: left; margin-right: 10px;">
<img class="cover" src="{{ edition.cover_image_small }}" />
</div>
<div>
<span class="publisher">{{ edition.publisher }}</span>,
<span class="publication-date">{{ edition.publication_date }}</span><br>
<span class="language">{{ edition.language }}</span><br>
ISBN-10:
<span class="isbn_10">{{ edition.isbn_10 }}</span><br>
ISBN-13:
<span class="isbn_13">{{ edition.isbn_13 }}</span>
</div>
</li>
{% endfor %}
</ul>
</div>
</div>
</div>
</div>
</div>
{% endblock %}

View File

@ -9,6 +9,7 @@ from regluit.frontend.views import CampaignFormView
urlpatterns = patterns(
"regluit.frontend.views",
url(r"^$", "home", name="home"),
url(r"work/(?P<work_id>.+)/$", "work", name="work"),
url(r"^supporter/(?P<supporter_username>.+)/$", "supporter", name="supporter"),
url(r"^search/$", "search", name="search"),
url(r"^privacy/$", TemplateView.as_view(template_name="privacy.html"),

View File

@ -1,31 +1,25 @@
from django.template import RequestContext
import logging
from decimal import Decimal as D
from django.conf import settings
from django.contrib.auth.models import User
# from django.contrib.auth.forms import UserChangeForm
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from django.views.generic import ListView, DetailView
from django.views.generic.base import TemplateView
from django.views.generic.edit import FormView
from django.contrib.auth.decorators import login_required
from django.shortcuts import render, render_to_response, get_object_or_404
from django.conf import settings
from regluit.core import tasks
from regluit.core import models, bookloader
from regluit.core.search import gluejar_search
from regluit.frontend.forms import UserData,ProfileForm
from regluit.frontend.forms import UserData, ProfileForm
from regluit.frontend.forms import CampaignPledgeForm
from regluit.payment.manager import PaymentManager
from regluit.payment.parameters import TARGET_TYPE_CAMPAIGN
from decimal import Decimal as D
import logging
logger = logging.getLogger(__name__)
from regluit.payment.models import Transaction
@ -36,6 +30,11 @@ def home(request):
args=[request.user.username]))
return render(request, 'home.html', {'suppress_search_box': True})
def work(request, work_id):
work = get_object_or_404(models.Work, id=work_id)
editions = work.editions.all().order_by('-publication_date')
return render(request, 'work.html', {'work': work, 'editions': editions})
def supporter(request, supporter_username):
supporter = get_object_or_404(User, username=supporter_username)
wishlist = supporter.wishlist
@ -137,6 +136,8 @@ def wishlist(request):
remove_work_id = request.POST.get('remove_work_id', None)
if googlebooks_id:
edition = bookloader.add_by_googlebooks_id(googlebooks_id)
# add related editions asynchronously
tasks.add_related.delay(edition.isbn_10)
request.user.wishlist.works.add(edition.work)
# TODO: redirect to work page, when it exists
return HttpResponseRedirect('/')
@ -158,6 +159,7 @@ class CampaignFormView(FormView):
'campaign': campaign
})
return context
def form_valid(self,form):
pk = self.kwargs["pk"]
pledge_amount = form.cleaned_data["pledge_amount"]

View File

@ -9,3 +9,6 @@ selenium
django-nose-selenium
nose
django-profiles
django-kombu
django-celery
redis

View File

@ -107,6 +107,7 @@ INSTALLED_APPS = (
'registration',
'social_auth',
'tastypie',
'djcelery',
)
# A sample logging configuration. The only tangible logging
@ -182,3 +183,6 @@ USER_AGENT = "unglue.it.bot v0.0.1 <http://unglue.it>"
SOUTH_TESTS_MIGRATE = True
AUTH_PROFILE_MODULE = "core.userprofile"
import djcelery
djcelery.setup_loader()

View File

@ -79,3 +79,7 @@ PAYPAL_TEST_RH_EMAIL = "rh1_1317336251_biz@gluejar.com"
PAYPAL_TEST_NONPROFIT_PARTNER_EMAIL = "nppart_1318957063_per@gluejar.com"
BASE_URL = 'http://0.0.0.0/'
# use database as queuing service in development
BROKER_TRANSPORT = "djkombu.transport.DatabaseTransport"
INSTALLED_APPS += ("djkombu",)

View File

@ -73,3 +73,7 @@ PAYPAL_BUYER_LOGIN =''
PAYPAL_BUYER_PASSWORD = ''
BASE_URL = 'http://0.0.0.0/'
# use database as queuing service in development
BROKER_TRANSPORT = "djkombu.transport.DatabaseTransport"
INSTALLED_APPS += ("djkombu",)

View File

@ -47,3 +47,31 @@ FACEBOOK_API_SECRET = '5eae483a0e92113d884c427b578ef23a'
GOOGLE_OAUTH2_CLIENT_ID = '989608723367.apps.googleusercontent.com'
GOOGLE_OAUTH2_CLIENT_SECRET = '3UqalKyNynnaaarumUIWh8vS'
GOOGLE_DISPLAY_NAME = 'unglue it!'
PAYPAL_USERNAME = ''
PAYPAL_PASSWORD = ''
PAYPAL_SIGNATURE = ''
PAYPAL_APPID = ''
PAYPAL_ENDPOINT = 'svcs.sandbox.paypal.com' # sandbox
PAYPAL_PAYMENT_HOST = 'http://www.sandbox.paypal.com' # sandbox
PAYPAL_SANDBOX_LOGIN = ''
PAYPAL_SANDBOX_PASSWORD = ''
PAYPAL_BUYER_LOGIN =''
PAYPAL_BUYER_PASSWORD = ''
PAYPAL_GLUEJAR_EMAIL = ""
# for test purposes have a single RH paypal email
PAYPAL_TEST_RH_EMAIL = "rh1_1317336251_biz@gluejar.com"
PAYPAL_TEST_NONPROFIT_PARTNER_EMAIL = ""
BASE_URL = 'http://0.0.0.0/'
# use redis for production queue
BROKER_TRANSPORT = "redis"
BROKER_HOST = "localhost"
BROKER_PORT = 6379
BROKER_VHOST = "0"

View File

@ -697,3 +697,21 @@ span.bounce-search {
border-bottom: 3px solid;
cursor: pointer;
}
/* work page */
ul.edition-list {
list-style: none;
}
ul.edition-list li {
clear: both;
padding: 10px;
height: 80px;
border-bottom: thin gray solid;
}
li.edition img.cover {
vertical-align: top;
}
li.edition span.publisher {
font-weight: bold;
}