EbookFoundation
/
autocat3
mirror of https://github.com/EbookFoundation/autocat3.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

initial commit

google-oauth
eric 2019-03-28 09:45:03 -04:00
parent 96a4230fda
commit 6849187c2a
52 changed files with 10630 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

310
BaseFormatter.py Normal file
View File

@ -0,0 +1,310 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: iso-8859-1 -*-
"""
BaseFormatter.py
Copyright 2009-2010 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Base class for output formatters.
"""
from __future__ import unicode_literals
import datetime
import re
from six.moves import urllib
import base64
import os
import genshi.output
import genshi.template
from genshi.core import _ensure
import cherrypy
from libgutenberg import GutenbergGlobals as gg
import BaseSearcher
# use a bit more aggressive whitespace removal than the standard whitespace filter
COLLAPSE_LINES = re.compile('\n[ \t\r\n]+').sub
WHITESPACE_FILTER = genshi.output.WhitespaceFilter ()
DATA_URL_CACHE = {}
class BaseFormatter (object):
""" Base class for formatters. """
CONTENT_TYPE = 'text/html; charset=UTF-8'
def __init__ (self):
self.templates = {}
def format (self, page, os):
""" Abstract method to override. """
pass
def get_serializer (self):
""" Abstract method to override.
Like this:
return genshi.output.XMLSerializer (doctype = self.DOCTYPE, strip_whitespace = False)
"""
pass
def send_headers (self):
""" Send HTTP content-type header. """
cherrypy.response.headers['Content-Type'] = self.CONTENT_TYPE
def render (self, page, os):
""" Render and send to browser. """
self.send_headers ()
template = self.templates[page]
ctxt = genshi.template.Context (cherrypy = cherrypy, os = os, bs = BaseSearcher)
stream = template.stream
for filter_ in template.filters:
stream = filter_ (iter (stream), ctxt)
# there's no easy way in genshi to pass collapse_lines to this filter
stream = WHITESPACE_FILTER (stream, collapse_lines = COLLAPSE_LINES)
return genshi.output.encode (self.get_serializer ()(_ensure (genshi.Stream (stream))),
encoding = 'utf-8')
def set_template (self, page, template):
""" Set template for page.
Override this for special handling of template, like adding filters. """
self.templates[page] = template
@staticmethod
def format_date (date):
""" Format a date. """
if date is None:
return ''
try:
# datetime
return date.replace (tzinfo = gg.UTC (), microsecond = 0).isoformat ()
except TypeError:
# date
return datetime.datetime.combine (
date, datetime.time (tzinfo = gg.UTC ())).isoformat ()
@staticmethod
def data_url (path):
""" Read and convert a file to a data url. """
if path in DATA_URL_CACHE:
return DATA_URL_CACHE[path]
abs_path = os.path.join ('http://' + cherrypy.config['file_host'], path.lstrip ('/'))
data_url = abs_path
try:
f = urllib.request.urlopen (abs_path)
retcode = f.getcode ()
if retcode is None or retcode == 200:
msg = f.info ()
mediatype = msg.get ('Content-Type')
if mediatype:
mediatype = mediatype.partition (';')[0]
data_url = ('data:' + mediatype + ';base64,' +
base64.b64encode (f.read ()).decode ('ascii'))
f.close ()
except IOError:
pass
DATA_URL_CACHE[path] = data_url
return data_url
def fix_dc (self, dc, os):
""" Add some info to dc for easier templating. """
# obsolete private marc codes for cover art
dc.marcs = [ marc for marc in dc.marcs if not marc.code.startswith ('9') ]
dc.cover_image = None
dc.cover_thumb = None
# cover image really should not be a property of opensearch,
# but it is accessed in many places and this way we can save a
# lot of iterations later
os.cover_image_url = None
os.cover_thumb_url = None
for file_ in dc.files:
# HACK for https://
if file_.url.startswith ('http://'):
file_.url = file_.url[5:] # to //
file_.dropbox_url = None
# file_.dropbox_filename = None
file_.gdrive_url = None
file_.msdrive_url = None
file_.honeypot_url = None
if file_.filetype == 'cover.medium':
dc.cover_image = file_
os.snippet_image_url = os.cover_image_url = file_.url
elif file_.filetype == 'cover.small':
dc.cover_thumb = file_
os.cover_thumb_url = file_.url
dc.xsd_release_date_time = self.format_date (dc.release_date)
if 'Sound' in dc.categories:
dc.icon = 'audiobook'
# lifted from genshi/output.py and fixed lang issue
# lang is not allowed in xhtml 1.1 which we must use
# because xhtml+rdfa is based on it
from genshi.core import escape, Attrs, Markup, Namespace, QName, StreamEventKind
from genshi.core import START, END, TEXT, XML_DECL, DOCTYPE, START_NS, END_NS, \
START_CDATA, END_CDATA, PI, COMMENT, XML_NAMESPACE
from genshi.output import EMPTY, EmptyTagFilter, WhitespaceFilter, \
NamespaceFlattener, DocTypeInserter
class XHTMLSerializer (genshi.output.XMLSerializer):
"""Produces XHTML text from an event stream.
>>> from genshi.builder import tag
>>> elem = tag.div(tag.a(href='foo'), tag.br, tag.hr(noshade=True))
>>> print(''.join(XHTMLSerializer()(elem.generate())))
<div><a href="foo"></a><br /><hr noshade="noshade" /></div>
"""
_EMPTY_ELEMS = frozenset(['area', 'base', 'basefont', 'br', 'col', 'frame',
'hr', 'img', 'input', 'isindex', 'link', 'meta',
'param'])
_BOOLEAN_ATTRS = frozenset(['selected', 'checked', 'compact', 'declare',
'defer', 'disabled', 'ismap', 'multiple',
'nohref', 'noresize', 'noshade', 'nowrap'])
_PRESERVE_SPACE = frozenset([
QName('pre'), QName('http://www.w3.org/1999/xhtml}pre'),
QName('textarea'), QName('http://www.w3.org/1999/xhtml}textarea')
])
def __init__(self, doctype=None, strip_whitespace=True,
namespace_prefixes=None, drop_xml_decl=True, cache=True):
super(XHTMLSerializer, self).__init__(doctype, False)
self.filters = [EmptyTagFilter()]
if strip_whitespace:
self.filters.append(WhitespaceFilter(self._PRESERVE_SPACE))
namespace_prefixes = namespace_prefixes or {}
namespace_prefixes['http://www.w3.org/1999/xhtml'] = ''
self.filters.append(NamespaceFlattener(prefixes=namespace_prefixes,
cache=cache))
if doctype:
self.filters.append(DocTypeInserter(doctype))
self.drop_xml_decl = drop_xml_decl
self.cache = cache
def __call__(self, stream):
boolean_attrs = self._BOOLEAN_ATTRS
empty_elems = self._EMPTY_ELEMS
drop_xml_decl = self.drop_xml_decl
have_decl = have_doctype = False
in_cdata = False
cache = {}
cache_get = cache.get
if self.cache:
def _emit(kind, input, output):
cache[kind, input] = output
return output
else:
def _emit(kind, input, output):
return output
for filter_ in self.filters:
stream = filter_(stream)
for kind, data, pos in stream:
cached = cache_get((kind, data))
if cached is not None:
yield cached
elif kind is START or kind is EMPTY:
tag, attrib = data
buf = ['<', tag]
for attr, value in attrib:
if attr in boolean_attrs:
value = attr
# this is the fix
# elif attr == 'xml:lang' and 'lang' not in attrib:
# buf += [' lang="', escape(value), '"']
elif attr == 'xml:space':
continue
buf += [' ', attr, '="', escape(value), '"']
if kind is EMPTY:
if tag in empty_elems:
buf.append(' />')
else:
buf.append('></%s>' % tag)
else:
buf.append('>')
yield _emit(kind, data, Markup(''.join(buf)))
elif kind is END:
yield _emit(kind, data, Markup('</%s>' % data))
elif kind is TEXT:
if in_cdata:
yield _emit(kind, data, data)
else:
yield _emit(kind, data, escape(data, quotes=False))
elif kind is COMMENT:
yield _emit(kind, data, Markup('<!--%s-->' % data))
elif kind is DOCTYPE and not have_doctype:
name, pubid, sysid = data
buf = ['<!DOCTYPE %s']
if pubid:
buf.append(' PUBLIC "%s"')
elif sysid:
buf.append(' SYSTEM')
if sysid:
buf.append(' "%s"')
buf.append('>\n')
yield Markup(''.join(buf)) % tuple([p for p in data if p])
have_doctype = True
elif kind is XML_DECL and not have_decl and not drop_xml_decl:
version, encoding, standalone = data
buf = ['<?xml version="%s"' % version]
if encoding:
buf.append(' encoding="%s"' % encoding)
if standalone != -1:
standalone = standalone and 'yes' or 'no'
buf.append(' standalone="%s"' % standalone)
buf.append('?>\n')
yield Markup(''.join(buf))
have_decl = True
elif kind is START_CDATA:
yield Markup('<![CDATA[')
in_cdata = True
elif kind is END_CDATA:
yield Markup(']]>')
in_cdata = False
elif kind is PI:
yield _emit(kind, data, Markup('<?%s %s?>' % data))

1019
BaseSearcher.py Normal file
View File

File diff suppressed because it is too large Load Diff

131
BibrecPage.py Normal file
View File

@ -0,0 +1,131 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
BibrecPage.py
Copyright 2009-2010 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
The bibrec page.
"""
from __future__ import unicode_literals
import cherrypy
from libgutenberg import GutenbergGlobals as gg
import BaseSearcher
import Page
class BibrecPage (Page.Page):
""" Implements the bibrec page. """
def index (self, **dummy_kwargs):
""" A bibrec page. """
os = BaseSearcher.OpenSearch ()
os.log_request ('bibrec')
dc = BaseSearcher.DC (cherrypy.engine.pool)
# the bulk of the work is done here
dc.load_from_database (os.id)
if not dc.files:
# NOTE: Error message
cherrypy.tools.rate_limiter.e404 ()
raise cherrypy.HTTPError (404, _('No ebook by that number.'))
# add these fields so we won't have to test for their existence later
dc.extra_info = None
dc.url = None
dc.translate ()
dc.header = gg.cut_at_newline (dc.title)
os.title = dc.make_pretty_title ()
dc.extra_info = ''
dc.class_ = BaseSearcher.ClassAttr ()
dc.order = 10
dc.icon = 'book'
if 'Sound' in dc.categories:
dc.icon = 'audiobook'
os.title_icon = dc.icon
os.twit = os.title
os.qrcode_url = '//%s/cache/epub/%d/pg%d.qrcode.png' % (os.file_host, os.id, os.id)
os.entries.append (dc)
s = cherrypy.session
last_visited = s.get ('last_visited', [])
last_visited.append (os.id)
s['last_visited'] = last_visited
# can we find some meaningful breadcrumbs ?
for a in dc.authors:
if a.marcrel in ('aut', 'cre'):
book_cnt = BaseSearcher.sql_get (
"select count (*) from mn_books_authors where fk_authors = %(aid)s", aid = a.id)
if book_cnt > 1:
os.breadcrumbs.append ((
__('One by {author}', '{count} by {author}', book_cnt).format (
count = book_cnt, author = dc.make_pretty_name (a.name)),
_('Find more ebooks by the same author.'),
os.url ('author', id = a.id)
))
if os.format in ('html', 'mobile'):
cat = BaseSearcher.Cat ()
cat.header = _('Similar Books')
cat.title = _('Readers also downloaded…')
cat.rel = 'related'
cat.url = os.url ('also', id = os.id)
cat.class_ += 'navlink grayed noprint'
cat.icon = 'suggestion'
cat.order = 30
os.entries.append (cat)
for bookshelf in dc.bookshelves:
cat = BaseSearcher.Cat ()
cat.title = _('In {bookshelf}').format (bookshelf = bookshelf.bookshelf)
cat.rel = 'related'
cat.url = os.url ('bookshelf', id = bookshelf.id)
cat.class_ += 'navlink grayed'
cat.icon = 'bookshelf'
cat.order = 33
os.entries.append (cat)
if os.format in ('mobile', ):
for author in dc.authors:
cat = BaseSearcher.Cat ()
cat.title = _('By {author}').format (author = author.name_and_dates)
cat.rel = 'related'
cat.url = os.url ('author', id = author.id)
cat.class_ += 'navlink grayed'
cat.icon = 'author'
cat.order = 31
os.entries.append (cat)
for subject in dc.subjects:
cat = BaseSearcher.Cat ()
cat.title = _('On {subject}').format (subject = subject.subject)
cat.rel = 'related'
cat.url = os.url ('subject', id = subject.id)
cat.class_ += 'navlink grayed'
cat.icon = 'subject'
cat.order = 32
os.entries.append (cat)
os.total_results = 1
os.template = 'results' if os.format == 'mobile' else 'bibrec'
os.page = 'bibrec'
os.og_type = 'book'
os.finalize ()
return self.format (os)

146
CaptchaPage.py Normal file
View File

@ -0,0 +1,146 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
CaptchaPage.py
Copyright 2013-14 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Serve a captcha page.
"""
from __future__ import unicode_literals
import requests
import cherrypy
import logging
import Page
import BaseSearcher
#
# reCaptcha API docs:
# https://developers.google.com/recaptcha/docs/verify
#
API = "http://www.google.com/recaptcha/api/verify"
class QuestionPage (Page.Page):
""" Output captcha page. """
def index (self, **kwargs):
""" Output captcha. """
cherrypy.lib.caching.expires (3600, True)
os = BaseSearcher.OpenSearch ()
os.template = 'recaptcha'
os.recaptcha_public_key = cherrypy.config['recaptcha_public_key']
os.error = kwargs.get ('error')
os.finalize ()
# Remove Session cookie, so that page can be cached.
name = cherrypy.serving.request.config.get ('tools.sessions.name', 'session_id')
del cherrypy.serving.response.cookie[name]
return self.format (os)
class AnswerPage (object):
""" Check answer with google. """
def index (self, **kwargs):
""" Check with google. """
cherrypy.lib.caching.expires (0, True)
os = BaseSearcher.OpenSearch ()
# Remove Session cookie.
name = cherrypy.serving.request.config.get ('tools.sessions.name', 'session_id')
del cherrypy.serving.response.cookie[name]
if 'recaptcha_challenge_field' in kwargs:
response = submit (
kwargs['recaptcha_challenge_field'],
kwargs['recaptcha_response_field'],
cherrypy.config['recaptcha_private_key'],
cherrypy.request.remote.ip)
cherrypy.ipsession.captcha_answer (response)
if not response.is_valid:
raise cherrypy.HTTPRedirect (
os.url ('captcha.question', error = 'incorrect-captcha-sol'))
for req in reversed (cherrypy.ipsession['requests']):
if 'captcha' not in req:
raise cherrypy.HTTPRedirect (req)
raise cherrypy.HTTPRedirect (os.url ('start'))
#
# Following is stolen from pypi package recaptcha-client 1.0.6
# http://code.google.com/p/recaptcha/
# to make it compatible with Python 3 requests.
#
class RecaptchaResponse (object):
""" The response from the reCaptcha server. """
def __init__ (self, is_valid, error_code = None):
self.is_valid = is_valid
self.error_code = error_code
def submit (recaptcha_challenge_field,
recaptcha_response_field,
private_key,
remoteip):
"""
Submits a reCAPTCHA request for verification. Returns RecaptchaResponse
for the request
recaptcha_challenge_field -- The value of recaptcha_challenge_field from the form
recaptcha_response_field -- The value of recaptcha_response_field from the form
private_key -- your reCAPTCHA private key
remoteip -- the user's ip address
"""
if not (recaptcha_response_field and recaptcha_challenge_field and
len (recaptcha_response_field) and len (recaptcha_challenge_field)):
return RecaptchaResponse (is_valid = False, error_code = 'incorrect-captcha-sol')
data = {
'privatekey': private_key,
'remoteip': remoteip,
'challenge': recaptcha_challenge_field,
'response': recaptcha_response_field,
}
headers = {
"User-agent": "reCAPTCHA Python"
}
cherrypy.log ('Data=' + repr (data), context = 'CAPTCHA', severity = logging.INFO)
try:
r = requests.post (API, data = data, headers = headers)
r.raise_for_status ()
lines = r.text.splitlines ()
cherrypy.log ('Response=' + "/".join (lines), context = 'CAPTCHA', severity = logging.INFO)
if lines[0] == "true":
return RecaptchaResponse (is_valid = True)
else:
return RecaptchaResponse (is_valid = False, error_code = lines[1])
except requests.exceptions.RequestException as what:
cherrypy.log (str (what), context = 'CAPTCHA', severity = logging.ERROR)
return RecaptchaResponse (is_valid = False, error_code = 'recaptcha-not-reachable')

343
CherryPyApp.py Normal file
View File

@ -0,0 +1,343 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
CherryPyApp.py
Copyright 2009-2014 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
The Project Gutenberg Catalog App Server.
Config and route setup.
"""
from __future__ import unicode_literals
import logging
import logging.handlers # rotating file handler
import os
import time
import traceback
import cherrypy
from cherrypy.process import plugins
import six
from six.moves import builtins
from libgutenberg import GutenbergDatabase
import i18n_tool
# Make translator functions available everywhere. Do this early, at
# least before Genshi starts loading templates.
builtins._ = i18n_tool.ugettext
builtins.__ = i18n_tool.ungettext
import ConnectionPool
import Page
import StartPage
import SuggestionsPage
from SearchPage import BookSearchPage, AuthorSearchPage, SubjectSearchPage, BookshelfSearchPage, \
AuthorPage, SubjectPage, BookshelfPage, AlsoDownloadedPage
from BibrecPage import BibrecPage
import CoverPages
import QRCodePage
import StatsPage
import CaptchaPage
import Sitemap
import Formatters
import RateLimiter
import Timer
import MyRamSession
import PostgresSession
cherrypy.lib.sessions.RamSession = MyRamSession.FixedRamSession
cherrypy.lib.sessions.MyramSession = MyRamSession.MyRamSession
cherrypy.lib.sessions.PostgresSession = PostgresSession.PostgresSession
plugins.Timer = Timer.TimerPlugin
if six.PY3:
CHERRYPY_CONFIG = ('/etc/autocat3.conf', os.path.expanduser ('~/.autocat3'))
# CCHERRYPY_CONFIG = ('/etc/autocat3.conf')
else:
CHERRYPY_CONFIG = ('/etc/autocat.conf', os.path.expanduser ('~/.autocat'))
class MyRoutesDispatcher (cherrypy.dispatch.RoutesDispatcher):
""" Dispatcher that tells us the matched route.
CherryPy makes it hard for us by forgetting the matched route object.
Here we add a 'route_name' parameter, that will tell us the route's name.
"""
def connect (self, name, route, controller, **kwargs):
""" Add a 'route_name' parameter that will tell us the matched route. """
kwargs['route_name'] = name
kwargs.setdefault ('action', 'index')
cherrypy.dispatch.RoutesDispatcher.connect (self, name, route, controller, **kwargs)
def main ():
""" Main function. """
# default config
cherrypy.config.update ({
'uid': 0,
'gid': 0,
'server_name': 'localhost',
'genshi.template_dir': os.path.join (
os.path.dirname (os.path.abspath (__file__)), 'templates'),
'daemonize': False,
'pidfile': None,
'host': 'localhost',
'host_mobile': 'localhost',
'file_host': 'localhost',
})
config_filename = None
for config_filename in CHERRYPY_CONFIG:
try:
cherrypy.config.update (config_filename)
break
except IOError:
pass
# Rotating Logs
#
# Remove the default FileHandlers if present.
error_file = cherrypy.log.error_file
access_file = cherrypy.log.access_file
cherrypy.log.error_file = ""
cherrypy.log.access_file = ""
max_bytes = getattr (cherrypy.log, "rot_max_bytes", 100 * 1024 * 1024)
backup_count = getattr (cherrypy.log, "rot_backup_count", 2)
#print(os.path.abspath(error_file)+": Filehandler cherrypy")
h = logging.handlers.RotatingFileHandler (error_file, 'a', max_bytes, backup_count, 'utf-8')
h.setLevel (logging.DEBUG)
h.setFormatter (cherrypy._cplogging.logfmt)
cherrypy.log.error_log.addHandler (h)
h = logging.handlers.RotatingFileHandler (access_file, 'a', max_bytes, backup_count, 'utf-8')
h.setLevel (logging.DEBUG)
h.setFormatter (cherrypy._cplogging.logfmt)
cherrypy.log.access_log.addHandler (h)
if not cherrypy.config['daemonize']:
ch = logging.StreamHandler ()
ch.setLevel (logging.DEBUG)
ch.setFormatter (cherrypy._cplogging.logfmt)
cherrypy.log.error_log.addHandler (ch)
# continue app init
#
cherrypy.log ('*' * 80, context = 'ENGINE', severity = logging.INFO)
cherrypy.log ("Using config file '%s'." % config_filename,
context = 'ENGINE', severity = logging.INFO)
# after cherrypy.config is parsed
Formatters.init ()
cherrypy.log ("Continuing App Init", context = 'ENGINE', severity = logging.INFO)
try:
cherrypy.tools.rate_limiter = RateLimiter.RateLimiterTool ()
except Exception as e:
tb = traceback.format_exc ()
cherrypy.log (tb, context = 'ENGINE', severity = logging.ERROR)
cherrypy.log ("Continuing App Init", context = 'ENGINE', severity = logging.INFO)
cherrypy.tools.I18nTool = i18n_tool.I18nTool ()
cherrypy.log ("Continuing App Init", context = 'ENGINE', severity = logging.INFO)
# Used to bust the cache on js and css files. This should be the
# files' mtime, but the files are not stored on the app server.
# This is a `good enough´ replacement though.
t = str (int (time.time ()))
cherrypy.config['css_mtime'] = t
cherrypy.config['js_mtime'] = t
cherrypy.config['all_hosts'] = (
cherrypy.config['host'], cherrypy.config['host_mobile'], cherrypy.config['file_host'])
if hasattr (cherrypy.engine, 'signal_handler'):
cherrypy.engine.signal_handler.subscribe ()
cherrypy.engine.pool = plugins.ConnectionPool (
cherrypy.engine, params = GutenbergDatabase.get_connection_params (cherrypy.config))
cherrypy.engine.pool.subscribe ()
plugins.RateLimiterReset (cherrypy.engine).subscribe ()
plugins.RateLimiterDatabase (cherrypy.engine).subscribe ()
plugins.Timer (cherrypy.engine).subscribe ()
cherrypy.log ("Daemonizing", context = 'ENGINE', severity = logging.INFO)
if cherrypy.config['daemonize']:
plugins.Daemonizer (cherrypy.engine).subscribe ()
uid = cherrypy.config['uid']
gid = cherrypy.config['gid']
if uid > 0 or gid > 0:
plugins.DropPrivileges (cherrypy.engine, uid = uid, gid = gid, umask = 0o22).subscribe ()
if cherrypy.config['pidfile']:
pid = plugins.PIDFile (cherrypy.engine, cherrypy.config['pidfile'])
# Write pidfile after privileges are dropped (prio == 77)
# or we will not be able to remove it.
cherrypy.engine.subscribe ('start', pid.start, 78)
cherrypy.engine.subscribe ('exit', pid.exit, 78)
cherrypy.log ("Setting up routes", context = 'ENGINE', severity = logging.INFO)
# setup 'routes' dispatcher
#
# d = cherrypy.dispatch.RoutesDispatcher (full_result = True)
d = MyRoutesDispatcher (full_result = True)
cherrypy.routes_mapper = d.mapper
def check_id (environ, result):
""" Check if id is a valid number. """
try:
return str (int (result['id'])) == result['id']
except:
return False
d.connect ('start', r'/ebooks{.format}/',
controller = StartPage.Start ())
d.connect ('suggest', r'/ebooks/suggest{.format}/',
controller = SuggestionsPage.Suggestions ())
# search pages
d.connect ('search', r'/ebooks/search{.format}/',
controller = BookSearchPage ())
d.connect ('author_search', r'/ebooks/authors/search{.format}/',
controller = AuthorSearchPage ())
d.connect ('subject_search', r'/ebooks/subjects/search{.format}/',
controller = SubjectSearchPage ())
d.connect ('bookshelf_search', r'/ebooks/bookshelves/search{.format}/',
controller = BookshelfSearchPage ())
# 'id' pages
d.connect ('author', r'/ebooks/author/{id:\d+}{.format}',
controller = AuthorPage (), conditions = dict (function = check_id))
d.connect ('subject', r'/ebooks/subject/{id:\d+}{.format}',
controller = SubjectPage (), conditions = dict (function = check_id))
d.connect ('bookshelf', r'/ebooks/bookshelf/{id:\d+}{.format}',
controller = BookshelfPage (), conditions = dict (function = check_id))
d.connect ('also', r'/ebooks/{id:\d+}/also/{.format}',
controller = AlsoDownloadedPage (), conditions = dict (function = check_id))
# bibrec pages
d.connect ('download', r'/ebooks/{id:\d+}/download{.format}',
controller = Page.NullPage (), _static = True)
d.connect ('bibrec', r'/ebooks/{id:\d+}{.format}',
controller = BibrecPage (), conditions = dict (function = check_id))
# legacy compatibility with /ebooks/123.bibrec
d.connect ('bibrec2', r'/ebooks/{id:\d+}.bibrec{.format}',
controller = BibrecPage (), conditions = dict (function = check_id))
d.connect ('cover', r'/covers/{size:small|medium}/{order:latest|popular}/{count}',
controller = CoverPages.CoverPages ())
d.connect ('qrcode', r'/qrcode/',
controller = QRCodePage.QRCodePage ())
d.connect ('iplimit', r'/iplimit/',
controller = Page.NullPage ())
d.connect ('stats', r'/stats/',
controller = StatsPage.StatsPage ())
d.connect ('block', r'/stats/block/',
controller = RateLimiter.BlockPage ())
d.connect ('unblock', r'/stats/unblock/',
controller = RateLimiter.UnblockPage ())
d.connect ('traceback', r'/stats/traceback/',
controller = RateLimiter.TracebackPage ())
d.connect ('honeypot_send', r'/ebooks/send/megaupload/{id:\d+}.{filetype}',
controller = Page.NullPage (), _static = True)
# /w/captcha/question/ so varnish will cache it
d.connect ('captcha.question', r'/w/captcha/question/',
controller = CaptchaPage.QuestionPage ())
d.connect ('captcha.answer', r'/w/captcha/answer/',
controller = CaptchaPage.AnswerPage ())
# sitemap protocol access control requires us to place sitemaps in /ebooks/
d.connect ('sitemap', r'/ebooks/sitemaps/',
controller = Sitemap.SitemapIndex ())
d.connect ('sitemap_index', r'/ebooks/sitemaps/{page:\d+}',
controller = Sitemap.Sitemap ())
if 'dropbox_client_id' in cherrypy.config:
import Dropbox
dropbox = Dropbox.Dropbox ()
cherrypy.log ("Dropbox Client Id: %s" % cherrypy.config['dropbox_client_id'],
context = 'ENGINE', severity = logging.INFO)
d.connect ('dropbox_send', r'/ebooks/send/dropbox/{id:\d+}.{filetype}',
controller = dropbox, conditions = dict (function = check_id))
d.connect ('dropbox_callback', r'/ebooks/send/dropbox/',
controller = dropbox)
if 'gdrive_client_id' in cherrypy.config:
import GDrive
gdrive = GDrive.GDrive ()
cherrypy.log ("GDrive Client Id: %s" % cherrypy.config['gdrive_client_id'],
context = 'ENGINE', severity = logging.INFO)
d.connect ('gdrive_send', r'/ebooks/send/gdrive/{id:\d+}.{filetype}',
controller = gdrive, conditions = dict (function = check_id))
d.connect ('gdrive_callback', r'/ebooks/send/gdrive/',
controller = gdrive)
if 'msdrive_client_id' in cherrypy.config:
import MSDrive
msdrive = MSDrive.MSDrive ()
cherrypy.log ("MSDrive Client Id: %s" % cherrypy.config['msdrive_client_id'],
context = 'ENGINE', severity = logging.INFO)
d.connect ('msdrive_send', r'/ebooks/send/msdrive/{id:\d+}.{filetype}',
controller = msdrive, conditions = dict (function = check_id))
d.connect ('msdrive_callback', r'/ebooks/send/msdrive/',
controller = msdrive)
# start http server
#
cherrypy.log ("Mounting root", context = 'ENGINE', severity = logging.INFO)
app = cherrypy.tree.mount (root = None, config = config_filename)
app.merge ({'/': {'request.dispatch': d}})
return app
if __name__ == '__main__':
main ()
cherrypy.engine.start ()
cherrypy.engine.block ()

303
CloudStorage.py Normal file
View File

@ -0,0 +1,303 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
CloudStorage.py
Copyright 2013-15 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Base classes for uploads to file hosting services.
"""
from __future__ import unicode_literals
from contextlib import closing
from six.moves import urllib
import logging
import re
import os
import cherrypy
import routes
import requests
import requests_oauthlib
from requests import RequestException
from oauthlib.oauth2.rfc6749.errors import OAuth2Error
import BaseSearcher
# pylint: disable=R0921
http_adapter = requests.adapters.HTTPAdapter ()
https_adapter = requests.adapters.HTTPAdapter ()
# Google Drive `bug´ see:
# https://github.com/idan/oauthlib/commit/ca4811b3087f9d34754d3debf839e247593b8a39
os.environ['OAUTHLIB_RELAX_TOKEN_SCOPE'] = '1'
def log (msg):
""" Log an informational message. """
cherrypy.log (msg, context = 'CLOUDSTORAGE', severity = logging.INFO)
def error_log (msg):
""" Log an error message. """
cherrypy.log ('Error: ' + msg, context = 'CLOUDSTORAGE', severity = logging.ERROR)
class CloudOAuth2Session (requests_oauthlib.OAuth2Session): # pylint: disable=R0904
""" An OAuth2 session. """
name_prefix = None
oauth2_auth_endpoint = None
oauth2_token_endpoint = None
oauth2_scope = None
def __init__ (self, **kwargs):
""" Initialize session from cherrypy config. """
config = cherrypy.config
prefix = self.name_prefix
host = config['file_host']
urlgen = routes.URLGenerator (cherrypy.routes_mapper, {
'HTTP_HOST': host,
'HTTPS': 1
})
client_id = config[prefix + '_client_id']
redirect_uri = urlgen (prefix + '_callback', host = host)
super (CloudOAuth2Session, self).__init__ (
client_id = client_id,
scope = self.oauth2_scope,
redirect_uri = redirect_uri,
**kwargs
)
self.client_secret = config[prefix + '_client_secret']
self.ebook = None
self.mount ("http://", http_adapter)
self.mount ("https://", https_adapter)
def oauth_dance (self, kwargs):
""" Do the OAuth2 dance. """
#
# OAuth 2.0 flow see:
# http://tools.ietf.org/html/rfc6749
#
if not self.token:
if 'code' not in kwargs:
# oauth step 1:
# redirect the user to the Authorization Endpoint
log ('Building auth url ...')
auth_url, dummy_state = self.authorization_url (
self.oauth2_auth_endpoint)
log ('Redirecting user to auth endpoint ...')
raise cherrypy.HTTPRedirect (auth_url)
else:
# oauth step 2
# the user's browser just came back with an authorization code
# get the access_token from the Token Endpoint
log ('Fetching access token ...')
self.fetch_token (self.oauth2_token_endpoint,
client_secret = self.client_secret,
code = kwargs['code'])
log ('Got access token.')
def unauthorized (self, msg = 'Unauthorized'):
""" Called on OAuth2 failure. """
pass
class CloudStorage (object):
""" Base class for uploads to cloud storage providers.
:param name: The name of the cloud service, eg. 'Dropbox'.
:param session_class: The class to use for the oauth session.
:param user_agent: The user agent to make requests to www.gutenberg.org.
"""
name = None
session_class = CloudOAuth2Session
user_agent = None
upload_endpoint = None
re_filename = re.compile (r'[/\<>:"|?*]')
def __init__ (self):
self.host = cherrypy.config['host']
self.urlgen = routes.URLGenerator (cherrypy.routes_mapper, {'HTTP_HOST': self.host})
def index (self, **kwargs):
""" Output the page. """
#
# OAuth 2.0 flow see:
# http://tools.ietf.org/html/rfc6749
#
session = self.get_or_create_session ()
if 'id' in kwargs:
session.ebook = EbookMetaData (kwargs)
if session.ebook is None:
raise cherrypy.HTTPError (400, "No ebook selected. Are your cookies enabled?")
name = self.name
if 'not_approved' in kwargs or 'error' in kwargs:
self._dialog (
_('Sorry. The file could not be sent to {name}.').format (name = name),
_('Error'))
self.redirect_done (session)
try:
session.oauth_dance (kwargs)
log ("Sending file %s to %s" % (
session.ebook.get_source_url (), name))
with closing (self.request_ebook (session)) as r:
r.raise_for_status ()
self.upload_file (session, r)
log ("File %s sent to %s" % (
session.ebook.get_source_url (), name))
self._dialog (
_('The file has been sent to {name}.').format (name = name),
_('Sent to {name}').format (name = name))
self.redirect_done (session)
except (OAuth2Error, ) as what:
session.unauthorized (what)
self.unauthorized ('OAuthError: ' + str (what.urlencoded))
except (RequestException, IOError, ValueError) as what:
session.unauthorized (what)
self.unauthorized ('RequestError: ' + str (what))
raise cherrypy.HTTPError (500, str (what))
def upload_file (self, oauth_session, response):
""" Upload the file. """
raise NotImplementedError
def get_or_create_session (self):
""" Retrieve an ongoing cloud session or create a new one. """
session_name = self.session_class.name_prefix + '_session'
session = cherrypy.session.get (session_name, self.session_class ())
cherrypy.session[session_name] = session
return session
def delete_session (self):
""" Delete cloud session. """
session_name = self.session_class.name_prefix + '_session'
# cherrypy.session[session_name].close ()
del cherrypy.session[session_name]
def request_ebook (self, session):
""" Return an open request object for the ebook file. """
url = session.ebook.get_source_url ()
# Caveat: use requests.get, not session.get, because it is an insecure
# transport. session.get would raise InsecureTransportError
return requests.get (
url, headers = { 'user-agent': self.user_agent }, stream = True)
def fix_filename (self, filename):
""" Replace characters unsupported by many OSs. """
return self.re_filename.sub ('_', filename)
def redirect_done (self, session):
""" Redirect user back to bibrec page. """
raise cherrypy.HTTPRedirect (self.urlgen (
'bibrec', id = session.ebook.id, host = self.host))
def unauthorized (self, msg = 'Unauthorized'):
""" Call on OAuth failure. """
msg = str (msg) # msg may be exception class
error_log (msg)
self.delete_session ()
raise cherrypy.HTTPError (401, msg)
@staticmethod
def _dialog (message, title):
""" Open a user-visible dialog on the next page. """
cherrypy.session['user_dialog'] = (message, title)
class EbookMetaData (object):
""" Helper class that holds ebook metadata. """
accepted_filetypes = (
'epub.images',
'epub.noimages',
'kindle.images',
'kindle.noimages',
'pdf')
def __init__ (self, kwargs):
self.id = None
self.filetype = None
try :
self.id = int (kwargs['id'])
self.filetype = kwargs['filetype']
if self.filetype not in self.accepted_filetypes:
self.filetype = None
raise ValueError
except (KeyError, ValueError):
raise cherrypy.HTTPError (400, 'Bad Request. Invalid parameters')
def get_dc (self):
""" Get a DublinCore struct for the ebook. """
dc = BaseSearcher.DC (cherrypy.engine.pool)
dc.load_from_database (self.id)
# dc.translate ()
return dc
def get_extension (self):
""" Get the ebook filename extension. """
ext = self.filetype.split ('.', 1)[0]
if ext == 'kindle':
ext = 'mobi'
return ext
def get_filename (self):
""" Get a suitable filename to store the ebook. """
filename = self.get_dc ().make_pretty_title () + '.' + self.get_extension ()
return filename.replace (':', '_')
def get_source_url (self):
""" Return the url of the ebook file on gutenberg.org. """
return urllib.parse.urljoin (
'http://' + cherrypy.config['file_host'],
'ebooks/%d.%s' % (self.id, self.filetype))

101
ConnectionPool.py Normal file
View File

@ -0,0 +1,101 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
ConnectionPool.py
Copyright 2010 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
"""
from __future__ import unicode_literals
import logging
import psycopg2
import sqlalchemy.pool as pool
import cherrypy
from cherrypy.process import plugins
class ConnectionCreator ():
""" Creates connections for the connection pool. """
def __init__ (self, params):
self.params = params
def __call__ (self):
cherrypy.log (
"Connecting to database '%(database)s' on '%(host)s:%(port)d' as user '%(user)s'."
% self.params, context = 'POSTGRES', severity = logging.INFO)
conn = psycopg2.connect (**self.params)
conn.cursor ().execute ('SET statement_timeout = 5000')
return conn
class ConnectionPool (plugins.SimplePlugin):
"""A WSPBus plugin that controls a SQLAlchemy engine/connection pool."""
def __init__ (self, bus, params = None):
plugins.SimplePlugin.__init__ (self, bus)
self.params = params
self.name = 'sqlalchemy'
self.pool = None
def _start (self):
""" Init the connection pool. """
pool_size = cherrypy.config.get ('sqlalchemy.pool_size', 5)
max_overflow = cherrypy.config.get ('sqlalchemy.max_overflow', 10)
timeout = cherrypy.config.get ('sqlalchemy.timeout', 30)
recycle = cherrypy.config.get ('sqlalchemy.recycle', 3600)
self.bus.log ("... pool_size = %d, max_overflow = %d" % (pool_size, max_overflow))
return pool.QueuePool (ConnectionCreator (self.params),
pool_size = pool_size,
max_overflow = max_overflow,
timeout = timeout,
recycle = recycle,
use_threadlocal = True)
def connect (self):
""" Return a connection. """
return self.pool.connect ()
def start (self):
""" Called on engine start. """
if self.pool is None:
self.bus.log ("Creating the SQL connection pool ...")
self.pool = self._start ()
else:
self.bus.log ("An SQL connection pool already exists.")
# start.priority = 80
def stop (self):
""" Called on engine stop. """
if self.pool is not None:
self.bus.log ("Disposing the SQL connection pool.")
self.pool.dispose ()
self.pool = None
def graceful (self):
""" Called on engine restart. """
if self.pool is not None:
self.bus.log ("Restarting the SQL connection pool ...")
self.pool.dispose ()
self.pool = self._start ()
cherrypy.process.plugins.ConnectionPool = ConnectionPool

103
CoverPages.py Normal file
View File

@ -0,0 +1,103 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
CoverPages.py
Copyright 2009-2010 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Serve cover images of most popular and latest ebooks.
"""
from __future__ import unicode_literals
import cherrypy
import six
from libgutenberg import GutenbergGlobals as gg
import BaseSearcher
class CoverPages (object):
""" Output a gallery of cover pages. """
orders = { 'latest': 'release_date',
'popular': 'downloads' }
@staticmethod
def serve (rows, size):
""" Output a gallery of coverpages. """
cherrypy.response.headers['Content-Type'] = 'text/html; charset=utf-8'
cherrypy.response.headers['Content-Language'] = 'en'
s = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en" xml:base="http://www.gutenberg.org">
<head>
<title>Cover Flow</title>
<style>
.cover-thumb {
display: inline-block;
background-position: center;
background-repeat: no-repeat;
}
.cover-thumb-small {
width: 76px;
height: 110px;
}
.cover-thumb-medium {
width: 210px;
height: 310px;
}
</style>
</head>
<body><div>"""
for row in rows:
url = '/' + row.filename
href = '/ebooks/%d' % row.pk
title = gg.xmlspecialchars (row.title)
title = title.replace ('"', '&quot;')
s += """<a href="{href}"
title="{title}"
class="cover-thumb cover-thumb-{size}" target="_top"
style="background-image: url({url})"> </a>\n""".format (
url = url, href = href, title = title, size = size)
return (s + '</div></body></html>\n').encode ('utf-8')
def index (self, count, size, order, **kwargs):
""" Internal help function. """
try:
count = int (count)
if count < 1:
raise ValueError ('count < 0')
if size not in ('small', 'medium'):
raise ValueError ('bogus size')
order = 'books.%s' % self.orders[order]
rows = BaseSearcher.SQLSearcher.execute (
"""SELECT files.filename, books.pk, books.title FROM files, books
WHERE files.fk_books = books.pk
AND files.diskstatus = 0
AND files.fk_filetypes = %%(size)s
ORDER BY %s DESC
OFFSET 1 LIMIT %%(count)s -- %s""" % (order, cherrypy.request.remote.ip),
{ 'count': count,
'size': 'cover.%s' % size,
})
if rows:
return self.serve (rows, size)
except (ValueError, KeyError) as what:
raise cherrypy.HTTPError (400, 'Bad Request. %s' % six.text_type (what))
except IOError:
pass
raise cherrypy.HTTPError (500, 'Internal Server Error.')

63
Dropbox.py Normal file
View File

@ -0,0 +1,63 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
Dropbox.py
Copyright 2012-17 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
The send-to-dropbox pages.
"""
from __future__ import unicode_literals
import json
import re
from contextlib import closing
import CloudStorage
class DropboxOAuth2Session (CloudStorage.CloudOAuth2Session):
""" Hold parameters for OAuth2. """
name_prefix = 'dropbox'
oauth2_auth_endpoint = 'https://www.dropbox.com/oauth2/authorize'
oauth2_token_endpoint = 'https://api.dropbox.com/oauth2/token'
oauth2_scope = None
class Dropbox (CloudStorage.CloudStorage):
""" Send files to dropbox using OAuth2 authentication. """
name = 'Dropbox'
session_class = DropboxOAuth2Session
user_agent = 'PG2Dropbox/0.3'
upload_endpoint = 'https://content.dropboxapi.com/2/files/upload'
# Incompatible characters see: https://www.dropbox.com/help/145/en
# also added ' and ,
re_filename = re.compile ('[/\\<>:"|?*\',]')
def upload_file (self, session, response):
""" Get the file from gutenberg.org and upload it to dropbox.
:param session: authorized OAuthlib session.
"""
parameters = {
'path': '/' + self.fix_filename (session.ebook.get_filename ())
}
headers = {
'Authorization' : 'Bearer ' + str (session.token),
'Content-Type' : 'application/octet-stream',
'Dropbox-API-Arg' : json.dumps (parameters)
}
with closing (session.post (self.upload_endpoint,
data = response.content,
headers = headers)) as r:
CloudStorage.error_log (r.text)
r.raise_for_status ()

72
DublinCoreI18n.py Normal file
View File

@ -0,0 +1,72 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: iso-8859-1 -*-
"""
DublinCoreI18n.py
Copyright 2009-2010 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Translate a DublinCore struct with Babel.
"""
from __future__ import unicode_literals
import cherrypy
import babel
class DublinCoreI18nMixin (object):
""" Translator Mixin for GutenbergDatabaseDublinCore class. """
def __init__ (self):
self.translated = False
self.hr_release_date = None
self.rights = None
@staticmethod
def dummy_text_holder ():
"""Never gets called.
Only holds some gettext messages to translate. Keep this in
sync with GutenbergDatabaseDublinCore.
"""
_('Copyrighted. Read the copyright notice inside this book for details.')
_('Public domain in the USA.')
def translate (self):
""" Translate DublinCore struct. """
if self.translated:
# already translated
return
self.hr_release_date = babel.dates.format_date (
self.release_date, locale = str (cherrypy.response.i18n.locale))
if cherrypy.response.i18n.locale.language == 'en':
# no translation required
return
self.rights = _(self.rights)
for author in self.authors:
author.role = _(author.role)
for marc in self.marcs:
marc.caption = _(marc.caption)
for dcmitype in self.dcmitypes:
dcmitype.description = _(dcmitype.description)
for lang in self.languages:
if lang.id in cherrypy.response.i18n.locale.languages:
lang.language = cherrypy.response.i18n.locale.languages[lang.id].capitalize ()
for file_ in self.files:
file_.hr_filetype = _(file_.hr_filetype)
for file_ in self.generated_files:
file_.hr_filetype = _(file_.hr_filetype)
self.translated = True

79
Formatters.py Normal file
View File

@ -0,0 +1,79 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: iso-8859-1 -*-
"""
Formatters.py
Copyright 2009-2010 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Formatters for all mediatypes.
"""
from __future__ import unicode_literals
import glob
import logging
import os.path
import genshi.template
import genshi.filters
import cherrypy
import HTMLFormatter
import OPDSFormatter
import JSONFormatter
def format (format_, page, os_):
""" Main entry point. """
return formatters[format_].format (page, os_)
formatters = {}
formatters['opds'] = OPDSFormatter.OPDSFormatter ()
formatters['stanza'] = formatters['opds']
formatters['mobile'] = HTMLFormatter.MobileFormatter ()
formatters['html'] = HTMLFormatter.HTMLFormatter ()
formatters['json'] = JSONFormatter.JSONFormatter ()
# FIXME: only needed to load sitemap.xml templates
formatters['xml'] = HTMLFormatter.XMLishFormatter ()
def on_template_loaded (template):
"""
Callback.
We need to use the callback because we are using includes.
The callback will also setup () the includes.
"""
genshi.filters.Translator (_).setup (template)
def init ():
""" Load all template files in template_dir. """
template_dir = cherrypy.config['genshi.template_dir']
for fn in glob.glob (os.path.join (template_dir, '*')):
if fn.endswith ('~') or fn.endswith ('#') or fn.endswith ('schemas.xml'):
# backup file or emacs temp file
continue
cherrypy.engine.autoreload.files.update (fn)
bn = os.path.basename (fn)
template = genshi.template.TemplateLoader (
template_dir,
callback = on_template_loaded).load (bn)
page, dot_format = os.path.splitext (bn)
formatters[dot_format[1:]].set_template (page, template)
cherrypy.log ("Template '%s' loaded." % fn,
context = 'GENSHI', severity = logging.INFO)

72
GDrive.py Normal file
View File

@ -0,0 +1,72 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
GDrive.py
Copyright 2013-15 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
The send-to-google-drive pages.
"""
from __future__ import unicode_literals
from contextlib import closing
import json
import CloudStorage
class GDriveSession (CloudStorage.CloudOAuth2Session):
""" Hold parameters for OAuth. """
#
# OAuth 2.0 flow see:
# http://tools.ietf.org/html/rfc6749
# https://developers.google.com/api-client-library/python/guide/aaa_oauth
#
name_prefix = 'gdrive'
oauth2_auth_endpoint = 'https://accounts.google.com/o/oauth2/auth'
oauth2_token_endpoint = 'https://accounts.google.com/o/oauth2/token'
# Check https://developers.google.com/drive/web/scopes for all available scopes
oauth2_scope = 'https://www.googleapis.com/auth/drive.file'
class GDrive (CloudStorage.CloudStorage):
""" Send files to Google Drive. """
name = 'Google Drive'
session_class = GDriveSession
user_agent = 'PG2GDrive/0.2'
upload_endpoint = 'https://www.googleapis.com/upload/drive/v2/files?uploadType=resumable'
def upload_file (self, session, request):
""" Upload a file to google drive. """
file_metadata = {
'title': self.fix_filename (session.ebook.get_filename ()),
'description': 'A Project Gutenberg Ebook',
}
headers = {
'X-Upload-Content-Type': request.headers['Content-Type'],
'X-Upload-Content-Length': request.headers['Content-Length'],
'Content-Type': 'application/json; charset=UTF-8',
}
with closing (session.post (self.upload_endpoint,
data = json.dumps (file_metadata),
headers = headers)) as r2:
r2.raise_for_status ()
session_uri = r2.headers['Location']
headers = {
'Content-Type': request.headers['Content-Type'],
}
with closing (session.put (session_uri,
data = request.iter_content (1024 * 1024),
headers = headers)) as r3:
r3.raise_for_status ()

208
HTMLFormatter.py Normal file
View File

@ -0,0 +1,208 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: iso-8859-1 -*-
"""
HTMLFormatter.py
Copyright 2009-2014 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Produce a HTML page.
"""
from __future__ import unicode_literals
import operator
import cherrypy
import genshi.output
import six
from six.moves import urllib
from libgutenberg.MediaTypes import mediatypes as mt
import BaseSearcher
import BaseFormatter
# filetypes ignored on desktop site
NO_DESKTOP_FILETYPES = 'plucker qioo rdf rst rst.gen rst.master tei cover.medium cover.small'.split ()
# filetypes shown on mobile site
MOBILE_TYPES = (mt.epub, mt.plucker, mt.mobi, mt.pdf, 'text/html', mt.html, mt.qioo)
# filetypes which are usually handed over to a separate app on mobile devices
HANDOVER_TYPES = (mt.epub, mt.plucker, mt.mobi, mt.pdf, mt.qioo)
# self-contained files we can send to dropbox
CLOUD_TYPES = (mt.epub, mt.mobi, mt.pdf)
class XMLishFormatter (BaseFormatter.BaseFormatter):
""" Produce XMLish output. """
def __init__ (self):
super (XMLishFormatter, self).__init__ ()
def fix_dc (self, dc, os):
""" Tweak dc. """
super (XMLishFormatter, self).fix_dc (dc, os)
for file_ in dc.generated_files:
file_.help_topic = file_.hr_filetype
file_.compression = 'none'
file_.encoding = None
for file_ in dc.files + dc.generated_files:
type_ = six.text_type (file_.mediatypes[0])
m = type_.partition (';')[0]
if m in CLOUD_TYPES:
file_.dropbox_url = os.url (
'dropbox_send', id = dc.project_gutenberg_id, filetype = file_.filetype)
file_.gdrive_url = os.url (
'gdrive_send', id = dc.project_gutenberg_id, filetype = file_.filetype)
file_.msdrive_url = os.url (
'msdrive_send', id = dc.project_gutenberg_id, filetype = file_.filetype)
if m in HANDOVER_TYPES:
file_.url = file_.url + '?' + urllib.parse.urlencode (
{ 'session_id': str (cherrypy.session.id) } )
for file_ in dc.files:
file_.honeypot_url = os.url (
'honeypot_send', id = dc.project_gutenberg_id, filetype = file_.filetype)
break
def format (self, page, os):
""" Format to HTML. """
for e in os.entries:
if isinstance (e, BaseSearcher.DC):
self.fix_dc (e, os)
# loop again because fix:dc appends things
for e in os.entries:
if isinstance (e, BaseSearcher.Cat):
if e.url:
e.icon2 = e.icon2 or 'next'
else:
e.class_ += 'grayed'
if os.title_icon:
os.class_ += 'icon_' + os.title_icon
os.entries.sort (key = operator.attrgetter ('order'))
return self.render (page, os)
class HTMLFormatter (XMLishFormatter):
""" Produce HTML output. """
CONTENT_TYPE = 'text/html; charset=UTF-8'
DOCTYPE = ('html',
'-//W3C//DTD XHTML+RDFa 1.0//EN',
'http://www.w3.org/MarkUp/DTD/xhtml-rdfa-1.dtd')
def __init__ (self):
super (HTMLFormatter, self).__init__ ()
def get_serializer (self):
# return BaseFormatter.XHTMLSerializer (doctype = self.DOCTYPE, strip_whitespace = False)
return genshi.output.XHTMLSerializer (doctype = self.DOCTYPE, strip_whitespace = False)
def fix_dc (self, dc, os):
""" Add some info to dc for easier templating.
Also make sure that dc `walks like a cat´. """
super (HTMLFormatter, self).fix_dc (dc, os)
#for author in dc.authors:
# author.authors_page_url = (
# "/browse/authors/%s#a%d" % (author.name[:1].lower (), author.id))
if dc.new_filesystem:
dc.base_dir = "/files/%d/" % dc.project_gutenberg_id
# dc.mirror_dir = gg.archive_dir (dc.project_gutenberg_id)
else:
dc.base_dir = None
# dc.mirror_dir = None
dc.magnetlink = None
# hide all txt files but the first one
txtcount = showncount = 0
for file_ in dc.files + dc.generated_files:
filetype = file_.filetype or ''
file_.hidden = False
if filetype in NO_DESKTOP_FILETYPES:
file_.hidden = True
if file_.compression != 'none':
file_.hidden = True
if filetype.startswith ('txt'):
if txtcount > 0:
file_.hidden = True
txtcount += 1
if filetype != 'txt':
file_.encoding = None
if file_.encoding:
file_.hr_filetype += ' ' + file_.encoding.upper ()
if filetype.startswith ('html') and file_.compression == 'none':
file_.hr_filetype = 'Read this book online: {}'.format (file_.hr_filetype)
if not file_.hidden:
showncount += 1
# if we happened to hide everything, show all files
if showncount == 0:
for file_ in dc.files + dc.generated_files:
file_.hidden = False
class MobileFormatter (XMLishFormatter):
""" Produce HTML output suitable for mobile devices. """
CONTENT_TYPE = mt.xhtml + '; charset=UTF-8'
DOCTYPE = ('html',
'-//WAPFORUM//DTD XHTML Mobile 1.2//EN',
'http://www.openmobilealliance.org/tech/DTD/xhtml-mobile12.dtd')
def __init__ (self):
super (MobileFormatter, self).__init__ ()
def get_serializer (self):
return genshi.output.XMLSerializer (doctype = self.DOCTYPE, strip_whitespace = False)
def fix_dc (self, dc, os):
""" Add some info to dc for easier templating.
Also make sure that dc `walks like a cat´. """
super (MobileFormatter, self).fix_dc (dc, os)
for file_ in dc.files + dc.generated_files:
if len (file_.mediatypes) == 1:
type_ = six.text_type (file_.mediatypes[0])
m = type_.partition (';')[0]
if m in MOBILE_TYPES:
cat = BaseSearcher.Cat ()
cat.type = file_.mediatypes[0]
cat.header = _('Download')
cat.title = file_.hr_filetype
cat.extra = file_.hr_extent
cat.charset = file_.encoding
cat.url = file_.url
cat.icon = dc.icon
cat.icon2 = 'download'
cat.class_ += 'filelink'
cat.order = 20
os.entries.append (cat)

70
Icons.py Normal file
View File

@ -0,0 +1,70 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: iso-8859-1 -*-
"""
Icons.py
Copyright 2010-2012 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Stock icons.
"""
from __future__ import unicode_literals
# This list is used to generate a sprite.
THUMBS = {
'alias' : 'pics/24/messagebox_info.png',
'alpha' : 'pics/24/applixware.png',
'audiobook' : 'pics/24/amarok.png',
'author' : 'pics/24/kuser.png',
'back' : 'pics/24/previous.png',
'bibrec' : 'pics/24/messagebox_info.png',
'bookshelf' : 'pics/24/bookcase.png',
'date' : 'pics/24/date.png',
'download' : 'pics/24/dock.png',
'external' : 'pics/24/browser.png',
'folder' : 'pics/24/folder_yellow_open.png',
'home' : 'pics/24/gohome.png',
'internal' : 'pics/24/pg-logo.png',
'language' : 'pics/24/locale.png',
'more' : 'pics/24/edit_add.png',
'popular' : 'pics/24/bookmark.png',
'print' : 'pics/24/print.png',
'quantity' : 'pics/24/licq.png',
'random' : 'pics/24/roll.png',
'rss' : 'pics/24/rss.png',
'search' : 'pics/24/viewmag.png',
'subject' : 'pics/24/edu_science.png',
'suggestion' : 'pics/24/ktip.png',
'book' : 'pics/24/kdict.png',
'book_no_images' : 'pics/24/book-cut.png',
'cancel' : 'pics/16/no.png',
'help' : 'pics/16/help.png',
'next' : 'pics/16/next.png',
'external_link' : 'pics/link.png',
'qrtab' : 'pics/qricon.png',
'smsearch' : 'pics/16/viewmag.png',
'logo' : 'pics/android-touch-icon.png',
'logo_yellow' : 'pics/android-touch-icon-yellow.png',
'paypal' : 'pics/paypal-donate-button-LG.gif',
'flattr' : 'pics/flattr-badge-large.png',
'facebook' : 'pics/facebook-20x20.png',
'dropbox' : 'pics/dropbox-22x22.png',
'gdrive' : 'pics/gdrive-24x21.png',
'msdrive' : 'pics/msdrive-24x24.png',
'twitter' : 'pics/twitter_whiteonblue-20x20.png',
'gplus' : 'pics/gplus-20x20.png',
# 'hosted' : 'pics/ibiblio-hosted-110x32.png',
# 'apache' : 'pics/apache-powered-118x41.gif',
# 'python' : 'pics/python-powered-w-81x32.png',
# 'postgres' : 'pics/postgres-powered-88x31.gif',
}

53
JSONFormatter.py Normal file
View File

@ -0,0 +1,53 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: iso-8859-1 -*-
"""
JSONFormatter.py
Copyright 2009-2012 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Produce a JSON response.
"""
from __future__ import unicode_literals
import json
import re
from libgutenberg.MediaTypes import mediatypes as mt
import BaseFormatter
RE_WORD = re.compile (r'\W+', re.U)
class JSONFormatter (BaseFormatter.BaseFormatter):
""" Produce JSON output. """
CONTENT_TYPE = mt.json + '; charset=UTF-8'
CONTENT_TYPE = 'application/json; charset=UTF-8'
DOCTYPE = None
def __init__ (self):
super (JSONFormatter, self).__init__ ()
def format (self, dummy_page, os):
# Specs:
# http://www.opensearch.org/Specifications/OpenSearch/Extensions/Suggestions/1.0
sugg0 = []
sugg1 = []
sugg2 = []
for e in os.entries:
if 'navlink' not in e.class_:
sugg0.append (e.title)
sugg1.append (e.subtitle)
sugg2.append (e.url)
self.send_headers ()
return json.dumps ( [os.query, sugg0, sugg1, sugg2] ).encode ('utf-8')

55
MSDrive.py Normal file
View File

@ -0,0 +1,55 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
MSDrive.py
Copyright 2014,15 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
The send-to-microsoft-drive pages.
"""
from __future__ import unicode_literals
from contextlib import closing
import CloudStorage
class MSDriveSession (CloudStorage.CloudOAuth2Session):
""" Hold parameters for OAuth. """
#
# OAuth 2.0 flow see:
# http://tools.ietf.org/html/rfc6749
# http://msdn.microsoft.com/en-us/library/live/hh243649
#
name_prefix = 'msdrive'
oauth2_auth_endpoint = 'https://login.live.com/oauth20_authorize.srf'
oauth2_token_endpoint = 'https://login.live.com/oauth20_token.srf'
oauth2_scope = 'wl.signin wl.basic wl.skydrive wl.skydrive_update'
class MSDrive (CloudStorage.CloudStorage):
""" Send files to Microsoft Drive. """
name = 'OneDrive'
session_class = MSDriveSession
user_agent = 'PG2MSDrive/0.2'
upload_endpoint = 'https://apis.live.net/v5.0/me/skydrive/files/'
def upload_file (self, session, request):
""" Upload a file to microsoft drive. """
url = self.upload_endpoint + self.fix_filename (session.ebook.get_filename ())
# MSDrive does not like such never-heard-of-before
# content-types like 'epub', so we just send it without
# content-type.
with closing (session.put (url, data = request.iter_content (1024 * 1024))) as r:
r.raise_for_status ()

141
MyRamSession.py Normal file
View File

@ -0,0 +1,141 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
MyRamSession.py
Copyright 2014 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
A quick Python3 fix for the cherrypy RamSession. May be removed when
RamSession is fixed upstream.
Usage:
import MyRamSession
cherrypy.lib.sessions.RamSession = MyRamSession.FixedRamSession
cherrypy.lib.sessions.MyramSession = MyRamSession.MyRamSession
"""
import threading
import cherrypy
import cherrypy.lib.sessions
class Struct (object):
""" Data store. """
def __init__ (self):
self.expires = None
self.data = None
self.cache_lock = threading.Lock ()
class MyRamSession (cherrypy.lib.sessions.Session):
""" A cherrypy session kept in ram. """
cache = {}
# all inserts/deletes in cache must be guarded by this lock
# or we will get 'RuntimeError: dictionary changed size during iteration'
# because you cannot atomically iterate a dict in Python3
cache_lock = threading.Lock ()
def __init__ (self, id_ = None, **kwargs):
super (MyRamSession, self).__init__ (id_, **kwargs)
def clean_up (self):
"""Clean up expired sessions."""
now = self.now ()
def expired (x):
return x[1].expires <= now
with self.cache_lock:
for id_, s in list (filter (expired, self.cache.items ())):
self.cache.pop (id_, None)
def _exists (self):
return self.id in self.cache
def _load (self):
try:
s = self.cache[self.id]
return s.data, s.expires
except KeyError:
return None
def _save (self, expires):
s = self.cache.get (self.id, Struct ())
s.expires = expires
s.data = self._data
with self.cache_lock:
self.cache[self.id] = s
def _delete (self):
with self.cache_lock:
self.cache.pop (self.id, None)
def acquire_lock (self):
"""Acquire an exclusive lock on the currently-loaded session data."""
try:
self.cache[self.id].lock.acquire ()
self.locked = True
except KeyError:
pass
def release_lock (self):
"""Release the lock on the currently-loaded session data."""
try:
self.cache[self.id].lock.release ()
self.locked = False
except KeyError:
pass
def __len__ (self):
"""Return the number of active sessions."""
return len (self.cache)
from cherrypy._cpcompat import copyitems
class FixedRamSession (cherrypy.lib.sessions.RamSession):
def clean_up(self):
"""Clean up expired sessions."""
now = self.now()
try:
for id, (data, expiration_time) in copyitems(self.cache):
if expiration_time <= now:
try:
del self.cache[id]
except KeyError:
pass
try:
del self.locks[id]
except KeyError:
pass
# added to remove obsolete lock objects
for id in list(self.locks):
if id not in self.cache:
self.locks.pop(id, None)
except RuntimeError:
# RuntimeError: dictionary changed size during iteration
# Do nothig. Keep cleanup thread running and maybe next time lucky.
pass

160
OPDSFormatter.py Normal file
View File

@ -0,0 +1,160 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: iso-8859-1 -*-
"""
OPDSFormatter.py
Copyright 2009-2012 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Produce an OPDS feed.
"""
from __future__ import unicode_literals
import copy
import re
import genshi.output
import cherrypy
import six
from libgutenberg.GutenbergGlobals import Struct, xmlspecialchars
from libgutenberg.MediaTypes import mediatypes as mt
from libgutenberg import DublinCore
import BaseFormatter
from Icons import THUMBS as th
# files a mobile can download
OPDS_TYPES = (mt.epub, mt.mobi, mt.pdf)
# domains allowed to XMLHttpRequest () our OPDS feed
CORS_DOMAINS = '*'
class OPDSFormatter (BaseFormatter.BaseFormatter):
""" Produces opds output. """
CONTENT_TYPE = mt.opds + '; charset=UTF-8'
DOCTYPE = None
def get_serializer (self):
return genshi.output.XMLSerializer (doctype = self.DOCTYPE, strip_whitespace = False)
def send_headers (self):
""" Send HTTP content-type header. """
cherrypy.response.headers['Access-Control-Allow-Origin'] = CORS_DOMAINS
super (OPDSFormatter, self).send_headers ()
def format (self, page, os):
""" Format os struct into opds output. """
entries = []
for dc in os.entries:
dc.thumbnail = None
if isinstance (dc, DublinCore.DublinCore):
dc.image_flags = 0
if dc.has_images ():
dc.pool = None
dc_copy = copy.deepcopy (dc)
dc.image_flags = 2
dc.icon = 'title_no_images'
self.fix_dc (dc, os)
entries.append (dc)
dc_copy.image_flags = 3
self.fix_dc (dc_copy, os, True)
entries.append (dc_copy)
else:
self.fix_dc (dc, os)
entries.append (dc)
else:
# actually not a dc
# throw out 'start over' link, FIXME: actually throw out all non-dc's ?
if page == 'bibrec' and dc.rel == 'start':
continue
dc.links = []
if dc.icon in th:
link = Struct ()
link.type = mt.png
link.rel = 'thumb'
link.url = self.data_url (th[dc.icon])
link.title = None
link.length = None
dc.links.append (link)
entries.append (dc)
os.entries = entries
if page == 'bibrec':
# we have just one template for both
page = 'results'
return self.render (page, os)
def fix_dc (self, dc, os, want_images = False):
""" Make fixes to dublincore struct. """
def to_html (text):
""" Turn plain text into html. """
return re.sub (r'[\r\n]+', '<br/>', xmlspecialchars (text))
def key_role (author):
""" Sort authors first, then other contributors. """
if author.marcrel in ('cre', 'aut'):
return ''
return author.marcrel
super (OPDSFormatter, self).fix_dc (dc, os)
if dc.icon in th:
dc.thumbnail = self.data_url (th[dc.icon])
dc.links = []
dc.title_html = to_html (dc.title)
dc.authors.sort (key = key_role)
for file_ in dc.files + dc.generated_files:
if len (file_.mediatypes) == 1:
type_ = six.text_type (file_.mediatypes[0])
filetype = file_.filetype or ''
if type_.partition (';')[0] in OPDS_TYPES:
if ((filetype.find ('.images') > -1) == want_images):
link = Struct ()
link.type = type_
link.title = file_.hr_filetype
link.url = file_.url
link.length = str (file_.extent)
link.rel = 'acquisition'
dc.links.append (link)
if filetype == 'cover.small':
link = Struct ()
link.type = six.text_type (file_.mediatypes[0])
link.title = None
link.url = file_.url
link.length = None
link.rel = 'thumb'
dc.links.append (link)
if filetype == 'cover.medium':
link = Struct ()
link.type = six.text_type (file_.mediatypes[0])
link.title = None
link.url = file_.url
link.length = None
link.rel = 'cover'
dc.links.append (link)

305
Page.py Normal file
View File

@ -0,0 +1,305 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
Page.py
Copyright 2009-2010 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Base class for all pages.
"""
from __future__ import unicode_literals
import logging
import cherrypy
from libgutenberg.MediaTypes import mediatypes as mt
from libgutenberg.GutenbergDatabase import DatabaseError
import BaseSearcher
import Formatters
class Page (object):
""" Base for all pages. """
def __init__ (self):
self.supported_book_mediatypes = [ mt.epub, mt.mobi ]
@staticmethod
def format (os):
""" Output page. """
return Formatters.formatters[os.format].format (os.template, os)
def client_book_mediatypes (self):
""" Return the book mediatypes accepted by the client. """
client_accepted_book_mediatypes = []
accept_header = cherrypy.request.headers.get ('Accept')
if accept_header is None:
client_accepted_book_mediatypes = self.supported_book_mediatypes
else:
#cherrypy.log ("Accept: %s" % accept_header,
# context = 'REQUEST', severity = logging.DEBUG)
client_accepted_book_mediatypes = []
accepts = cherrypy.request.headers.elements ('Accept')
for accept in accepts:
if accept.value in self.supported_book_mediatypes:
if accept.qvalue > 0:
client_accepted_book_mediatypes.append (accept.value)
return client_accepted_book_mediatypes
class NullPage (Page):
""" An empty page. """
def index (self, **dummy_kwargs):
""" Output an empty page. """
return '<html/>'
class SearchPage (Page):
""" Abstract base class for all search page classes. """
def setup (self, dummy_os, dummy_sql):
""" Let derived classes setup the query. """
raise NotImplementedError
def fixup (self, os):
""" Give derived classes a chance to further manipulate database results. """
pass
def finalize (self, os):
""" Give derived classes a chance to fix default finalization. """
pass
def nothing_found (self, os):
""" Give derived class a chance to react if no records were found. """
os.entries.insert (0, self.no_records_found (os))
def output_suggestions (self, os, max_suggestions_per_word = 3):
""" Make suggestions. """
# similarity == matching_trigrams / (len1 + len2 - matching_trigrams)
sql_query = """
SELECT
word,
nentry,
similarity (word, %(word)s) AS sml
FROM terms
WHERE word %% %(word)s
ORDER BY sml DESC, nentry DESC LIMIT %(suggestions)s;"""
q = os.query.lower ()
sugg = []
for word in q.split ():
if len (word) > 3:
try:
rows = BaseSearcher.SQLSearcher().execute (
sql_query,
{ 'word': word, 'suggestions': max_suggestions_per_word + 1})
for i, row in enumerate (rows):
if i >= max_suggestions_per_word:
break
corr = row.word
if corr != word:
sugg.append ( (word, corr) )
except DatabaseError:
pass
for word, corr in reversed (sugg):
os.entries.insert (0, self.did_you_mean (os, corr, q.replace (word, corr)))
def index (self, **kwargs):
""" Output a search result page. """
os = BaseSearcher.OpenSearch ()
os.log_request ('search')
if 'default_prefix' in kwargs:
raise cherrypy.HTTPError (400, 'Bad Request. Unknown parameter: default_prefix')
if os.start_index > BaseSearcher.MAX_RESULTS:
raise cherrypy.HTTPError (400, 'Bad Request. Parameter start_index too high')
sql = BaseSearcher.SQLStatement ()
sql.query = 'SELECT *'
sql.from_ = ['v_appserver_books_4 as books']
# let derived classes prepare the query
try:
self.setup (os, sql)
except ValueError as what:
raise cherrypy.HTTPError (400, 'Bad Request. ' + str (what))
os.fix_sortorder ()
# execute the query
try:
BaseSearcher.SQLSearcher ().search (os, sql)
except DatabaseError as what:
cherrypy.log ("SQL Error: " + str (what),
context = 'REQUEST', severity = logging.ERROR)
raise cherrypy.HTTPError (400, 'Bad Request. Check your query.')
# sync os.title and first entry header
if os.entries:
entry = os.entries[0]
if os.title and not entry.header:
entry.header = os.title
elif entry.header and not os.title:
os.title = entry.header
os.template = os.page = 'results'
# give derived class a chance to tweak result set
self.fixup (os)
# warn user about no records found
if os.total_results == 0:
self.nothing_found (os)
# suggest alternate queries
if os.total_results < 5:
self.output_suggestions (os)
# add sort by links
if os.start_index == 1 and os.total_results > 1:
if 'downloads' in os.alternate_sort_orders:
self.sort_by_downloads (os)
if 'release_date' in os.alternate_sort_orders:
self.sort_by_release_date (os)
if 'title' in os.alternate_sort_orders:
self.sort_by_title (os)
if 'alpha' in os.alternate_sort_orders:
self.sort_alphabetically (os)
if 'quantity' in os.alternate_sort_orders:
self.sort_by_quantity (os)
os.finalize ()
self.finalize (os)
if os.total_results > 0:
# call this after finalize ()
os.entries.insert (0, self.status_line (os))
return self.format (os)
@staticmethod
def sort_by_downloads (os):
""" Append the sort by downloads link. """
cat = BaseSearcher.Cat ()
cat.rel = 'popular'
cat.title = _('Sort by Popularity')
cat.url = os.url_carry (sort_order = 'downloads')
cat.class_ += 'navlink grayed'
cat.icon = 'popular'
cat.order = 4.0
os.entries.insert (0, cat)
@staticmethod
def sort_alphabetically (os):
""" Append the sort alphabetically link. """
cat = BaseSearcher.Cat ()
cat.rel = 'alphabethical'
cat.title = _('Sort Alphabetically')
cat.url = os.url_carry (sort_order = 'alpha')
cat.class_ += 'navlink grayed'
cat.icon = 'alpha'
cat.order = 4.1
os.entries.insert (0, cat)
@staticmethod
def sort_by_title (os):
""" Append the sort alphabetically link. """
cat = BaseSearcher.Cat ()
cat.rel = 'alphabethical'
cat.title = _('Sort Alphabetically')
cat.url = os.url_carry (sort_order = 'title')
cat.class_ += 'navlink grayed'
cat.icon = 'alpha'
cat.order = 4.1
os.entries.insert (0, cat)
@staticmethod
def sort_by_quantity (os):
""" Append the sort by quantity link. """
cat = BaseSearcher.Cat ()
cat.rel = 'numerous'
cat.title = _('Sort by Quantity')
cat.url = os.url_carry (sort_order = 'quantity')
cat.class_ += 'navlink grayed'
cat.icon = 'quantity'
cat.order = 4.2
os.entries.insert (0, cat)
@staticmethod
def sort_by_release_date (os):
""" Append the sort by release date link. """
cat = BaseSearcher.Cat ()
cat.rel = 'new'
cat.title = _('Sort by Release Date')
cat.url = os.url_carry (sort_order = 'release_date')
cat.class_ += 'navlink grayed'
cat.icon = 'date'
cat.order = 4.3
os.entries.insert (0, cat)
@staticmethod
def status_line (os):
""" Placeholder for status line. """
cat = BaseSearcher.Cat ()
cat.rel = '__statusline__'
cat.class_ += 'grayed'
cat.icon = 'bibrec'
cat.order = 10
cat.header = os.title
cat.title = _(u"Displaying results {from_}–{to}").format (
from_ = os.start_index, to = os.end_index)
return cat
@staticmethod
def no_records_found (os):
""" Message. """
cat = BaseSearcher.Cat ()
cat.rel = '__notfound__'
cat.title = _('No records found.')
cat.url = os.url ('start')
cat.class_ += 'navlink grayed'
cat.icon = 'bibrec'
cat.order = 11
return cat
@staticmethod
def did_you_mean (os, corr, corrected_query):
""" Message. """
cat = BaseSearcher.Cat ()
cat.rel = '__didyoumean__'
cat.title = _('Did you mean: {correction}').format (correction = corr)
cat.url = os.url ('search', query = corrected_query)
cat.class_ += 'navlink'
cat.icon = 'suggestion'
cat.order = 12
return cat

49
Pipfile Normal file
View File

@ -0,0 +1,49 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
[packages]
"repoze.lru" = "*"
Babel = "*"
chardet = "*"
CherryPy = "*"
colorama = "*"
#command-not-found = "*"
Genshi = "*"
html5lib = "*"
isodate = "*"
#language-selector = "*"
libgutenberg = "*"
netaddr = "*"
oauth = "*"
oauthlib = "*"
Pillow = "*"
psycopg2 = "*"
#pycurl = "*"
#PyGObject = "*"
pyparsing = "*"
#python-apt = "*"
pytz = "*"
qrcode = "*"
rdflib = "*"
recaptcha-client = "*"
regex = "*"
requests = "*"
requests-oauthlib = "*"
Routes = "*"
simplejson = "*"
six = "*"
SQLAlchemy = "*"
#ufw = "*"
#unattended-upgrades = "*"
urllib3 = "*"
adns = "==1.4-py0"
[requires]
python_version = "3.6"
[pipenv]
allow_prereleases = true

413
Pipfile.lock generated Normal file
View File

@ -0,0 +1,413 @@
{
"_meta": {
"hash": {
"sha256": "ee8fb05228d77d9d0e14c78195d3ead551470bbfb9d44c0cbe88444aefe72370"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.6"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"adns": {
"hashes": [
"sha256:1176e63b66db007e8798150a9a7cfcca5d8a810ae488905f8b2b6b07e678f2ef"
],
"index": "pypi",
"version": "==1.4-py0"
},
"babel": {
"hashes": [
"sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669",
"sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23"
],
"index": "pypi",
"version": "==2.6.0"
},
"backports.functools-lru-cache": {
"hashes": [
"sha256:9d98697f088eb1b0fa451391f91afb5e3ebde16bbdb272819fd091151fda4f1a",
"sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd"
],
"version": "==1.5"
},
"certifi": {
"hashes": [
"sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5",
"sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae"
],
"version": "==2019.3.9"
},
"chardet": {
"hashes": [
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
],
"index": "pypi",
"version": "==3.0.4"
},
"cheroot": {
"hashes": [
"sha256:beb8eb9eeff5746059607e81b72efd6f4ca099111dc13f8961ae9e4f63f7786b",
"sha256:c52f8df52c461351b91ce9a7769208c064bae74bd61b531312ffc62bff667650"
],
"version": "==6.5.4"
},
"cherrypy": {
"hashes": [
"sha256:4dd2f59b5af93bd9ca85f1ed0bb8295cd0f5a8ee2b84d476374d4e070aa5c615",
"sha256:626e305bca3c5d56a16e5f7d64bc8a4e25d26c41be1779f585fad2608edbc4c8"
],
"index": "pypi",
"version": "==18.1.0"
},
"colorama": {
"hashes": [
"sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d",
"sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48"
],
"index": "pypi",
"version": "==0.4.1"
},
"genshi": {
"hashes": [
"sha256:0d87ae62cf2ed92133f35725da51e02d09f79bb4cb986f0d948408a0279dd3f8",
"sha256:8dd0628da7898c625d487a676992490c8222e47bb56f36a3dcb0cfe28159997f"
],
"index": "pypi",
"version": "==0.7.1"
},
"html5lib": {
"hashes": [
"sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3",
"sha256:66cb0dcfdbbc4f9c3ba1a63fdb511ffdbd4f513b2b6d81b80cd26ce6b3fb3736"
],
"index": "pypi",
"version": "==1.0.1"
},
"idna": {
"hashes": [
"sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
"sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
],
"version": "==2.8"
},
"isodate": {
"hashes": [
"sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8",
"sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81"
],
"index": "pypi",
"version": "==0.6.0"
},
"jaraco.functools": {
"hashes": [
"sha256:35ba944f52b1a7beee8843a5aa6752d1d5b79893eeb7770ea98be6b637bf9345",
"sha256:e9e377644cee5f6f9128b4dab1631fca74981236e95a255f80e4292bcd2b5284"
],
"version": "==2.0"
},
"libgutenberg": {
"hashes": [
"sha256:6afff1b3067f3a862aad28b981fb72d4ff0ee33a9bad0893d570c9cec3fcef20"
],
"index": "pypi",
"version": "==0.3.1"
},
"lxml": {
"hashes": [
"sha256:0358b9e9642bc7d39aac5cffe9884a99a5ca68e5e2c1b89e570ed60da9139908",
"sha256:091a359c4dafebbecd3959d9013f1b896b5371859165e4e50b01607a98d9e3e2",
"sha256:1998e4e60603c64bcc35af61b4331ab3af087457900d3980e18d190e17c3a697",
"sha256:2000b4088dee9a41f459fddaf6609bba48a435ce6374bb254c5ccdaa8928c5ba",
"sha256:2afb0064780d8aaf165875be5898c1866766e56175714fa5f9d055433e92d41d",
"sha256:2d8f1d9334a4e3ff176d096c14ded3100547d73440683567d85b8842a53180bb",
"sha256:2e38db22f6a3199fd63675e1b4bd795d676d906869047398f29f38ca55cb453a",
"sha256:3181f84649c1a1ca62b19ddf28436b1b2cb05ae6c7d2628f33872e713994c364",
"sha256:37462170dfd88af8431d04de6b236e6e9c06cda71e2ca26d88ef2332fd2a5237",
"sha256:3a9d8521c89bf6f2a929c3d12ad3ad7392c774c327ea809fd08a13be6b3bc05f",
"sha256:3d0bbd2e1a28b4429f24fd63a122a450ce9edb7a8063d070790092d7343a1aa4",
"sha256:483d60585ce3ee71929cea70949059f83850fa5e12deb9c094ed1c8c2ec73cbd",
"sha256:4888be27d5cba55ce94209baef5bcd7bbd7314a3d17021a5fc10000b3a5f737d",
"sha256:64b0d62e4209170a2a0c404c446ab83b941a0003e96604d2e4f4cb735f8a2254",
"sha256:68010900898fdf139ac08549c4dba8206c584070a960ffc530aebf0c6f2794ef",
"sha256:872ecb066de602a0099db98bd9e57f4cfc1d62f6093d94460c787737aa08f39e",
"sha256:88a32b03f2e4cd0e63f154cac76724709f40b3fc2f30139eb5d6f900521b44ed",
"sha256:b1dc7683da4e67ab2bebf266afa68098d681ae02ce570f0d1117312273d2b2ac",
"sha256:b29e27ce9371810250cb1528a771d047a9c7b0f79630dc7dc5815ff828f4273b",
"sha256:ce197559596370d985f1ce6b7051b52126849d8159040293bf8b98cb2b3e1f78",
"sha256:d45cf6daaf22584eff2175f48f82c4aa24d8e72a44913c5aff801819bb73d11f",
"sha256:e2ff9496322b2ce947ba4a7a5eb048158de9d6f3fe9efce29f1e8dd6878561e6",
"sha256:f7b979518ec1f294a41a707c007d54d0f3b3e1fd15d5b26b7e99b62b10d9a72e",
"sha256:f9c7268e9d16e34e50f8246c4f24cf7353764affd2bc971f0379514c246e3f6b",
"sha256:f9c839806089d79de588ee1dde2dae05dc1156d3355dfeb2b51fde84d9c960ad",
"sha256:ff962953e2389226adc4d355e34a98b0b800984399153c6678f2367b11b4d4b8"
],
"version": "==4.3.2"
},
"more-itertools": {
"hashes": [
"sha256:0125e8f60e9e031347105eb1682cef932f5e97d7b9a1a28d9bf00c22a5daef40",
"sha256:590044e3942351a1bdb1de960b739ff4ce277960f2425ad4509446dbace8d9d1"
],
"version": "==6.0.0"
},
"netaddr": {
"hashes": [
"sha256:38aeec7cdd035081d3a4c306394b19d677623bf76fa0913f6695127c7753aefd",
"sha256:56b3558bd71f3f6999e4c52e349f38660e54a7a8a9943335f73dfc96883e08ca"
],
"index": "pypi",
"version": "==0.7.19"
},
"oauth": {
"hashes": [
"sha256:e769819ff0b0c043d020246ce1defcaadd65b9c21d244468a45a7f06cb88af5d"
],
"index": "pypi",
"version": "==1.0.1"
},
"oauthlib": {
"hashes": [
"sha256:0ce32c5d989a1827e3f1148f98b9085ed2370fc939bf524c9c851d8714797298",
"sha256:3e1e14f6cde7e5475128d30e97edc3bfb4dc857cb884d8714ec161fdbb3b358e"
],
"index": "pypi",
"version": "==3.0.1"
},
"pillow": {
"hashes": [
"sha256:051de330a06c99d6f84bcf582960487835bcae3fc99365185dc2d4f65a390c0e",
"sha256:0ae5289948c5e0a16574750021bd8be921c27d4e3527800dc9c2c1d2abc81bf7",
"sha256:0b1efce03619cdbf8bcc61cfae81fcda59249a469f31c6735ea59badd4a6f58a",
"sha256:163136e09bd1d6c6c6026b0a662976e86c58b932b964f255ff384ecc8c3cefa3",
"sha256:18e912a6ccddf28defa196bd2021fe33600cbe5da1aa2f2e2c6df15f720b73d1",
"sha256:24ec3dea52339a610d34401d2d53d0fb3c7fd08e34b20c95d2ad3973193591f1",
"sha256:267f8e4c0a1d7e36e97c6a604f5b03ef58e2b81c1becb4fccecddcb37e063cc7",
"sha256:3273a28734175feebbe4d0a4cde04d4ed20f620b9b506d26f44379d3c72304e1",
"sha256:4c678e23006798fc8b6f4cef2eaad267d53ff4c1779bd1af8725cc11b72a63f3",
"sha256:4d4bc2e6bb6861103ea4655d6b6f67af8e5336e7216e20fff3e18ffa95d7a055",
"sha256:505738076350a337c1740a31646e1de09a164c62c07db3b996abdc0f9d2e50cf",
"sha256:5233664eadfa342c639b9b9977190d64ad7aca4edc51a966394d7e08e7f38a9f",
"sha256:5d95cb9f6cced2628f3e4de7e795e98b2659dfcc7176ab4a01a8b48c2c2f488f",
"sha256:7eda4c737637af74bac4b23aa82ea6fbb19002552be85f0b89bc27e3a762d239",
"sha256:801ddaa69659b36abf4694fed5aa9f61d1ecf2daaa6c92541bbbbb775d97b9fe",
"sha256:825aa6d222ce2c2b90d34a0ea31914e141a85edefc07e17342f1d2fdf121c07c",
"sha256:9c215442ff8249d41ff58700e91ef61d74f47dfd431a50253e1a1ca9436b0697",
"sha256:a3d90022f2202bbb14da991f26ca7a30b7e4c62bf0f8bf9825603b22d7e87494",
"sha256:a631fd36a9823638fe700d9225f9698fb59d049c942d322d4c09544dc2115356",
"sha256:a6523a23a205be0fe664b6b8747a5c86d55da960d9586db039eec9f5c269c0e6",
"sha256:a756ecf9f4b9b3ed49a680a649af45a8767ad038de39e6c030919c2f443eb000",
"sha256:b117287a5bdc81f1bac891187275ec7e829e961b8032c9e5ff38b70fd036c78f",
"sha256:ba04f57d1715ca5ff74bb7f8a818bf929a204b3b3c2c2826d1e1cc3b1c13398c",
"sha256:cd878195166723f30865e05d87cbaf9421614501a4bd48792c5ed28f90fd36ca",
"sha256:cee815cc62d136e96cf76771b9d3eb58e0777ec18ea50de5cfcede8a7c429aa8",
"sha256:d1722b7aa4b40cf93ac3c80d3edd48bf93b9208241d166a14ad8e7a20ee1d4f3",
"sha256:d7c1c06246b05529f9984435fc4fa5a545ea26606e7f450bdbe00c153f5aeaad",
"sha256:e9c8066249c040efdda84793a2a669076f92a301ceabe69202446abb4c5c5ef9",
"sha256:f227d7e574d050ff3996049e086e1f18c7bd2d067ef24131e50a1d3fe5831fbc",
"sha256:fc9a12aad714af36cf3ad0275a96a733526571e52710319855628f476dcb144e"
],
"index": "pypi",
"version": "==5.4.1"
},
"portend": {
"hashes": [
"sha256:b7ce7d35ea262415297cbfea86226513e77b9ee5f631d3baa11992d663963719",
"sha256:f5c99a1aa1655733736bb0283fee6a1e115e18db500332bec8e24c43f320d8e8"
],
"version": "==2.3"
},
"psycopg2": {
"hashes": [
"sha256:02445ebbb3a11a3fe8202c413d5e6faf38bb75b4e336203ee144ca2c46529f94",
"sha256:0e9873e60f98f0c52339abf8f0339d1e22bfe5aae0bcf7aabd40c055175035ec",
"sha256:1148a5eb29073280bf9057c7fc45468592c1bb75a28f6df1591adb93c8cb63d0",
"sha256:259a8324e109d4922b0fcd046e223e289830e2568d6f4132a3702439e5fd532b",
"sha256:28dffa9ed4595429e61bacac41d3f9671bb613d1442ff43bcbec63d4f73ed5e8",
"sha256:314a74302d4737a3865d40ea50e430ce1543c921ba10f39d562e807cfe2edf2a",
"sha256:36b60201b6d215d7658a71493fdf6bd5e60ad9a0cffed39906627ff9f4f3afd3",
"sha256:3f9d532bce54c4234161176ff3b8688ff337575ca441ea27597e112dfcd0ee0c",
"sha256:5d222983847b40af989ad96c07fc3f07e47925e463baa5de716be8f805b41d9b",
"sha256:6757a6d2fc58f7d8f5d471ad180a0bd7b4dd3c7d681f051504fbea7ae29c8d6f",
"sha256:6a0e0f1e74edb0ab57d89680e59e7bfefad2bfbdf7c80eb38304d897d43674bb",
"sha256:6ca703ccdf734e886a1cf53eb702261110f6a8b0ed74bcad15f1399f74d3f189",
"sha256:8513b953d8f443c446aa79a4cc8a898bd415fc5e29349054f03a7d696d495542",
"sha256:9262a5ce2038570cb81b4d6413720484cb1bc52c064b2f36228d735b1f98b794",
"sha256:97441f851d862a0c844d981cbee7ee62566c322ebb3d68f86d66aa99d483985b",
"sha256:a07feade155eb8e69b54dd6774cf6acf2d936660c61d8123b8b6b1f9247b67d6",
"sha256:a9b9c02c91b1e3ec1f1886b2d0a90a0ea07cc529cb7e6e472b556bc20ce658f3",
"sha256:ae88216f94728d691b945983140bf40d51a1ff6c7fe57def93949bf9339ed54a",
"sha256:b360ffd17659491f1a6ad7c928350e229c7b7bd83a2b922b6ee541245c7a776f",
"sha256:b4221957ceccf14b2abdabef42d806e791350be10e21b260d7c9ce49012cc19e",
"sha256:b90758e49d5e6b152a460d10b92f8a6ccf318fcc0ee814dcf53f3a6fc5328789",
"sha256:c669ea986190ed05fb289d0c100cc88064351f2b85177cbfd3564c4f4847d18c",
"sha256:d1b61999d15c79cf7f4f7cc9021477aef35277fc52452cf50fd13b713c84424d",
"sha256:de7bb043d1adaaf46e38d47e7a5f703bb3dab01376111e522b07d25e1a79c1e1",
"sha256:e393568e288d884b94d263f2669215197840d097c7e5b0acd1a51c1ea7d1aba8",
"sha256:ed7e0849337bd37d89f2c2b0216a0de863399ee5d363d31b1e5330a99044737b",
"sha256:f153f71c3164665d269a5d03c7fa76ba675c7a8de9dc09a4e2c2cdc9936a7b41",
"sha256:f1fb5a8427af099beb7f65093cbdb52e021b8e6dbdfaf020402a623f4181baf5",
"sha256:f36b333e9f86a2fba960c72b90c34be6ca71819e300f7b1fc3d2b0f0b2c546cd",
"sha256:f4526d078aedd5187d0508aa5f9a01eae6a48a470ed678406da94b4cd6524b7e"
],
"index": "pypi",
"version": "==2.7.7"
},
"pyparsing": {
"hashes": [
"sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a",
"sha256:f6c5ef0d7480ad048c054c37632c67fca55299990fff127850181659eea33fc3"
],
"index": "pypi",
"version": "==2.3.1"
},
"pytz": {
"hashes": [
"sha256:32b0891edff07e28efe91284ed9c31e123d84bea3fd98e1f72be2508f43ef8d9",
"sha256:d5f05e487007e29e03409f9398d074e158d920d36eb82eaf66fb1136b0c5374c"
],
"index": "pypi",
"version": "==2018.9"
},
"qrcode": {
"hashes": [
"sha256:3996ee560fc39532910603704c82980ff6d4d5d629f9c3f25f34174ce8606cf5",
"sha256:505253854f607f2abf4d16092c61d4e9d511a3b4392e60bff957a68592b04369"
],
"index": "pypi",
"version": "==6.1"
},
"rdflib": {
"hashes": [
"sha256:58d5994610105a457cff7fdfe3d683d87786c5028a45ae032982498a7e913d6f",
"sha256:da1df14552555c5c7715d8ce71c08f404c988c58a1ecd38552d0da4fc261280d"
],
"index": "pypi",
"version": "==4.2.2"
},
"recaptcha-client": {
"hashes": [
"sha256:28c6853c1d13d365b7dc71a6b05e5ffb56471f70a850de318af50d3d7c0dea2f"
],
"index": "pypi",
"version": "==1.0.6"
},
"regex": {
"hashes": [
"sha256:0306149889c1a1bec362511f737bc446245ddfcdbe4b556abdfc506ed46dfa47",
"sha256:4b08704a5939c698d2d5950b5dc950597613216cc8c01048efc0da860a0c3db9",
"sha256:6ba0eb777ada6887062c2620e6d644b011078d5d3dc09119ae7107285f6f95e9",
"sha256:7789cc323948792c4c62b269a56f2f2f9bc77d44e54fd81e01b12a967dd7244c",
"sha256:825143aadca0da7d26eeaf2ab0f8bc33921a5642e570ded92dde08c5aaebc65f",
"sha256:8fbd057faab28ce552d89c46f7a968e950f07e80752dfb93891dd11c6b0ee3b4",
"sha256:a41aabb0b9072a14f1e2e554f959ed6439b83610ed656edace9096a0b27e378e",
"sha256:d8807231aed332a1d0456d2088967b87e8c664222bed8e566384ca0ec0b43bfd",
"sha256:dfd89b642fe71f4e8a9906455d4147d453061377b650e6233ddd9ea822971360"
],
"index": "pypi",
"version": "==2019.3.12"
},
"repoze.lru": {
"hashes": [
"sha256:0429a75e19380e4ed50c0694e26ac8819b4ea7851ee1fc7583c8572db80aff77",
"sha256:f77bf0e1096ea445beadd35f3479c5cff2aa1efe604a133e67150bc8630a62ea"
],
"index": "pypi",
"version": "==0.7"
},
"requests": {
"hashes": [
"sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
"sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
],
"index": "pypi",
"version": "==2.21.0"
},
"requests-oauthlib": {
"hashes": [
"sha256:bd6533330e8748e94bf0b214775fed487d309b8b8fe823dc45641ebcd9a32f57",
"sha256:d3ed0c8f2e3bbc6b344fa63d6f933745ab394469da38db16bdddb461c7e25140"
],
"index": "pypi",
"version": "==1.2.0"
},
"routes": {
"hashes": [
"sha256:26ee43340fca5a32769ffe0c58edcb396ccce6bc1dfa689ddf844d50877355fd",
"sha256:d64b8ae22bef127d856afd9266a3e4cfc9e0dda0e120195e38268a95d20de135"
],
"index": "pypi",
"version": "==2.4.1"
},
"simplejson": {
"hashes": [
"sha256:067a7177ddfa32e1483ba5169ebea1bc2ea27f224853211ca669325648ca5642",
"sha256:2fc546e6af49fb45b93bbe878dea4c48edc34083729c0abd09981fe55bdf7f91",
"sha256:354fa32b02885e6dae925f1b5bbf842c333c1e11ea5453ddd67309dc31fdb40a",
"sha256:37e685986cf6f8144607f90340cff72d36acf654f3653a6c47b84c5c38d00df7",
"sha256:3af610ee72efbe644e19d5eaad575c73fb83026192114e5f6719f4901097fce2",
"sha256:3b919fc9cf508f13b929a9b274c40786036b31ad28657819b3b9ba44ba651f50",
"sha256:3dd289368bbd064974d9a5961101f080e939cbe051e6689a193c99fb6e9ac89b",
"sha256:6c3258ffff58712818a233b9737fe4be943d306c40cf63d14ddc82ba563f483a",
"sha256:75e3f0b12c28945c08f54350d91e624f8dd580ab74fd4f1bbea54bc6b0165610",
"sha256:b1f329139ba647a9548aa05fb95d046b4a677643070dc2afc05fa2e975d09ca5",
"sha256:ee9625fc8ee164902dfbb0ff932b26df112da9f871c32f0f9c1bcf20c350fe2a",
"sha256:fb2530b53c28f0d4d84990e945c2ebb470edb469d63e389bf02ff409012fe7c5"
],
"index": "pypi",
"version": "==3.16.0"
},
"six": {
"hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
],
"index": "pypi",
"version": "==1.12.0"
},
"sqlalchemy": {
"hashes": [
"sha256:781fb7b9d194ed3fc596b8f0dd4623ff160e3e825dd8c15472376a438c19598b"
],
"index": "pypi",
"version": "==1.3.1"
},
"tempora": {
"hashes": [
"sha256:4951da790bd369f718dbe2287adbdc289dc2575a09278e77fad6131bcfe93097",
"sha256:f8abbbd486eca3340bd3d242417b203c861d4e113ef778cd5fb9535b2b32ae54"
],
"version": "==1.14"
},
"urllib3": {
"hashes": [
"sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
"sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
],
"index": "pypi",
"version": "==1.24.1"
},
"webencodings": {
"hashes": [
"sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
"sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"
],
"version": "==0.5.1"
},
"zc.lockfile": {
"hashes": [
"sha256:95a8e3846937ab2991b61703d6e0251d5abb9604e18412e2714e1b90db173253"
],
"version": "==1.4"
}
},
"develop": {}
}

151
PostgresSession.py Normal file
View File

@ -0,0 +1,151 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
PostgresSession.py
Copyright 2014 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
A rewrite of the cherrypy PostgresqlSession.
Usage:
import PostgresSession
cherrypy.lib.sessions.PostgresSession = PostgresSession.PostgresSession
"""
import datetime
import logging
import pickle
import cherrypy
import cherrypy.lib.sessions
class PostgresSession (cherrypy.lib.sessions.Session):
""" Implementation of the PostgreSQL backend for sessions. It assumes
a table like this::
create table <table_name> (
id varchar (40) primary key,
expires timestamp,
data bytea
)
You must provide your own `get_dbapi20_connection ()` function.
"""
pickle_protocol = pickle.HIGHEST_PROTOCOL
select = 'select expires, data from table_name where id=%(id)s for update'
def __init__ (self, id_ = None, **kwargs):
self.table_name = kwargs.get ('table', 'session')
# Session.__init__ () may need working connection
self.connection = self.get_dbapi20_connection ()
super (PostgresSession, self).__init__ (id_, **kwargs)
@staticmethod
def get_dbapi20_connection ():
""" Return a dbapi 2.0 compatible connection. """
return cherrypy.engine.pool.connect ()
@classmethod
def setup (cls, **kwargs):
"""Set up the storage system for Postgres-based sessions.
This should only be called once per process; this will be done
automatically when using sessions.init (as the built-in Tool does).
"""
cherrypy.log ("Using PostgresSession",
context = 'SESSION', severity = logging.INFO)
for k, v in kwargs.items ():
setattr (cls, k, v)
def now (self):
"""Generate the session specific concept of 'now'.
Other session providers can override this to use alternative,
possibly timezone aware, versions of 'now'.
"""
return datetime.datetime.utcnow ()
def _exec (self, sql, **kwargs):
""" Internal helper to execute sql statements. """
kwargs['id'] = self.id
cursor = self.connection.cursor ()
cursor.execute (sql.replace ('table_name', self.table_name), kwargs)
return cursor
def _exists (self):
""" Return true if session data exists. """
cursor = self._exec (self.select)
return bool (cursor.fetchall ())
def _load (self):
""" Load the session data. """
cursor = self._exec (self.select)
rows = cursor.fetchall ()
if not rows:
return None
expires, pickled_data = rows[0]
data = pickle.loads (pickled_data)
return data, expires
def _save (self, expires):
""" Save the session data. """
pickled_data = pickle.dumps (self._data, self.pickle_protocol)
self._delete ()
self._exec (
"""\
insert into table_name (id, expires, data)
values (%(id)s, %(expires)s, %(data)s)
""",
data = pickled_data,
expires = expires
)
def _delete (self):
""" Delete the session data. """
self._exec ('delete from table_name where id=%(id)s')
def acquire_lock (self):
"""Acquire an exclusive lock on the currently-loaded session data."""
self._exec (self.select)
self.locked = True
def release_lock (self):
"""Release the lock on the currently-loaded session data."""
self.connection.commit ()
self.locked = False
def clean_up (self):
"""Clean up expired sessions."""
self._exec (
'delete from table_name where expires < %(now)s',
now = self.now ()
)

72
QRCodePage.py Normal file
View File

@ -0,0 +1,72 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
QRCodePage.py
Copyright 2014 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
A page to generate QR-codes.
"""
from __future__ import unicode_literals
import six
from six.moves import urllib
import cherrypy
import qrcode
class QRCodePage (object):
""" Serve a QR-code as PNG image. """
def index (self, **kwargs):
""" Output QR-Code.
Parameters are:
data: the data to encode (url quoted)
ec_level: error correction level. One of: L M Q H
version: QR code version
box_size: size of one QR code box in pixel
border: width of border in boxes (should be at least 4)
"""
qr = qrcode.QRCode (
error_correction = self._get_ecl (kwargs),
version = kwargs.get ('version', None),
box_size = kwargs.get ('box_size', 10),
border = kwargs.get ('border', 4),
)
qr.add_data (urllib.parse.unquote (kwargs['data']))
qr.make (fit = True)
img = qr.make_image ()
cherrypy.response.headers['Content-Type'] = 'image/png'
buf = six.BytesIO ()
img._img.save (buf, 'PNG')
return buf.getvalue ()
@staticmethod
def _get_ecl (kwargs):
""" Get and decode error correction paramter. """
ecl = {
'L': 1,
'M': 0,
'Q': 3,
'H': 2,
}
if 'ec_level' in kwargs and kwargs['ec_level'] in ecl:
return ecl[kwargs['ec_level']]
return ecl['M']

32
README.md Normal file
View File

@ -0,0 +1,32 @@
# autocat3_original
## Intruduction
**autocat3** is a python-based application used for supporting [Project Gutenberg](gutenberg.org).
CherryPy is used as the web framwork which is easy to develop.
If mainly implemented the search functionality and rate limiter. Also return results pages based on templates.
## How it works.
Currently, the dev environment and production environment share the same server **login1**.
The production version of autocat3 is on **gutenberg2**.
This application in this repository is on **gutenberg1**.
Previously, the old version of autocat3 relies on dependencies installed directly on the system. To make it more flexible and easy to deploy, we tend to use virtual env rather than the previous method. To use virtual env, we use pipenv instead of using pip and virtual env separately.
The virtual env directory is on the default directory while we run ```pipenv --three```. So it's not in this directory. (we are only using python3 for this project because CherryPy will discard the python2 in the future).
To start the service/application, we use **systemd** to do that. the ```autocat3.service``` file is written under ```/etc/systemd/system```directory.
*To start*:
1. make sure ```sudo systemctl daemon-reload``` everytime we edit the systemd unit file
2. ```sudo systemctl start autocat3.service``` to start service
3. ```sudo systemctl stop autocat3.service``` to stop service
4. ```sudo systemctl status autocat3.service``` to check the running status of the service
## How to install
Currently, we use the following steps to deploy autocat3 on a different server.
1. **Create Virtual env**: ```pipenv --three``` to create a virtual env for current working directory(current project)
2. **Install packages/python modules**: ```pipenv install``` to install all the packages in the Pipfile. If there is a requirements.txt file output from ```pip freeze```, the command will automatically add the package names into Pipfile and install the packages and keep them in the Popfile for later use.
3. **lock the packages**: ```pipenv lock``` to be used to produce deterministic builds.
4. **Check the virtual env path**: ```pipenv --venv```
5. **Start virtual env**: ```pipenv shell```
Copyright 2009-2010 by Marcello Perathoner

1124
RateLimiter-new.py Normal file
View File

File diff suppressed because it is too large Load Diff

1131
RateLimiter.py Normal file
View File

File diff suppressed because it is too large Load Diff

365
SearchPage.py Normal file
View File

@ -0,0 +1,365 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
SearchPage.py
Copyright 2009-2012 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
The various flavors of search page.
"""
from __future__ import unicode_literals
import cherrypy
from libgutenberg.MediaTypes import mediatypes as mt
from libgutenberg.DublinCore import DublinCore
import BaseSearcher
from Page import SearchPage
class BookSearchPage (SearchPage):
""" search term => list of books """
def setup (self, os, sql):
os.sort_orders = ('downloads', 'release_date', 'title', 'random')
os.icon = 'book'
os.class_ += 'booklink'
os.f_format_icon = os.format_icon_titles
if os.sort_order == 'random':
sql.where.append ("""
pk in (select floor (random () * maxbook)::integer
from generate_series (1, 30), (select max (pk) as maxbook
from books) xbks1)
""")
if len (os.query):
sql.fulltext ('books.tsvec', os.query)
os.title = _("Books: {title}").format (title = os.query)
else:
os.title = _('All Books')
def fixup (self, os):
""" strip marc subfields, add social media hints and facet links """
for e in os.entries:
if '$' in e.title:
e.title = DublinCore.strip_marc_subfields (e.title)
if (os.sort_order == 'release_date' and os.total_results > 0 and os.start_index == 1):
cat = BaseSearcher.Cat ()
cat.title = _('Follow new books on Twitter')
cat.subtitle = _("Follow our new books on Twitter.")
cat.url = 'https://twitter.com/gutenberg_new'
cat.class_ += 'navlink grayed'
cat.icon = 'twitter'
cat.order = 5
os.entries.insert (0, cat)
cat = BaseSearcher.Cat ()
cat.title = _('Follow new books on Facebook')
cat.subtitle = _("Follow the link and like the page to have us post new books to your wall.")
cat.url = 'https://www.facebook.com/gutenberg.new'
cat.class_ += 'navlink grayed'
cat.icon = 'facebook'
cat.order = 5
os.entries.insert (0, cat)
if (len (os.query) and os.start_index == 1):
sql2 = BaseSearcher.SQLStatement ()
sql2.query = "select count (*) from bookshelves"
sql2.fulltext ('bookshelves.tsvec', os.query)
rows = BaseSearcher.SQLSearcher.execute (*sql2.build ())
if rows[0][0] > 0:
cat = BaseSearcher.Cat ()
cat.rel = 'related'
cat.title = _('Bookshelves')
cat.subtitle = __('One bookshelf matches your query.',
'{count} bookshelves match your search.',
rows[0][0]).format (count = rows[0][0])
cat.url = os.url ('bookshelf_search', query = os.query)
cat.class_ += 'navlink grayed'
cat.icon = 'bookshelf'
cat.order = 3
os.entries.insert (0, cat)
sql2 = BaseSearcher.SQLStatement ()
sql2.query = "select count (*) from subjects"
sql2.fulltext ('subjects.tsvec', os.query)
rows = BaseSearcher.SQLSearcher.execute (*sql2.build ())
if rows[0][0] > 0:
cat = BaseSearcher.Cat ()
cat.rel = 'related'
cat.title = _('Subjects')
cat.subtitle = __('One subject heading matches your search.',
'{count} subject headings match your search.',
rows[0][0]).format (count = rows[0][0])
cat.url = os.url ('subject_search', query = os.query)
cat.class_ += 'navlink grayed'
cat.icon = 'subject'
cat.order = 3
os.entries.insert (0, cat)
sql2 = BaseSearcher.SQLStatement ()
sql2.query = "select count (*) from authors"
sql2.fulltext ('authors.tsvec', os.query)
rows = BaseSearcher.SQLSearcher.execute (*sql2.build ())
if rows[0][0] > 0:
cat = BaseSearcher.Cat ()
cat.rel = 'related'
cat.title = _('Authors')
cat.subtitle = __('One author name matches your search.',
'{count} author names match your search.',
rows[0][0]).format (count = rows[0][0])
cat.url = os.url ('author_search', query = os.query)
cat.class_ += 'navlink grayed'
cat.icon = 'author'
cat.order = 3
os.entries.insert (0, cat)
class AuthorSearchPage (SearchPage):
""" name => list of authors """
def setup (self, os, sql):
os.f_format_subtitle = os.format_subtitle
os.f_format_url = BaseSearcher.SearchUrlFormatter ('author')
os.f_format_thumb_url = os.format_none
os.sort_orders = ('downloads', 'quantity', 'alpha', 'release_date')
os.icon = 'author'
os.class_ += 'navlink'
os.title = _('All Authors')
sql.query = """
SELECT
authors.author as title,
coalesce (authors.born_floor || '', '') || '-' ||
coalesce (authors.died_floor || '', '') as subtitle,
authors.pk as pk,
max (books.release_date) as release_date,
sum (books.downloads) as downloads,
count (books.pk) as quantity"""
sql.from_ = ('authors', 'mn_books_authors as mn', 'books')
sql.groupby += ('authors.author', 'subtitle', 'authors.pk')
sql.where.append ('authors.pk = mn.fk_authors')
sql.where.append ('books.pk = mn.fk_books')
if len (os.query):
sql.fulltext ('authors.tsvec', os.query)
os.title = _("Authors: {author}").format (author = os.query)
else:
sql.where.append ("authors.author not in ('Various', 'Anonymous', 'Unknown')")
class SubjectSearchPage (SearchPage):
""" term => list of subects """
def setup (self, os, sql):
os.f_format_url = BaseSearcher.SearchUrlFormatter ('subject')
os.f_format_thumb_url = os.format_none
os.sort_orders = ('downloads', 'quantity', 'alpha', 'release_date')
os.icon = 'subject'
os.class_ += 'navlink'
os.title = _('All Subjects')
sql.query = """
SELECT
subjects.subject as title,
subjects.pk as pk,
max (books.release_date) as release_date,
sum (books.downloads) as downloads,
count (books.pk) as quantity"""
sql.from_ = ('subjects', 'mn_books_subjects as mn', 'books')
sql.groupby += ('subjects.subject', 'subjects.pk')
sql.where.append ('subjects.pk = mn.fk_subjects')
sql.where.append ('books.pk = mn.fk_books')
if len (os.query):
sql.fulltext ('subjects.tsvec', os.query)
os.title = _("Subjects: {subject}").format (subject = os.query)
class BookshelfSearchPage (SearchPage):
""" term => list of bookshelves """
def setup (self, os, sql):
os.f_format_url = BaseSearcher.SearchUrlFormatter ('bookshelf')
os.f_format_thumb_url = os.format_none
os.sort_orders = ('downloads', 'quantity', 'alpha', 'release_date')
os.icon = 'bookshelf'
os.class_ += 'navlink'
os.title = _('All Bookshelves')
sql.query = """
SELECT
bookshelves.bookshelf as title,
bookshelves.pk as pk,
max (books.release_date) as release_date,
sum (books.downloads) as downloads,
count (books.pk) as quantity"""
sql.from_ = ('bookshelves', 'mn_books_bookshelves as mn', 'books')
sql.groupby += ('bookshelves.bookshelf', 'bookshelves.pk')
sql.where.append ('bookshelves.pk = mn.fk_bookshelves')
sql.where.append ('books.pk = mn.fk_books')
if len (os.query):
sql.fulltext ('bookshelves.tsvec', os.query)
os.title = _("Bookshelves: {bookshelf}").format (bookshelf = os.query)
class AuthorPage (SearchPage):
""" author id => books by author """
def setup (self, os, sql):
os.sort_orders = ('downloads', 'title', 'release_date')
os.title_icon = 'author'
os.icon = 'book'
os.class_ += 'booklink'
os.f_format_icon = os.format_icon_titles
os.author = BaseSearcher.sql_get (
"select author from authors where pk = %(pk)s", pk = os.id)
os.title = _('Books by {author}').format (author = os.author)
sql.from_.append ('mn_books_authors as mn')
sql.where.append ('books.pk = mn.fk_books')
sql.where.append ("mn.fk_authors = %(fk_authors)s")
sql.params['fk_authors'] = os.id
def fixup (self, os):
if (os.start_index == 1 and len (os.entries) > 1):
# browse-by-author page for maintainers
if 'is-catalog-maintainer' in cherrypy.request.cookie:
cat = BaseSearcher.Cat ()
cat.type = mt.html
cat.rel = 'related'
cat.title = _('Browse by Author')
cat.url = "/browse/authors/%s#a%d" % (os.author[:1].lower (), os.id)
cat.class_ += 'navlink grayed'
cat.icon = 'internal'
cat.order = 9
os.entries.insert (0, cat)
# wikipedia links etc.
rows = BaseSearcher.SQLSearcher.execute (
"""SELECT url, description AS title FROM author_urls
WHERE fk_authors = %(fk_authors)s""",
{ 'fk_authors': os.id } )
for row in rows:
cat = BaseSearcher.Cat ()
cat.type = mt.html
cat.rel = 'related'
cat.title = _('See also: {title}').format (title = row.title)
cat.url = row.url
cat.class_ += 'navlink grayed'
cat.icon = 'external'
cat.order = 8
os.entries.insert (0, cat)
# author aliases
if os.format in ('html', 'mobile'):
rows = BaseSearcher.SQLSearcher.execute (
"""SELECT alias AS title FROM aliases
WHERE fk_authors = %(fk_authors)s AND alias_heading = 1""",
{ 'fk_authors': os.id }
)
for row in rows:
cat = BaseSearcher.Cat ()
cat.title = _('Alias {alias}').format (alias = row.title)
cat.class_ += 'grayed'
cat.icon = 'alias'
cat.order = 7
os.entries.insert (0, cat)
class SubjectPage (SearchPage):
""" subject id => books about subject """
def setup (self, os, sql):
os.sort_orders = ('downloads', 'title', 'release_date')
os.title_icon = 'subject'
os.icon = 'book'
os.class_ += 'booklink'
os.f_format_icon = os.format_icon_titles
os.subject = BaseSearcher.sql_get (
"select subject from subjects where pk = %(pk)s", pk = os.id)
os.title = _('Books about {subject}').format (subject = os.subject)
sql.from_.append ('mn_books_subjects as mn')
sql.where.append ('books.pk = mn.fk_books')
sql.where.append ("mn.fk_subjects = %(fk_subjects)s")
sql.params['fk_subjects'] = os.id
class BookshelfPage (SearchPage):
""" bookshelf id => books on bookshelf """
def setup (self, os, sql):
os.sort_orders = ('downloads', 'title', 'release_date')
os.title_icon = 'bookshelf'
os.icon = 'book'
os.class_ += 'booklink'
os.f_format_icon = os.format_icon_titles
os.bookshelf = BaseSearcher.sql_get (
"select bookshelf from bookshelves where pk = %(pk)s", pk = os.id)
os.title = _('Books in {bookshelf}').format (bookshelf = os.bookshelf)
sql.from_.append ('mn_books_bookshelves as mn')
sql.where.append ('books.pk = mn.fk_books')
sql.where.append ("mn.fk_bookshelves = %(fk_bookshelves)s")
sql.params['fk_bookshelves'] = os.id
class AlsoDownloadedPage (SearchPage):
""" ebook id => books people also downloaded """
def setup (self, os, sql):
os.sort_orders = ('downloads', )
os.icon = 'book'
os.class_ += 'booklink'
os.f_format_icon = os.format_icon_titles
os.title = _('Readers also downloaded')
sql.query = """
SELECT
books.pk,
books.title,
books.filing,
books.author,
books.release_date,
books.fk_categories,
books.fk_langs,
books.coverpages,
d.dl as downloads
FROM
v_appserver_books_4 as books
JOIN (
SELECT
s1.fk_books as pk, count (s1.id) as dl
FROM
scores.also_downloads as s1,
scores.also_downloads as s2
WHERE s2.fk_books = %(fk_books)s
AND s1.fk_books != %(fk_books)s
AND s1.id = s2.id
GROUP BY s1.fk_books) as d
ON d.pk = books.pk"""
sql.from_ = ()
sql.params['fk_books'] = os.id
def finalize (self, os):
# one page is enough
os.show_next_page_link = False

81
Sitemap.py Normal file
View File

@ -0,0 +1,81 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: iso-8859-1 -*-
"""
Sitemap.py
Copyright 2009-2013 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Output a Google sitemap.
"""
from __future__ import unicode_literals
from __future__ import division
import datetime
import cherrypy
from libgutenberg.GutenbergGlobals import Struct
import TemplatedPage
import BaseSearcher
SITEMAP_SIZE = 1000 # max no. of urls to put into one sitemap
class Sitemap (TemplatedPage.TemplatedPage):
""" Output Google sitemap. """
def index (self, **kwargs):
""" Output sitemap. """
urls = []
start = int (kwargs['page']) * SITEMAP_SIZE
rows = BaseSearcher.SQLSearcher.execute (
'select pk from books where pk >= %(start)s and pk < %(end)s order by pk',
{ 'start': str (start), 'end': str (start + SITEMAP_SIZE) })
os = BaseSearcher.OpenSearch ()
host = cherrypy.config['host']
for row in rows:
url = Struct ()
url.loc = os.url ('bibrec', id = row[0], host = host, format = None)
urls.append (url)
data = Struct ()
data.urls = urls
return self.output ('sitemap', data = data)
class SitemapIndex (TemplatedPage.TemplatedPage):
""" Output Google sitemap index. """
def index (self, **dummy_kwargs):
""" Output sitemap index. """
sitemaps = []
now = datetime.datetime.utcnow ().replace (microsecond = 0).isoformat () + 'Z'
# 99999 is safeguard against bogus ebook numbers
lastbook = BaseSearcher.sql_get ('select max (pk) as lastbook from books where pk < 99999')
os = BaseSearcher.OpenSearch ()
host = cherrypy.config['host']
for n in range (0, lastbook // SITEMAP_SIZE + 1):
sitemap = Struct ()
sitemap.loc = os.url ('sitemap_index', page = n, host = host, format = None)
sitemap.lastmod = now
sitemaps.append (sitemap)
data = Struct ()
data.sitemaps = sitemaps
return self.output ('sitemap-index', data = data)

75
StartPage.py Normal file
View File

@ -0,0 +1,75 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
StartPage.py
Copyright 2009-2012 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
The Search Start Page.
"""
from __future__ import unicode_literals
import BaseSearcher
import Page
class Start (Page.Page):
""" The start page. """
def index (self, **dummy_kwargs):
""" Output the start page. """
os = BaseSearcher.OpenSearch ()
os.log_request ('start')
os.search_terms = ''
os.title = {
'mobile': _('PG Mobile'),
'opds': _('Project Gutenberg'),
'stanza': _('Project Gutenberg')
}.get (os.format, _('Search Project Gutenberg'))
cat = BaseSearcher.Cat ()
cat.header = _(
'Welcome to Project Gutenberg. Use the search box to find your book or pick a link.')
cat.title = _('Popular')
cat.subtitle = _('Our most popular books.')
cat.url = os.url ('search', sort_order = 'downloads')
cat.class_ += 'navlink'
cat.icon = 'popular'
cat.order = 2
os.entries.append (cat)
cat = BaseSearcher.Cat ()
cat.title = _('Latest')
cat.subtitle = _('Our latest releases.')
cat.url = os.url ('search', sort_order = 'release_date')
cat.class_ += 'navlink'
cat.icon = 'date'
cat.order = 3
os.entries.append (cat)
cat = BaseSearcher.Cat ()
cat.title = _('Random')
cat.subtitle = _('Random books.')
cat.url = os.url ('search', sort_order = 'random')
cat.class_ += 'navlink'
cat.icon = 'random'
cat.order = 4
os.entries.append (cat)
os.total_results = 0
os.template = 'results'
os.page = 'start'
os.url_share = os.url ('/', host = os.file_host)
os.twit = os.tagline
os.finalize ()
return self.format (os)

82
StatsPage.py Normal file
View File

@ -0,0 +1,82 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
StatsPage.py
Copyright 2009-2014 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
The appserver stats page.
"""
from __future__ import unicode_literals
import cherrypy
import BaseSearcher
import TemplatedPage
import asyncdns
import ipinfo
class StatsPage (TemplatedPage.TemplatedPage):
""" Output some statistics. """
CONTENT_TYPE = 'application/xhtml+xml; charset=UTF-8'
FORMATTER = 'html'
def index (self, **kwargs):
""" Output stats. """
backends = int (BaseSearcher.sql_get ("SELECT count (*) from pg_stat_activity"))
active_backends = int (BaseSearcher.sql_get (
"SELECT count (*) - 1 from pg_stat_activity where current_query !~ '^<IDLE>'"))
ipsessions = list (cherrypy.tools.rate_limiter.cache.values ()) # pylint: disable=E1101
adns = asyncdns.AsyncDNS ()
# blocked IPs
blocked = sorted ([s for s in ipsessions if s.get ('blocked', 0) >= 2],
key = lambda s: s.ips.sort_key ())
if 'resolve' in kwargs:
for d in blocked:
if d.ips.ipinfo is None:
d.ips.ipinfo = ipinfo.IPInfo (adns, d.ips.get_ip_to_block ())
# active IPs
active = sorted ([s for s in ipsessions if s.get ('active', False)],
key = lambda s: s.ips.sort_key ())
# busiest IPs
busiest = sorted ([s for s in active if s.get ('blocked', 0) < 2],
key = lambda x: -x.get ('rhits'))[:10]
if 'resolve' in kwargs:
for d in busiest:
if d.ips.ipinfo is None:
d.ips.ipinfo = ipinfo.IPInfo (adns, d.ips.get_ip_to_block ())
# IPs with most sessions
most_sessions = sorted ([s for s in active
if not s.ips.whitelisted and len (s.sessions) > 1],
key = lambda s: -len (s.sessions))[:10]
if 'resolve' in kwargs:
for d in most_sessions:
if d.ips.ipinfo is None:
d.ips.ipinfo = ipinfo.IPInfo (adns, d.ips.get_ip_to_block ())
adns.wait ()
adns.cancel ()
return self.output ('stats',
active = active,
blocked = blocked,
busiest = busiest,
most_sessions = most_sessions,
resolve = 'resolve' in kwargs,
rl = cherrypy.tools.rate_limiter, # pylint: disable=E1101
backends = backends,
active_backends = active_backends)

86
SuggestionsPage.py Normal file
View File

@ -0,0 +1,86 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
SuggestionsPage.py
Copyright 2012 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
The search suggestions page.
"""
from __future__ import unicode_literals
import logging
import cherrypy
from libgutenberg.GutenbergDatabase import DatabaseError
import BaseSearcher
import Page
class Suggestions (Page.Page):
""" Output the search suggestions page. """
sql_searcher = BaseSearcher.SQLSearcher ()
def index (self, **dummy_kwargs):
""" Output the suggestions page. """
cherrypy.request.params['format'] = 'json' # override user
os = BaseSearcher.OpenSearch ()
os.sort_order = 'nentry'
os.start_index = 1
os.items_per_page = 5
if os.format != 'json':
raise cherrypy.HTTPError (400, 'Bad Request. Unknown format.')
if len (os.query) == 0:
raise cherrypy.HTTPError (400, 'Bad Request. No query.')
last_word = os.query.split ()[-1]
if len (last_word) < 4:
raise cherrypy.HTTPError (400, 'Bad Request. Query too short.')
# ok. request looks sane. process it
os.log_request ('suggestions')
os.f_format_title = os.format_suggestion
os.f_format_subtitle = os.format_none
os.f_format_extra = os.format_none
os.f_format_url = os.format_none
os.f_format_thumb_url = os.format_none
os.f_format_icon = os.format_none
sql = BaseSearcher.SQLStatement ()
# prepare inner query
sql.query = 'SELECT tsvec'
sql.from_ = ('books', )
sql.fulltext ('books.tsvec', os.query)
inner_sql_query = self.sql_searcher.mogrify (os, sql)
sql.query = "SELECT substr (word, 2) AS title FROM ts_stat ( %(inner)s )"
sql.from_ = ()
sql.params['inner'] = inner_sql_query
sql.where = ["word ~* %(re_word)s"]
sql.params['re_word'] = '^0' + last_word
try:
os = self.sql_searcher.search (os, sql)
except DatabaseError as what:
cherrypy.log ("SQL Error: " + str (what),
context = 'REQUEST', severity = logging.ERROR)
raise cherrypy.HTTPError (500, 'Internal Server Error.')
os.template = os.page = 'results'
os.finalize ()
return self.format (os)

272
SupportedLocales.py Normal file
View File

@ -0,0 +1,272 @@
""" Locales supported by 3rd party apps. Generated file. Do not edit! """
FB_LANGS = set ((
'af_ZA',
'ar_AR',
'az_AZ',
'be_BY',
'bg_BG',
'bn_IN',
'bs_BA',
'ca_ES',
'cs_CZ',
'cx_PH',
'cy_GB',
'da_DK',
'de_DE',
'el_GR',
'en_GB',
'en_PI',
'en_UD',
'en_US',
'eo_EO',
'es_ES',
'es_LA',
'et_EE',
'eu_ES',
'fa_IR',
'fb_LT',
'fi_FI',
'fo_FO',
'fr_CA',
'fr_FR',
'fy_NL',
'ga_IE',
'gl_ES',
'gn_PY',
'gu_IN',
'he_IL',
'hi_IN',
'hr_HR',
'hu_HU',
'hy_AM',
'id_ID',
'is_IS',
'it_IT',
'ja_JP',
'ja_KS',
'jv_ID',
'ka_GE',
'kk_KZ',
'km_KH',
'kn_IN',
'ko_KR',
'ku_TR',
'la_VA',
'lt_LT',
'lv_LV',
'mk_MK',
'ml_IN',
'mn_MN',
'mr_IN',
'ms_MY',
'nb_NO',
'ne_NP',
'nl_NL',
'nn_NO',
'pa_IN',
'pl_PL',
'ps_AF',
'pt_BR',
'pt_PT',
'ro_RO',
'ru_RU',
'si_LK',
'sk_SK',
'sl_SI',
'sq_AL',
'sr_RS',
'sv_SE',
'sw_KE',
'ta_IN',
'te_IN',
'tg_TJ',
'th_TH',
'tl_PH',
'tr_TR',
'uk_UA',
'ur_PK',
'uz_UZ',
'vi_VN',
'zh_CN',
'zh_HK',
'zh_TW',
))
FLATTR_LANGS = set ((
'ar_DZ',
'be_BY',
'bg_BG',
'br_FR',
'ca_ES',
'cs_CZ',
'da_DK',
'de_DE',
'el_GR',
'en_GB',
'eo_EO',
'es_ES',
'es_GL',
'et_EE',
'fa_FA',
'fi_FI',
'fr_FR',
'ga_IE',
'hi_IN',
'hr_HR',
'hu_HU',
'in_ID',
'is_IS',
'it_IT',
'iw_IL',
'ja_JP',
'ko_KR',
'lt_LT',
'lv_LV',
'mk_MK',
'ms_MY',
'mt_MT',
'nl_NL',
'nn_NO',
'no_NO',
'pl_PL',
'pt_PT',
'ro_RO',
'ru_RU',
'sk_SK',
'sl_SI',
'sq_AL',
'sr_RS',
'sv_SE',
'th_TH',
'tr_TR',
'uk_UA',
'vi_VN',
'zh_CN',
))
GOOGLE_LANGS = set ((
'af',
'am',
'ar',
'eu',
'bn',
'bg',
'ca',
'zh_HK',
'zh_CN',
'zh_TW',
'hr',
'cs',
'da',
'nl',
'en_GB',
'en_US',
'et',
'fil',
'fi',
'fr',
'fr_CA',
'gl',
'de',
'el',
'gu',
'iw',
'hi',
'hu',
'is',
'id',
'it',
'ja',
'kn',
'ko',
'lv',
'lt',
'ms',
'ml',
'mr',
'no',
'fa',
'pl',
'pt_BR',
'pt_PT',
'ro',
'ru',
'sr',
'sk',
'sl',
'es',
'es_419',
'sw',
'sv',
'ta',
'te',
'th',
'tr',
'uk',
'ur',
'vi',
'zu',
))
TWITTER_LANGS = set ((
'fr',
'en',
'ar',
'ja',
'es',
'de',
'it',
'id',
'pt',
'ko',
'tr',
'ru',
'nl',
'fil',
'msa',
'zh-tw',
'zh-cn',
'hi',
'no',
'sv',
'fi',
'da',
'pl',
'hu',
'fa',
'he',
'th',
'uk',
'cs',
'ro',
'en-gb',
'vi',
'bn',
))
PAYPAL_LANGS = set ((
'da_DK',
'de_DE',
'en_GB',
'en_US',
'es_ES',
'fr_CA',
'fr_FR',
'he_IL',
'id_ID',
'it_IT',
'ja_JP',
'nl_NL',
'pl_PL',
'pt_BR',
'pt_PT',
'ru_RU',
'sv_SE',
'tr_TR',
'zh_CN',
'zh_HK',
'zh_TW',
))

68
TemplatedPage.py Normal file
View File

@ -0,0 +1,68 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: iso-8859-1 -*-
"""
TemplatedPage.py
Copyright 2013 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Output a Genshi page.
"""
from __future__ import unicode_literals
import genshi.output
import genshi.template
from genshi.core import _ensure
import cherrypy
import Formatters
import BaseSearcher
class TemplatedPage (object):
""" Output a page from a genshi template. """
CONTENT_TYPE = 'application/xml; charset=UTF-8'
FORMATTER = 'xml'
def get_serializer (self):
""" Override to get a different serializer. """
return genshi.output.XMLSerializer (strip_whitespace = False)
def output (self, template, **kwargs):
""" Output the page. """
# Send HTTP content-type header.
cherrypy.response.headers['Content-Type'] = self.CONTENT_TYPE
template = Formatters.formatters[self.FORMATTER].templates[template]
ctxt = genshi.template.Context (cherrypy = cherrypy, bs = BaseSearcher, **kwargs)
stream = template.stream
for filter_ in template.filters:
stream = filter_ (iter (stream), ctxt)
serializer = self.get_serializer ()
return genshi.output.encode (serializer (
_ensure (genshi.Stream (stream))), encoding = 'utf-8')
class TemplatedPageXHTML (TemplatedPage):
""" Output a page from a genshi template. """
CONTENT_TYPE = 'text/html; charset=UTF-8'
FORMATTER = 'html'
DOCTYPE = ('html',
'-//W3C//DTD XHTML 1.0 Strict//EN',
'http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd')
def get_serializer (self):
return genshi.output.XHTMLSerializer (doctype = self.DOCTYPE, strip_whitespace = False)

51
Timer.py Normal file
View File

@ -0,0 +1,51 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
Timer.py
Copyright 2010-2013 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
A cron-like process that runs at fixed intervals.
Usage:
import Timer
cherrypy.process.plugins.Timer = TimerPlugin
"""
from __future__ import unicode_literals
import cherrypy
import BaseSearcher
class TimerPlugin (cherrypy.process.plugins.Monitor):
""" Plugin to start the timer thread.
We cannot start any threads before daemonizing,
so we must start the timer thread by this plugin.
"""
def __init__ (self, bus):
# interval in seconds
frequency = 300
super (TimerPlugin, self).__init__ (bus, self.tick, frequency)
self.name = 'timer'
def start (self):
super (TimerPlugin, self).start ()
self.tick ()
start.priority = 80
def tick (self):
""" Do things here. """
try:
BaseSearcher.books_in_archive = BaseSearcher.sql_get ('select count (*) from books')
except:
pass

17
UserAgents.py Normal file
View File

@ -0,0 +1,17 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
UserAgent.py
Copyright 2010 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Mobile Browser Detection.
"""
from __future__ import unicode_literals
UA = "iPhone MIDP BlackBerry Android Mobile webOS Blazer Kindle nook"

2
__init__.py Normal file
View File

@ -0,0 +1,2 @@
""" Package """
name = "autocat3"

200
asyncdns.py Normal file
View File

@ -0,0 +1,200 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
asyncdns.py
Copyright 2013-14 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Higher level interface to the GNU asynchronous DNS library.
"""
from __future__ import unicode_literals
import sys
import time
import adns
# pass this to __init__ to use Google Public DNS
RESOLV_CONF = 'nameserver 8.8.8.8'
# http://www.ietf.org/rfc/rfc1035.txt Domain Names
# http://www.ietf.org/rfc/rfc3490.txt IDNA
# http://www.ietf.org/rfc/rfc3492.txt Punycode
class AsyncDNS (object):
""" An asynchronous DNS resolver. """
def __init__ (self, resolv_conf = None):
if resolv_conf:
self.resolver = adns.init (
adns.iflags.noautosys + adns.iflags.noerrprint,
sys.stderr, # FIXME: adns version 1.2.2 will allow keyword params
resolv_conf)
else:
self.resolver = adns.init (
adns.iflags.noautosys + adns.iflags.noerrprint)
self._queries = {} # keeps query objects alive
def query (self, query, callback, rr = adns.rr.A):
""" Queue a query.
:param query: the query string (may contain unicode characters)
:param callback: function taking a tuple of answers
:param rr: the query rr type code
"""
if rr not in (adns.rr.PTR, adns.rr.PTRraw):
query = self.encode (query)
if rr in (adns.rr.PTR, adns.rr.PTRraw):
self._queries [self.resolver.submit_reverse (query, rr)] = callback, rr
else:
self._queries [self.resolver.submit (query, rr)] = callback, rr
def query_dnsbl (self, query, zone, callback, rr = adns.rr.A):
""" Queue a reverse dnsbl-type query. """
self._queries [self.resolver.submit_reverse_any (query, zone, rr)] = callback, rr
def done (self):
""" Are all queued queries answered? """
return not self._queries
def wait (self, timeout = 10):
""" Wait for the queries to complete. """
timeout += time.time ()
while self._queries and time.time () < timeout:
for q in self.resolver.completed (1):
answer = q.check ()
callback, rr = self._queries[q]
del self._queries[q]
# print (answer)
a0 = answer[0]
if a0 == 0:
callback (self.decode_answer (rr, answer[3]))
elif a0 == 101 and rr == adns.rr.A:
# got CNAME, wanted A: resubmit
self.query (answer[1], callback, rr)
# else
# pass
def decode_answer (self, rr, answers):
""" Decode the answer to unicode.
Supports only some rr types. You may override this to support
some more.
"""
if rr in (adns.rr.A, adns.rr.TXT):
# A records are ip addresses that need no decoding.
# TXT records may be anything, even binary data,
# so leave decoding to the caller.
return answers
if rr in (adns.rr.PTR, adns.rr.PTRraw, adns.rr.CNAME, adns.rr.NSraw):
return [ self.decode (host) for host in answers ]
if rr == adns.rr.MXraw:
return [ (prio, self.decode (host)) for (prio, host) in answers ]
if rr == adns.rr.SRVraw:
return [ (prio, weight, port, self.decode (host))
for (prio, weight, port, host) in answers ]
if rr in (adns.rr.SOA, adns.rr.SOAraw):
return [ (self.decode (mname), self.decode (rname),
serial, refresh, retry, expire, minimum)
for (mname, rname, serial, refresh,
retry, expire, minimum) in answers ]
# unsupported HINFO, RP, RPraw, NS, SRV, MX
return answers
@staticmethod
def encode (query):
""" Encode a unicode query to idna.
Result will still be of type unicode/str.
"""
return query.encode ('idna').decode ('ascii')
@staticmethod
def decode (answer):
""" Decode an answer to unicode. """
try:
return answer.decode ('idna')
except ValueError:
return answer.decode ('ascii', 'replace')
def cancel (self):
""" Cancel all pending queries. """
for q in self._queries.keys ():
q.cancel ()
self._queries.clear ()
def bulk_query (self, query_dict, rr):
""" Bulk lookup.
:param dict: on entry { query1: None, query2: None }
on exit { query1: (answer1, ), query2: (answer2a, answer2b) }
Note: you must call wait () after bulk_query () for the answers to appear
"""
def itemsetter (query):
""" Return a callable object that puts the answer into
the dictionary under the right key. """
def g (answer):
""" Put the answer into the dictionary. """
query_dict[query] = answer
# print "put: " + answer
return g
for query in query_dict.keys ():
if query:
self.query (query, itemsetter (query), rr)
def bulk_query (dict_, rr = adns.rr.A, timeout = 10):
""" Perform bulk lookup. """
a = AsyncDNS ()
a.bulk_query (dict_, rr)
a.wait (timeout)
a.cancel ()
if __name__ == '__main__':
import netaddr
queries = dict ()
for i in range (64, 64 + 32):
ip = '66.249.%d.42' % i # google assigned netblock
queries[ip] = None
bulk_query (queries, adns.rr.PTR)
ipset = netaddr.IPSet ()
for ip in sorted (queries):
if queries[ip] and 'proxy' in queries[ip][0]:
print (ip)
ipset.add (ip + '/24')
for cidr in ipset.iter_cidrs ():
print (cidr)

341
i18n_tool.py Normal file
View File

@ -0,0 +1,341 @@
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""Internationalization and Localization for CherryPy
**Tested with CherryPy 3.1.2**
This tool provides locales and loads translations based on the
HTTP-ACCEPT-LANGUAGE header. If no header is send or the given language
is not supported by the application, it falls back to
`tools.I18nTool.default`. Set `default` to the native language used in your
code for strings, so you must not provide a .mo file for it.
The tool uses `babel<http://babel.edgewall.org>`_ for localization and
handling translations. Within your Python code you can use four functions
defined in this module and the loaded locale provided as
`cherrypy.response.i18n.locale`.
Example::
from i18n_tool import ugettext as _, ungettext
class MyController(object):
@cherrypy.expose
def index(self):
loc = cherrypy.response.i18n.locale
s1 = _('Translateable string')
s2 = ungettext('There is one string.',
'There are more strings.', 2)
return '<br />'.join([s1, s2, loc.display_name])
If you have code (e.g. database models) that is executed before the response
object is available, use the *_lazy functions to mark the strings
translateable. They will be translated later on, when the text is used (and
hopefully the response object is available then).
Example::
from i18n_tool import ugettext_lazy
class Model:
def __init__(self):
name = ugettext_lazy('Name of the model')
For your templates read the documentation of your template engine how to
integrate babel with it. I think `Genshi<http://genshi.edgewall.org>`_ and
`Jinja 2<http://jinja.pocoo.org`_ support it out of the box.
Settings for the CherryPy configuration::
[/]
tools.I18nTool.on = True
tools.I18nTool.default = Your language with territory (e.g. 'en_US')
tools.I18nTool.mo_dir = Directory holding the locale directories
tools.I18nTool.domain = Your gettext domain (e.g. application name)
The mo_dir must contain subdirectories named with the language prefix
for all translations, containing a LC_MESSAGES dir with the compiled
catalog file in it.
Example::
[/]
tools.I18nTool.on = True
tools.I18nTool.default = 'en_US'
tools.I18nTool.mo_dir = '/home/user/web/myapp/i18n'
tools.I18nTool.domain = 'myapp'
Now the tool will look for a file called myapp.mo in
/home/user/web/myapp/i18n/en/LC_MESSACES/
or generic: <mo_dir>/<language>/LC_MESSAGES/<domain>.mo
That's it.
:License: BSD
:Author: Thorsten Weimann <thorsten.weimann (at) gmx (dot) net>
:Date: 2010-02-08
"""
from __future__ import unicode_literals
import gettext
import cherrypy
import six
from babel.core import Locale, UnknownLocaleError
from babel.support import Translations, LazyProxy
# Cache for Translations objects
_trans_cache = {}
class Struct (object):
""" Empty class to pin attributes on later. """
pass
if six.PY2:
# Public translation functions
def ugettext(message):
"""Standard translation function. You can use it in all your exposed
methods and everywhere where the response object is available.
:parameters:
message : Unicode
The message to translate.
:returns: The translated message.
:rtype: Unicode
"""
if message:
return cherrypy.response.i18n.trans.ugettext(message)
return ''
def ugettext_lazy(message):
"""Like ugettext, but lazy.
:returns: A proxy for the translation object.
:rtype: LazyProxy
"""
def get_translation():
return cherrypy.response.i18n.trans.ugettext(message)
return LazyProxy(get_translation)
def ungettext(singular, plural, num):
"""Like ugettext, but considers plural forms.
:parameters:
singular : Unicode
The message to translate in singular form.
plural : Unicode
The message to translate in plural form.
num : Integer
Number to apply the plural formula on. If num is 1 or no
translation is found, singular is returned.
:returns: The translated message as singular or plural.
:rtype: Unicode
"""
return cherrypy.response.i18n.trans.ungettext(singular, plural, num)
def ungettext_lazy(singular, plural, num):
"""Like ungettext, but lazy.
:returns: A proxy for the translation object.
:rtype: LazyProxy
"""
def get_translation():
return cherrypy.response.i18n.trans.ungettext(singular, plural, num)
return LazyProxy(get_translation)
else: # PY3
# Public translation functions
def ugettext(message):
"""Standard translation function. You can use it in all your exposed
methods and everywhere where the response object is available.
:parameters:
message : Unicode
The message to translate.
:returns: The translated message.
:rtype: Unicode
"""
if message:
return cherrypy.response.i18n.trans.gettext(message)
return ''
def ugettext_lazy(message):
"""Like ugettext, but lazy.
:returns: A proxy for the translation object.
:rtype: LazyProxy
"""
def get_translation():
return cherrypy.response.i18n.trans.gettext(message)
return LazyProxy(get_translation)
def ungettext(singular, plural, num):
"""Like ugettext, but considers plural forms.
:parameters:
singular : Unicode
The message to translate in singular form.
plural : Unicode
The message to translate in plural form.
num : Integer
Number to apply the plural formula on. If num is 1 or no
translation is found, singular is returned.
:returns: The translated message as singular or plural.
:rtype: Unicode
"""
return cherrypy.response.i18n.trans.ngettext(singular, plural, num)
def ungettext_lazy(singular, plural, num):
"""Like ungettext, but lazy.
:returns: A proxy for the translation object.
:rtype: LazyProxy
"""
def get_translation():
return cherrypy.response.i18n.trans.ngettext(singular, plural, num)
return LazyProxy(get_translation)
def load_translation(languages, dirname, domain, default):
"""Loads the first existing translations for known locale and saves the
`Lang` object in a global cache for faster lookup on the next request.
:parameters:
langs : List
List of languages as returned by `parse_accept_language_header`.
dirname : String
Directory of the translations (`tools.I18nTool.mo_dir`).
domain : String
Gettext domain of the catalog (`tools.I18nTool.domain`).
:returns: Lang object with two attributes (Lang.trans = the translations
object, Lang.locale = the corresponding Locale object).
:rtype: Lang
:raises: ImproperlyConfigured if no locale where known.
"""
res = Struct ()
res.trans = gettext.NullTranslations ()
try:
# use the preferred locale for date formatting
# even if we have no translation for it
res.locale = Locale.parse (languages [0])
except (IndexError, ValueError, UnknownLocaleError):
res.locale = Locale.parse (default)
for language in languages:
try:
#cherrypy.log ("trying %s" % str (language),
# context = 'REQUEST', severity = logging.DEBUG)
locale = str (Locale.parse (language))
# cached ?
if (domain, locale) in _trans_cache:
res.trans = _trans_cache[(domain, locale)]
return res
# not cached
trans = Translations.load (dirname, locale, domain)
if isinstance (trans, Translations):
res.trans = _trans_cache [(domain, locale)] = trans
break
except (ValueError, UnknownLocaleError):
continue
return res
def get_lang (mo_dir, default, domain):
"""Main function which will be invoked during the request by `I18nTool`.
If the SessionTool is on and has a lang key, this language get the
highest priority. Default language get the lowest priority.
The `Lang` object will be saved as `cherrypy.response.i18n` and the
language string will also saved as `cherrypy.session['_lang_']` (if
SessionTool is on).
:parameters:
mo_dir : String
`tools.I18nTool.mo_dir`
default : String
`tools.I18nTool.default`
domain : String
`tools.I18nTool.domain`
"""
# try explicit lang param, then session
lang = cherrypy.request.params.get ('lang', None)
if not lang:
try:
lang = cherrypy.session['_lang_']
except (AttributeError, KeyError):
pass
if lang:
lang = lang.replace ('-', '_')
langs = (lang, )
else:
langs = cherrypy.request.headers.elements ('Accept-Language') or []
langs = [x.value.replace ('-', '_') for x in langs]
loc = load_translation (langs, mo_dir, domain, default)
cherrypy.response.i18n = loc
try:
cherrypy.session['_lang_'] = str (loc.locale)
except AttributeError:
pass
def set_lang ():
"""Sets the Content-Language response header (if not already set) to the
language of `cherrypy.response.i18n.locale`.
"""
if 'Content-Language' not in cherrypy.response.headers:
if hasattr (cherrypy.response, 'i18n'):
cherrypy.response.headers['Content-Language'] = str (
cherrypy.response.i18n.locale)
class I18nTool (cherrypy.Tool):
"""Tool to integrate babel translations in CherryPy."""
def __init__ (self):
# cherrypy.Tool.__init__ (self)
self._name = 'I18nTool'
self._point = 'before_handler'
self.callable = get_lang
# Make sure, session tool (priority 50) is loaded before
self._priority = 100
def _setup (self):
c = cherrypy.request.config
if c.get ('tools.staticdir.on', False) or \
c.get ('tools.staticfile.on', False):
return
cherrypy.Tool._setup (self)
dirname = c.get ('tools.I18nTool.mo_dir', 'i18n')
default = c.get ('tools.I18nTool.default', 'en')[:2].lower ()
domain = c.get ('tools.I18nTool.domain', 'messages')
trans = Translations.load (dirname, default, domain)
if isinstance (trans, Translations):
_trans_cache [(domain, default)] = trans
else:
_trans_cache [(domain, default)] = gettext.NullTranslations ()
cherrypy.request.hooks.attach ('before_finalize', set_lang)

321
ipinfo.py Normal file
View File

@ -0,0 +1,321 @@
#!/usr/bin/env python
# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
"""
ipinfo.py
Copyright 2013-14 by Marcello Perathoner
Distributable under the GNU General Public License Version 3 or newer.
Find information about an IP, eg. hostname, whois, DNS blocklists.
The Spamhaus Block List (SBL) Advisory is a database of IP
addresses from which Spamhaus does not recommend the acceptance of
electronic mail.
The Spamhaus Exploits Block List (XBL) is a realtime database
of IP addresses of hijacked PCs infected by illegal 3rd party
exploits, including open proxies (HTTP, socks, AnalogX, wingate,
etc), worms/viruses with built-in spam engines, and other types of
trojan-horse exploits.
The Spamhaus PBL is a DNSBL database of end-user IP address
ranges which should not be delivering unauthenticated SMTP email
to any Internet mail server except those provided for specifically
by an ISP for that customer's use. The PBL helps networks enforce
their Acceptable Use Policy for dynamic and non-MTA customer IP
ranges.
"""
from __future__ import unicode_literals
import asyncdns
# pylint: disable=R0903
class DNSBL (object):
""" Base class for DNS blocklists. """
zone = ''
blackhat_tags = {}
dialup_tags = {}
### TOR ###
# see:
# https://www.torproject.org/projects/tordnsel.html.en
# https://www.dan.me.uk/dnsbl
class TorProject (DNSBL):
""" A TOR exitnode list. """
# note: reverse IP of www.gutenberg.org:80
zone = '80.47.134.19.152.ip-port.exitlist.torproject.org'
blackhat_tags = {
'127.0.0.2': 'TOR',
}
class TorDanme (DNSBL):
""" A TOR exitnode list. """
zone = 'torexit.dan.me.uk'
blackhat_tags = {
'127.0.0.100': 'TOR',
}
### SPAMHAUS ###
# see: http://www.spamhaus.org/faq/answers.lasso?section=DNSBL%20Usage#202
class Spamhaus (DNSBL):
""" A DNS blocklist. """
zone = 'zen.spamhaus.org'
blackhat_tags = {
'127.0.0.2': 'SPAMHAUS_SBL',
'127.0.0.3': 'SPAMHAUS_SBL_CSS',
'127.0.0.4': 'SPAMHAUS_XBL_CBL',
}
dialup_tags = {
'127.0.0.10': 'SPAMHAUS_PBL_ISP',
'127.0.0.11': 'SPAMHAUS_PBL',
}
lookup = 'http://www.spamhaus.org/query/ip/{ip}'
### SORBS ###
# see: http://www.sorbs.net/using.shtml
class SORBS (DNSBL):
""" A DNS blocklist. """
zone = 'dnsbl.sorbs.net'
blackhat_tags = {
'127.0.0.2': 'SORBS_HTTP_PROXY',
'127.0.0.3': 'SORBS_SOCKS_PROXY',
'127.0.0.4': 'SORBS_MISC_PROXY',
'127.0.0.5': 'SORBS_SMTP_RELAY',
'127.0.0.6': 'SORBS_SPAMMER',
'127.0.0.7': 'SORBS_WEB', # formmail etc.
'127.0.0.8': 'SORBS_BLOCK',
'127.0.0.9': 'SORBS_ZOMBIE',
'127.0.0.11': 'SORBS_BADCONF',
'127.0.0.12': 'SORBS_NOMAIL',
}
dialup_tags = {
'127.0.0.10': 'SORBS_DUL',
}
### mailspike.net ###
# see: http://mailspike.net/usage.html
class MailSpike (DNSBL):
""" A DNS blocklist. """
zone = 'bl.mailspike.net'
blackhat_tags = {
'127.0.0.2': 'MAILSPIKE_DISTRIBUTED_SPAM',
'127.0.0.10': 'MAILSPIKE_WORST_REPUTATION',
'127.0.0.11': 'MAILSPIKE_VERY_BAD_REPUTATION',
'127.0.0.12': 'MAILSPIKE_BAD_REPUTATION',
}
### shlink.org ###
# see: http://shlink.org/
class BlShlink (DNSBL):
""" A DNS blocklist. """
zone = 'bl.shlink.org'
blackhat_tags = {
'127.0.0.2': 'SHLINK_SPAM_SENDER',
'127.0.0.4': 'SHLINK_SPAM_ORIGINATOR',
'127.0.0.5': 'SHLINK_POLICY_BLOCK',
'127.0.0.6': 'SHLINK_ATTACKER',
}
class DynShlink (DNSBL):
""" A DNS dul list. """
zone = 'dyn.shlink.org'
dialup_tags = {
'127.0.0.3': 'SHLINK_DUL',
}
### barracudacentral.org ###
# see: http://www.barracudacentral.org/rbl/how-to-usee
class Barracuda (DNSBL):
""" A DNS blocklist. """
zone = 'b.barracudacentral.org'
blackhat_tags = {
'127.0.0.2': 'BARRACUDA_BLOCK',
}
### SHADOWSERVER ###
# http://www.shadowserver.org/wiki/pmwiki.php/Services/IP-BGP
class ShadowServer (DNSBL):
""" A DNS-based whois service. """
zone = 'origin.asn.shadowserver.org'
peer_zone = 'peer.asn.shadowserver.org'
fields = 'asn cidr org2 country org1 org'.split ()
# TEAMCYMRU
# http://www.team-cymru.org/Services/ip-to-asn.html
class TeamCymru (DNSBL):
""" A DNS-based whois service. """
zone = 'origin.asn.cymru.com'
asn_zone = 'asn.cymru.com'
fields = 'asn cidr country registry date'.split ()
class IPInfo (object):
""" Holds DNSBL information for one IP. """
dnsbl = [ Spamhaus, SORBS, MailSpike, BlShlink, DynShlink, Barracuda,
TorProject, TorDanme ]
""" Which blocklists to consider. """
def __init__ (self, aresolver, ip):
self.hostname = None
self.whois = {}
self.blackhat_tags = set ()
self.dialup_tags = set ()
ip = str (ip)
rr = asyncdns.adns.rr
try:
aresolver.query (ip, self._hostnamesetter (), rr.PTR)
for dnsbl in self.dnsbl:
aresolver.query_dnsbl (ip, dnsbl.zone, self._tagsetter (dnsbl))
# ShadowServer seems down: March 2014
aresolver.query_dnsbl (ip, ShadowServer.zone, self._whoissetter_ss (), rr.TXT)
# aresolver.query_dnsbl (ip, TeamCymru.zone, self._whoissetter_tc (aresolver), rr.TXT)
except:
pass
@property
def tags (self):
""" All tags (bad and dialup). """
return self.blackhat_tags | self.dialup_tags
def is_blackhat (self):
""" Return true if this is probably a blackhat IP. """
return bool (self.blackhat_tags)
def is_dialup (self):
""" Test if this IP is a dialup. """
return bool (self.dialup_tags)
def is_tor_exit (self):
""" Test if this is a Tor exit node. """
return 'TOR' in self.blackhat_tags
def _hostnamesetter (self):
""" Return a callable object that puts the answer into
the hostname attribute. """
def g (answer):
""" Store answer. """
self.hostname = answer[0]
return g
@staticmethod
def _filter (answers, tag_dict):
""" Lookup answers in tag_dict, return values of matches. """
return [ tag_dict[ip] for ip in answers if ip in tag_dict ]
def _tagsetter (self, dnsbl):
""" Return a callable object that puts the answer into
our *tags attributes. """
def g (answer):
""" Store answer. """
self.blackhat_tags.update (self._filter (answer, dnsbl.blackhat_tags))
self.dialup_tags.update (self._filter (answer, dnsbl.dialup_tags))
return g
@staticmethod
def _decode_txt (answer):
""" Helper: decode / unpack whois answer. """
try:
answer = answer[0][0].decode ('utf-8')
except UnicodeError:
answer = answer[0][0].decode ('iso-8859-1')
answer = answer.strip ('"').split ('|')
return [ a.strip () for a in answer if a ]
def _whoissetter_ss (self):
""" Return a callable object that puts the answer into
the whois dict. """
def g (answer):
""" Store answer. """
self.whois = dict (zip (ShadowServer.fields, self._decode_txt (answer)))
return g
def _whoissetter_tc (self, aresolver):
""" Return a callable object that puts the answer into
the right attribute. """
def g (answer):
""" Store answer. """
self.whois = dict (zip (TeamCymru.fields, self._decode_txt (answer)))
self.whois['org'] = None
# maybe there's still more info?
aresolver.query ('AS' + self.whois['asn'] + '.' + TeamCymru.asn_zone,
self._whoissetter_tc2 (), asyncdns.adns.rr.TXT)
return g
def _whoissetter_tc2 (self):
""" Return a callable object that puts the answer into
the right attribute. """
def g (answer):
""" Store answer. """
self.whois['org'] = self._decode_txt (answer)[-1]
return g
if __name__ == '__main__':
import sys
# test IP 127.0.0.2 should give all positives
a = asyncdns.AsyncDNS (asyncdns.RESOLV_CONF)
i = IPInfo (a, sys.argv[1])
a.wait ()
a.cancel ()
print ('hostname: %s' % i.hostname)
for k in sorted (i.whois.keys ()):
print ("%s: %s" % (k, i.whois[k]))
for tag in sorted (i.tags):
print (tag)
if i.is_blackhat ():
print ('BLACKHAT')
if i.is_dialup ():
print ('DUL')

405
templates/bibrec.html Normal file
View File

@ -0,0 +1,405 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
DON'T USE THIS PAGE FOR SCRAPING.
Seriously. You'll only get your IP blocked.
Read http://www.gutenberg.org/feeds/ to learn how to download Project
Gutenberg metadata much faster than by scraping.
-->
<html xmlns="http://www.w3.org/1999/xhtml"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:py="http://genshi.edgewall.org/"
xmlns:i18n="http://genshi.edgewall.org/i18n"
xmlns:dcterms="http://purl.org/dc/terms/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:ebook="http://www.gutenberg.org/ebooks/"
xmlns:marcrel="http://www.loc.gov/loc.terms/relators/"
xmlns:xsd="http://www.w3.org/2001/XMLSchema#"
xml:lang="${os.lang}"
version="XHTML+RDFa 1.0">
<?python
import re
import urllib
from libgutenberg.MediaTypes import mediatypes as mt
from libgutenberg import GutenbergGlobals as gg
os.description = "Free kindle book and epub digitized and proofread by %s." % os.pg
old_header = os.entries[0].header # suppress first header as already in <h1>
def help_page (s = ''):
s = s.replace (' ', '_')
return '%s#%s' % ('/wiki/Gutenberg:Help_on_Bibliographic_Record_Page', s)
def explain (s):
return _('Explain {topic}.').format (topic = _(s))
def _ (s):
return s
i = 0
maintainer = 'is-catalog-maintainer' in cherrypy.request.cookie
?>
<xi:include href="site-layout.html" />
<xi:include href="social-functions.html" />
<py:def function="help(topic)">
<a href="${help_page (topic)}"
title="${explain (topic)}"><span class="icon icon_help noprint" /></a>
</py:def>
<head profile="http://a9.com/-/spec/opensearch/1.1/"
xmlns:og="http://opengraphprotocol.org/schema/">
${site_head ()}
<title>${os.title} - Free Ebook</title>
<link rel="alternate nofollow" type="${mt.rdf}"
href="${os.url ('bibrec', id = os.id, format = 'rdf')}" />
<style type="text/css">
.qrcode { background: transparent url(${os.qrcode_url}) 0 0 no-repeat; }
</style>
<meta name="google" content="notranslate" />
</head>
<body>
<div id="mw-head-dummy" class="noprint" />
<div id="content" itemscope="itemscope" itemtype="http://schema.org/Book" i18n:comment="On the 'bibrec' page.">
<div class="breadcrumbs noprint">
<ul>
<py:for each="n, bc in enumerate (os.breadcrumbs)">
<li class="breadcrumb ${'first' if n == 0 else 'next'}"
itemscope="itemscope" itemtype="http://data-vocabulary.org/Breadcrumb">
<span class="breadcrumb-separator"></span>
<a href="${bc[2]}" title="${bc[1]}" itemprop="url"><span itemprop="title">${bc[0]}</span></a>
</li>
</py:for>
</ul>
</div>
<div class="header">
<h1 itemprop="name">${os.title}</h1>
</div>
<div class="body">
<div property="dcterms:publisher" itemprop="publisher" content="Project Gutenberg" />
<div itemprop="bookFormat" content="EBook" />
<div id="cover-social-wrapper">
<div py:if="os.cover_image_url" id="cover">
<img class="cover-art"
src="${os.cover_image_url}"
title="Book Cover"
alt="Book Cover"
itemprop="image" />
</div>
<div py:if="not os.cover_image_url" id="no-cover">
<div class="icon icon_${os.title_icon}" />
<div class="text">No cover available</div>
</div>
<div id="social" class="noprint">
<ul>
<!--! Broken. Dialog opens and closes immediately. Disabled for now.
<li>
${fb_share (os.canonical_url, os.title.encode ('utf-8'),
os.description.encode ('utf-8'), os.cover_thumb_url)}
</li>
-->
<li>
${gplus_share (os.canonical_url)}
</li>
<li>
${tw_share (os.canonical_url, os.twit)}
</li>
<li>
<a onclick="printpage ()" title="Print this page"><span class="icon icon_print" /></a>
</li>
</ul>
</div>
<div id="qr">
<!--! qr code of desktop page for print -->
<span class="qrcode qrcode-desktop noscreen" />
<!--! qr code of mobile page for screen -->
<span class="qrcode qrcode-mobile noprint" title="Go to our mobile site." />
</div>
</div>
<div id="tabs-wrapper">
<div id="tabs">
<ul class="noprint">
<li><a href="#download">Download</a></li>
<li><a href="#bibrec">Bibrec</a></li>
</ul>
<div id="bibrec" i18n:comment="On the 'bibrec' tab of the 'bibrec' page.">
<py:for each="e in os.entries">
<py:if test="isinstance (e, bs.DC)">
<div typeof="pgterms:ebook" about="[ebook:$e.project_gutenberg_id]">
<h2>Bibliographic Record <span>${help (_('Table: Bibliographic Record'))}</span></h2>
<table class="bibrec" summary="Bibliographic data of author and book.">
<colgroup>
<col class="narrow" />
<col />
</colgroup>
<tr py:for="author in e.authors">
<th>${author.role}</th>
<td>
<a href="${os.url ('author', id = author.id)}"
rel="marcrel:${author.marcrel}" about="/authors/${author.id}" typeof="pgterms:agent"
itemprop="creator">${author.name_and_dates}</a></td>
</tr>
<tr py:for="marc in e.marcs">
<th>${marc.caption}</th>
<py:choose test="">
<td py:when="marc.code == '010'">
<a class="external"
href="http://lccn.loc.gov/${marc.text}"
title="Look up this book in the Library of Congress catalog.">${marc.text} <span class="icon icon_external_link"/></a>
</td>
<td py:when="marc.code[0]=='5'">
<?python
text = gg.xmlspecialchars (marc.text)
text = re.sub (r'(//\S+)', r'<a href="\1">\1</a>', text)
text = re.sub (r'#(\d+)', r'<a href="/ebooks/\1">#\1</a>', text)
?>
${ Markup (gg.insert_breaks (text)) }
</td>
<td py:when="marc.code=='245'" itemprop="headline">
${ Markup (gg.insert_breaks (gg.xmlspecialchars (marc.text))) }
</td>
<td py:when="marc.code=='240'" itemprop="alternativeHeadline">
${ Markup (gg.insert_breaks (gg.xmlspecialchars (marc.text))) }
</td>
<td py:when="marc.code=='246'" itemprop="alternativeHeadline">
${ Markup (gg.insert_breaks (gg.xmlspecialchars (marc.text))) }
</td>
<td py:otherwise="">
${ Markup (gg.insert_breaks (gg.xmlspecialchars (marc.text))) }
</td>
</py:choose>
</tr>
<tr py:for="language in e.languages"
property="dcterms:language" datatype="dcterms:RFC4646" itemprop="inLanguage" content="${language.id}">
<th>Language</th>
<td py:if="language.id != 'en'"><a href="/browse/languages/${language.id}">${language.language}</a></td>
<td py:if="language.id == 'en'">${language.language}</td>
</tr>
<tr py:for="locc in e.loccs"
property="dcterms:subject" datatype="dcterms:LCC" content="${locc.id}">
<th>LoC Class</th>
<td>
<a href="/browse/loccs/${locc.id.lower ()}">${locc.id}: ${locc.locc}</a>
</td>
</tr>
<tr py:for="subject in e.subjects">
<th>Subject</th>
<td property="dcterms:subject" datatype="dcterms:LCSH">
<a class="block" href="${os.url ('subject', id = subject.id)}">
${subject.subject}
</a>
</td>
</tr>
<tr py:for="dcmitype in e.dcmitypes">
<th>Category</th>
<td property="dcterms:type" datatype="dcterms:DCMIType">${dcmitype.description}</td>
</tr>
<tr>
<th>EBook-No.</th>
<td>${e.project_gutenberg_id}</td>
</tr>
<tr property="dcterms:issued" datatype="xsd:date" content="${e.xsd_release_date_time}">
<th>Release Date</th>
<td itemprop="datePublished">${e.hr_release_date}</td>
</tr>
<tr>
<th>Copyright Status</th>
<td property="dcterms:rights">${e.rights}</td>
</tr>
<tr>
<th>Downloads</th>
<td itemprop="interactionCount" i18n:msg="count">${e.downloads} downloads in the last 30 days.</td>
</tr>
<tr itemprop="offers" itemscope="itemscope" itemtype="http://schema.org/Offer">
<th>Price</th>
<td><span itemprop="priceCurrency" content="USD" /><span itemprop="price">$0.00</span><span itemprop="availability" href="http://schema.org/InStock" content="In Stock" /></td>
</tr>
</table>
</div>
</py:if>
</py:for>
<div id="more_stuff">
<py:for each="n, e in enumerate (os.entries)">
<py:if test="isinstance (e, bs.Cat) and e.rel not in ('start', )">
<py:if test="e.header and old_header != e.header">
<h2 class="header">${e.header}</h2>
</py:if>
<?python old_header = e.header ?>
<div class="${e.class_}">
<a rel="nofollow"
href="${e.url}"
type="${e.type}"
charset="${e.charset}"
accesskey="${str (n % 10)}">
<span class="cell leftcell">
<span class="icon icon_${e.icon}" />
</span>
<span class="cell content">
<span class="title">${e.title}</span>
<span py:if="e.subtitle" class="subtitle">${e.subtitle}</span>
<span py:if="e.extra" class="extra">${e.extra}</span>
</span>
<span class="hstrut" />
</a>
</div>
</py:if>
</py:for>
</div> <!-- more stuff -->
</div> <!--! bibrec -->
<div id="download" i18n:comment="On the 'Download' tab of the 'bibrec' page.">
<py:for each="e in os.entries">
<py:if test="isinstance (e, bs.DC)">
<div about="[ebook:$e.project_gutenberg_id]" rel="dcterms:hasFormat" rev="dcterms:isFormatOf">
<h2>Download This eBook</h2>
<table class="files" summary="Table of available file types and sizes.">
<colgroup>
<col class="narrow" />
<col />
<col />
<col class="narrow noprint" />
<col class="narrow noprint" />
<col class="narrow noprint" />
</colgroup>
<tr>
<th />
<th>Format <span>${help ('Format')}</span></th>
<th class="noscreen">Url</th>
<th i18n:comment="Size of a file." class="right">Size</th>
<th class="noprint"><span>${help ('Dropbox')}</span></th>
<th class="noprint"><span>${help ('Google Drive')}</span></th>
<th class="noprint"><span>${help ('OneDrive')}</span></th>
</tr>
<tr py:for="i, file_ in enumerate (e.files)"
py:if="not file_.hidden"
class="${i%2 and 'odd' or 'even'}"
about="${file_.url}" typeof="pgterms:file">
<td><span class="icon icon_${e.icon}" /></td>
<td property="dcterms:format" content="${file_.mediatypes[-1]}" datatype="dcterms:IMT"
class="unpadded icon_save"
><a href="${file_.url}" type="${file_.mediatypes[-1]}" charset="${file_.encoding}"
class="link"
title="Download">${file_.hr_filetype}</a></td>
<td class="noscreen">${file_.url}</td>
<td class="right"
property="dcterms:extent" content="${file_.extent}">${file_.hr_extent}</td>
<td class="noprint">
<a py:if="file_.dropbox_url"
href="${file_.dropbox_url}"
title="Send to Dropbox." rel="nofollow"><span class="icon icon_dropbox" /></a>
</td>
<td class="noprint">
<a py:if="file_.gdrive_url"
href="${file_.gdrive_url}"
title="Send to Google Drive." rel="nofollow"><span class="icon icon_gdrive" /></a>
</td>
<td class="noprint">
<a py:if="file_.honeypot_url"
href="${file_.honeypot_url}"
title="Send to MegaUpload." rel="nofollow" />
<a py:if="file_.msdrive_url"
href="${file_.msdrive_url}"
title="Send to OneDrive." rel="nofollow"><span class="icon icon_msdrive" /></a>
</td>
</tr>
<!--! more files ... -->
<tr py:if="e.base_dir"
class="${i%2 and 'odd' or 'even'}">
<td><span class="icon icon_folder" /></td>
<td class="unpadded icon_file"><a href="${e.base_dir}" class="link"
i18n:comment="Link to the directory containing all files.">More Files…</a></td>
<td class="noscreen">http:${os.qualify (e.base_dir)}</td>
<td/>
<td class="noprint"><!--! dropbox column --></td>
<td class="noprint"><!--! gdrive column --></td>
<td class="noprint"><!--! msdrive column --></td>
</tr>
</table>
</div>
</py:if>
</py:for>
</div> <!-- download -->
</div> <!--! tabs -->
</div> <!--! tabs-wrapper -->
</div> <!--! body -->
<div id="dialog" class="hidden">
</div>
<!--!
<a href="http://validator.w3.org/check?uri=referer" rel="nofollow"
title="This page contains RDFa metadata."><img
src="http://www.w3.org/Icons/valid-xhtml-rdfa-blue"
width="88"
height="31"
alt="Valid XHTML + RDFa"
about=""
rel="dcterms:conformsTo"
resource="http://www.w3.org/TR/rdfa-syntax" /></a>
<a href="${os.url_carry ('bibrec', format='rdf')}"
title="Download RDF/XML Metadata for this ebook."><img
src="http://www.w3.org/RDF/icons/rdf_metadata_button.32"
width="76"
height="32"
alt="RDF/XML Metadata" /></a>
-->
${site_footer ()}
</div>
${site_top ()}
</body>
</html>

111
templates/help.html Normal file
View File

@ -0,0 +1,111 @@
<?xml version="1.0" encoding="utf-8"?>
<div xmlns="http://www.w3.org/1999/xhtml"
xmlns:i18n="http://genshi.edgewall.org/i18n"
i18n:comment="The help menu for the search box.">
<p>Enter your search terms separated by spaces,
then press &lt;Enter&gt;.
Avoid punctuation except as indicated below:</p>
<table class="helpbox">
<tr>
<th>Suffixes</th>
<th xml:lang="en">.</th>
<td>exact match</td>
</tr>
<tr>
<th rowspan="7">Prefixes</th>
<th xml:lang="en">a.</th>
<td>author</td>
</tr>
<tr>
<th xml:lang="en">t.</th>
<td>title</td>
</tr>
<tr>
<th xml:lang="en">s.</th>
<td>subject</td>
</tr>
<tr>
<th xml:lang="en">l.</th>
<td>language</td>
</tr>
<tr>
<th xml:lang="en">#</th>
<td>ebook no.</td>
</tr>
<tr>
<th xml:lang="en">n.</th>
<td>ebook no.</td>
</tr>
<tr>
<th xml:lang="en">cat.</th>
<td>category</td>
</tr>
<tr>
<th rowspan="3" style="width: 8em">
Operators
<small>Always put spaces around these.</small>
</th>
<th xml:lang="en">|</th>
<td>or</td>
</tr>
<tr>
<th xml:lang="en">!</th>
<td>not</td>
</tr>
<tr>
<th xml:lang="en">( )</th>
<td>grouping</td>
</tr>
</table>
<table class="helpbox">
<tr>
<th>this query</th>
<th>finds</th>
</tr>
<tr>
<td xml:lang="en">shakespeare hamlet</td>
<td>"Hamlet" by Shakespeare</td>
</tr>
<tr>
<td xml:lang="en">qui.</td>
<td>"qui", not "Quixote"</td>
</tr>
<tr>
<td xml:lang="en">love stories</td>
<td>love stories</td>
</tr>
<tr>
<td xml:lang="en">a.shakespeare</td>
<td>by Shakespeare</td>
</tr>
<tr>
<td xml:lang="en">s.shakespeare</td>
<td>about Shakespeare</td>
</tr>
<tr>
<td xml:lang="en">#74</td>
<td>ebook no. 74</td>
</tr>
<tr>
<td xml:lang="en">juvenile l.german</td>
<td>juvenile lit in German</td>
</tr>
<tr>
<td xml:lang="en">verne ( l.fr | l.it )</td>
<td>by Verne in French or Italian</td>
</tr>
<tr>
<td xml:lang="en">love stories ! austen</td>
<td>love stories not by Austen</td>
</tr>
<tr>
<td xml:lang="en">jane austen cat.audio</td>
<td>audio books by Jane Austen</td>
</tr>
</table>
</div>

107
templates/recaptcha.html Normal file
View File

@ -0,0 +1,107 @@
<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:py="http://genshi.edgewall.org/"
xmlns:i18n="http://genshi.edgewall.org/i18n">
<xi:include href="site-layout.html" />
<head>
${site_head ()}
<title>Captcha</title>
<style type="text/css">
.ui-dialog-titlebar-close {
display: none;
}
#recaptcha_response_field {
margin-bottom: 1em;
width: 300px;
}
.recaptcha_only_if_incorrect_sol {
color: red;
}
</style>
<script type="text/javascript" src="//www.google.com/recaptcha/api/js/recaptcha_ajax.js" />
<script type="text/javascript">
function on_recaptcha_loaded () {
$('#need_javascript').hide ();
Recaptcha.focus_response_field ();
}
var RecaptchaOptions = {
theme: "custom",
custom_theme_widget: "recaptcha_widget",
callback: on_recaptcha_loaded
};
require (
[
'jquery',
'jquery-ui/dialog',
'jquery.cookie/jquery.cookie'
],
function (jquery, dialog, cookie) {
if (jquery.cookie ('session_id')) {
jquery ('#need_cookies').hide ();
}
Recaptcha.create ("${os.recaptcha_public_key}", "id_captcha", RecaptchaOptions);
var dlg = jquery ('#dialog');
dlg.dialog ({
width: 350,
resizable: false,
modal: true,
closeOnEscape: false
});
});
</script>
</head>
<body>
<div id="mw-head-dummy" class="noprint" />
<div id="content">
<div class="body">
<div id="dialog" title="Are you human?" class="hidden">
<p py:if="os.error is not None" style="color: red">Incorrect please try again</p>
<p>You have used Project Gutenberg quite a lot today or clicked through it really fast. Email webmaster2017@pglaf.org to report problems. To make sure you are human, we ask you to resolve this captcha:</p>
<form method="post" action="/w/captcha/answer/">
<div id="recaptcha_widget">
<div id="recaptcha_image"></div>
<p class="recaptcha_only_if_incorrect_sol">Incorrect please try again</p>
<p class="recaptcha_only_if_image">Enter the words you see:</p>
<p class="recaptcha_only_if_audio">Enter the numbers you hear:</p>
<input type="text" id="recaptcha_response_field" name="recaptcha_response_field" />
<input type="submit" name="SubmitButton" value="Submit" />
<input type="button" name="ReloadButton" value="Get another captcha"
onclick="Recaptcha.reload ()" />
<input type="button" name="AudioButton" value="Get an audio captcha"
class="recaptcha_only_if_image" onclick="Recaptcha.switch_type ('audio')" />
<input type="button" name="ImageButton" value="Get an image captcha"
class="recaptcha_only_if_audio" onclick="Recaptcha.switch_type ('image')" />
<input type="button" name="HelpButton" value="Help"
onclick="Recaptcha.showhelp ()" />
<p id="need_cookies">Project Gutenberg works better with cookies enabled.</p>
</div>
</form>
</div>
<p id="need_javascript">You need javascript for this.</p>
</div>
${site_footer ()}
</div>
${site_top ()}
</body>
</html>

89
templates/recaptcha.mobile Normal file
View File

@ -0,0 +1,89 @@
<?xml version="1.0" encoding="utf-8"?>
<!--! recaptcha is broken with application/xhtml+xml -->
<html xmlns="http://www.w3.org/1999/xhtml"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:py="http://genshi.edgewall.org/"
xmlns:i18n="http://genshi.edgewall.org/i18n"
xml:lang="${os.lang}">
<xi:include href="site-layout.mobile" />
<head>
${site_head ()}
<title>Captcha</title>
<style type="text/css">
#recaptcha_response_field {
margin-top: 1em;
margin-bottom: 1em;
width: 300px;
}
.recaptcha_only_if_incorrect_sol {
color: red;
}
</style>
<script type="application/javascript" src="//www.google.com/recaptcha/api/js/recaptcha_ajax.js" />
<script type="application/javascript">
function on_recaptcha_loaded () {
$('#need_javascript').hide ();
Recaptcha.focus_response_field ();
}
var RecaptchaOptions = {
theme: "custom",
custom_theme_widget: "recaptcha_widget",
callback: on_recaptcha_loaded
};
require (
[
'jquery',
'jquery.cookie/jquery.cookie'
],
function (jquery, cookie) {
if (jquery.cookie ('session_id')) {
jquery ('#need_cookies').hide ();
}
Recaptcha.create ("${os.recaptcha_public_key}", "id_captcha", RecaptchaOptions);
});
</script>
</head>
<body>
<div class="content">
<p>You have used Project Gutenberg quite a lot today or clicked through it really fast. To make sure you are human, we ask you to resolve this captcha:</p>
<form method="post" action="/w/captcha/answer/">
<div id="recaptcha_widget">
<div id="recaptcha_image"></div>
<p class="recaptcha_only_if_incorrect_sol">Incorrect please try again</p>
<p class="recaptcha_only_if_image">Enter the words you see:</p>
<p class="recaptcha_only_if_audio">Enter the numbers you hear:</p>
<input type="text" id="recaptcha_response_field" name="recaptcha_response_field" />
<input type="submit" name="SubmitButton" value="Submit" />
<input type="button" name="ReloadButton" value="Get another captcha"
onclick="Recaptcha.reload ()" />
<input type="button" name="AudioButton" value="Get an audio captcha"
class="recaptcha_only_if_image" onclick="Recaptcha.switch_type ('audio')" />
<input type="button" name="ImageButton" value="Get an image captcha"
class="recaptcha_only_if_audio" onclick="Recaptcha.switch_type ('image')" />
<input type="button" name="HelpButton" value="Help"
onclick="Recaptcha.showhelp ()" />
<p id="need_cookies">Project Gutenberg works better with cookies enabled.</p>
</div>
</form>
<p id="need_javascript">You need javascript for this.</p>
</div>
${site_footer ()}
</body>
</html>

142
templates/results.html Normal file
View File

@ -0,0 +1,142 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
DON'T USE THIS PAGE FOR SCRAPING.
Seriously. You'll only get your IP blocked.
Download http://www.gutenberg.org/feeds/catalog.rdf.bz2 instead,
which contains *all* Project Gutenberg metadata in one RDF/XML file.
-->
<html xmlns="http://www.w3.org/1999/xhtml"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:py="http://genshi.edgewall.org/"
xmlns:i18n="http://genshi.edgewall.org/i18n"
xmlns:dcterms="http://purl.org/dc/terms/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:ebook="http://www.gutenberg.org/ebooks/"
xmlns:marcrel="http://www.loc.gov/loc.terms/relators/"
xmlns:xsd="http://www.w3.org/2001/XMLSchema#"
xmlns:og="http://opengraphprotocol.org/schema/"
xmlns:fb="http://www.facebook.com/2008/fbml"
xml:lang="${os.lang}"
version="XHTML+RDFa 1.0">
<?python
old_header = os.title
os.status_line = ''
?>
<xi:include href="site-layout.html" />
<xi:include href="social-functions.html" />
<head profile="http://a9.com/-/spec/opensearch/1.1/">
${site_head ()}
<title>${os.title} - ${os.pg}</title>
<link rel="self"
i18n:comment="Link pointing to the same page the user is viewing."
title="This Page"
href="${os.url ('search', start_index = os.start_index)}" />
<link py:if="os.show_prev_page_link"
rel="first"
title="First Page"
href="${os.url ('search')}" />
<link py:if="os.show_prev_page_link"
rel="previous"
title="Previous Page"
href="${os.url ('search', start_index = os.prev_page_index)}" />
<link py:if="os.show_next_page_link"
rel="next"
title="Next Page"
href="${os.url ('search', start_index = os.next_page_index)}" />
<meta name="totalResults" content="${os.total_results}" />
<meta name="startIndex" content="${os.start_index}" />
<meta name="itemsPerPage" content="${os.items_per_page}" />
</head>
<body>
<div id="mw-head-dummy" class="noprint" />
<div id="content" i18n:comment="On the page of results of a search.">
<div class="header">
<h1><span class="icon icon_${os.title_icon}" />${os.title}</h1>
</div>
<div class="body">
<div>
<ul class="results">
<py:for each="n, e in enumerate (os.entries)">
<py:if test="e.header and old_header != e.header">
<li class="h2">
<h2 class="padded">${e.header}</h2>
</li>
<?python old_header = e.header ?>
</py:if>
<py:if test="isinstance (e, bs.Cat) and e.rel not in ('start', )">
<py:choose test="">
<py:when test="e.rel == '__statusline__'">
<li class="statusline">
<div class="padded">
<span>${e.title}</span> ${prev_next_links ()}
</div>
</li>
<?python os.status_line = e.title ?>
</py:when>
<py:otherwise>
<li class="${e.class_}">
<a class="link" href="${e.url}" accesskey="${str (n % 10)}">
<span class="cell leftcell${' with-cover' if e.thumb_url else ' without-cover'}">
<span py:if="not e.thumb_url" class="icon-wrapper">
<span class="icon icon_${e.icon}" />
</span>
<img py:if="e.thumb_url"
class="cover-thumb" src="${e.thumb_url}" alt="" />
</span>
<span class="cell content">
<span py:if="e.title" class="title">${e.title}</span>
<span py:if="e.subtitle" class="subtitle">${e.subtitle}</span>
<span py:if="e.extra" class="extra">${e.extra}</span>
</span>
<span class="hstrut" />
</a>
</li>
</py:otherwise>
</py:choose>
</py:if>
</py:for>
<py:if test="os.status_line != ''">
<li class="statusline">
<div class="padded">
${os.status_line} ${prev_next_links ()}
</div>
</li>
</py:if>
</ul>
</div>
</div> <!--! body -->
${site_footer ()}
</div>
${site_top ()}
</body>
</html>

151
templates/results.mobile Normal file
View File

@ -0,0 +1,151 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
DON'T USE THIS PAGE FOR SCRAPING.
Seriously. You'll only get your IP blocked.
Download http://www.gutenberg.org/feeds/catalog.rdf.bz2 instead,
which contains *all* Project Gutenberg metadata in one RDF/XML file.
-->
<html xmlns="http://www.w3.org/1999/xhtml"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:py="http://genshi.edgewall.org/"
xmlns:i18n="http://genshi.edgewall.org/i18n"
xml:lang="${os.lang}">
<?python
import re
from libgutenberg import GutenbergGlobals as gg
old_header = ''
def help_page (s = ''):
s = s.replace (' ', '_')
return '%s#%s' % ('/wiki/Gutenberg:Help_on_Bibliographic_Record_Page', s)
?>
<xi:include href="site-layout.mobile" />
<head profile="http://a9.com/-/spec/opensearch/1.1/">
${site_head ()}
<title>${os.title}</title>
<meta name="totalResults" content="${os.total_results}" />
<meta name="startIndex" content="${os.start_index}" />
<meta name="itemsPerPage" content="${os.items_per_page}" />
</head>
<body>
<div class="content" i18n:comment="On the page of results of a search.">
<ol class="results">
${search_box ()}
<py:for each="n, e in enumerate (os.entries)">
<py:if test="e.header and old_header != e.header">
<li class="header">${e.header}</li>
</py:if>
<?python old_header = e.header ?>
<py:if test="isinstance (e, bs.DC)">
<li class="bibrec">
<div class="table">
<div class="row">
<div class="cell leftcell">
<div class="icon icon_${e.icon}" />
</div>
<div class="cell content">
<!--! get coverpage first, it floats to the right -->
<img py:if="e.cover_image" class="coverpage"
src="${e.cover_image.url}" alt="[Coverpage]" />
<p py:for="author in e.authors">${author.role}: ${author.name_and_dates}</p>
<py:for each="marc in e.marcs">
<py:choose test="">
<p py:when="marc.code[0]=='5'">
<?python
text = gg.xmlspecialchars (marc.text)
text = re.sub (r'(//\S+)', r'<a href="\1">\1</a>', text)
text = re.sub (r'#(\d+)',
r'<a href="/ebooks/\1.mobile">#\1</a>', text)
?>
${marc.caption}:
${ Markup (gg.insert_breaks (text)) }
</p>
<p py:otherwise="">
${marc.caption}:
${ Markup (gg.insert_breaks (gg.xmlspecialchars (marc.text))) }
</p>
</py:choose>
</py:for>
<p>Ebook No.: ${e.project_gutenberg_id}</p>
<p>Published: ${e.hr_release_date}</p>
<p>Downloads: ${e.downloads}</p>
<p py:for="language in e.languages">Language: ${language.language}</p>
<p py:for="subject in e.subjects">Subject: ${subject.subject}</p>
<p py:for="locc in e.loccs">LoCC: ${locc.locc}</p>
<p py:for="category in e.categories">Category: ${category}</p>
<p>Rights: ${e.rights}</p>
</div>
</div>
</div>
</li>
</py:if>
<py:if test="isinstance (e, bs.Cat)">
<py:choose test="">
<py:when test="e.rel == '__statusline__'" />
<py:otherwise>
<li class="${e.class_}">
<a class="table link" href="${e.url}" accesskey="${str (n % 10)}">
<span class="row">
<span class="cell leftcell">
<span class="icon icon_${e.icon}" />
</span>
<span class="cell content">
<span py:if="e.title" class="title">${e.title}</span>
<span py:if="e.subtitle" class="subtitle">${e.subtitle}</span>
<span py:if="e.extra" class="extra">${e.extra}</span>
</span>
<span py:if="e.icon2" class="cell rightcell">
<span class="icon icon_${e.icon2}" />
</span>
</span>
</a>
</li>
</py:otherwise>
</py:choose>
</py:if>
</py:for>
<py:if test="os.show_next_page_link">
<li class="navlink more grayed">
<a class="table link"
href="${os.url_carry (start_index = os.next_page_index)}">
<span class="row">
<span class="cell leftcell">
<span class="icon icon_more" />
</span>
<span class="cell content">
<span class="title"><span>Next Page</span>…</span>
</span>
<span class="cell rightcell">
<span class="icon spinner" />
</span>
</span>
</a>
</li>
</py:if>
</ol>
</div>
${site_footer ()}
</body>
</html>

293
templates/results.opds Normal file
View File

@ -0,0 +1,293 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
DON'T USE THIS PAGE FOR SCRAPING.
Seriously. You'll only get your IP blocked.
Download http://www.gutenberg.org/feeds/catalog.rdf.bz2 instead,
which contains *all* Project Gutenberg metadata in one RDF/XML file.
-->
<?python
import re
from libgutenberg import GutenbergGlobals as gg
if os.format == 'stanza':
os.type_opds = "application/atom+xml"
opds_relations = {
'cover': 'x-stanza-cover-image',
'thumb': 'x-stanza-cover-image-thumbnail',
}
else:
opds_relations = {
'new': 'http://opds-spec.org/sort/new',
'popular': 'http://opds-spec.org/sort/popular',
'cover': 'http://opds-spec.org/image',
'thumb': 'http://opds-spec.org/image/thumbnail',
'acquisition': 'http://opds-spec.org/acquisition',
}
?>
<feed xmlns="http://www.w3.org/2005/Atom"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:opds="http://opds-spec.org/2010/catalog"
xmlns:dcterms="http://purl.org/dc/terms/"
xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
xmlns:relevance="http://a9.com/-/opensearch/extensions/relevance/1.0/"
xmlns:py="http://genshi.edgewall.org/">
<id>${os.url_carry (host = os.host, start_index = os.start_index)}</id>
<updated>${os.now}</updated>
<title>${os.title}</title>
<subtitle>Free ebooks since 1971.</subtitle>
<author>
<name>Marcello Perathoner</name>
<uri>http://www.gutenberg.org</uri>
<email>webmaster@gutenberg.org</email>
</author>
<icon>${os.qualify ('/pics/favicon.png')}</icon>
<py:choose test="os.opensearch_support">
<py:when test="2">
<!--! fake opensearch support in Stanza and Aldiko -->
<!--! The next 2 links are for Stanza that can't read the standard opensearch description.
Stanza even requires unescaped '{' and '}' which are not valid characters in urls! AARGH!!
Aldiko needs fully qualified urls here!
-->
<link rel="search"
type="${os.type_opds}"
title="${os.placeholder}"
href="${os.add_amp (os.url ('search', host = os.host))}query={searchTerms}" />
<!--! routes would quote the invalid '{' and '}' -->
<link rel="x-stanza-search-suggestions"
type="application/x-suggestions+json"
href="${os.add_amp (os.url ('suggest', host = os.host, format = None))}query={searchTerms}" />
<!--! routes would quote '{' and '}' -->
</py:when>
<py:when test="1">
<!--! real opensearch support -->
<link rel="search"
type="application/opensearchdescription+xml"
title="Project Gutenberg Catalog Search"
href="${os.osd_url}" />
</py:when>
</py:choose>
<link rel="self"
title="This Page"
type="${os.type_opds}"
href="${os.url_carry (start_index = os.start_index)}" />
<link rel="alternate"
type="text/html"
title="HTML Page"
href="${os.url_carry (format = 'html', start_index = os.start_index)}" />
<link rel="start"
title="Start Page"
type="${os.type_opds}"
href="${os.url ('start')}" />
<link py:if="os.show_prev_page_link"
rel="first"
title="First Page"
type="${os.type_opds}"
href="${os.url_carry (start_index = 1)}" />
<link py:if="os.show_prev_page_link"
rel="previous"
title="Previous Page"
type="${os.type_opds}"
href="${os.url_carry (start_index = os.prev_page_index)}" />
<!--! Coolreader sucks up to 1000 entries without user paging. See:
http://crengine.git.sourceforge.net/git/gitweb.cgi?p=crengine/crengine;a=blob;f=android/src/org/coolreader/crengine/OPDSUtil.java
We give it one page. That's enough.
-->
<link py:if="os.show_next_page_link and not os.user_agent.startswith ('CoolReader/')"
rel="next"
title="Next Page"
type="${os.type_opds}"
href="${os.url_carry (start_index = os.next_page_index)}" />
<py:for each="e in os.entries">
<py:if test="isinstance (e, bs.Cat) and e.rel in opds_relations">
<link rel="${opds_relations[e.rel]}"
title="${e.title}"
type="${e.type or os.type_opds}"
href="${e.url}" />
</py:if>
</py:for>
<opensearch:itemsPerPage>${os.items_per_page}</opensearch:itemsPerPage>
<opensearch:startIndex>${os.start_index}</opensearch:startIndex>
<!--!
<opensearch:totalResults>${os.total_results}</opensearch:totalResults>
<opensearch:Query role="request"
searchTerms="${os.search_terms}"
startIndex="${os.start_index}" />
-->
<py:for each="e in os.entries">
<!--! Navigation feed entry -->
<py:if test="isinstance (e, bs.Cat)">
<py:choose>
<py:when test="e.rel == '__statusline__'" />
<py:otherwise>
<entry>
<updated>${os.now}</updated>
<id>${os.qualify (e.url)}</id>
<title>${e.title}</title>
<!--! according to spec type defaults to text but quickreader doesn't think so -->
<content py:if="e.subtitle or e.extra" type="text">${e.subtitle or e.extra}</content>
<category py:if="os.format == 'stanza' and e.header" label="${e.header}"
scheme="http://lexcycle.com/stanza/header" term="free" />
<link type="${e.type or os.type_opds}"
rel="${opds_relations.get (e.rel, 'subsection')}"
href='${e.url}' />
<py:for each="link in e.links">
<link type="${link.type}"
rel="${opds_relations.get (link.rel, 'subsection')}"
title="${link.title}"
length="${link.length}"
href="${link.url}" />
</py:for>
</entry>
</py:otherwise>
</py:choose>
</py:if>
<!--! Acquisition feed entry -->
<py:if test="isinstance (e, bs.DC)">
<entry>
<updated>${os.now}</updated>
<title>${re.sub (r'[\r\n].*', '', e.title)}</title>
<content type="xhtml">
<div xmlns="http://www.w3.org/1999/xhtml" >
<py:choose test="e.image_flags">
<p py:when="3">This edition has images.</p>
<p py:when="2">This edition had all images removed.</p>
</py:choose>
<py:for each="marc in e.marcs">
<py:choose test="">
<p py:when="marc.code[0]=='5'">
<?python
text = gg.xmlspecialchars (marc.text)
text = re.sub (r'(//\S+)', r'<a href="\1">\1</a>', text)
text = re.sub (r'#(\d+)',
r'<a href="/ebooks/\1.bibrec.mobile">#\1</a>', text)
?>
${marc.caption}:
${ Markup (gg.insert_breaks (text)) }
</p>
<p py:otherwise="">
${marc.caption}:
${ Markup (gg.insert_breaks (gg.xmlspecialchars (marc.text))) }
</p>
</py:choose>
</py:for>
<p py:for="author in e.authors">${author.role}: ${author.name_and_dates}</p>
<p>Ebook No.: ${e.project_gutenberg_id}</p>
<p>Published: ${e.hr_release_date}</p>
<p>Downloads: ${e.downloads}</p>
<p py:for="language in e.languages">Language: ${language.language}</p>
<p py:for="subject in e.subjects">Subject: ${subject.subject}</p>
<p py:for="locc in e.loccs">LoCC: ${locc.locc}</p>
<p py:for="category in e.categories">Category: ${category}</p>
<p>Rights: ${e.rights}</p>
</div>
</content>
<id>urn:gutenberg:${e.project_gutenberg_id}:${e.image_flags}</id>
<published>${e.xsd_release_date_time}</published>
<rights>${e.rights}</rights>
<py:for each="author in reversed (e.authors)">
<author py:if="author.marcrel in ('cre', 'aut')">
<name>${author.name}</name>
</author>
<contributor py:if="author.marcrel not in ('cre', 'aut')">
<name>${author.name}</name>
</contributor>
</py:for>
<category py:if="os.format == 'stanza' and e.header"
scheme="http://lexcycle.com/stanza/header"
term="free"
label="${e.header}" />
<category py:for="subject in e.subjects"
scheme="http://purl.org/dc/terms/LCSH"
term="${subject.subject}" />
<category py:for="locc in e.loccs"
scheme="http://purl.org/dc/terms/LCC"
term="${locc.id}"
label="${locc.locc}" />
<category py:for="category in e.categories"
scheme="http://purl.org/dc/terms/DCMIType"
term="$category" />
<dcterms:language py:for="language in e.languages">${language.id}</dcterms:language>
<py:for each="marc in e.marcs">
<dcterms:identifier py:if="marc.code == '010'">urn:lccn:${marc.text}</dcterms:identifier>
</py:for>
<relevance:score py:if="hasattr (e, 'score')">${e.score}</relevance:score>
<py:for each="link in e.links">
<link type="${link.type}"
rel="${opds_relations.get (link.rel, 'acquisition')}"
title="${link.title}"
length="${link.length}"
href="${link.url}" />
</py:for>
<link type="${os.type_opds}"
rel="related"
href="${os.url ('also', id = e.project_gutenberg_id)}"
title="Readers also downloaded…" />
<py:for each="author in e.authors">
<link type="${os.type_opds}"
rel="related"
href="${os.url ('author', id = author.id)}"
title="${_('By {author}').format (author = author.name)}…"/>
</py:for>
<py:for each="subject in e.subjects">
<link type="${os.type_opds}"
rel="related"
href="${os.url ('subject', id = subject.id)}"
title="${_('On {subject}').format (subject = subject.subject)}…"/>
</py:for>
<py:for each="bookshelf in e.bookshelves">
<link type="${os.type_opds}"
rel="related"
href="${os.url ('bookshelf', id = bookshelf.id)}"
title="${_('In {bookshelf}').format (bookshelf = bookshelf.bookshelf)}…"/>
</py:for>
</entry>
</py:if>
</py:for>
</feed>

257
templates/site-layout.html Normal file
View File

@ -0,0 +1,257 @@
<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:py="http://genshi.edgewall.org/"
xmlns:i18n="http://genshi.edgewall.org/i18n"
xmlns:og="http://opengraphprotocol.org/schema/"
xmlns:fb="http://www.facebook.com/2008/fbml"
py:strip="">
<py:def function="site_head">
<style type="text/css">
.icon { background: transparent url(/pics/sprite.png?${cherrypy.config['css_mtime']}) 0 0 no-repeat; }
</style>
<link rel="stylesheet" type="text/css"
href="/css/pg-desktop-one.css?${cherrypy.config['css_mtime']}" />
<!--! IE8 does not recognize application/javascript -->
<script type="text/javascript">//<![CDATA[
var json_search = "${os.json_search}";
var mobile_url = "${os.mobile_url}";
var canonical_url = "${os.canonical_url}";
var lang = "${os.lang}";
var fb_lang = "${os.fb_lang}"; /* FB accepts only xx_XX */
var msg_load_more = "${_('Load More Results…')}";
var page_mode = "${os.page_mode}";
var dialog_title = "${os.user_dialog[1]}";
var dialog_message = "${os.user_dialog[0]}";
//]]></script>
<script type="text/javascript"
src="/js/pg-desktop-one.js?${cherrypy.config['js_mtime']}" />
<link rel="shortcut icon" href="/pics/favicon" />
<link rel="canonical" href="${os.canonical_url}" />
<link rel="search"
type="application/opensearchdescription+xml"
title="Search Project Gutenberg"
href="${os.osd_url}" />
<link rel="alternate nofollow"
type="${os.type_opds}"
title="OPDS feed"
href="${os.url_carry (format = 'opds')}" />
<link py:if="os.touch_icon" rel="apple-touch-icon" href="${os.touch_icon}" />
<link py:if="os.touch_icon_precomposed" rel="apple-touch-icon-precomposed" href="${os.touch_icon_precomposed}" />
<meta py:if="os.viewport" name="viewport" content="${os.viewport}" />
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta http-equiv="Content-Style-Type" content="text/css" />
<meta http-equiv="Content-Language" content="${os.lang}" />
<!--! plain old metadata -->
<meta name="title" content="${os.title}" />
<meta name="description" content="${os.description}" />
<meta name="keywords" content="ebook, ebooks, free ebooks, free books, book, books, audio books" />
<meta name="classification" content="public" />
<!--! facebook open graph -->
<meta property="og:title" content="${os.title}" />
<meta property="og:description" content="${os.description}" />
<meta property="og:type" content="${os.og_type}" />
<meta property="og:image" content="${os.snippet_image_url}" />
<meta property="og:url" content="${os.canonical_url}" />
<meta property="og:site_name" content="Project Gutenberg" />
<meta property="fb:app_id" content="${cherrypy.config['facebook_app_id']}" />
<!--! <meta property="fb:admins" content="615269807" /> -->
<!--! twitter card -->
<meta name="twitter:card" content="summary" />
<meta name="twitter:site" content="@gutenberg_new" />
</py:def>
<py:def function="credits">
<div id="credits">
<div class="h1"
title="Credits">Credits</div>
<a href="http://www.ibiblio.org" rel="nofollow"
title="Project Gutenberg is hosted by ibiblio.">
<span class ="icon icon_hosted" />
</a>
<p>This web site uses only free software.</p>
<a href="http://httpd.apache.org/" rel="nofollow"
title="Powered by Apache">
<span class="icon icon_apache" />
</a>
<a href="http://www.python.org" rel="nofollow"
title="Powered by Python">
<span class="icon icon_python" />
</a>
<a href="http://www.postgresql.org" rel="nofollow"
title="Powered by PostgreSQL">
<span class="icon icon_postgres" />
</a>
</div>
</py:def>
<py:def function="copyright">
<div class="copyright" i18n:comment="The copyright notice on the footer of every page.">
© 2003–2012 Project Gutenberg Literary Archive Foundation — All Rights Reserved.
</div>
</py:def>
<py:def function="site_top">
<div id="fb-root" />
<div id="print-head" class="noscreen">
<div class="center">http:${os.desktop_url}<br/><br/>${os.tagline}</div>
</div>
<div id="screen-head" class="noprint">
<table>
<tr i18n:comment="The logo, tagline and badges at the very top of every page.">
<td rowspan="2" id="logo" i18n:comment="The PG logo at the top left of every page.">
<a href="/wiki/Main_Page" title="Go to the Main Page.">
<span id="${cherrypy.tools.rate_limiter.get_challenge ()}" class="icon icon_logo" />
</a>
</td>
<td id="tagline-badges" colspan="2">
<table>
<tr>
<td id="tagline">${os.tagline}</td>
<td id="paypal-badge" class="badge">${paypal ()}</td>
<td id="flattr-badge" class="badge">${flattr ()}</td>
</tr>
</table>
</td>
</tr>
<tr id="menubar-search">
<td id="menubar" i18n:comment="The menu bar at the top of every page.">
<a py:if="os.page != 'start'"
id="menubar-first"
tabindex="11" accesskey="1"
title="Start a new search."
href="${os.url ('start')}">Search</a>
<span py:if="os.page == 'start'"
id="menubar-first" class="grayed">Search</span>
<a tabindex="22"
title="Our latest releases."
href="/ebooks/search/?sort_order=release_date">Latest</a>
<a tabindex="31"
title="Read the Project Gutenberg terms of use."
href="/terms_of_use/">Terms of Use</a>
<a tabindex="32"
href="/wiki/Gutenberg:Project_Gutenberg_Needs_Your_Donation"
title="Learn why we need some money.">Donate?</a>
<a tabindex="33" accesskey="m" href="${os.mobile_url}"
title="Go to our mobile site.">Mobile</a>
</td>
<td id="search" i18n:comment="The search box at the top right of every page.">
<form method="get" action="${os.desktop_search}"
enctype="multipart/form-data">
<table class="borderless">
<tr>
<td id="search-button-cell">
<button id="search-button" type="submit" title="Execute the search. &lt;enter&gt;">
<span class="icon icon_smsearch" />
</button>
</td>
<td id="search-input-cell">
<input id="search-input" name="query" type="text" title="${os.placeholder} &lt;s&gt;"
accesskey="s" value="${os.search_terms}" />
</td>
<td id="help-button-cell">
<button id="help-button" type="button" title="Open the help menu. &lt;h&gt;"
accesskey="h">Help</button>
</td>
</tr>
</table>
</form>
</td>
</tr>
</table>
<div id="helpbox" class="hide">
<xi:include href="help.html" />
</div>
</div>
</py:def>
<py:def function="site_footer">
<div class="footer" i18n:comment="On the footer of every page.">
</div>
</py:def>
<py:def function="prev_next_links">
<span class="links" i18n:comment="The links to move between paginated search results."
xmlns="http://www.w3.org/1999/xhtml"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:py="http://genshi.edgewall.org/">
<py:if test="os.show_prev_page_link">|
<a title="Go to the first page of results."
accesskey="a"
href="${os.url_carry (start_index = 1)}">First</a>
</py:if>
<py:if test="os.show_prev_page_link">|
<a title="Go to the previous page of results."
accesskey="-"
href="${os.url_carry (start_index = os.prev_page_index)}">Previous</a>
</py:if>
<py:if test="os.show_next_page_link">|
<a title="Go to the next page of results."
accesskey="+"
href="${os.url_carry (start_index = os.next_page_index)}">Next</a>
</py:if>
</span>
</py:def>
<py:def function="paypal">
<form class="paypal-button"
action="https://www.paypal.com/cgi-bin/webscr" method="post">
<div>
<!--! xml:lang avoids extraction for translation -->
<input xml:lang="en" type="hidden" name="cmd" value="_s-xclick" />
<input xml:lang="en" type="hidden" name="hosted_button_id" value="XKAL6BZL3YPSN" />
<input type="image" name="submit"
src="//${cherrypy.config['file_host']}/pics/paypal/${os.paypal_lang}.gif"
title="Send us money through PayPal." />
</div>
</form>
</py:def>
<py:def function="flattr">
<a class="flattr-button" target="_blank"
href="https://flattr.com/thing/509045/Project-Gutenberg"
title="Send us money through Flattr.">
<span class="icon icon_flattr" />
</a>
</py:def>
</html>

121
templates/site-layout.mobile Normal file
View File

@ -0,0 +1,121 @@
<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:py="http://genshi.edgewall.org/"
xmlns:i18n="http://genshi.edgewall.org/i18n"
py:strip="">
<py:def function="site_head">
<style type="text/css">
.icon { background: transparent url(/pics/sprite.png?${cherrypy.config['css_mtime']}) 0 0 no-repeat; }
</style>
<link rel="stylesheet" type="text/css"
href="/css/pg-mobile-one.css?${cherrypy.config['css_mtime']}" />
<script type="application/javascript"><![CDATA[
var mobile_search = "${os.add_amp (os.mobile_search)}query=";
var json_search = "${os.json_search}";
var msg_load_more = "${_('Load More Results…')}";
var do_animations = ${'true' if os.do_animations else 'false'};
]]>
</script>
<script type="application/javascript"
src="/js/pg-mobile-one.js?${cherrypy.config['js_mtime']}" />
<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=UTF-8" />
<meta http-equiv="Content-Style-Type" content="text/css" />
<meta http-equiv="Content-Language" content="${os.lang}" />
<meta name="description" content="The Project Gutenberg ebook catalog for mobile devices." />
<meta name="keywords" content="free ebooks, free books, free audio books" />
<meta name="classification" content="public" />
<link rel="icon" href="/pics/favicon" />
<link rel="canonical" href="${os.canonical_url}" />
<link rel="search"
type="application/opensearchdescription+xml"
title="Search Project Gutenberg"
href="${os.osd_url}" />
<link rel="alternate nofollow"
type="${os.type_opds}"
title="OPDS feed"
href="${os.url_carry (format = 'opds')}" />
<link py:if="os.touch_icon" rel="apple-touch-icon" href="${os.touch_icon}" />
<link py:if="os.touch_icon_precomposed" rel="apple-touch-icon-precomposed" href="${os.touch_icon_precomposed}" />
<meta py:if="os.viewport" name="viewport" content="${os.viewport}" />
</py:def>
<py:def function="search_box">
<li class="grayed" id="searchlist">
<div class="table link">
<div class="row">
<div class="cell leftcell">
<div class="icon icon_search" id="${cherrypy.tools.rate_limiter.get_challenge ()}" />
</div>
<div class="cell content">
<form id="search" method="get" action="${os.mobile_search}"
enctype="multipart/form-data">
<div id="query-clear-wrapper">
<div id="query-wrapper">
<input id="query" name="query"
type="text"
inputmode="latin"
value="${os.search_terms}"
title="${os.placeholder}" />
</div>
</div>
</form>
</div>
<!--! <div class="cell cancelcell">
<button type="reset" id="clear" class="icon icon_cancel"
title="Clear" i18n:comment="Reset Form Button" />
</div> -->
<div class="cell rightcell">
<button type="button" id="help"
title="Help" onclick="toggle_help ()" i18n:comment="Help about search button">
<div class="icon icon_help" />
</button>
</div>
</div>
</div>
</li>
<li class="grayed" id="helpbox"
style="display: none" onclick="clear_help ()">
<div class="helpbox">
<xi:include href="help.html" />
</div>
</li>
<py:if test="os.page != 'start' and os.start_index == 1">
<li class="navlink grayed">
<a class="table link" href="${os.url ('start')}" accesskey="h">
<span class="row">
<span class="cell leftcell">
<span class="icon icon_internal" />
</span>
<span class="cell content">
<span class="title">Search Start Page</span>
</span>
<span class="cell rightcell">
<span class="icon icon_next" />
</span>
</span>
</a>
</li>
</py:if>
</py:def>
<py:def function="site_footer">
<div class="footer">
<div class="copyright">
© 2003–2012 Project Gutenberg Literary Archive Foundation — All Rights Reserved.
</div>
</div>
</py:def>
</html>

15
templates/sitemap-index.xml Normal file
View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--! support.google.com/webmasters/bin/answer.py?hl=en&answer=71453&topic=8476&ctx=topic -->
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:py="http://genshi.edgewall.org/"
>
<sitemap py:for="s in data.sitemaps">
<loc>${s.loc}</loc>
<lastmod>${s.lastmod}</lastmod>
</sitemap>
</sitemapindex>

15
templates/sitemap.xml Normal file
View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--! support.google.com/webmasters/bin/answer.py?hl=en&answer=183668&topic=8476&ctx=topic -->
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:mobile="http://www.google.com/schemas/sitemap-mobile/1.0"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:py="http://genshi.edgewall.org/"
>
<url py:for="u in data.urls">
<loc>${u.loc}</loc>
</url>
</urlset>

92
templates/social-functions.html Normal file
View File

@ -0,0 +1,92 @@
<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:py="http://genshi.edgewall.org/"
xmlns:i18n="http://genshi.edgewall.org/i18n"
xmlns:og="http://opengraphprotocol.org/schema/"
py:strip="">
<?python
from six.moves import urllib
def p (params):
return urllib.parse.urlencode (params).replace ('+', '%20')
?>
<py:def function="fb_share(url, title, description, picture)">
<?python
params = {
'link': url,
'app_id': cherrypy.config['facebook_app_id'],
'name': title,
'description': description,
'redirect_uri': 'https://www.gutenberg.org/fb_redirect.html',
}
if picture is not None:
params['picture'] = picture
?>
<div class="social-button fb-share-button" i18n:comment="Share on Facebook.">
<a href="https://www.facebook.com/dialog/feed?${p (params)}"
title="Share on Facebook"
onclick="open_share_popup(this.href, this.target, 1024, 560)" target="_fb_share_popup">
<span class="icon icon_facebook" />
</a>
</div>
</py:def>
<py:def function="gplus_share(url)">
<!-- share without javascript -->
<?python
params = {
'url': url,
}
?>
<div class="social-button gplus-share-button">
<!--! https://developers.google.com/+/web/share/#sharelink -->
<a href="https://plus.google.com/share?${p (params)}"
title="Share on Google+"
onclick="open_share_popup(this.href, this.target, 640, 320)"
target="_gplus_share_popup">
<span class="icon icon_gplus" />
</a>
</div>
</py:def>
<py:def function="tw_share(url, text)">
<!-- tweet without javascript -->
<?python
params = {
'url': url,
'text': text.encode ('utf-8'),
'count': 'none',
'lang': os.twitter_lang,
'related': "gutenberg_new:Project Gutenberg New Books"
}
?>
<div class="social-button twitter-share-button">
<!--! https://dev.twitter.com/docs/tweet-button -->
<a href="https://twitter.com/share?${p (params)}"
title="Share on Twitter"
onclick="open_share_popup(this.href, this.target, 640, 320)"
target="_tw_share_popup">
<span class="icon icon_twitter" />
</a>
</div>
</py:def>
<py:def function="rss_follow()">
<div class="social-button rss-follow-button">
<a href="/feeds/today.rss"
title="Subscribe to the New Books RSS feed.">
<span class="icon icon_rss" />
<span class="alt" i18n:comment="Subscribe to RSS feed.">Subscribe</span>
</a>
</div>
</py:def>
</html>

165
templates/stats.html Normal file
View File

@ -0,0 +1,165 @@
<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:py="http://genshi.edgewall.org/"
xmlns:i18n="http://genshi.edgewall.org/i18n"
xml:lang="en">
<!--! xml:lang avoids extraction for translation -->
<?python
from libgutenberg.GutenbergGlobals import xmlspecialchars as esc
?>
<py:def function="print_session">
<?python
from __future__ import unicode_literals
hostname = ''
whois = ''
tags = []
bad_tags = []
ips = d.ips
info = ips.ipinfo
if info is not None:
hostname = info.hostname
whois = ' - '.join (filter (None, info.whois.values ()))
if info.is_dialup ():
tags.append ('DUL')
bad_tags = sorted (info.tags & info.blackhat_tags)
if d['blocked'] == 9:
tags.append ('->DB')
if d['css_ok'] == False:
bad_tags.append ('NO_CSS')
session_cnt = len (d.sessions)
?>
<span class='ip'>${str (ips)}</span>
-
<span class='hits'>${d['hits']}</span> hits
<span py:if="d['dhits'] > 0"><span class='dhits'>${d['dhits']}</span> denied</span>
<span class='rhits'>(${d['rhits']}/${d['rhits_max']})</span>
<span py:if='session_cnt > 1' class='sessions'>${session_cnt} sess.</span>
<span class='host'>${hostname}</span>
<span class='tags'>${' '.join (sorted (tags))}</span>
<span class='bad_tags'>${' '.join (sorted (bad_tags))}</span>
<span class='whois'>${whois}</span>
<span class="ua">${' -- '.join (sorted (
[rl.ua_decode (ua) for ua in d['user_agents']]
))[:1000]}</span>
<span class="ua-sig">${' -- '.join (sorted (
[rl.ua_decode (sig) for sig in d['signatures']]
))[:1000]}</span>
<a href="/webmaster/stats/block/?ip=${ips.get_ip_to_block ()}">block</a>
<a href="/webmaster/stats/unblock/?ip=${ips.get_ip_to_block ()}">unblock</a>
</py:def>
<py:def function="print_requests">
<ul class="requests">
<li py:for="request in d['requests']">${request}</li>
</ul>
</py:def>
<head>
<title>AppServer Status</title>
<style type="text/css">
.ip { font-weight: bold }
.hits { font-weight: bold }
.dhits { font-weight: bold; color: red }
.rhits { color: blue }
.host { font-weight: bold }
.sessions { font-weight: bold; color: red }
.whois { color: grey }
.tags { font-weight: bold; color: green }
.bad_tags { font-weight: bold; color: red }
.ua { color: blue }
.ua-sig { color: #808 }
.requests { list-style-type: none; font-size: small; color: grey }
.requests li { display: inline }
.requests li:before { content: ' - ' }
.whitelisted { color: green }
.nagios { display: none }
</style>
</head>
<body>
<h1>Stats</h1>
<p>
RHits Max: ${rl.rhits_max}.
Captchas: ${rl.captchas_max}.
Cleanup Frequency: ${rl.frequency}s.
</p>
<p>Active Users: ${rl.users}</p>
<p>Hits: ${rl.hits}</p>
<p>Whitelisted: ${rl.whitelist_hits}</p>
<p>Denied: ${rl.denied_hits}</p>
<p>Blocked IPs: ${len (blocked)}</p>
<p>Total Backends: ${backends}</p>
<p>Active Backends: ${active_backends}</p>
<p>Formats Sum:</p>
<table>
<tr py:for="key, value, percent in bs.formats_sum_acc.iter_results ()">
<td>${key}</td><td>${value}</td><td>${"%.3f" % percent}</td>
<td class="nagios">nagios-${key}-sum: ${value} ${"%.3f" % percent}</td>
</tr>
</table>
<p>Formats Detail:</p>
<table>
<tr py:for="key, value, percent in bs.formats_acc.iter_results ()">
<td>${key}</td><td>${value}</td><td>${"%.3f" % percent}</td>
<td class="nagios">nagios-${key}: ${value} ${"%.3f" % percent}</td>
</tr>
</table>
<py:if test="resolve">
<p>Blocked IPs:</p>
<ul>
<!--! the blocked IPs sorted by IP -->
<li py:for="d in blocked">
${print_session ()}
${print_requests ()}
</li>
</ul>
<p>Busiest IPs:</p>
<ul>
<!--! the top 10 active IPs sorted by hits desc -->
<li py:for="d in busiest" class="${'whitelisted' if d.ips.whitelisted else ''}">
${print_session ()}
${print_requests ()}
</li>
</ul>
<p>IPs with most sessions:</p>
<ul>
<!--! the top 10 IPs with most sessions sorted by sessions desc -->
<li py:for="d in most_sessions" class="${'whitelisted' if d.ips.whitelisted else ''}">
${print_session ()}
${print_requests ()}
</li>
</ul>
<p>Active IPs: ${len (active)}</p>
<ul>
<!--! the active IPs sorted by IP -->
<li py:for="d in active" class="${'whitelisted' if d.ips.whitelisted else ''}">
${print_session ()}
</li>
</ul>
</py:if> <!--! resolve -->
</body>
</html>