386 lines
13 KiB
Python
386 lines
13 KiB
Python
import datetime
|
|
from itertools import islice
|
|
import logging
|
|
from urllib.parse import urlparse, urlunparse
|
|
|
|
from bs4 import BeautifulSoup
|
|
import pytz
|
|
|
|
from django.urls import reverse
|
|
from django.utils.http import urlquote
|
|
|
|
|
|
|
|
from regluit.core import models, facets
|
|
import regluit.core.cc as cc
|
|
|
|
licenses = cc.LICENSE_LIST
|
|
logger = logging.getLogger(__name__)
|
|
|
|
soup = None
|
|
FORMAT_TO_MIMETYPE = {'pdf':"application/pdf",
|
|
'epub':"application/epub+zip",
|
|
'mobi':"application/x-mobipocket-ebook",
|
|
'html':"text/html",
|
|
'text':"text/html"}
|
|
|
|
UNGLUEIT_URL = 'https://unglue.it'
|
|
ACQUISITION = "application/atom+xml;profile=opds-catalog;kind=acquisition"
|
|
FACET_RELATION = "http://opds-spec.org/facet"
|
|
|
|
old_facets = ["creative_commons", "active_campaigns"]
|
|
|
|
|
|
def feeds():
|
|
for facet in old_facets:
|
|
yield globals()[facet]
|
|
for facet_path in facets.get_all_facets('Format'):
|
|
yield get_facet_facet(facet_path)
|
|
for facet_path in facets.get_all_facets('Keyword'):
|
|
yield get_facet_facet(facet_path)
|
|
|
|
def get_facet_class(name):
|
|
if name in old_facets:
|
|
return globals()[name]
|
|
return get_facet_facet(name)
|
|
|
|
|
|
def text_node(tag, text):
|
|
node = soup.new_tag(tag)
|
|
if text:
|
|
node.string = text
|
|
return node
|
|
|
|
def html_node(tag, html):
|
|
node = text_node(tag, html)
|
|
node.attrs.update({"type":'html'})
|
|
return node
|
|
|
|
def add_query_component(url, qc):
|
|
"""
|
|
add component qc to the querystring of url
|
|
"""
|
|
m = list(urlparse(url))
|
|
if m[4]:
|
|
m[4] = "&".join([m[4], qc])
|
|
else:
|
|
m[4] = qc
|
|
return urlunparse(m)
|
|
|
|
def isbn_node(isbn):
|
|
node = soup.new_tag("dcterms:identifier")
|
|
node.attrs.update({"xsi:type":'dcterms:URI'})
|
|
node.string = 'urn:ISBN:'+ isbn
|
|
return node
|
|
|
|
def work_node(work, facet=None):
|
|
|
|
node = soup.new_tag("entry")
|
|
# title
|
|
node.append(text_node("title", work.title))
|
|
|
|
# id
|
|
node.append(text_node(
|
|
'id',
|
|
"{base}{url}".format(
|
|
base=UNGLUEIT_URL,
|
|
url=reverse('work_identifier', kwargs={'work_id': work.id})
|
|
)
|
|
))
|
|
|
|
updated = None
|
|
|
|
# links for all ebooks
|
|
ebooks = facet.filter_model("Ebook", work.ebooks()) if facet else work.ebooks()
|
|
versions = set()
|
|
for ebook in ebooks:
|
|
if updated is None:
|
|
# most recent ebook, first ebook in loop
|
|
updated = ebook.created.isoformat()
|
|
node.append(text_node('updated', updated))
|
|
if not ebook.version_label in versions:
|
|
versions.add(ebook.version_label)
|
|
link_node = soup.new_tag("link")
|
|
|
|
# ebook.download_url is an absolute URL with the protocol, domain, and path baked in
|
|
link_rel = "http://opds-spec.org/acquisition/open-access"
|
|
link_node.attrs.update({
|
|
"href":add_query_component(ebook.download_url, "feed=opds"),
|
|
"rel":link_rel,
|
|
"dcterms:rights": str(ebook.rights)
|
|
})
|
|
if ebook.is_direct():
|
|
link_node["type"] = FORMAT_TO_MIMETYPE.get(ebook.format, "")
|
|
else:
|
|
# indirect acquisition, i.e. google books
|
|
link_node["type"] = "text/html"
|
|
indirect = soup.new_tag("opds:indirectAcquisition",)
|
|
indirect["type"] = FORMAT_TO_MIMETYPE.get(ebook.format, "")
|
|
link_node.append(indirect)
|
|
if ebook.version_label:
|
|
link_node.attrs.update({"schema:version": ebook.version_label})
|
|
node.append(link_node)
|
|
|
|
# get the cover -- assume jpg?
|
|
|
|
cover_node = soup.new_tag("link")
|
|
cover_node.attrs.update({
|
|
"href": work.cover_image_small(),
|
|
"type": "image/" + work.cover_filetype(),
|
|
"rel": "http://opds-spec.org/image/thumbnail"
|
|
})
|
|
node.append(cover_node)
|
|
cover_node = soup.new_tag("link")
|
|
cover_node.attrs.update({
|
|
"href": work.cover_image_thumbnail(),
|
|
"type": "image/" + work.cover_filetype(),
|
|
"rel": "http://opds-spec.org/image"
|
|
})
|
|
node.append(cover_node)
|
|
|
|
|
|
# <dcterms:issued>2012</dcterms:issued>
|
|
node.append(text_node("dcterms:issued", work.publication_date))
|
|
|
|
# author
|
|
# TO DO: include all authors?
|
|
author_node = soup.new_tag("author")
|
|
author_node.append(text_node("name", work.author()))
|
|
node.append(author_node)
|
|
|
|
# publisher
|
|
#<dcterms:publisher>Open Book Publishers</dcterms:publisher>
|
|
if work.publishers().count():
|
|
for publisher in work.publishers():
|
|
node.append(text_node("dcterms:publisher", publisher.name.name))
|
|
|
|
# language
|
|
#<dcterms:language>en</dcterms:language>
|
|
node.append(text_node("dcterms:language", work.language))
|
|
|
|
# description
|
|
node.append(html_node("content", work.description))
|
|
|
|
# identifiers
|
|
if work.identifiers.filter(type='isbn'):
|
|
for isbn in work.identifiers.filter(type='isbn')[0:9]: #10 should be more than enough
|
|
node.append(isbn_node(isbn.value))
|
|
|
|
# subject tags
|
|
# [[subject.name for subject in work.subjects.all()] for work in ccworks if work.subjects.all()]
|
|
for subject in work.subjects.all():
|
|
if subject.is_visible:
|
|
category_node = soup.new_tag("category")
|
|
try:
|
|
category_node["term"] = subject.name
|
|
node.append(category_node)
|
|
try:
|
|
subject.works.filter(is_free=True)[1]
|
|
# only show feed if there's another work in it
|
|
node.append(navlink('related', 'kw.' + subject.name, 0,
|
|
'popular', title=subject.name))
|
|
except:
|
|
pass
|
|
except ValueError:
|
|
# caused by control chars in subject.name
|
|
logger.warning('Deleting subject: %s' % subject.name)
|
|
subject.delete()
|
|
|
|
# age level
|
|
# <category term="15-18" scheme="http://schema.org/typicalAgeRange"
|
|
# label="Teen - Grade 10-12, Age 15-18"/>
|
|
if work.age_level:
|
|
category_node = soup.new_tag("category")
|
|
category_node["scheme"] = 'http://schema.org/typicalAgeRange'
|
|
category_node["term"] = work.age_level
|
|
category_node["label"] = work.get_age_level_display()
|
|
node.append(category_node)
|
|
|
|
|
|
# rating
|
|
rating_node = soup.new_tag("schema:Rating")
|
|
rating_node.attrs.update({"schema:ratingValue":"{:}".format(work.priority())})
|
|
node.append(rating_node)
|
|
return node
|
|
|
|
class Facet:
|
|
title = ''
|
|
works = None
|
|
feed_path = ''
|
|
description = ''
|
|
|
|
def feed(self, page=None, order_by='newest'):
|
|
self.works = self.works.order_by(*facets.get_order_by(order_by))
|
|
return opds_feed_for_works(self, page=page, order_by=order_by)
|
|
|
|
def updated(self):
|
|
# return the creation date for most recently added item
|
|
if not self.works:
|
|
return pytz.utc.localize(datetime.datetime.utcnow()).isoformat()
|
|
return pytz.utc.localize(self.works[0].created).isoformat()
|
|
|
|
def get_facet_facet(facet_path):
|
|
class Facet_Facet(Facet):
|
|
|
|
def __init__(self, facet_path=facet_path):
|
|
self.feed_path = facet_path
|
|
self.facet_object = facets.get_facet_object(facet_path)
|
|
self.title = "Unglue.it"
|
|
for facet in self.facet_object.facets():
|
|
self.title = self.title + " " + facet.title
|
|
self.works = self.facet_object.get_query_set().distinct()
|
|
self.description = self.facet_object.description
|
|
return Facet_Facet
|
|
|
|
class creative_commons(Facet):
|
|
def __init__(self):
|
|
self.title = "Unglue.it Catalog: Creative Commons Books"
|
|
self.feed_path = "creative_commons"
|
|
self.works = models.Work.objects.filter(
|
|
editions__ebooks__isnull=False,
|
|
editions__ebooks__rights__in=cc.LICENSE_LIST
|
|
).distinct()
|
|
self.description = """These Creative Commons licensed ebooks are free to read - the people
|
|
who created them want you to read and share them."""
|
|
self.facet_object = facets.get_facet_object(self.feed_path)
|
|
|
|
class active_campaigns(Facet):
|
|
"""
|
|
return opds feed for works associated with active campaigns
|
|
"""
|
|
def __init__(self):
|
|
self.title = "Unglue.it Catalog: Books under Active Campaign"
|
|
self.feed_path = "active_campaigns"
|
|
self.works = models.Work.objects.filter(campaigns__status='ACTIVE', is_free=True)
|
|
self.description = """With your help we're raising money
|
|
to make these books free to the world."""
|
|
self.facet_object = facets.get_facet_object(self.feed_path)
|
|
|
|
def opds_feed_for_work(work_id):
|
|
class single_work_facet:
|
|
def __init__(self, work_id):
|
|
try:
|
|
works = models.Work.objects.filter(id=work_id)
|
|
except models.Work.DoesNotExist:
|
|
works = models.Work.objects.none()
|
|
except ValueError:
|
|
# not a valid work_id
|
|
works = models.Work.objects.none()
|
|
self.works = works
|
|
self.title = 'Unglue.it work #%s' % work_id
|
|
self.feed_path = ''
|
|
self.facet_object = facets.BaseFacet(None)
|
|
return opds_feed_for_works(single_work_facet(work_id))
|
|
|
|
def opds_feed_for_works(the_facet, page=None, order_by='newest'):
|
|
global soup
|
|
if not soup:
|
|
soup = BeautifulSoup('', 'lxml')
|
|
works = the_facet.works
|
|
feed_path = the_facet.feed_path
|
|
title = the_facet.title
|
|
feed_header = """<?xml version="1.0" encoding="UTF-8"?>
|
|
<feed xmlns:dcterms="http://purl.org/dc/terms/"
|
|
xmlns:opds="http://opds-spec.org/"
|
|
xmlns="http://www.w3.org/2005/Atom"
|
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
xmlns:schema="http://schema.org/"
|
|
xsi:noNamespaceSchemaLocation="http://www.kbcafe.com/rss/atom.xsd.xml"
|
|
xsi:schemaLocation="http://purl.org/dc/elements/1.1/ http://dublincore.org/schemas/xmls/qdc/2008/02/11/dc.xsd
|
|
http://purl.org/dc/terms/ http://dublincore.org/schemas/xmls/qdc/2008/02/11/dcterms.xsd">
|
|
"""
|
|
|
|
yield feed_header
|
|
|
|
# add title
|
|
# TO DO: will need to calculate the number items and where in the feed we are
|
|
|
|
yield text_node('title', title + ' - sorted by ' + order_by).prettify()
|
|
|
|
# id
|
|
|
|
feed = text_node(
|
|
'id',
|
|
"{url}/api/opds/{feed_path}/?order_by={order_by}".format(
|
|
url=UNGLUEIT_URL,
|
|
feed_path=urlquote(feed_path),
|
|
order_by=order_by,
|
|
),
|
|
)
|
|
yield feed.prettify()
|
|
|
|
# updated
|
|
# TO DO: fix time zone?
|
|
# also use our wrapped datetime code
|
|
|
|
feed = text_node('updated', pytz.utc.localize(datetime.datetime.utcnow()).isoformat())
|
|
yield feed.prettify()
|
|
|
|
# author
|
|
|
|
author_node = soup.new_tag("author")
|
|
author_node.append(text_node('name', 'unglue.it'))
|
|
author_node.append(text_node('uri', UNGLUEIT_URL))
|
|
yield author_node.prettify()
|
|
|
|
# links: start, self, next/prev (depending what's necessary -- to start with put all CC books)
|
|
|
|
# start link
|
|
yield navlink('start', feed_path, None, order_by, title="First 10").prettify()
|
|
|
|
# next link
|
|
|
|
if not page:
|
|
page = 0
|
|
else:
|
|
try:
|
|
page = int(page)
|
|
except TypeError:
|
|
page = 0
|
|
|
|
try:
|
|
works[10 * page + 10]
|
|
yield navlink('next', feed_path, page+1, order_by, title="Next 10").prettify()
|
|
except IndexError:
|
|
pass
|
|
|
|
# sort facets
|
|
yield navlink(FACET_RELATION, feed_path, None, 'popular', group="Order",
|
|
active=order_by == 'popular', title="Sorted by popularity").prettify()
|
|
yield navlink(FACET_RELATION, feed_path, None, 'newest', group="Order",
|
|
active=order_by == 'newest', title="Sorted by newest").prettify()
|
|
|
|
#other facets
|
|
if feed_path not in old_facets:
|
|
for other_group in the_facet.facet_object.get_other_groups():
|
|
for facet_object in other_group.get_facets():
|
|
yield navlink(FACET_RELATION, feed_path + '/' + facet_object.facet_name,
|
|
None, order_by, group=other_group.title,
|
|
title=facet_object.title).prettify()
|
|
|
|
works = islice(works, 10 * page, 10 * page + 10)
|
|
if page > 0:
|
|
yield navlink('previous', feed_path, page-1, order_by, title="Previous 10").prettify()
|
|
|
|
for work in works:
|
|
yield work_node(work, facet=the_facet.facet_object).prettify()
|
|
|
|
yield '</feed>'
|
|
|
|
def navlink(rel, path, page, order_by, group=None, active=None, title=""):
|
|
link = soup.new_tag("link")
|
|
link.attrs.update({
|
|
"rel":rel,
|
|
"href": UNGLUEIT_URL + "/api/opds/" + urlquote(path) + '/?order_by=' + order_by + (
|
|
'&page=' + str(page) if page is not None else ''
|
|
),
|
|
"type": ACQUISITION,
|
|
"title": title,
|
|
})
|
|
if rel == FACET_RELATION:
|
|
if group:
|
|
link['opds:facetGroup'] = group
|
|
if active:
|
|
link['opds:activeFacet'] = 'true'
|
|
return link
|