From f4a9697971f4b6dfd431809a194941058aa00680 Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 1 Apr 2020 17:18:37 -0400 Subject: [PATCH 1/8] fix some problems --- api/opds.py | 3 ++- core/models/bibmodels.py | 8 ++++---- settings/common.py | 2 ++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/api/opds.py b/api/opds.py index 13c0a176..2c3da9a8 100644 --- a/api/opds.py +++ b/api/opds.py @@ -47,7 +47,8 @@ def get_facet_class(name): def text_node(tag, text): node = soup.new_tag(tag) - node.string = text + if text: + node.string = text return node def html_node(tag, html): diff --git a/core/models/bibmodels.py b/core/models/bibmodels.py index 7ee3be3f..6c3b8580 100644 --- a/core/models/bibmodels.py +++ b/core/models/bibmodels.py @@ -891,7 +891,7 @@ class Edition(models.Model): im = get_thumbnail(self.cover_image, 'x550', crop='noop', quality=95) if im.exists(): return im.url - except IOError: + except (IOError, OSError): pass elif self.googlebooks_id: url = "https://encrypted.google.com/books?id=%s&printsec=frontcover&img=1&zoom=0" % self.googlebooks_id @@ -902,7 +902,7 @@ class Edition(models.Model): im = get_thumbnail(url, 'x550', crop='noop', quality=95) if im.exists(): return im.url - except IOError: + except (IOError, OSError): pass return '' @@ -913,7 +913,7 @@ class Edition(models.Model): im = get_thumbnail(self.cover_image, 'x80', crop='noop', quality=95) if im.exists(): return im.url - except IOError: + except (IOError, OSError): pass if self.googlebooks_id: return "https://encrypted.google.com/books?id=%s&printsec=frontcover&img=1&zoom=5" % self.googlebooks_id @@ -926,7 +926,7 @@ class Edition(models.Model): im = get_thumbnail(self.cover_image, '128', crop='noop', quality=95) if im.exists(): return im.url - except IOError: + except (IOError, OSError): pass if self.googlebooks_id: return "https://encrypted.google.com/books?id=%s&printsec=frontcover&img=1&zoom=1" % self.googlebooks_id diff --git a/settings/common.py b/settings/common.py index 8a8f37c4..1345a5ab 100644 --- a/settings/common.py +++ b/settings/common.py @@ -433,6 +433,8 @@ NOTIFICATION_QUEUE_ALL = True # amazon or paypal for now. PAYMENT_PROCESSOR = 'stripelib' +# allow application code to catch thumbnailing errors +THUMBNAIL_DEBUG = True # we should suppress Google Analytics outside of production SHOW_GOOGLE_ANALYTICS = False From f77f8b4006ce0048555b2511b72ca851328ddc5a Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 1 Apr 2020 20:24:36 -0400 Subject: [PATCH 2/8] try being careful instantiating soup --- api/onix.py | 10 +++++----- api/opds.py | 18 +++++++++--------- api/views.py | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/api/onix.py b/api/onix.py index fb04a7ef..e88a1842 100644 --- a/api/onix.py +++ b/api/onix.py @@ -17,8 +17,8 @@ feed_header = """ """ feed_xml = feed_header + '' +soup = BeautifulSoup('', 'xml') bisac = Bisac() -soup = BeautifulSoup(feed_xml, 'xml') def text_node(tag, text, attrib=None): node = soup.new_tag(tag) @@ -56,13 +56,13 @@ def onix_feed(facet, max=None, page_number=None): yield '' def onix_feed_for_work(work): - soup = BeautifulSoup(feed_xml, 'xml') - soup.ONIXMessage.append(header(work)) + feed = BeautifulSoup(feed_xml, 'xml') + feed.ONIXMessage.append(header(work)) for edition in models.Edition.objects.filter(work=work, ebooks__isnull=False).distinct(): edition_prod = product(edition) if edition_prod is not None: - soup.ONIXMessage.append(product(edition)) - return str(soup) + feed.ONIXMessage.append(product(edition)) + return str(feed) def header(facet=None): header_node = soup.new_tag("Header") diff --git a/api/opds.py b/api/opds.py index 2c3da9a8..d4e6b45c 100644 --- a/api/opds.py +++ b/api/opds.py @@ -16,8 +16,8 @@ import regluit.core.cc as cc licenses = cc.LICENSE_LIST logger = logging.getLogger(__name__) -soup = BeautifulSoup('', 'xml') +soup = BeautifulSoup('', 'xml') FORMAT_TO_MIMETYPE = {'pdf':"application/pdf", 'epub':"application/epub+zip", 'mobi':"application/x-mobipocket-ebook", @@ -125,16 +125,16 @@ def work_node(work, facet=None): cover_node = soup.new_tag("link") cover_node.attrs.update({ - "href":work.cover_image_small(), - "type":"image/"+work.cover_filetype(), - "rel":"http://opds-spec.org/image/thumbnail" + "href": work.cover_image_small(), + "type": "image/" + work.cover_filetype(), + "rel": "http://opds-spec.org/image/thumbnail" }) node.append(cover_node) cover_node = soup.new_tag("link") cover_node.attrs.update({ - "href":work.cover_image_thumbnail(), - "type":"image/"+work.cover_filetype(), - "rel":"http://opds-spec.org/image" + "href": work.cover_image_thumbnail(), + "type": "image/" + work.cover_filetype(), + "rel": "http://opds-spec.org/image" }) node.append(cover_node) @@ -276,7 +276,8 @@ def opds_feed_for_works(the_facet, page=None, order_by='newest'): works = the_facet.works feed_path = the_facet.feed_path title = the_facet.title - feed_header = """ + 0: yield navlink('previous', feed_path, page-1, order_by, title="Previous 10").prettify() - for work in works: yield work_node(work, facet=the_facet.facet_object).prettify() diff --git a/api/views.py b/api/views.py index 51fae3d5..55a9b5e6 100755 --- a/api/views.py +++ b/api/views.py @@ -225,7 +225,7 @@ class OnixView(View): work = models.safe_get_work(work) except models.Work.DoesNotExist: raise Http404 - return StreamingHttpResponse(onix.onix_feed_for_work(work), content_type="text/xml") + return HttpResponse(onix.onix_feed_for_work(work), content_type="text/xml") facet = kwargs.get('facet', 'all') From 6d083cea8319023c75ffafe39971b477a9628f96 Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 2 Apr 2020 10:27:08 -0400 Subject: [PATCH 3/8] flailing here Trying to isolate deployment problem --- api/views.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/api/views.py b/api/views.py index 55a9b5e6..7ba55188 100755 --- a/api/views.py +++ b/api/views.py @@ -17,7 +17,7 @@ from django.http import ( HttpResponseBadRequest, HttpResponseRedirect, Http404, - StreamingHttpResponse, + #StreamingHttpResponse, ) @@ -194,10 +194,12 @@ class OPDSAcquisitionView(View): work = request.GET.get('work', None) if work: if self.json: - return StreamingHttpResponse(opds_json.opds_feed_for_work(work), + #return StreamingHttpResponse(opds_json.opds_feed_for_work(work), + return HttpResponse(opds_json.opds_feed_for_work(work), content_type="application/opds-publication+json") else: - return StreamingHttpResponse(opds.opds_feed_for_work(work), + #return StreamingHttpResponse(opds.opds_feed_for_work(work), + return HttpResponse(opds.opds_feed_for_work(work), content_type="application/atom+xml;profile=opds-catalog;kind=acquisition") facet = kwargs.get('facet') page = request.GET.get('page', None) @@ -212,7 +214,8 @@ class OPDSAcquisitionView(View): content_type="application/opds+json") else: facet_class = opds.get_facet_class(facet)() - return StreamingHttpResponse(facet_class.feed(page,order_by), + #return StreamingHttpResponse(facet_class.feed(page,order_by), + return HttpResponse(facet_class.feed(page,order_by), content_type="application/atom+xml;profile=opds-catalog;kind=acquisition") @@ -247,5 +250,6 @@ class OnixView(View): page = None feed = onix.onix_feed(facet_class, max_records, page_number=page) - return StreamingHttpResponse(feed, content_type="text/xml") + #return StreamingHttpResponse(feed, content_type="text/xml") + return HttpResponse(feed, content_type="text/xml") From bfa957429fb2676a8bd39cb53b02c19df2fc3a5c Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 2 Apr 2020 12:34:06 -0400 Subject: [PATCH 4/8] maybe its the xml parser??? --- api/onix.py | 2 +- api/opds.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/onix.py b/api/onix.py index e88a1842..6b6488cb 100644 --- a/api/onix.py +++ b/api/onix.py @@ -17,7 +17,7 @@ feed_header = """ """ feed_xml = feed_header + '' -soup = BeautifulSoup('', 'xml') +soup = BeautifulSoup('', 'lxml') bisac = Bisac() def text_node(tag, text, attrib=None): diff --git a/api/opds.py b/api/opds.py index d4e6b45c..1ebc3981 100644 --- a/api/opds.py +++ b/api/opds.py @@ -17,7 +17,7 @@ import regluit.core.cc as cc licenses = cc.LICENSE_LIST logger = logging.getLogger(__name__) -soup = BeautifulSoup('', 'xml') +soup = BeautifulSoup('', 'lxml') FORMAT_TO_MIMETYPE = {'pdf':"application/pdf", 'epub':"application/epub+zip", 'mobi':"application/x-mobipocket-ebook", From 9d7c780488f8e3725b7de2f48a173f3e2651e8a6 Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 2 Apr 2020 12:42:10 -0400 Subject: [PATCH 5/8] Revert "maybe its the xml parser???" This reverts commit bfa957429fb2676a8bd39cb53b02c19df2fc3a5c. --- api/onix.py | 2 +- api/opds.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/onix.py b/api/onix.py index 6b6488cb..e88a1842 100644 --- a/api/onix.py +++ b/api/onix.py @@ -17,7 +17,7 @@ feed_header = """ """ feed_xml = feed_header + '' -soup = BeautifulSoup('', 'lxml') +soup = BeautifulSoup('', 'xml') bisac = Bisac() def text_node(tag, text, attrib=None): diff --git a/api/opds.py b/api/opds.py index 1ebc3981..d4e6b45c 100644 --- a/api/opds.py +++ b/api/opds.py @@ -17,7 +17,7 @@ import regluit.core.cc as cc licenses = cc.LICENSE_LIST logger = logging.getLogger(__name__) -soup = BeautifulSoup('', 'lxml') +soup = BeautifulSoup('', 'xml') FORMAT_TO_MIMETYPE = {'pdf':"application/pdf", 'epub':"application/epub+zip", 'mobi':"application/x-mobipocket-ebook", From 5579308e4dc2a5b4fc0fa5a4e2804fc5c297702d Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 2 Apr 2020 12:42:17 -0400 Subject: [PATCH 6/8] Revert "flailing here" This reverts commit 6d083cea8319023c75ffafe39971b477a9628f96. --- api/views.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/api/views.py b/api/views.py index 7ba55188..55a9b5e6 100755 --- a/api/views.py +++ b/api/views.py @@ -17,7 +17,7 @@ from django.http import ( HttpResponseBadRequest, HttpResponseRedirect, Http404, - #StreamingHttpResponse, + StreamingHttpResponse, ) @@ -194,12 +194,10 @@ class OPDSAcquisitionView(View): work = request.GET.get('work', None) if work: if self.json: - #return StreamingHttpResponse(opds_json.opds_feed_for_work(work), - return HttpResponse(opds_json.opds_feed_for_work(work), + return StreamingHttpResponse(opds_json.opds_feed_for_work(work), content_type="application/opds-publication+json") else: - #return StreamingHttpResponse(opds.opds_feed_for_work(work), - return HttpResponse(opds.opds_feed_for_work(work), + return StreamingHttpResponse(opds.opds_feed_for_work(work), content_type="application/atom+xml;profile=opds-catalog;kind=acquisition") facet = kwargs.get('facet') page = request.GET.get('page', None) @@ -214,8 +212,7 @@ class OPDSAcquisitionView(View): content_type="application/opds+json") else: facet_class = opds.get_facet_class(facet)() - #return StreamingHttpResponse(facet_class.feed(page,order_by), - return HttpResponse(facet_class.feed(page,order_by), + return StreamingHttpResponse(facet_class.feed(page,order_by), content_type="application/atom+xml;profile=opds-catalog;kind=acquisition") @@ -250,6 +247,5 @@ class OnixView(View): page = None feed = onix.onix_feed(facet_class, max_records, page_number=page) - #return StreamingHttpResponse(feed, content_type="text/xml") - return HttpResponse(feed, content_type="text/xml") + return StreamingHttpResponse(feed, content_type="text/xml") From 3bb738dbf3db30a8ea56c19a7b9bb41a642928a8 Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 2 Apr 2020 13:47:50 -0400 Subject: [PATCH 7/8] Revert "Revert "maybe its the xml parser???"" This reverts commit 9d7c780488f8e3725b7de2f48a173f3e2651e8a6. --- api/onix.py | 2 +- api/opds.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/onix.py b/api/onix.py index e88a1842..6b6488cb 100644 --- a/api/onix.py +++ b/api/onix.py @@ -17,7 +17,7 @@ feed_header = """ """ feed_xml = feed_header + '' -soup = BeautifulSoup('', 'xml') +soup = BeautifulSoup('', 'lxml') bisac = Bisac() def text_node(tag, text, attrib=None): diff --git a/api/opds.py b/api/opds.py index d4e6b45c..1ebc3981 100644 --- a/api/opds.py +++ b/api/opds.py @@ -17,7 +17,7 @@ import regluit.core.cc as cc licenses = cc.LICENSE_LIST logger = logging.getLogger(__name__) -soup = BeautifulSoup('', 'xml') +soup = BeautifulSoup('', 'lxml') FORMAT_TO_MIMETYPE = {'pdf':"application/pdf", 'epub':"application/epub+zip", 'mobi':"application/x-mobipocket-ebook", From 8efe0b012e4ec4bc2a1cd88af43e3437eda309e1 Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 2 Apr 2020 14:09:34 -0400 Subject: [PATCH 8/8] instantiate soup inside methods --- api/onix.py | 10 +++++++++- api/opds.py | 5 ++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/api/onix.py b/api/onix.py index 6b6488cb..10a4ff6e 100644 --- a/api/onix.py +++ b/api/onix.py @@ -17,7 +17,7 @@ feed_header = """ """ feed_xml = feed_header + '' -soup = BeautifulSoup('', 'lxml') +soup = None bisac = Bisac() def text_node(tag, text, attrib=None): @@ -36,6 +36,10 @@ def sub_element(node, tag, attrib=None): def onix_feed(facet, max=None, page_number=None): + global soup + if not soup: + soup = BeautifulSoup('', 'lxml') + yield feed_header + str(header(facet)) works = facet.works[0:max] if max else facet.works @@ -56,6 +60,10 @@ def onix_feed(facet, max=None, page_number=None): yield '' def onix_feed_for_work(work): + global soup + if not soup: + soup = BeautifulSoup('', 'lxml') + feed = BeautifulSoup(feed_xml, 'xml') feed.ONIXMessage.append(header(work)) for edition in models.Edition.objects.filter(work=work, ebooks__isnull=False).distinct(): diff --git a/api/opds.py b/api/opds.py index 1ebc3981..b74f9ea2 100644 --- a/api/opds.py +++ b/api/opds.py @@ -17,7 +17,7 @@ import regluit.core.cc as cc licenses = cc.LICENSE_LIST logger = logging.getLogger(__name__) -soup = BeautifulSoup('', 'lxml') +soup = None FORMAT_TO_MIMETYPE = {'pdf':"application/pdf", 'epub':"application/epub+zip", 'mobi':"application/x-mobipocket-ebook", @@ -273,6 +273,9 @@ def opds_feed_for_work(work_id): return opds_feed_for_works(single_work_facet(work_id)) def opds_feed_for_works(the_facet, page=None, order_by='newest'): + global soup + if not soup: + soup = BeautifulSoup('', 'lxml') works = the_facet.works feed_path = the_facet.feed_path title = the_facet.title