EbookFoundation
/
regluit
mirror of https://github.com/EbookFoundation/regluit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #983 from Gluejar/maintenance2022 

fix thumbnailing
pull/94/head
Eric Hellman 2022-02-28 19:53:27 -05:00 committed by GitHub
parent 2e6476a612 fd1178d1ae
commit a54b9e3f94
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 161 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/views.py
View File

@ -26,6 +26,7 @@ import regluit.core.isbn
from regluit.api import opds, onix, opds_json
from regluit.api.models import repo_allowed
from regluit.core.bookloader import load_from_yaml
from regluit.core.covers import DEFAULT_COVER
from regluit.core import models
from regluit.core.parameters import ORDER_BY_KEYS
@ -83,7 +84,7 @@ def widget(request, isbn):
def featured_cover(request):
work = featured_work()
tn = work.cover_image_thumbnail()
return HttpResponseRedirect(tn if tn else "/static/images/generic_cover_larger.png")
return HttpResponseRedirect(tn if tn else DEFAULT_COVER)
def featured_url(request):
work = featured_work()

101
core/covers.py Normal file
View File

@ -0,0 +1,101 @@
""" handle caching and thumbnailing of covers """
import logging
from django.utils.functional import LazyObject
import sorl
from sorl.thumbnail import get_thumbnail as sorl_get_thumbnail
from sorl.thumbnail.base import ThumbnailBackend
from sorl.thumbnail.conf import settings, defaults as default_settings
from sorl.thumbnail.helpers import get_module_class
from sorl.thumbnail.images import BaseImageFile, ImageFile
from sorl.thumbnail import default
import regluit
logger = logging.getLogger(__name__)
DEFAULT_COVER = '/static/images/generic_cover_larger.png'
_storage = None
class Storage(LazyObject):
'''
Monkey patch to fix S3 backend slowness in sorl.thumbnail
https://github.com/jazzband/sorl-thumbnail/issues/301
'''
def _setup(self):
global _storage
if not _storage:
_storage = get_module_class(settings.THUMBNAIL_STORAGE)()
self._wrapped = _storage
sorl.thumbnail.default.storage = Storage()
class DefaultImageFile(BaseImageFile):
url = DEFAULT_COVER
size = (131, 192)
is_default = True
def exists(self):
return True
class ReadOnlyThumbnailBackend(ThumbnailBackend):
"""
A backend that never makes a new thumbnail, but adds missing thumbnails to a task queue
"""
def get_thumbnail(self, file_, geometry_string, **options):
"""
Returns thumbnail as an ImageFile instance for file with geometry and
options given. It will try to get it from the key value store,
otherwise return a Dummy.
"""
logger.debug('Getting thumbnail for file [%s] at [%s]', file_, geometry_string)
if file_:
source = ImageFile(file_)
else:
raise ValueError('falsey file_ argument in get_thumbnail()')
# preserve image filetype
if settings.THUMBNAIL_PRESERVE_FORMAT:
options.setdefault('format', self._get_format(source))
for key, value in self.default_options.items():
options.setdefault(key, value)
for key, attr in self.extra_options:
value = getattr(settings, attr)
if value != getattr(default_settings, attr):
options.setdefault(key, value)
name = self._get_thumbnail_filename(source, geometry_string, options)
thumbnail = ImageFile(name, default.storage)
cached = default.kvstore.get(thumbnail)
if cached:
setattr(cached, 'is_default', True)
return cached
regluit.core.tasks.make_cover_thumbnail.delay(file_, geometry_string, **options)
return DefaultImageFile()
backend = ReadOnlyThumbnailBackend()
get_thumbnail = backend.get_thumbnail
def make_cover_thumbnail(url, geometry_string, **options):
try:
im = sorl_get_thumbnail(url, geometry_string, **options)
except (IOError, OSError):
return False
logger.error('couldnt make thumbnail for %s', url)
if im.exists():
return True
return False

57
core/models/bibmodels.py
View File

@ -33,6 +33,7 @@ from questionnaire.models import Landing
from regluit.bisac.models import interpret_notation
from regluit.core import mobi
import regluit.core.cc as cc
from regluit.core.covers import get_thumbnail, DEFAULT_COVER
from regluit.core.epub import test_epub
from regluit.core.links import id_url
from regluit.core.loaders.harvest import dl_online
@ -52,7 +53,6 @@ from regluit.core.parameters import (
WORK_IDENTIFIERS,
DOMAIN_TO_PROVIDER,
)
from regluit.core.thumbnail import get_thumbnail
# fix truncated file problems per https://stackoverflow.com/questions/12984426/python-pil-ioerror-image-file-truncated-with-big-images
ImageFile.LOAD_TRUNCATED_IMAGES = True
@ -269,12 +269,12 @@ class Work(models.Model):
def cover_image_large(self):
if self.preferred_edition and self.preferred_edition.has_cover_image():
return self.preferred_edition.cover_image_large()
return "/static/images/generic_cover_larger.png"
return DEFAULT_COVER
def cover_image_small(self):
if self.preferred_edition and self.preferred_edition.has_cover_image():
return self.preferred_edition.cover_image_small()
return "/static/images/generic_cover_larger.png"
return DEFAULT_COVER
def cover_image_thumbnail(self):
try:
@ -282,7 +282,7 @@ class Work(models.Model):
return self.preferred_edition.cover_image_thumbnail()
except IndexError:
pass
return "/static/images/generic_cover_larger.png"
return DEFAULT_COVER
def authors(self):
# assumes that they come out in the same order they go in!
@ -877,51 +877,38 @@ class Edition(models.Model):
def cover_image_large(self):
#550 pixel high image
if self.cover_image:
try:
im = get_thumbnail(self.cover_image, 'x550', crop='noop', quality=95)
if im.exists():
return im.url
except (IOError, OSError, CertificateError):
pass
elif self.googlebooks_id:
im = get_thumbnail(self.cover_image, 'x550', crop='noop', quality=95)
if not im.is_default:
return im.url
if self.googlebooks_id:
url = "https://encrypted.google.com/books?id=%s&printsec=frontcover&img=1&zoom=0" % self.googlebooks_id
try:
im = get_thumbnail(url, 'x550', crop='noop', quality=95)
if im.is_default or im.storage.size(im.name) == 16392: # check for "image not available" image
url = "https://encrypted.google.com/books?id=%s&printsec=frontcover&img=1&zoom=1" % self.googlebooks_id
im = get_thumbnail(url, 'x550', crop='noop', quality=95)
if not im.exists() or im.storage.size(im.name) == 16392: # check for "image not available" image
url = "https://encrypted.google.com/books?id=%s&printsec=frontcover&img=1&zoom=1" % self.googlebooks_id
im = get_thumbnail(url, 'x550', crop='noop', quality=95)
if im.exists():
return im.url
except (IOError, OSError, CertificateError):
pass
return ''
if not im.is_default:
return im.url
return DEFAULT_COVER
def cover_image_small(self):
#80 pixel high image
if self.cover_image:
try:
im = get_thumbnail(self.cover_image, 'x80', crop='noop', quality=95)
if im.exists():
return im.url
except (IOError, OSError, CertificateError):
pass
im = get_thumbnail(self.cover_image, 'x80', crop='noop', quality=95)
if not im.is_default:
return im.url
if self.googlebooks_id:
return "https://encrypted.google.com/books?id=%s&printsec=frontcover&img=1&zoom=5" % self.googlebooks_id
return ''
return DEFAULT_COVER
def cover_image_thumbnail(self):
#128 pixel wide image
if self.cover_image:
try:
im = get_thumbnail(self.cover_image, '128', crop='noop', quality=95)
if im.exists():
return im.url
except (IOError, OSError, CertificateError):
pass
im = get_thumbnail(self.cover_image, '128', crop='noop', quality=95)
if not im.is_default:
return im.url
if self.googlebooks_id:
return "https://encrypted.google.com/books?id=%s&printsec=frontcover&img=1&zoom=1" % self.googlebooks_id
else:
return ''
return DEFAULT_COVER
def has_cover_image(self):
if self.cover_image:

3
core/search.py
View File

@ -4,6 +4,7 @@ import json
import requests
from django.conf import settings
from regluit.core.covers import DEFAULT_COVER
import regluit.core.isbn
def gluejar_search(q, user_ip='69.243.24.29', page=1):
@ -50,7 +51,7 @@ def gluejar_search(q, user_ip='69.243.24.29', page=1):
)
r['cover_image_thumbnail'] = url
else:
r['cover_image_thumbnail'] = "/static/images/generic_cover_larger.png"
r['cover_image_thumbnail'] = DEFAULT_COVER
access_info = item.get('accessInfo')
if access_info:

10
core/tasks.py
View File

@ -27,6 +27,7 @@ from mailchimp3 import MailChimp
#
from regluit.core import (
bookloader,
covers,
models,
librarything,
mobigen
@ -242,3 +243,12 @@ def feature_new_work():
work.featured = now()
work.save()
@task
def make_cover_thumbnail(url, geom_string, **options):
success = covers.make_cover_thumbnail(url, geom_string, **options)
if not success and url != '':
for ed in models.Edition.objects.filter(cover_image=url):
logger.error('bad cover image %s for edition: %s', (url, ed.id))
#ed.cover_image = ''
#ed.save()

27
core/tests.py
View File

@ -37,13 +37,14 @@ from regluit.payment.parameters import PAYMENT_TYPE_AUTHORIZATION
from regluit.utils.localdatetime import date_today
from . import (
isbn,
bookloader,
models,
search,
covers,
isbn,
librarything,
tasks,
models,
parameters,
search,
tasks,
)
from .epub import test_epub
from .loaders.utils import (load_from_books, loaded_book_ok, )
@ -972,6 +973,24 @@ class MailingListTests(TestCase):
self.user = User.objects.create_user('chimp_test', 'eric@gluejar.com', 'chimp_test')
self.assertTrue(self.user.profile.on_ml)
class CoverTests(TestCase):
test_image = 'https://unglue.it/static/images/logo.png'
test_bad_image = 'https://example.com/static/images/logo.png'
def setUp(self):
self.work = Work.objects.create(title="Cover Work")
self.edition = Edition.objects.create(title=self.work.title, work=self.work)
covers.sorl_get_thumbnail(self.test_image, 'x550', crop='noop', quality=95)
def test_cached_cover(self):
thumb = covers.get_thumbnail(self.test_image, 'x550', crop='noop', quality=95)
self.assertTrue(thumb.exists())
self.assertTrue(thumb.width, 550)
def test_bad_cover(self):
thumb = covers.get_thumbnail(self.test_bad_image, 'x550', crop='noop', quality=95)
self.assertEqual(thumb.url, covers.DEFAULT_COVER)
@override_settings(LOCAL_TEST=True)
class EbookFileTests(TestCase):
fixtures = ['initial_data.json']

23
core/thumbnail.py
View File

@ -1,23 +0,0 @@
'''
Monkey patch to fix S3 backend slowness in sorl.thumbnail
https://github.com/jazzband/sorl-thumbnail/issues/301
'''
from django.utils.functional import LazyObject
import sorl.thumbnail.default
from sorl.thumbnail.conf import settings
from sorl.thumbnail.helpers import get_module_class
from sorl.thumbnail import get_thumbnail
_storage = None
class Storage(LazyObject):
def _setup(self):
global _storage
if not _storage:
_storage = get_module_class(settings.THUMBNAIL_STORAGE)()
self._wrapped = _storage
sorl.thumbnail.default.storage = Storage()

2
settings/common.py
View File

@ -451,7 +451,7 @@ PAYMENT_PROCESSOR = 'stripelib'
# allow application code to catch thumbnailing errors
THUMBNAIL_DEBUG = True
THUMBNAIL_FORCE_OVERWRITE = True
THUMBNAIL_FORCE_OVERWRITE = False
# use redis
# THUMBNAIL_KVSTORE = 'sorl.thumbnail.kvstores.redis_kvstore.KVStore'