management command to push books
eric
parent
11ea9b4d93
commit
3e61822111
|
|
@ -0,0 +1,48 @@
|
|||
from datetime import datetime
|
||||
from django.core.management.base import BaseCommand
|
||||
from regluit.distro.models import Target
|
||||
from regluit.distro.push import push_all, push_books
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "ftp <max> books to <target> or 'all' <since> date . "
|
||||
args = "<max> <target> <since> <new>"
|
||||
|
||||
def handle(self, max=0, target=None, since=None, new=None, *args, **options):
|
||||
try:
|
||||
max=int(max)
|
||||
except:
|
||||
self.stderr.write("max should be number (0 for all available) ")
|
||||
return
|
||||
new = new=='new'
|
||||
if new:
|
||||
self.stdout.write( "previously deposited books will not be pushed")
|
||||
try:
|
||||
since = datetime.strptime(since, '%Y-%m-%d')
|
||||
except:
|
||||
since=None
|
||||
|
||||
if since:
|
||||
try:
|
||||
target = Target.objects.get(name=target)
|
||||
self.stdout.write( "pushing {} new books since {} to {}".format(max if max else 'all', since, target))
|
||||
push_books(target, max=max, start=since, new=new)
|
||||
except Target.DoesNotExist:
|
||||
if target == "all":
|
||||
self.stdout.write( "pushing {} to all targets".format(max if max else 'all'))
|
||||
push_all(start=since, max=max, new=new)
|
||||
else:
|
||||
self.stderr.write("'{}' is not a defined target".format(target))
|
||||
else:
|
||||
try:
|
||||
self.stdout.write( "pushing {} books to {}".format(max if max else 'all', target))
|
||||
target = Target.objects.get(name=target)
|
||||
push_books(target, max=max, new=new)
|
||||
except Target.DoesNotExist:
|
||||
if target == "all":
|
||||
self.stdout.write("pushing {} books to all targets".format(max if max else 'all'))
|
||||
push_all(max=max, new=new)
|
||||
else:
|
||||
self.stderr.write("'{}' is not a defined target".format(target))
|
||||
|
||||
|
||||
|
|
@ -25,26 +25,32 @@ class Target(models.Model):
|
|||
_ftp = FTP(self.host, self.user, self.pw)
|
||||
return _ftp
|
||||
|
||||
def push(self, book):
|
||||
def push(self, book, new=False):
|
||||
pushed_formats = []
|
||||
pushed_isbns = set()
|
||||
for ebook in book.ebooks():
|
||||
isbn = ebook.edition.isbn_13 if ebook.edition.isbn_13 else ebook.edition.work.first_isbn_13()
|
||||
isbn = ebook.edition.isbn_13 if ebook.edition.isbn_13 else book.first_isbn_13()
|
||||
if isbn and self.formats.filter(name=ebook.format).exists() and ebook.format not in pushed_formats:
|
||||
ebfile = ebook.get_archive()
|
||||
self.push_file(u'{}.{}'.format(isbn,ebook.format), ebfile)
|
||||
ebfile.close()
|
||||
pushed_formats.append(ebook.format)
|
||||
pushed_isbns.add(isbn)
|
||||
Deposit.objects.create(target=self, isbn=isbn, format=ebook.format)
|
||||
|
||||
cover = book.cover_image_large()
|
||||
if cover:
|
||||
r = requests.get(cover)
|
||||
for isbn in pushed_isbns:
|
||||
self.push_file(u'{}.{}'.format(isbn,'jpg'), StringIO(r.content))
|
||||
else:
|
||||
logger('no cover available for {}'.format(book))
|
||||
dont_push = new and Deposit.objects.filter(isbn=isbn, target=self, format=ebook.format).exists()
|
||||
if not dont_push:
|
||||
ebfile = ebook.get_archive()
|
||||
if ebfile:
|
||||
self.push_file(u'{}.{}'.format(isbn,ebook.format), ebfile)
|
||||
ebfile.close()
|
||||
pushed_formats.append(ebook.format)
|
||||
pushed_isbns.add(isbn)
|
||||
Deposit.objects.create(target=self, isbn=isbn, format=ebook.format)
|
||||
else:
|
||||
logger.error('no file available for ebook {}'.format(ebook.id))
|
||||
if pushed_isbns:
|
||||
cover = book.cover_image_large()
|
||||
if cover:
|
||||
r = requests.get(cover)
|
||||
for isbn in pushed_isbns:
|
||||
self.push_file(u'{}.{}'.format(isbn,'jpg'), StringIO(r.content))
|
||||
else:
|
||||
logger.error('no cover available for {}'.format(book))
|
||||
return pushed_isbns
|
||||
|
||||
def push_file(self, filename, file_to_push):
|
||||
self.get_ftp().storbinary(u'STOR {}'.format(filename), file_to_push)
|
||||
|
|
|
|||
|
|
@ -11,15 +11,25 @@ from .models import Target
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def push_books(target, start=datetime(1900,1,1), end=datetime(2100,1,1),max=0):
|
||||
def push_books(target, start=datetime(1900,1,1), new=False, max=0):
|
||||
"""given a list of books this task will push the books, metadata and covers to the target
|
||||
"""
|
||||
facet_class = get_target_facet(target, start=start, end=end,max=max)
|
||||
facet_class = get_target_facet(target, start=start, new=new)
|
||||
pushed_books = []
|
||||
for book in facet_class.works:
|
||||
target.push(book)
|
||||
logger.info(u'{} pushed to {}'.format(book, target))
|
||||
pushed = target.push(book, new=new)
|
||||
if pushed:
|
||||
pushed_books.append(book)
|
||||
logger.info(u'{} pushed to {}'.format(book, target))
|
||||
else:
|
||||
logger.info(u'{} was not pushed to {}'.format(book, target))
|
||||
if max and len(pushed_books) >= max:
|
||||
break
|
||||
facet_class.works = pushed_books
|
||||
if len(pushed_books)>0:
|
||||
push_onix(target, facet_class)
|
||||
|
||||
def get_target_facet(target, start=datetime(1900,1,1), end=datetime(2100,1,1),max=0):
|
||||
def get_target_facet(target, start=datetime(1900,1,1), new=False):
|
||||
formats = [ format.name for format in target.formats.all() ]
|
||||
|
||||
def format_filter(query_set):
|
||||
|
|
@ -31,17 +41,12 @@ def get_target_facet(target, start=datetime(1900,1,1), end=datetime(2100,1,1),ma
|
|||
class TargetFacet(BaseFacet):
|
||||
def __init__(self):
|
||||
self.facet_object = self
|
||||
works = Work.objects.filter(
|
||||
editions__ebooks__created__lt = end,
|
||||
self.works = Work.objects.filter(
|
||||
editions__ebooks__created__gt = start,
|
||||
identifiers__type="isbn",
|
||||
editions__ebooks__format__in = formats,
|
||||
editions__ebooks__provider__in = ('Internet Archive', 'Unglue.it', 'Github', 'OAPEN Library'),
|
||||
).distinct().order_by('-featured')
|
||||
if max > 0 :
|
||||
self.works = works[0:max]
|
||||
else:
|
||||
self.works = works
|
||||
|
||||
model_filters = {"Ebook": format_filter, "Edition": edition_format_filter}
|
||||
outer_facet = None
|
||||
|
|
@ -50,6 +55,9 @@ def get_target_facet(target, start=datetime(1900,1,1), end=datetime(2100,1,1),ma
|
|||
|
||||
return TargetFacet()
|
||||
|
||||
def push_onix(target, start=datetime(1900,1,1), end=datetime(2100,1,1),max=0):
|
||||
facet_class = get_target_facet(target, start=start, end=end,max=max)
|
||||
target.push_file('unglueit_onix_{:%Y%m%d}.xml'.format(datetime.now()),StringIO(onix_feed(facet_class)))
|
||||
def push_onix(target, facet_class):
|
||||
target.push_file('unglueit_onix_{:%Y%m%d}.xml'.format(datetime.now()),StringIO(onix_feed(facet_class)))
|
||||
|
||||
def push_all(start=datetime(1900,1,1), new=False, max=0):
|
||||
for target in Target.objects.all():
|
||||
push_books(target, start=start, new=new, max=max)
|
||||
|
|
|
|||
|
|
@ -169,6 +169,7 @@ INSTALLED_APPS = (
|
|||
# this must appear *after* django.frontend or else it overrides the
|
||||
# registration templates in frontend/templates/registration
|
||||
'django.contrib.admin',
|
||||
'regluit.distro',
|
||||
'regluit.booxtream',
|
||||
'regluit.pyepub',
|
||||
'regluit.libraryauth',
|
||||
|
|
|
|||
Loading…
Reference in New Issue