diff --git a/distro/management/__init__.py b/distro/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/distro/management/commands/__init__.py b/distro/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/distro/management/commands/push_books.py b/distro/management/commands/push_books.py new file mode 100644 index 00000000..df1a3880 --- /dev/null +++ b/distro/management/commands/push_books.py @@ -0,0 +1,48 @@ +from datetime import datetime +from django.core.management.base import BaseCommand +from regluit.distro.models import Target +from regluit.distro.push import push_all, push_books + + +class Command(BaseCommand): + help = "ftp books to or 'all' date . " + args = " " + + def handle(self, max=0, target=None, since=None, new=None, *args, **options): + try: + max=int(max) + except: + self.stderr.write("max should be number (0 for all available) ") + return + new = new=='new' + if new: + self.stdout.write( "previously deposited books will not be pushed") + try: + since = datetime.strptime(since, '%Y-%m-%d') + except: + since=None + + if since: + try: + target = Target.objects.get(name=target) + self.stdout.write( "pushing {} new books since {} to {}".format(max if max else 'all', since, target)) + push_books(target, max=max, start=since, new=new) + except Target.DoesNotExist: + if target == "all": + self.stdout.write( "pushing {} to all targets".format(max if max else 'all')) + push_all(start=since, max=max, new=new) + else: + self.stderr.write("'{}' is not a defined target".format(target)) + else: + try: + self.stdout.write( "pushing {} books to {}".format(max if max else 'all', target)) + target = Target.objects.get(name=target) + push_books(target, max=max, new=new) + except Target.DoesNotExist: + if target == "all": + self.stdout.write("pushing {} books to all targets".format(max if max else 'all')) + push_all(max=max, new=new) + else: + self.stderr.write("'{}' is not a defined target".format(target)) + + diff --git a/distro/models.py b/distro/models.py index 6371de47..5e9adb4b 100644 --- a/distro/models.py +++ b/distro/models.py @@ -25,26 +25,32 @@ class Target(models.Model): _ftp = FTP(self.host, self.user, self.pw) return _ftp - def push(self, book): + def push(self, book, new=False): pushed_formats = [] pushed_isbns = set() for ebook in book.ebooks(): - isbn = ebook.edition.isbn_13 if ebook.edition.isbn_13 else ebook.edition.work.first_isbn_13() + isbn = ebook.edition.isbn_13 if ebook.edition.isbn_13 else book.first_isbn_13() if isbn and self.formats.filter(name=ebook.format).exists() and ebook.format not in pushed_formats: - ebfile = ebook.get_archive() - self.push_file(u'{}.{}'.format(isbn,ebook.format), ebfile) - ebfile.close() - pushed_formats.append(ebook.format) - pushed_isbns.add(isbn) - Deposit.objects.create(target=self, isbn=isbn, format=ebook.format) - - cover = book.cover_image_large() - if cover: - r = requests.get(cover) - for isbn in pushed_isbns: - self.push_file(u'{}.{}'.format(isbn,'jpg'), StringIO(r.content)) - else: - logger('no cover available for {}'.format(book)) + dont_push = new and Deposit.objects.filter(isbn=isbn, target=self, format=ebook.format).exists() + if not dont_push: + ebfile = ebook.get_archive() + if ebfile: + self.push_file(u'{}.{}'.format(isbn,ebook.format), ebfile) + ebfile.close() + pushed_formats.append(ebook.format) + pushed_isbns.add(isbn) + Deposit.objects.create(target=self, isbn=isbn, format=ebook.format) + else: + logger.error('no file available for ebook {}'.format(ebook.id)) + if pushed_isbns: + cover = book.cover_image_large() + if cover: + r = requests.get(cover) + for isbn in pushed_isbns: + self.push_file(u'{}.{}'.format(isbn,'jpg'), StringIO(r.content)) + else: + logger.error('no cover available for {}'.format(book)) + return pushed_isbns def push_file(self, filename, file_to_push): self.get_ftp().storbinary(u'STOR {}'.format(filename), file_to_push) diff --git a/distro/push.py b/distro/push.py index 15c37ff5..a367b4f7 100644 --- a/distro/push.py +++ b/distro/push.py @@ -11,15 +11,25 @@ from .models import Target logger = logging.getLogger(__name__) -def push_books(target, start=datetime(1900,1,1), end=datetime(2100,1,1),max=0): +def push_books(target, start=datetime(1900,1,1), new=False, max=0): """given a list of books this task will push the books, metadata and covers to the target """ - facet_class = get_target_facet(target, start=start, end=end,max=max) + facet_class = get_target_facet(target, start=start, new=new) + pushed_books = [] for book in facet_class.works: - target.push(book) - logger.info(u'{} pushed to {}'.format(book, target)) + pushed = target.push(book, new=new) + if pushed: + pushed_books.append(book) + logger.info(u'{} pushed to {}'.format(book, target)) + else: + logger.info(u'{} was not pushed to {}'.format(book, target)) + if max and len(pushed_books) >= max: + break + facet_class.works = pushed_books + if len(pushed_books)>0: + push_onix(target, facet_class) -def get_target_facet(target, start=datetime(1900,1,1), end=datetime(2100,1,1),max=0): +def get_target_facet(target, start=datetime(1900,1,1), new=False): formats = [ format.name for format in target.formats.all() ] def format_filter(query_set): @@ -31,17 +41,12 @@ def get_target_facet(target, start=datetime(1900,1,1), end=datetime(2100,1,1),ma class TargetFacet(BaseFacet): def __init__(self): self.facet_object = self - works = Work.objects.filter( - editions__ebooks__created__lt = end, + self.works = Work.objects.filter( editions__ebooks__created__gt = start, identifiers__type="isbn", editions__ebooks__format__in = formats, editions__ebooks__provider__in = ('Internet Archive', 'Unglue.it', 'Github', 'OAPEN Library'), ).distinct().order_by('-featured') - if max > 0 : - self.works = works[0:max] - else: - self.works = works model_filters = {"Ebook": format_filter, "Edition": edition_format_filter} outer_facet = None @@ -50,6 +55,9 @@ def get_target_facet(target, start=datetime(1900,1,1), end=datetime(2100,1,1),ma return TargetFacet() -def push_onix(target, start=datetime(1900,1,1), end=datetime(2100,1,1),max=0): - facet_class = get_target_facet(target, start=start, end=end,max=max) - target.push_file('unglueit_onix_{:%Y%m%d}.xml'.format(datetime.now()),StringIO(onix_feed(facet_class))) \ No newline at end of file +def push_onix(target, facet_class): + target.push_file('unglueit_onix_{:%Y%m%d}.xml'.format(datetime.now()),StringIO(onix_feed(facet_class))) + +def push_all(start=datetime(1900,1,1), new=False, max=0): + for target in Target.objects.all(): + push_books(target, start=start, new=new, max=max) diff --git a/settings/common.py b/settings/common.py index 4f0810be..4bc55779 100644 --- a/settings/common.py +++ b/settings/common.py @@ -169,6 +169,7 @@ INSTALLED_APPS = ( # this must appear *after* django.frontend or else it overrides the # registration templates in frontend/templates/registration 'django.contrib.admin', + 'regluit.distro', 'regluit.booxtream', 'regluit.pyepub', 'regluit.libraryauth',