core/bookloader.py

@@ -1113,14 +1113,13 @@ def add_from_bookdatas(bookdatas):
     editions = []
     for bookdata in bookdatas:
         edition = work = None
-        if bookdata and bookdata.metadata:
-            loader = BasePandataLoader(bookdata.base)
-            pandata = Pandata()
-            pandata.metadata = bookdata.metadata
-            for metadata in pandata.get_edition_list():
-                edition = loader.load_from_pandata(metadata, work)
-                work = edition.work
-            loader.load_ebooks(pandata, edition)
-            if edition:
-                editions.append(edition)
+        loader = BasePandataLoader(bookdata.base)
+        pandata = Pandata()
+        pandata.metadata = bookdata.metadata
+        for metadata in pandata.get_edition_list():
+            edition = loader.load_from_pandata(metadata, work)
+            work = edition.work
+        loader.load_ebooks(pandata, edition)
+        if edition:
+            editions.append(edition)
     return editions

core/loaders/doab_utils.py

@@ -85,7 +85,6 @@ STOREPROVIDERS = [
     'manchesteruniversitypress.co.uk',
     'mitpress.mit.edu',
     'munishop.muni.cz',
-    'nai010.com',
     'nomos-shop.de',
     'palgrave.com',
     'placedeslibraires.fr',

core/loaders/harvest.py

@@ -138,7 +138,6 @@ CMPPROVIDERS = [
     'omp.ub.rub.de',
     'penerbit.brin.go.id',
     'press.uni.lodz.pl',
-    'redliclibros.com',
     'Scholars Portal',
     'teiresias-supplements.mcgill.ca',
     'textbooks.open.tudelft.nl',
@@ -939,11 +938,9 @@ def harvest_muse(ebook):
 
 
 def harvest_mitpress(ebook):
-    def selector(doc):
-        return doc.select('a.book-pdfLink[href]')
     def chap_selector(doc):
         return doc.select('a.section-pdfLink[href]')
-    return harvest_stapled_generic(ebook, selector, chap_selector, strip_covers=0)
+    return harvest_stapled_generic(ebook, None, chap_selector, strip_covers=0)
 
 
 def harvest_ios(ebook):

core/loaders/scrape.py

@@ -65,16 +65,13 @@ class BaseScraper(object):
                 for review in self.doc.find_all(itemtype="http://schema.org/Review"):
                     review.clear()
                 self.get_all()
-                if not self.metadata.get('title', None):
-                    self.set('title', '!!! missing title !!!')
-                if not self.metadata.get('language', None):
-                    self.set('language', 'en')
-                self.metadata['identifiers'] = self.identifiers
-            else:
-                self.metadata = None
+            if not self.metadata.get('title', None):
+                self.set('title', '!!! missing title !!!')
+            if not self.metadata.get('language', None):
+                self.set('language', 'en')
         else:
-            self.metadata = None
-        
+            self.metadata = {}
+        self.metadata['identifiers'] = self.identifiers
 
     #
     # utilities

core/loaders/springer.py

@@ -146,9 +146,9 @@ class SpringerScraper(BaseScraper):
             self.set('rights_url', lic_url)
 
     def get_pubdate(self):
-        pubinfo = self.doc.find(attrs={"data-test": "electronic_isbn_publication_date"})
+        pubinfo = self.doc.select_one('#copyright-info')
         if not pubinfo:
-            pubinfo = self.doc.find(attrs={"data-test": "softcover_isbn_publication_date"})
+            pubinfo = self.doc.select_one('header .c-article-identifiers')
         if pubinfo:
             for yearstring in pubinfo.stripped_strings:
                 yearmatch = HAS_YEAR.search(yearstring)

core/migrations/0029_auto_20241122_1525.py

@@ -1,25 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.29 on 2024-11-22 15:25
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0028_auto_20240819_1450'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='subject',
-            name='name',
-            field=models.CharField(db_index=True, max_length=200, unique=True),
-        ),
-        migrations.AlterField(
-            model_name='work',
-            name='is_free',
-            field=models.BooleanField(db_index=True, default=False),
-        ),
-    ]

core/models/bibmodels.py

@@ -130,7 +130,7 @@ class Work(models.Model):
     # repurposed earliest_publication to actually be publication range
     publication_range = models.CharField(max_length=50, null=True, blank=True)
     featured = models.DateTimeField(null=True, blank=True, db_index=True,)
-    is_free = models.BooleanField(default=False, db_index=True)
+    is_free = models.BooleanField(default=False)
     related = models.ManyToManyField('self', symmetrical=False, blank=True, through='WorkRelation', related_name='reverse_related')
     age_level = models.CharField(max_length=5, choices=AGE_LEVEL_CHOICES, default='', blank=True) 
 
@@ -787,7 +787,7 @@ AUTHMATCH = re.compile(r'\s*!([a-z]+):?\s+(.*)')
 
 class Subject(models.Model):
     created = models.DateTimeField(auto_now_add=True)
-    name = models.CharField(max_length=200, unique=True, db_index=True)
+    name = models.CharField(max_length=200, unique=True)
     works = models.ManyToManyField("Work", related_name="subjects")
     is_visible = models.BooleanField(default=True)
     authority = models.CharField(max_length=10, blank=False, default="")