Merge pull request #916 from Gluejar/maintenance2020

clean up line endings

core/bookloader.py

@@ -665,7 +665,7 @@ def despam_description(description):
     pieces = description.split("a million books for free.")
     if len(pieces) > 1:
         return pieces[1]
-    return description
+    return description.replace('\r\n', '\n')
 
 def add_openlibrary(work, hard_refresh=False):
     if (not hard_refresh) and work.openlibrary_lookup is not None:
@@ -958,7 +958,7 @@ class BasePandataLoader(object):
         if metadata.description and len(metadata.description) > len(work.description):
             # don't over-write reasonably long descriptions
             if len(work.description) < 500:
-                work.description = metadata.description
+                work.description = metadata.description.replace('\r\n', '\n')
 
         if metadata.creator and not edition.authors.exists():
             edition.authors.clear()

core/loaders/doab.py

@@ -119,8 +119,8 @@ def attach_more_doab_metadata(edition, description, subjects,
 
     # attach description to work if it's not empty
     work = edition.work
-    if not work.description:
-        work.description = description
+    if description and not work.description:
+        work.description = description.replace('\r\n', '\n')
 
     # update subjects
     for s in subjects:

core/loaders/utils.py

@@ -244,7 +244,7 @@ def load_from_books(books):
         # possibly replace work.description
         description = get_description(book)
         if len(description) > len(work.description):
-            work.description = description
+            work.description = description.replace('\r\n', '\n')
             work.save()
 
         # set language

core/management/commands/clean_db_strings.py

@@ -16,10 +16,17 @@ class Command(BaseCommand):
                 work.title = sanitize_line(work.title)
                 work.save()
                 work_titles_fixed +=1
-            if work.description and remove_badxml(work.description) != work.description:
-                work.description = remove_badxml(work.description)
-                work.save()
-                work_descriptions_fixed +=1
+            if work.description:
+                save =  False
+                if '\r\n' in work.description:
+                    work.description = work.description.replace('\r\n', '\n')
+                    save = True
+                if work.description and remove_badxml(work.description) != work.description:
+                    work.description = remove_badxml(work.description)
+                    save = True
+                if save:
+                    work.save()
+                    work_descriptions_fixed +=1
         self.stdout.write("work_titles_fixed = {}".format(work_titles_fixed))
         self.stdout.write("work_descriptions_fixed = {}".format(work_descriptions_fixed))
         for edition in models.Edition.objects.all():