From e1409ceac1c06ebeefa01f7fda480cb621d5cfa5 Mon Sep 17 00:00:00 2001 From: eric Date: Mon, 27 Oct 2014 16:57:35 -0400 Subject: [PATCH] redo marc file upload --- admin.py | 14 ++- frontend/forms.py | 9 -- frontend/templates/edition_display.html | 13 +- frontend/templates/libraries.html | 8 +- frontend/templates/marc.html | 4 +- frontend/templates/marc/upload.html | 62 ++++++++++ frontend/templates/marcungluify.html | 42 ------- frontend/urls.py | 2 - frontend/views.py | 35 ------ marc/forms.py | 22 ++++ marc/load.py | 9 ++ marc/management/commands/migrate_records.py | 11 +- ...cord_user__add_field_marcrecord_created.py | 116 ++++++++++++++++++ marc/models.py | 18 ++- marc/urls.py | 4 + marc/views.py | 44 ++++++- 16 files changed, 295 insertions(+), 118 deletions(-) create mode 100644 frontend/templates/marc/upload.html delete mode 100644 frontend/templates/marcungluify.html create mode 100644 marc/forms.py create mode 100644 marc/migrations/0002_auto__add_field_marcrecord_user__add_field_marcrecord_created.py diff --git a/admin.py b/admin.py index bf72172c..e1e16bd4 100644 --- a/admin.py +++ b/admin.py @@ -35,6 +35,7 @@ regluit imports """ from regluit import payment from regluit.core import models +from regluit.marc.models import MARCRecord from regluit.core.lookups import ( PublisherNameLookup, WorkLookup, @@ -207,12 +208,17 @@ class MARCRecordAdminForm(forms.ModelForm): widget=AutoCompleteSelectWidget(EditionLookup), required=True, ) - + user = AutoCompleteSelectField( + OwnerLookup, + widget=AutoCompleteSelectWidget(OwnerLookup), + required=True, + ) class Meta(object): - model = models.MARCRecord + model = MARCRecord class MARCRecordAdmin(ModelAdmin): - list_display = ('edition',) + list_display = ('edition', 'user') + date_hierarchy = 'created' form = MARCRecordAdminForm admin_site = RegluitAdmin("Admin") @@ -238,7 +244,7 @@ admin_site.register(models.Wishlist, WishlistAdmin) admin_site.register(models.UserProfile, UserProfileAdmin) admin_site.register(models.CeleryTask, CeleryTaskAdmin) admin_site.register(models.Press, PressAdmin) -admin_site.register(models.MARCRecord, MARCRecordAdmin) +admin_site.register(MARCRecord, MARCRecordAdmin) # payments diff --git a/frontend/forms.py b/frontend/forms.py index e1945c06..527d00ad 100644 --- a/frontend/forms.py +++ b/frontend/forms.py @@ -770,15 +770,6 @@ class PressForm(forms.ModelForm): class KindleEmailForm(forms.Form): kindle_email = forms.EmailField() -class MARCUngluifyForm(forms.Form): - edition = AutoCompleteSelectField( - EditionLookup, - label='Edition', - widget=AutoCompleteSelectWidget(EditionLookup), - required=True, - error_messages={'required': 'Please specify an edition.'}, - ) - file = forms.FileField(label='Download a MARCXML file from Library of Congress; then upload it here.') class MARCFormatForm(forms.ModelForm): class Meta: diff --git a/frontend/templates/edition_display.html b/frontend/templates/edition_display.html index 6a1b0d23..c5e3110f 100644 --- a/frontend/templates/edition_display.html +++ b/frontend/templates/edition_display.html @@ -43,17 +43,8 @@ See this edition on Google Books
{% endif %} {% endif %} - {% if edition.MARCrecords.all %} - {% for record in edition.MARCrecords.all %} - Download {{record.link_target}} MARC record for this edition: (XML) (mrc)
- {% endfor %} - {% else %} - {% if edition.ebooks.all %} - Download {{record.link_target}} MARC record for this edition: (XML) (mrc)
- {% endif %} - {% endif %} - {% if user.is_staff %}{% if edition.ebooks.0 or edition.ebook_files.0 %} - Upload a MARC record for this edition.
+ {% if user.libpref %}{% if edition.ebooks.all or edition.ebook_files.all %} + Upload a MARC record for this edition.
{% endif %} {% endif %} diff --git a/frontend/templates/libraries.html b/frontend/templates/libraries.html index 6d517a00..e20fb085 100644 --- a/frontend/templates/libraries.html +++ b/frontend/templates/libraries.html @@ -31,6 +31,8 @@ We’ve added "Buy-to-Unglue" campaigns to so that libraries can lend ebooks on
Sign up. It's Free!
Starting an account, for yourself or your library, is free. Use it to help us distribute unglued and ungluing ebooks.
+
Add free ebooks to your collection with our MARC records!
+
We're building a comprehensive database of freely licensed ebooks. We provide MARC records for these ebooks wherever we can. They can live in the catalog in your ILS; they link to an epub file in the Internet Archive. Of course, because of unglued ebooks' Creative Commons licenses, you're free to shift that epub to other formats, host the file on your own servers, and edit the MARC record accordingly. And if you want, help us improve our records.
Become an Unglue.it participating library
  1. Make your library page on Unglue.it. It takes just a few minutes. It's the first step towards becoming an Unglue.it participating library.
  2. Review our LIBRARY LICENSE AGREEMENT.
    3. @@ -42,10 +44,6 @@ The library license gives download access to one library member at a time for 14
Stay in touch.
You can follow us on Twitter (@unglueit), Facebook, and our blog, and subscribe to our newsletter (1-2 emails per month).
-
Add unglued ebooks to your collection.
-
We provide MARC records for unglued ebooks wherever we can. We also strive to have MARC records available for all unglued ebooks. They can live in the catalog in your ILS; they link to an epub file in the Internet Archive. Of course, because of unglued ebooks' Creative Commons licenses, you're free to shift that epub to other formats, host the file on your own servers, and edit the MARC record accordingly.
-
Join our catalogers list.
-
We're able to get MARC records into OCLC and SkyRiver because of the support of catalogers like you. Want to help? Add yourself to the list. Usually all you'll need to do is modify an existing record for an earlier edition and upload the record. We'll supply you with the link and the unglued edition. We'll only contact you when there's an unglued book to be cataloged.
Spread the word.
There are social media sharing links on most pages on the site. There are some right here!
Copy/paste this into your site:
@@ -59,7 +57,7 @@ The library license gives download access to one library member at a time for 14
Educate yourself and your patrons about ebook issues and Creative Commons licenses.
Checkout limits, publishers who won't sell ebooks to libraries, DRM, companies tracking readers' behavior, library prices far in excess of consumer costs, incompatible technologies and formats, cataloging silos...you know why it's hard for you to deliver a seamless ereading experience to your patrons. Make sure they know, too. And make sure everyone knows how solutions, like Creative Commons licenses, can help libraries and readers while respecting copyright and protecting creators' rights.
Support our active campaigns.
-
Ultimately ebooks can't be unglued unless authors and publishers are paid for their work. Many of our staunchest supporters are librarians. There are also several libraries which have supported campaigns, including Leddy Library (University of Windsor, Ontario); the University of Alberta library ; and the Z. Smith Reynolds library (Wake Forest University.
+
Ultimately ebooks can't be unglued unless authors and publishers are paid for their work. Many of our staunchest supporters are librarians. There are also several libraries which have supported campaigns, including Leddy Library (University of Windsor, Ontario); the University of Alberta library ; and the Z. Smith Reynolds library (Wake Forest University).
Give feedback and ask questions.
Want to know more? Need help? Have ideas for how we could improve the site or make it more library-friendly? Contact us at: libraries@gluejar.com.
diff --git a/frontend/templates/marc.html b/frontend/templates/marc.html index 03de81de..b61f18fa 100644 --- a/frontend/templates/marc.html +++ b/frontend/templates/marc.html @@ -44,8 +44,8 @@ ul.local li {

Go ahead: add unglued ebooks to your library catalog!

-{% if request.user.is_staff %} -

Hi, {{ request.user.username }}. Unglue.it staffers can also add new records.

+{% if request.user.libpref %} +

Hi, {{ request.user.username }}. Librarians can also add new records.

{% endif %} {% if messages %} diff --git a/frontend/templates/marc/upload.html b/frontend/templates/marc/upload.html new file mode 100644 index 00000000..7546a671 --- /dev/null +++ b/frontend/templates/marc/upload.html @@ -0,0 +1,62 @@ +{% extends "basedocumentation.html" %} + +{% block title %}Add new MARC records{% endblock %} + +{% block extra_extra_head %} +{{ block.super }} + + + + +{% endblock %} + +{% block doccontent %} +{% if not request.user.libpref %} +

If you want to load MARC records, you probably want to enable Unglue.it's librarian tools first!

+{% endif %} +

Adding MARC records to Unglue.it

+{% if messages %} + +{% endif %} +{% if not form.initial.edition %} +

Selecting an edition

+

+MARC records are associated with unglue.it edition records. Look for links in the "more" tab of any unglue.it work page that has a "Download" button on it. +{% endif %} +

Adapting records from LoC

+

+ For ebooks which have existing print editions cataloged in the Library of Congress, we can automatically convert those to Unglue.it ebook MARC records, which will then be automatically affixed to the ebook edition in our database and provided to ungluers. +

+ +

+ To get the XML record, search http://catalog.loc.gov/, select the print edition, and click on the permalink (will look something like http://lccn.loc.gov/63008219). This page has a MARCXML link. +

+ +

+ The record loader will automatically add links to all the unglued ebook files known to the Unglue.it database. Make sure those links are in the database before adding the record.

+ +

Editing stub records from LoC

+{% if form.initial.edition %} +The current records for this edition are here. They may be auto-generated stubs: +
+
+ {% include 'marc_form.html' %} + + +
+
+{% endif %} + +

+ Edit the record, then upload it as a "record prepared for Unglue.it". The record loader strips 001, 003, 005, 006, 007, 856 and >900 fields, and adds them back in on download. +

+

Load a record

+
{% csrf_token %} + {{ form.as_p }} + +
+{% endblock %} \ No newline at end of file diff --git a/frontend/templates/marcungluify.html b/frontend/templates/marcungluify.html deleted file mode 100644 index 623968d1..00000000 --- a/frontend/templates/marcungluify.html +++ /dev/null @@ -1,42 +0,0 @@ -{% extends "basedocumentation.html" %} - -{% block title %}Add new MARC records{% endblock %} - -{% block extra_extra_head %} -{{ block.super }} - - - - -{% endblock %} - -{% block doccontent %} -

Make your unglued MARC records here

-{% if messages %} - -{% endif %} -

- For unglued ebooks which have existing print editions cataloged in the Library of Congress, we can automatically convert those to unglued ebook MARC records, which will then be automatically affixed to the ebook edition in our database and provided to ungluers. -

- -

- To get the XML record, search http://catalog.loc.gov/, select the print edition, and click on the permalink (will look something like http://lccn.loc.gov/63008219). This page has a MARCXML link. -

- -

- The robot cataloger will automatically add links to all the unglued ebook files known to the Unglue.it database. Make sure you have added those links to the database before creating the record. Likewise, it will autofill the ISBN, and you should have already provided that. -

- -

- If you want to add a MARC record for a non-unglued but publicly available book, create a MARCRecord instance in the admin interface and populate its fields there. (This assumes that the MARC record is for the freely available ebook version and exists in a linkable form. If it does not, upload it to S3 first, by analogy with the django-storages documentation, and use that link.) -

- -
{% csrf_token %} - {{ form.as_p }} - -
-{% endblock %} \ No newline at end of file diff --git a/frontend/urls.py b/frontend/urls.py index 57b21147..a735c800 100644 --- a/frontend/urls.py +++ b/frontend/urls.py @@ -36,7 +36,6 @@ from regluit.frontend.views import ( ByPubView, kindle_config, send_to_kindle, - MARCUngluifyView, MARCConfigView, DownloadView, ) @@ -157,7 +156,6 @@ urlpatterns = patterns( url(r"^accounts/edit/kindle_config/(?P\d+)/$", "kindle_config", name="kindle_config_download"), url(r"^send_to_kindle/(?P\d+)/(?P\d)/$", "send_to_kindle", name="send_to_kindle"), url(r"^marc/$", direct_to_template, {'template': 'marc.html'}, name="marc"), - url(r"^marc/ungluify/$", staff_member_required(MARCUngluifyView.as_view()), name="MARCUngluify"), url(r"^accounts/edit/marc_config/$", login_required(MARCConfigView.as_view()), name="marc_config"), ) diff --git a/frontend/views.py b/frontend/views.py index d1123372..523448b0 100755 --- a/frontend/views.py +++ b/frontend/views.py @@ -17,7 +17,6 @@ from notification import models as notification from random import randint from re import sub from xml.etree import ElementTree as ET -from xml.sax import SAXParseException from tastypie.models import ApiKey ''' @@ -116,7 +115,6 @@ from regluit.frontend.forms import ( MsgForm, PressForm, KindleEmailForm, - MARCUngluifyForm, MARCFormatForm, DateCalculatorForm ) @@ -3110,39 +3108,6 @@ def work_marc(request, work_id): work = safe_get_work(work_id) return qs_marc_records(request, qs=[ work ]) -class MARCUngluifyView(FormView): - template_name = 'marcungluify.html' - form_class = MARCUngluifyForm - success_url = reverse_lazy('MARCUngluify') - - # allow a get param to specify the edition - def get_initial(self): - if self.request.method == 'GET': - edition = self.request.GET.get('edition',None) - if models.Edition.objects.filter(id=edition).count(): - edition = models.Edition.objects.filter(id=edition)[0] - if edition.ebooks.count() or edition.ebook_files.count(): - return {'edition':edition.id} - return {} - - def form_valid(self, form): - edition = form.cleaned_data['edition'] - - try: - marc.makemarc( - marcfile=self.request.FILES['file'], - edition=edition - ) - messages.success( - self.request, - "You have successfully added a MARC record. Hooray! Add another?" - ) - except SAXParseException: - messages.error( - self.request, - "Sorry, couldn't parse that file." - ) - return super(MARCUngluifyView,self).form_valid(form) class MARCConfigView(FormView): template_name = 'marc_config.html' diff --git a/marc/forms.py b/marc/forms.py new file mode 100644 index 00000000..09a7e951 --- /dev/null +++ b/marc/forms.py @@ -0,0 +1,22 @@ +from django import forms + +from selectable.forms import ( + AutoCompleteSelectWidget, + AutoCompleteSelectField +) + +from regluit.core.lookups import EditionLookup + +class MARCUploadForm(forms.Form): + edition = AutoCompleteSelectField( + EditionLookup, + label='Edition', + widget=AutoCompleteSelectWidget(EditionLookup), + required=True, + error_messages={'required': 'Please specify an edition.'}, + ) + file = forms.FileField(label='Select a MARCXML file.') + source = forms.ChoiceField(label='This file is ...', choices=[ + ( 'loc' , 'from Library of Congress (print)'), + ( 'raw' , 'prepared for Unglue.it'), + ]) diff --git a/marc/load.py b/marc/load.py index c9bdffe4..a22e7903 100644 --- a/marc/load.py +++ b/marc/load.py @@ -117,6 +117,15 @@ def stub(edition): return record +#load a with minimal change +def raw(marcfile, edition): + record = pymarc.parse_xml_to_array(marcfile)[0] + for field in record: + if field.tag in ('001', '003', '005', '006', '007', '856') or int( field.tag ) > 900: + record.remove_field(field) + add_stuff(record) + return record + #load a record from library of Congress def from_lc(marcfile, edition): diff --git a/marc/management/commands/migrate_records.py b/marc/management/commands/migrate_records.py index a627f9f1..79d94e51 100644 --- a/marc/management/commands/migrate_records.py +++ b/marc/management/commands/migrate_records.py @@ -5,12 +5,15 @@ from regluit.marc.models import MARCRecord from regluit.core.models import MARCRecord as OldRecord class Command(BaseCommand): - help = "migrate records from files" - args = "" + help = "migrate records (id' '