Merge pull request #404 from Gluejar/marc_package

Marc package

.gitignore

@@ -10,5 +10,4 @@ build
 deploy/last-update
 logs/*
 celerybeat.pid
-marc/*
 .gitignore~

admin.py

@@ -35,6 +35,7 @@ regluit imports
 """
 from regluit import payment
 from regluit.core import models
+from regluit.marc.models import MARCRecord
 from regluit.core.lookups import (
     PublisherNameLookup,
     WorkLookup,
@@ -207,12 +208,17 @@ class MARCRecordAdminForm(forms.ModelForm):
             widget=AutoCompleteSelectWidget(EditionLookup),
             required=True,
     )
-
+    user = AutoCompleteSelectField(
+            OwnerLookup,
+            widget=AutoCompleteSelectWidget(OwnerLookup),
+            required=True,
+        )
     class Meta(object):
-        model = models.MARCRecord
+        model = MARCRecord
 
 class MARCRecordAdmin(ModelAdmin):
-    list_display = ('edition',)
+    list_display = ('edition', 'user')
+    date_hierarchy = 'created'
     form = MARCRecordAdminForm
 
 admin_site = RegluitAdmin("Admin")
@@ -238,7 +244,7 @@ admin_site.register(models.Wishlist, WishlistAdmin)
 admin_site.register(models.UserProfile, UserProfileAdmin)
 admin_site.register(models.CeleryTask, CeleryTaskAdmin)
 admin_site.register(models.Press, PressAdmin)
-admin_site.register(models.MARCRecord, MARCRecordAdmin)
+admin_site.register(MARCRecord, MARCRecordAdmin)
 
 # payments
 

core/bookloader.py

@@ -202,8 +202,7 @@ def update_edition(edition):
     models.Identifier.get_or_add(type='goog',value=googlebooks_id,edition=edition,work=edition.work)
 
     for a in d.get('authors', []):
-        a, created = models.Author.objects.get_or_create(name=a)
+        edition.add_author(a)
-        a.editions.add(edition)
     
     add_ebooks(item, edition)
             

core/marc.py

@@ -1,367 +0,0 @@
-"""
-This takes a MARCXML filename as an argument and converts it into
-MARC records for the unglued edition (in .xml and .mrc formats).
-Consider it a catalogolem: http://commons.wikimedia.org/wiki/File:Arcimboldo_Librarian_Stokholm.jpg
-Use the MARCXML file for the non-unglued edition from Library of Congress.
-"""
-
-import pymarc
-import logging
-from copy import deepcopy
-from datetime import datetime
-from StringIO import StringIO
-
-from django.conf import settings
-from django.core.files.storage import default_storage
-from django.core.urlresolvers import reverse
-
-import regluit.core.cc as cc
-from regluit.core import models
-
-def makestub(edition):
-    return makemarc(None,  edition)
-
-
-def makemarc(marcfile,  edition):
-    logger = logging.getLogger(__name__)
-    
-    try:
-        license = edition.ebooks.all()[0].rights
-        ebf = None
-    except IndexError:
-        license = None
-        try:
-            ebf = edition.ebook_files.all()[0]
-        except IndexError:
-            # no record if no ebooks
-            return None
-    
-    logger.info("Making MARC records for edition %s " % edition)
-    
-    # save lccn for later (if there is one) before deleting it
-    print_lccn = None
-    if marcfile:
-        record = pymarc.parse_xml_to_array(marcfile)[0]
-        for lccn in record.get_fields('010'):
-            for validlccn in lccn.get_subfields('a'):
-                print_lccn = validlccn
-        fields_to_delete = []
-        fields_to_delete += record.get_fields('001')
-        fields_to_delete += record.get_fields('003')
-        fields_to_delete += record.get_fields('005')
-        fields_to_delete += record.get_fields('006')
-        fields_to_delete += record.get_fields('007')
-        fields_to_delete += record.get_fields('010')
-        fields_to_delete += record.get_fields('040')
-        for field in fields_to_delete:
-            record.remove_field(field)
-    else:
-        record = pymarc.Record()
-
-        
-    # create accession number and write 001 field 
-    # (control field syntax is special)
-    if ebf:
-        (marc_record, created) = models.MARCRecord.objects.get_or_create(edition=edition,link_target='B2U')
-    else:
-        (marc_record, created) = models.MARCRecord.objects.get_or_create(edition=edition,link_target='UNGLUE')
-    field001 = pymarc.Field(tag='001', data=marc_record.accession)
-    record.add_ordered_field(field001)
-
-    # add field indicating record originator
-    field003 = pymarc.Field(tag='003', data='UnglueIt')
-    record.add_ordered_field(field003)
-    
-    # update timestamp of record
-    now = datetime.now()
-    datestamp = now.strftime('%Y%m%d%H%M%S') + '.0'
-    field005 = pymarc.Field(tag='005', data=datestamp)
-    record.add_ordered_field(field005)
-
-    # change 006, 007, 008 because this is an online resource
-    field006 = pymarc.Field(
-        tag='006',
-        data='m     o  d        '
-    )
-    record.add_ordered_field(field006)
-    
-    field007 = pymarc.Field(
-        tag='007',
-        data='cr'
-    )
-    record.add_ordered_field(field007)
-    
-    try:
-        field008 = record.get_fields('008')[0]
-        record.remove_field(field008)
-        old_field_value = field008.value()
-        new_field_value = old_field_value[:23] + 'o' + old_field_value[24:]
-    except IndexError:
-        # fun fun fun 
-        new_field_value= now.strftime('%y%m%d')+'s'
-        if edition.publication_date and len(edition.publication_date)>3:
-            new_field_value += edition.publication_date[0:4]
-        else:
-            new_field_value += '||||'
-        new_field_value += '||||xx |||||o|||||||||||eng||'
-    field008 = pymarc.Field(tag='008', data=new_field_value)
-    record.add_ordered_field(field008)   
-        
-    # add IBSN for ebook where applicable; relegate print ISBN to $z
-    isbn = ''
-    try:
-        isbn = edition.identifiers.filter(type='isbn')[0].value
-    except IndexError:
-        pass
-    try:
-        field020 = record.get_fields('020')[0]
-        print_isbn = field020.get_subfields('a')[0]
-        field020.delete_subfield('a')
-        if isbn:
-            field020.add_subfield('a', isbn)
-        field020.add_subfield('z', print_isbn)
-    except IndexError:
-        print_isbn = None
-
-    # change 050 and 082 indicators because LOC is no longer responsible for these
-    # no easy indicator change function, so we'll just reconstruct the fields
-    try:
-        field050 = record.get_fields('050')[0]
-        field050_new = field050
-        field050_new.indicators = [' ', '4']
-        record.remove_field(field050)
-        record.add_ordered_field(field050_new)
-    except:
-        pass # if no 050 field, don't need to change indicator
-    
-    try:
-        field082 = record.get_fields('082')[0]
-        field082_new = field082
-        field082_new.indicators = [' ', '4']
-        record.remove_field(field082)
-        record.add_ordered_field(field082_new)
-    except:
-        pass # if no 082 field, don't need to change indicator
-    
-    # author name
-    try:
-        field100 = record.get_fields('100')[0]
-    except IndexError:
-        num_auths = edition.authors.count()
-        if num_auths:
-            field100 = pymarc.Field(
-                tag='100',
-                indicators = ['1', ' '],
-                subfields = [
-                    'a', edition.authors.all()[0].last_name_first,
-                ]
-            )
-            record.add_ordered_field(field100)
-        if num_auths > 1:
-            for auth in edition.authors.all()[1:]:
-                field = pymarc.Field(
-                    tag='700',
-                    indicators = ['1', ' '],
-                    subfields = [
-                        'a', auth.last_name_first,
-                        'e', 'joint author.',
-                    ]
-                )
-                record.add_ordered_field(field)
-    # add subfield to 245 indicating format
-    try:
-        field245 = record.get_fields('245')[0]
-    except IndexError:
-        field245 = pymarc.Field(
-            tag='245',
-            indicators = ['1', '0'],
-            subfields = [
-                'a', edition.title,
-            ]
-        )
-        record.add_ordered_field(field245)
-    field245.add_subfield('a', '[electronic resource]')
-    
-    # publisher, date
-    try:
-        field260 = record.get_fields('260')[0]
-    except IndexError:
-        field260 = pymarc.Field(
-            tag='260',
-            indicators = [' ', ' '],
-            subfields = [
-                'b', edition.publisher_name.name if edition.publisher_name is not None else "",
-                'c', unicode(edition.publication_date),
-            ]
-        )
-        record.add_ordered_field(field260)
-    
-    # modify 300 field (physical description)
-    try:
-        field300 = record.get_fields('300')[0]
-        subfield_a = field300.get_subfields('a')[0]
-        if (
-            subfield_a[-2:] == ' ;' or 
-            subfield_a[-2:] == ' :' or 
-            subfield_a[-2:] == ' +'
-        ):
-            subfield_a = subfield_a[:-2]
-        new300a = '1 online resource (' + subfield_a + ')'
-        if field300.get_subfields('b'):
-            new300a += ' :'
-        field300.delete_subfield('a')
-        field300.add_subfield('a', new300a)
-        field300.delete_subfield('c')
-    except:
-        pass
-    
-    if license:
-        # add 536 field (funding information)
-        if edition.unglued:
-            funding_info = 'The book is available as a free download thanks to the generous support of interested readers and organizations, who made donations using the crowd-funding website Unglue.it.'
-        else:
-            if edition.ebooks.all()[0].rights in cc.LICENSE_LIST:
-                funding_info = 'The book is available as a free download thanks to a Creative Commons license.'
-            else:
-                funding_info = 'The book is available as a free download because it is in the Public Domain.'
-        field536 = pymarc.Field(
-            tag='536',
-            indicators = [' ', ' '],
-            subfields = [
-                'a', funding_info,
-            ]
-        )
-        record.add_ordered_field(field536)
-    
-        # add 540 field (terms governing use)
-        field540 = pymarc.Field(
-            tag='540',
-            indicators = [' ', ' '],
-            subfields = [
-                'a', dict(cc.CHOICES)[license],
-                'u', dict(cc.GRANTS)[license], 
-            ]
-        )
-        record.add_ordered_field(field540)
-
-    # add 588 field (source of description) - credit where credit is due
-    if print_lccn:
-        field588 = pymarc.Field(
-            tag='588',
-            indicators = [' ', ' '],
-            subfields = [
-                'a', 'Description based on print version record from the Library of Congress.',
-            ]
-        )
-        record.add_ordered_field(field588)
-    
-    # add 776 field (related editions) - preserve pISBN, LCCN, OCLCnum
-    title = record.get_fields('245')[0].get_subfields('a')[0]
-    title = title.split('/')[0]
-    try:
-        oclcnum = edition.identifiers.filter(type='oclc')[0].value
-    except IndexError:
-        oclcnum = None
-    
-    subfields = ['i', 'Print version: ','t', title,]
-    
-    if print_isbn:
-        subfields.extend(['z', print_isbn])
-    elif isbn:
-        subfields.extend(['z', isbn])
-    if print_lccn:
-        subfields.extend(['w', '(DLC) ' + print_lccn, ])
-    if oclcnum:
-        subfields.extend(['w', '(OCoLC) ' + oclcnum,])
-
-    field776 = pymarc.Field(
-        tag='776',
-        indicators = ['0', '8'],
-        subfields = subfields
-    )
-    
-    record.add_ordered_field(field776)
-    """
-    add 776 fields
-    indicators: 0 8
-    '$i Print version: '
-    $t Title. <--note space
-    $d is optional
-    $z pISBN goes here
-        harvest from 020 (was moved from $a to $z)
-    $w (DLC) LCCN_goes_here
-        harvest from 010 field before deletion
-    $w (OCoLC) OCLCnum_goes_here
-        harvest from identifiers db
-    """
-
-    # strip any 9XX fields (they're for local use)    
-    for i in range(900, 1000):
-        fields = record.get_fields(str(i))
-        for field in fields:
-            record.remove_field(field)
-    
-    # add 856 fields with links for each available file
-    # doing this out of order as it's the only thing that differs
-    # between direct-link and via-unglue.it versions
-    if not ebf:
-        # need deepcopy() because omg referential transparency!
-        record_direct = deepcopy(record)  # 2 records for unglued stuff
-    
-        for format_tuple in settings.FORMATS:
-            format = format_tuple[0]
-            ebooks = edition.ebooks.filter(format=format)
-            if ebooks:
-                for book in ebooks:
-                    field856 = pymarc.Field(
-                        tag='856',
-                        indicators = ['4', '0'],
-                        subfields = [
-                            '3', format + ' version',
-                            'q', settings.CONTENT_TYPES[format],
-                            'u', book.url,
-                        ]
-                    )
-                    record_direct.add_ordered_field(field856)
-                
-    unglued_url = settings.BASE_URL_SECURE + reverse('download', args=[edition.work.id])
-    field856_via = pymarc.Field(
-        tag='856',
-        indicators = ['4', '0'],
-        subfields = [
-            'u', unglued_url,
-        ]
-    )
-    record.add_ordered_field(field856_via)
-
-    if not ebf:
-        # this via_unglueit record needs its own accession number
-        field001 = record_direct.get_fields('001')[0]
-        record_direct.remove_field(field001)
-        (marc_record_direct, created) = models.MARCRecord.objects.get_or_create(edition=edition,link_target='DIRECT')
-        field001 = pymarc.Field(tag='001', data=marc_record_direct.accession)
-        record_direct.add_ordered_field(field001)
-
-        # write the unglued MARCxml records
-        xmlrecord = pymarc.record_to_xml(record_direct)
-        xml_file = default_storage.open(marc_record_direct.xml_record, 'w')
-        xml_file.write(xmlrecord)
-        xml_file.close()
-    
-        # write the unglued .mrc records, then save to s3
-        mrc_file = default_storage.open(marc_record_direct.mrc_record, 'w')
-        writer = pymarc.MARCWriter(mrc_file)
-        writer.write(record_direct)
-        mrc_file.close()
-
-    xmlrecord = pymarc.record_to_xml(record)
-    xml_file = default_storage.open(marc_record.xml_record, 'w')
-    xml_file.write(xmlrecord)
-    xml_file.close()
-
-    mrc_file = default_storage.open(marc_record.mrc_record, 'w')
-    writer = pymarc.MARCWriter(mrc_file)
-    writer.write(record)
-    mrc_file.close()
-    
-    return marc_record.pk

core/models.py

@@ -37,7 +37,7 @@ import regluit.core.isbn
 import regluit.core.cc as cc
 from regluit.core.epub import personalize, ungluify, test_epub, ask_epub
 from regluit.core.pdf import ask_pdf, pdf_append
-
+from regluit.marc.models import MARCRecord as NewMARC
 from regluit.core.signals import (
     successful_campaign,
     unsuccessful_campaign,
@@ -1008,6 +1008,10 @@ class Campaign(models.Model):
     def user_to_pay(self):
         return self.rh.owner
     
+    ### for compatibility with MARC output
+    def marc_records(self):
+        return self.work.marc_records()
+
 class Identifier(models.Model):
     # olib, ltwk, goog, gdrd, thng, isbn, oclc, olwk, olib, gute, glue
     type = models.CharField(max_length=4, null=False)
@@ -1518,6 +1522,20 @@ class Work(models.Model):
             # assume it's several users
             return self.user_license(self.acqs.filter(user__in=user))
 
+    ### for compatibility with MARC output
+    def marc_records(self):
+        record_list = []
+        record_list.extend(NewMARC.objects.filter(edition__work=self))
+        for obj in record_list:
+            break
+        else:
+            for ebook in self.ebooks():
+                record_list.append(ebook.edition)
+                break
+        return record_list
+            
+        
+                
 class Author(models.Model):
     created = models.DateTimeField(auto_now_add=True)
     name = models.CharField(max_length=500)
@@ -1640,6 +1658,14 @@ class Edition(models.Model):
         except Identifier.DoesNotExist:
             return None
     
+    def add_author(self, author_name):
+        if author_name:
+            try:
+                author= Author.objects.get(name=author_name)
+            except Author.DoesNotExist:
+                author= Author.objects.create(name=author_name)
+            author.editions.add(self)
+
     def set_publisher(self,publisher_name):
         if publisher_name and publisher_name != '':
             try:
@@ -1652,6 +1678,40 @@ class Edition(models.Model):
             self.publisher_name = pub_name
             self.save()
 
+    #### following methods for compatibility with marc outputter
+    def downloads(self):
+        return self.ebooks.all()
+
+    def download_via_url(self):
+        return settings.BASE_URL_SECURE + reverse('download', args=[self.work.id])
+        
+    def authnames(self):
+        return [auth.last_name_first for auth in self.authors.all()]
+    
+    @property
+    def license(self):
+        try:
+            return self.ebooks.all()[0].rights
+        except:
+            return None
+    
+    @property
+    def funding_info(self): 
+        if self.ebooks.all().count()==0:
+            return ''  
+        if self.unglued:
+            return 'The book is available as a free download thanks to the generous support of interested readers and organizations, who made donations using the crowd-funding website Unglue.it.'
+        else:
+            if self.ebooks.all()[0].rights in cc.LICENSE_LIST:
+                return 'The book is available as a free download thanks to a Creative Commons license.'
+            else:
+                return 'The book is available as a free download because it is in the Public Domain.'
+    
+    @property
+    def description(self): 
+        return self.work.description
+
+
 class Publisher(models.Model):
     created = models.DateTimeField(auto_now_add=True)
     name = models.ForeignKey('PublisherName', related_name='key_publisher')

frontend/forms.py

@@ -770,17 +770,8 @@ class PressForm(forms.ModelForm):
 class KindleEmailForm(forms.Form):
     kindle_email = forms.EmailField()
     
-class MARCUngluifyForm(forms.Form):
-    edition = AutoCompleteSelectField(
-            EditionLookup,
-            label='Edition',
-            widget=AutoCompleteSelectWidget(EditionLookup),
-            required=True,
-            error_messages={'required': 'Please specify an edition.'},
-        )    
-    file = forms.FileField(label='Download a MARCXML file from Library of Congress; then upload it here.')
         
-class MARCFormatForm(forms.ModelForm):
+class LibModeForm(forms.ModelForm):
     class Meta:
         model = Libpref
-        fields = ('marc_link_target',)
+        fields = ()

frontend/templates/bypub_list.html

@@ -20,3 +20,10 @@
 {% endblock %}
 
 {% block add_more %}{% endblock %}
+
+{% block marcform %}
+<form method="GET" action="{% url bypubname_list_marc facet pubname.id %}">
+    {% include 'marc_form.html' %}
+    <input type="submit" name="submit" value="download MARC" id="submit">
+</form>
+{% endblock %}

frontend/templates/campaign_list.html

@@ -102,6 +102,17 @@ location.hash = "#2";
                     {% endifequal %}
                 </div>
             </div>    
+            {% if request.user.libpref %}
+            <div id="libtools">
+                <p>for libraries...</p>
+                {% block marcform %}
+                <form method="GET" action="{% url campaign_list_marc facet %}">
+                    {% include 'marc_form.html' %}
+                    <input type="submit" name="submit" value="download MARC" id="submit">
+                </form>
+                {% endblock %}
+            </div> 
+            {% endif %}  
         </div>
         </div>
     </div>

frontend/templates/cc_list.html

@@ -94,6 +94,15 @@
                     </div>
                 </div>
             </div> 
+            {% if request.user.libpref %}
+            <div id="libtools">
+                <p>for libraries...</p>
+                <form method="GET" action="{% url cc_list_marc facet %}">
+                    {% include 'marc_form.html' %}
+                    <input type="submit" name="submit" value="download MARC" id="submit">
+                </form>
+            </div> 
+            {% endif %}  
         </div>
         </div>
     </div>

frontend/templates/edition_display.html

@@ -43,17 +43,8 @@
                     See <a href="https://encrypted.google.com/books?id={{ edition.googlebooks_id }}">this edition on Google Books</a><br />
                 {% endif %} 
             {% endif %}  
-            {% if edition.MARCrecords.all %}                         	    
+            {% if user.libpref %}{% if edition.ebooks.all or edition.ebook_files.all %}
-            {% for record in edition.MARCrecords.all %}
+                <a href="{% url upload_marc %}?edition={{ edition.id }}">Upload</a> a MARC record for this edition. <br />
-                Download {{record.link_target}} MARC record for this edition: (<a href="{% url marc_concatenate %}?record_{{ record.id }}=on&amp;format=xml">XML</a>) (<a href="{% url marc_concatenate %}?record_{{ record.id }}=on&amp;format=mrc">mrc</a>)<br />
-            {% endfor %}
-            {% else %}
-                {% if edition.ebooks.all %}
-                    Download {{record.link_target}} MARC record for this edition: (<a href="{% url marc_concatenate %}?edition_{{ edition.id }}=on&amp;format=xml">XML</a>) (<a href="{% url marc_concatenate %}?edition_{{ edition.id }}=on&amp;format=mrc">mrc</a>)<br />
-                {% endif %}
-            {% endif %}
-            {% if user.is_staff %}{% if edition.ebooks.0 or edition.ebook_files.0 %}
-                <a href="{% url MARCUngluify %}?edition={{ edition.id }}">Upload</a> a MARC record for this edition. <br />
             {% endif %} {% endif %}
         </div>
 

frontend/templates/libraries.html

@@ -31,6 +31,8 @@ We’ve added "Buy-to-Unglue" campaigns to so that libraries can lend ebooks on
 <dl>
 <dt><a href="{% url registration_register %}?next={{ request.get_full_path|urlencode }}">Sign up.</a> It's Free!</dt>
 <dd>Starting an account, for yourself or your library, is free. Use it to help us distribute unglued and ungluing ebooks.   </dd>
+<dt>Add free ebooks to your collection with our <a href="{% url marc %}">MARC records</a>!</dt>
+<dd>We're building a comprehensive database of freely licensed ebooks. We provide <a href="{% url marc %}">MARC records for these ebooks</a> wherever we can.  They can live in the catalog in your ILS; they link to an epub file in the Internet Archive.  Of course, because of unglued ebooks' Creative Commons licenses, you're free to shift that epub to other formats, host the file on your own servers, and edit the MARC record accordingly. And if you want, help us <a href="{% url marc %}">improve</a> our records.</dd>
 <dt>Become an Unglue.it participating library</dt>
 <dd><ol><li><a href="{% url library_create %}">Make your library page</a> on Unglue.it. It takes just a few minutes.  It's the first step towards becoming an Unglue.it participating library.</li>
 <li>Review our <a href="https://www.docracy.com/0_uyw26qv9c/unglue-it-library-license-agreement">LIBRARY LICENSE AGREEMENT</a>. </li>
@@ -42,10 +44,6 @@ The library license gives download access to one library member at a time for 14
 </dd>
 <dt>Stay in touch.</dt>
 <dd>You can follow us on Twitter (<a href="http://twitter.com/unglueit">@unglueit</a>), <a href="http://facebook/com/unglueit">Facebook</a>, and our <a href="http://blog.unglue.it">blog</a>, and <a href="http://eepurl.com/fKLfI">subscribe to our newsletter</a> (1-2 emails per month).</dd>
-<dt>Add unglued ebooks to your collection.</dt>
-<dd>We provide <a href="{% url marc %}">MARC records for unglued ebooks</a> wherever we can.  We also strive to have MARC records available for all unglued ebooks.  They can live in the catalog in your ILS; they link to an epub file in the Internet Archive.  Of course, because of unglued ebooks' Creative Commons licenses, you're free to shift that epub to other formats, host the file on your own servers, and edit the MARC record accordingly.</dd>
-<dt>Join our <a href="http://goo.gl/lCTLI">catalogers list</a>.</dt>
-<dd>We're able to get MARC records into OCLC and SkyRiver because of the support of catalogers like you.  Want to help?  <a href="http://goo.gl/lCTLI">Add yourself to the list.</a>  Usually all you'll need to do is modify an existing record for an earlier edition and upload the record.  We'll supply you with the link and the unglued edition.  We'll only contact you when there's an unglued book to be cataloged.</dd>
 <dt>Spread the word.</dt>
 <dd>There are social media sharing links on most pages on the site.  There are some right here!
 	<div id="widgetcode">Copy/paste this into your site:<br /><textarea rows="7" cols="22">&lt;iframe src="https://{{request.META.HTTP_HOST}}/api/widget/{{work.first_isbn_13}}/" width="152" height="325" frameborder="0"&gt;&lt;/iframe&gt;</textarea></div>
@@ -59,7 +57,7 @@ The library license gives download access to one library member at a time for 14
 <dt>Educate yourself and your patrons about ebook issues and Creative Commons licenses.</dt>
 <dd>Checkout limits, publishers who won't sell ebooks to libraries, DRM, companies tracking readers' behavior, library prices far in excess of consumer costs, incompatible technologies and formats, cataloging silos...you know why it's hard for you to deliver a seamless ereading experience to your patrons.  Make sure they know, too.  And make sure everyone knows how solutions, like <a href="http://creativecommons.org">Creative Commons</a> licenses, can help libraries and readers while respecting copyright and protecting creators' rights.</dd>
 <dt>Support <a href="{% url campaign_list 'ending' %}">our active campaigns</a>.</dt>
-<dd>Ultimately ebooks can't be unglued unless authors and publishers are paid for their work.  Many of our staunchest supporters are librarians.  There are also several libraries which have supported campaigns, including Leddy Library (University of Windsor, Ontario); the University of Alberta library ; and the Z. Smith Reynolds library (Wake Forest University.</dd>
+<dd>Ultimately ebooks can't be unglued unless authors and publishers are paid for their work.  Many of our staunchest supporters are librarians.  There are also several libraries which have supported campaigns, including Leddy Library (University of Windsor, Ontario); the University of Alberta library ; and the Z. Smith Reynolds library (Wake Forest University).</dd>
 <dt>Give feedback and ask questions.</dt>
 <dd>Want to know more?  Need help?  Have ideas for how we could improve the site or make it more library-friendly?  Contact us at: <a href="mailto:libraries@gluejar.com">libraries@gluejar.com</a>.</dd>
 </dl>

frontend/templates/marc.html

@@ -44,8 +44,8 @@ ul.local li {
 
 <p>Go ahead: add unglued ebooks to your library catalog!</p>
 
-{% if request.user.is_staff %}
+{% if request.user.libpref %}
-    <p>Hi, {{ request.user.username }}.  Unglue.it staffers can also <a href="{% url MARCUngluify %}">add new records</a>.</p>
+    <p>Hi, {{ request.user.username }}.  Librarians can also <a href="{% url upload_marc %}">add new records</a>.</p>
 {% endif %}
 
 {% if messages %}
@@ -55,53 +55,38 @@ ul.local li {
         {% endfor %}
     </ul>
 {% endif %}
-{% if userlist %}
-<h2>Records for  <a href="{% url supporter userlist %}">{{ userlist }}</a></h2>
-{% else %}
-{% if request.user.is_authenticated %}
-<p> To restrict this list to works on your list <a href="{% url user_marc request.user.username %}">click here</a> </p>
-{% endif %}
-{% endif %}
-{% if records %}
 <div class="marc">
-    <form method="POST" id="record_form" action="{% url marc_concatenate %}">
+    <form method="POST" id="record_form" action="{% url marc_all %}">
-        {% csrf_token %}
+        {% include 'marc_form.html' %}
-        Record format:
+        <input type="submit" name="submit" value="Download All Records" id="submit">
-        <select name="format">
-            <option value="xml">xml</option>
-            <option value="mrc">mrc</option>
-        </select>
-        <br /><br />
-        {% for record in records %}
-            <input type="checkbox" name="record_{{ record.id }}" id="record_{{ record.id }}" />
-            <label for="record_{{ record.id }}">
-                Record for <a href="{% url work record.edition.work.id %}" class="title clearfix">{{ record.edition.work.title }}{% if record.edition.isbn_13 %} (ISBN {{ record.edition.isbn_13 }}){% endif %}</a>
-            </label>
-            <br />
-        {% endfor %}
-        <input type="submit" name="submit" value="Download" id="submit">
         <span id="unsubmit"><img src="/static/images/loading.gif" alt="spinning wheel" /> Fetching record(s)...</span>
     </form>
-</div>
+    <br />
-{% else %}
+{% if request.user.is_authenticated %}
-<p>Sorry; we don't have any records meeting your needs at this time.  </p>
+    <p>...or</p>
+<div class="marc">
+    <form method="POST" id="record_form" action="{% url user_marc request.user.username %}">
+        {% include 'marc_form.html' %}
+        <input type="submit" name="submit" value="Download {{request.user.username}}'s Records" id="submit">
+        <span id="unsubmit"><img src="/static/images/loading.gif" alt="spinning wheel" /> Fetching record(s)...</span>
+    </form>
+    <br /><hr>
 {% endif %}
+    <p>The 856 link options are:</p>
+    <ul class="local">
+            <li> Download page link. MARC records link through Unglue.it download page ( library user authentication, context sensitive help for getting files onto user's device)</li>
+            <li> Links to each file type if available. MARC records link directly to files ( less help, not available for "Buy-to-unglue" titles) </li>
+    </ul>
+</div>
+
 
-<hr>
 {% if request.user.is_authenticated %}
 <div>
-    <p>Your preference for the links in the 856 field is:</p>
+    {% if request.user.libpref %}
-    <ul class="local">
+    <p>You have enabled librarian tools. You can <a href="{% url marc_config %}">change your librarian status here</a>.</p>
-        <li>{{ libpref.get_marc_link_target_display }}</li>
+    {% else %}
-    </ul>
+    <p>You have not enabled librarian tools. You can <a href="{% url marc_config %}">change your librarian status here</a>.</p>
-    
+    {% endif %}
-    <p>The options are:</p>
-    <ul class="local">
-            <li> Raw link if available. MARC records link direct to provider (one click, no help, not available for "Buy-to-unglue" titles) </li>
-            <li> Unglue.it link. MARC records link through Unglue.it download page (extra click, library user authentication, context sensitive help for getting files onto user's device)</li>
-    </ul>
-    
-    <p>You can <a href="{% url marc_config %}">change your preferences here</a>.</p>
 </div>
 {% endif %}
 {% endblock %}

frontend/templates/marc/upload.html

@@ -0,0 +1,62 @@
+{% extends "basedocumentation.html" %}
+
+{% block title %}Add new MARC records{% endblock %}
+
+{% block extra_extra_head %}
+{{ block.super }}
+<link rel="stylesheet" href="https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.13/themes/ui-lightness/jquery-ui.css" type="text/css" media="screen">
+<link href="/static/selectable/css/dj.selectable.css" type="text/css" media="all" rel="stylesheet" />
+<script type="text/javascript" src="{{ jquery_ui_home }}"></script>
+<script type="text/javascript" src="/static/selectable/js/jquery.dj.selectable.js"></script>
+{% endblock %}
+
+{% block doccontent %}
+{% if not request.user.libpref %}
+<p><i>If you want to load MARC records, you probably want to <a href="{% url marc_config %}">enable Unglue.it's librarian tools</a> first!</i></p>
+{% endif %}
+<h2>Adding MARC records to Unglue.it</h2>
+{% if messages %}
+<ul class="messages">
+    {% for message in messages %}
+    <li{% if message.tags %} class="{{ message.tags }}"{% endif %}>{{ message }}</li>
+    {% endfor %}
+</ul>
+{% endif %}
+{% if not form.initial.edition %}
+<h3>Selecting an edition</h3>
+<p>
+MARC records are associated with unglue.it edition records. Look for links in the "more" tab of any unglue.it work page that has a "Download" button on it.
+{% endif %}
+<h3>Adapting records from LoC</h3>
+<p>
+    For ebooks which have existing print editions cataloged in the Library of Congress, we can automatically convert those to Unglue.it ebook MARC records, which will then be automatically affixed to the ebook edition in our database and <a href="{% url marc %}">provided to ungluers</a>.
+</p>
+
+<p>
+    To get the XML record, search <a href="http://catalog.loc.gov/">http://catalog.loc.gov/</a>, select the print edition, and click on the permalink (will look something like <a href="http://lccn.loc.gov/63008219">http://lccn.loc.gov/63008219</a>).  This page has a MARCXML link.
+</p>
+
+<p>
+    The record loader will automatically add links to all the unglued ebook files known to the Unglue.it database.  <I>Make sure  those links are in the database before adding the record.</i>  </p>
+
+<h3>Editing stub records from LoC</h3>
+{% if form.initial.edition %}
+The current records for this edition are here. They may be auto-generated stubs: 
+    <div id="libtools">
+        <form method="POST" id="record_form" action="{% url marc_concatenate  %}">
+            {% include 'marc_form.html' %}
+            <input type="hidden" name="edition_{{form.initial.edition}}" value="on">
+            <input type="submit" name="submit" value="Download MARC" id="submit">
+        </form>
+     </div>
+{% endif %}
+
+<p>
+    Edit the record, then upload it as a "record prepared for Unglue.it". The record loader strips 001,  003, 005, 006, 007, 856 and >900 fields, and adds them back in on download.
+</p>
+<h3>Load a record</h3>
+<form action="" enctype="multipart/form-data" method="post">{% csrf_token %}
+    {{ form.as_p }}
+    <input type="submit" value="Add record" />
+</form>
+{% endblock %}

frontend/templates/marc_config.html

@@ -10,7 +10,7 @@
 </style>
 {% endblock %}
 
-{% block title %}Change your MARC record preferences{% endblock %}
+{% block title %}Change your librarian preferences{% endblock %}
 
 {% block ce_content %}
 {% if messages %}
@@ -20,14 +20,33 @@
     {% endfor %}
 </ul>
 {% endif %}
-Back to <a href="{% url marc %}">record download page</a><br /><br />
+<h3>Librarian Tools</h3>
+If you enable Librarian tools, you'll find links to download MARC record(s) at the bottom of many Unglue.it pages.
+<ul>
+<li> work pages</li>
+<li> lists of works</li>
+<li> supporter pages </li>
+</ul>
 
-Your current MARC record link target preference is:<br />
+{% if request.user.is_authenticated %}
-{{ libpref.get_marc_link_target_display }}
 <form method="post" action="{% url marc_config %}">
     {% csrf_token %}
+<div>
+    {% if request.user.libpref %}
+        <p>You have enabled librarian tools. </p>
+        {{ form }}
+        <input type="submit" value="Disable Librarian Tools" name="disable" />
+    {% else %}
+        <p>You have not enabled librarian tools. 
         {{ form }}
         <br />
-    <input type="submit" value="Change" />
+        <input type="submit" value="Enable Librarian Tools"  name="enable" />
+    
+    {% endif %}
+</div>
 </form>
+<p>
+Back to <a href="{% url marc %}">record download page</a>
+</p>
+{% endif %}
 {% endblock %}

frontend/templates/marc_form.html

@@ -0,0 +1,12 @@
+{% csrf_token %}
+<span style="padding: 5px;">record format:
+<select name="format">
+    <option value="xml" selected>xml</option>
+    <option value="mrc">mrc</option>
+</select></span>&nbsp;&nbsp;
+<span style="padding: 5px;">856 links:
+<select name="link_target">
+    <option value="via" selected>to download page</option>
+    <option value="direct">to each file type</option>
+</select>
+</span>

frontend/templates/marcungluify.html

@@ -1,42 +0,0 @@
-{% extends "basedocumentation.html" %}
-
-{% block title %}Add new MARC records{% endblock %}
-
-{% block extra_extra_head %}
-{{ block.super }}
-<link rel="stylesheet" href="https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.13/themes/ui-lightness/jquery-ui.css" type="text/css" media="screen">
-<link href="/static/selectable/css/dj.selectable.css" type="text/css" media="all" rel="stylesheet" />
-<script type="text/javascript" src="{{ jquery_ui_home }}"></script>
-<script type="text/javascript" src="/static/selectable/js/jquery.dj.selectable.js"></script>
-{% endblock %}
-
-{% block doccontent %}
-<h2>Make your unglued MARC records here</h2>
-{% if messages %}
-<ul class="messages">
-    {% for message in messages %}
-    <li{% if message.tags %} class="{{ message.tags }}"{% endif %}>{{ message }}</li>
-    {% endfor %}
-</ul>
-{% endif %}
-<p>
-    For unglued ebooks which have existing print editions cataloged in the Library of Congress, we can automatically convert those to unglued ebook MARC records, which will then be automatically affixed to the ebook edition in our database and <a href="{% url marc %}">provided to ungluers</a>.
-</p>
-
-<p>
-    To get the XML record, search <a href="http://catalog.loc.gov/">http://catalog.loc.gov/</a>, select the print edition, and click on the permalink (will look something like <a href="http://lccn.loc.gov/63008219">http://lccn.loc.gov/63008219</a>).  This page has a MARCXML link.
-</p>
-
-<p>
-    The robot cataloger will automatically add links to all the unglued ebook files known to the Unglue.it database.  <I>Make sure you have added those links to the database before creating the record.</i>  Likewise, it will autofill the ISBN, and you should have already provided that.
-</p>
-
-<p>
-    If you want to add a MARC record for a non-unglued but publicly available book, create a MARCRecord instance in the admin interface and populate its fields there.  (This assumes that the MARC record is for the freely available ebook version and exists in a linkable form.  If it does not, upload it to S3 first, by analogy with the <a href="http://django-storages.readthedocs.org/en/latest/backends/amazon-S3.html#tests">django-storages documentation</a>, and use that link.)
-</p>
-
-<form action="" enctype="multipart/form-data" method="post">{% csrf_token %}
-    {{ form.as_p }}
-    <input type="submit" value="Add record" />
-</form>
-{% endblock %}

frontend/templates/supporter.html

@@ -389,6 +389,17 @@ function highlightTarget(targetdiv) {
                     {% endifequal %}
                 </div>
             </div>    
+            {% if request.user.libpref %}
+            <div id="libtools">
+                <p>for libraries...</p>
+                {% block marcform %}
+                <form method="GET" action="{% url user_marc supporter.username %}">
+                    {% include 'marc_form.html' %}
+                    <input type="submit" name="submit" value="download MARC for {{supporter.username}} " id="submit">
+                </form>
+                {% endblock %}
+            </div> 
+            {% endif %}  
         </div>
         </div>
     </div>

frontend/templates/unglued_list.html

@@ -105,6 +105,17 @@
                     </div>
                 </div>
             </div>    
+            {% if request.user.libpref %}
+            <div id="libtools">
+                <p>for libraries...</p>
+                {% block marcform %}
+                <form method="GET" action="{% url unglued_list_marc facet %}">
+                    {% include 'marc_form.html' %}
+                    <input type="submit" name="submit" value="download MARC" id="submit">
+                </form>
+                {% endblock %}
+            </div> 
+            {% endif %}  
         </div>
         </div>
     </div>

frontend/templates/work.html

@@ -711,6 +711,15 @@
                         </div>
                     {% endif %}
                 </div>
+                {% if request.user.libpref and work.first_ebook %}
+                <div id="libtools">
+                    <p>for libraries...</p>
+                    <form method="POST" id="record_form" action="{% url work_marc work.id %}">
+                        {% include 'marc_form.html' %}
+                        <input type="submit" name="submit" value="Download MARC" id="submit">
+                    </form>
+                 </div>
+                {% endif %}
             </div>
         </div>
     </div>

frontend/templates/work_list.html

@@ -131,6 +131,17 @@
                     {% endifequal %}
                 </div>
             </div>    
+            {% if request.user.libpref %}
+            <div id="libtools">
+                <p>for libraries...</p>
+                {% block marcform %}
+                <form method="GET" action="{% url work_list_marc facet %}">
+                    {% include 'marc_form.html' %}
+                    <input type="submit" name="submit" value="download MARC" id="submit">
+                </form>
+                {% endblock %}
+            </div> 
+            {% endif %}  
         </div>
         </div>
     </div>

frontend/urls.py

@@ -36,8 +36,7 @@ from regluit.frontend.views import (
     ByPubView,
     kindle_config,
     send_to_kindle,
-    MARCUngluifyView,
+    LibModeView,
-    MARCConfigView,
     DownloadView,
 )
 
@@ -47,7 +46,7 @@ urlpatterns = patterns(
     url(r"^landing/$", "home", {'landing': True}, name="landing"),
     url(r"^next/$", "next", name="next"),
     url(r"^supporter/(?P<supporter_username>[^/]+)/$", "supporter", {'template_name': 'supporter.html'}, name="supporter"),
-    url(r"^supporter/(?P<userlist>[^/]+)/marc/$", "marc_admin", name="user_marc"),
+    url(r"^supporter/(?P<userlist>[^/]+)/marc/$", "userlist_marc", name="user_marc"),
     url(r"^library/(?P<library_name>[^/]+)/$", "library", name="library"),
     url(r"^accounts/manage/$", login_required(ManageAccount.as_view()), name="manage_account"),
     url(r"^search/$", "search", name="search"),
@@ -71,14 +70,19 @@ urlpatterns = patterns(
     url(r"^wishlist/$", "wishlist", name="wishlist"),
     url(r"^msg/$", "msg", name="msg"),
     url(r"^campaigns/(?P<facet>\w*)$", CampaignListView.as_view(), name='campaign_list'),
+    url(r"^campaigns/(?P<facet>\w*)/marc/$", CampaignListView.as_view(send_marc=True), name='campaign_list_marc'),
     url(r"^lists/(?P<facet>\w*)$", WorkListView.as_view(),  name='work_list'),
+    url(r"^lists/(?P<facet>\w*)/marc/$", WorkListView.as_view(send_marc=True),  name='work_list_marc'),
     url(r"^pid/all/(?P<pubname>\d+)$", ByPubView.as_view(),  name='bypubname_list'),
     url(r"^pid/(?P<facet>\w*)/(?P<pubname>\d+)$", ByPubView.as_view(),  name='bypubname_list'),
+    url(r"^pid/(?P<facet>\w*)/(?P<pubname>\d+)/marc/$", ByPubView.as_view(send_marc=True),  name='bypubname_list_marc'),
     url(r"^bypub/all/(?P<pubname>.*)$", ByPubListView.as_view(),  name='bypub_list'),
     url(r"^bypub/(?P<facet>\w*)/(?P<pubname>.*)$", ByPubListView.as_view(),  name='bypub_list'),
     url(r"^unglued/(?P<facet>\w*)$", UngluedListView.as_view(),  name='unglued_list'),
+    url(r"^unglued/(?P<facet>\w*)/marc/$", UngluedListView.as_view(send_marc=True),  name='unglued_list_marc'),
     url(r"^creativecommons/$", CCListView.as_view(),  name='cc_list'),
     url(r"^creativecommons/(?P<facet>[\w\-]*)$", CCListView.as_view(),  name='cc_list_detail'),
+    url(r"^creativecommons/(?P<facet>[\w\-]*)/marc/$", CCListView.as_view(send_marc=True),  name='cc_list_marc'),
     url(r"^goodreads/auth/$", "goodreads_auth", name="goodreads_auth"),
     url(r"^goodreads/auth_cb/$", "goodreads_cb", name="goodreads_cb"),
     url(r"^goodreads/flush/$","goodreads_flush_assoc", name="goodreads_flush_assoc"),
@@ -94,6 +98,7 @@ urlpatterns = patterns(
     url(r"^work/(?P<work_id>\d+)/download/$", DownloadView.as_view(), name="thank"),
     url(r"^work/(?P<work_id>\d+)/unglued/(?P<format>\w+)/$", "download_campaign", name="download_campaign"),
     url(r"^work/(?P<work_id>\d+)/borrow/$", "borrow", name="borrow"),
+    url(r"^work/(?P<work_id>\d+)/marc/$", "work_marc", name="work_marc"),
     url(r"^work/(?P<work_id>\d+)/reserve/$", "reserve", name="reserve"),
     url(r"^work/(?P<work_id>\d+)/feature/$", "feature", name="feature"),
     url(r"^work/(?P<work_id>\d+)/merge/$", login_required(MergeView.as_view()), name="merge"),
@@ -150,10 +155,8 @@ urlpatterns = patterns(
     url(r"^accounts/edit/kindle_config/$", "kindle_config",  name="kindle_config"),
     url(r"^accounts/edit/kindle_config/(?P<work_id>\d+)/$", "kindle_config",  name="kindle_config_download"),
     url(r"^send_to_kindle/(?P<work_id>\d+)/(?P<javascript>\d)/$", "send_to_kindle",  name="send_to_kindle"),
-    url(r"^marc/$", "marc_admin", name="marc"),
+    url(r"^marc/$", direct_to_template, {'template': 'marc.html'}, name="marc"),
-    url(r"^marc/ungluify/$", staff_member_required(MARCUngluifyView.as_view()), name="MARCUngluify"),
+    url(r"^accounts/edit/marc_config/$", login_required(LibModeView.as_view()),  name="marc_config"),
-    url(r"^marc/concatenate/$", "marc_concatenate", name="marc_concatenate"),
-    url(r"^accounts/edit/marc_config/$", login_required(MARCConfigView.as_view()),  name="marc_config"),
 )
 
 if settings.DEBUG:

frontend/views.py

@@ -17,7 +17,6 @@ from notification import models as notification
 from random import randint
 from re import sub
 from xml.etree import ElementTree as ET
-from xml.sax import SAXParseException
 from tastypie.models import ApiKey
 
 '''
@@ -72,7 +71,6 @@ from regluit.core import (
     librarything,
     userlists,
     goodreads,
-    marc
 )
 import regluit.core.cc as cc
 from regluit.core.bookloader import merge_works, detach_edition
@@ -116,8 +114,7 @@ from regluit.frontend.forms import (
     MsgForm,
     PressForm,
     KindleEmailForm,
-    MARCUngluifyForm,
+    LibModeForm,
-    MARCFormatForm,
     DateCalculatorForm
 )
 
@@ -145,6 +142,7 @@ from regluit.booxtream.exceptions import BooXtreamError
 from regluit.pyepub import InvalidEpub
 from regluit.libraryauth.views import Authenticator
 from regluit.libraryauth.models import Library
+from regluit.marc.views import qs_marc_records
 
 logger = logging.getLogger(__name__)
 
@@ -559,10 +557,10 @@ def new_edition(request, work_id, edition_id, by=None):
         elif request.POST.has_key('add_subject_submit') and admin:
             new_subject = request.POST['add_subject'].strip()
             try:
-                author= models.Subject.objects.get(name=new_subject)
+                subject= models.Subject.objects.get(name=new_subject)
             except models.Subject.DoesNotExist:
-                author=models.Subject.objects.create(name=new_subject)
+                subject=models.Subject.objects.create(name=new_subject)
-            edition.new_subjects.append(new_subject)
+            edition.new_subjects.append(subject)
             form = EditionForm(instance=edition, data=request.POST, files=request.FILES)
             edition.ebook_form = EbookForm( instance= models.Ebook(user = request.user, edition = edition, provider = 'x' ), prefix = 'ebook_%d'%edition.id)
 
@@ -605,11 +603,7 @@ def new_edition(request, work_id, edition_id, by=None):
                         else:
                             models.Identifier.set(type=id_type, value=id_val, edition=edition, work=work)
                 for author_name in edition.new_author_names:
-                    try:
+                    edition.add_author(author_name)
-                        author= models.Author.objects.get(name=author_name)
-                    except models.Author.DoesNotExist:
-                        author=models.Author.objects.create(name=author_name)
-                    author.editions.add(edition)
                 for subject_name in edition.new_subjects:
                     try:
                         subject= models.Subject.objects.get(name=subject_name)
@@ -783,6 +777,7 @@ def subjects(request):
     return render(request, 'subjects.html', {'subjects': subjects})
 
 class FilterableListView(ListView):
+    send_marc = False
     def get_queryset(self):
         if self.request.GET.has_key('pub_lang'):
             if self.model is models.Campaign:
@@ -802,6 +797,12 @@ class FilterableListView(ListView):
         context['WISHED_LANGS']=settings.WISHED_LANGS
         return context
     
+    def render_to_response(self, context, **response_kwargs):
+        if self.send_marc:
+            return qs_marc_records(self.request, qs=self.object_list)
+        else:
+            return super(FilterableListView,self).render_to_response(context, **response_kwargs)
+        
 recommended_user = User.objects.filter( username=settings.UNGLUEIT_RECOMMENDED_USERNAME)
 
 class WorkListView(FilterableListView):
@@ -881,6 +882,8 @@ class ByPubView(WorkListView):
             context = super(ByPubView, self).get_context_data(**kwargs)
             context['pubname'] = self.publisher_name
             context['publisher'] = self.publisher
+            context['facet'] = self.kwargs.get('facet','all')
+
             return context
 
 class ByPubListView(ByPubView):
@@ -913,7 +916,7 @@ class CCListView(FilterableListView):
 
     def get_context_data(self, **kwargs):
         context = super(CCListView, self).get_context_data(**kwargs)
-        facet = self.kwargs.get('facet','')
+        facet = self.kwargs.get('facet','all')
         qs=self.get_queryset()
         context['ungluers'] = userlists.work_list_users(qs,5)
         context['activetab'] = "#1"
@@ -3091,144 +3094,44 @@ def send_to_kindle(request, work_id, javascript='0'):
     return local_response(request, javascript,  context, 2)
     
     
-def marc_admin(request, userlist=None):
+def userlist_marc(request, userlist=None):
-    link_target = 'UNGLUE'
-    libpref = {'marc_link_target': settings.MARC_CHOICES[1]}
-    try:
-        libpref = request.user.libpref
-        link_target = libpref.marc_link_target
-    except:
-        if not request.user.is_anonymous():
-            libpref = models.Libpref(user=request.user)
-    unwanted = 'UNGLUE' if link_target == 'DIRECT' else 'DIRECT'
     if userlist:
-        records = []
         user = get_object_or_404(User,username=userlist)
-        for work in user.wishlist.works.all():
+        return qs_marc_records(request, qs=user.wishlist.works.all())
-            records.extend(models.MARCRecord.objects.filter(edition__work=work,link_target=link_target))
-            records.extend(models.MARCRecord.objects.filter(edition__work=work,link_target='B2U'))
     else:
-        records = models.MARCRecord.objects.exclude(link_target=unwanted)
+        return qs_marc_records(request, qs=request.user.wishlist.works.all())
-    return render(
-        request,
-        'marc.html',
-        {'records': records,  'libpref' : libpref , 'userlist' : userlist}
-    )
 
-class MARCUngluifyView(FormView):
+    return render( request,'marc.html',{'userlist' : [] })
-    template_name = 'marcungluify.html'
-    form_class = MARCUngluifyForm
-    success_url = reverse_lazy('MARCUngluify')
 
-    # allow a get param to specify the edition
+def work_marc(request, work_id):
-    def get_initial(self):
+    work = safe_get_work(work_id)
-        if self.request.method == 'GET':
+    return qs_marc_records(request, qs=[ work ])
-            edition = self.request.GET.get('edition',None)
-            if models.Edition.objects.filter(id=edition).count():
-                edition = models.Edition.objects.filter(id=edition)[0]
-                if edition.ebooks.count() or edition.ebook_files.count():
-                    return {'edition':edition.id}
-        return {}
 
-    def form_valid(self, form):
-        edition = form.cleaned_data['edition']
         
-        try:
+class LibModeView(FormView):
-            marc.makemarc(
-                marcfile=self.request.FILES['file'],
-                edition=edition
-            )
-            messages.success(
-                self.request,
-                "You have successfully added a MARC record. Hooray! Add another?"
-            )
-        except SAXParseException:
-            messages.error(
-                self.request,
-                "Sorry, couldn't parse that file."
-            )
-        return super(MARCUngluifyView,self).form_valid(form)
-        
-class MARCConfigView(FormView):
     template_name = 'marc_config.html'
-    form_class = MARCFormatForm
+    form_class = LibModeForm
     success_url = reverse_lazy('marc')
     
     def form_valid(self, form):
-        marc_link_target = form.cleaned_data['marc_link_target']
+        enable= form.data.has_key('enable')
+        if enable:
             try:
                 libpref = self.request.user.libpref
             except:
                 libpref = models.Libpref(user=self.request.user)
-        libpref.marc_link_target = marc_link_target
             libpref.save()
-        messages.success(
+            messages.success(self.request,"Tools are enabled.")
-            self.request,
+        else:
-            "Your preferences have been changed."
+            try:
-        )
+                self.request.user.libpref.delete()
+            except:
+                pass
+            messages.success(self.request,"Tools are disabled." )
         if reverse('marc_config', args=[]) in self.request.META['HTTP_REFERER']:
             return HttpResponseRedirect(reverse('marc_config', args=[]))
         else:
-            return super(MARCConfigView, self).form_valid(form)
+            return super(LibModeView, self).form_valid(form)
-            
+            
-def marc_concatenate(request):
-    if request.method == 'POST':
-        params=request.POST
-    elif request.method == 'GET':
-        params=request.GET
-    else:
-        return HttpResponseNotFound
-    format = params['format']
-    
-    # extract the user-selected records from the params QueryDict
-    selected_records = list(
-        k for k in params if params[k] == u'on'
-    )
-    if not selected_records:
-        messages.error(
-            request,
-            "Please select at least one record to download."
-        )
-        return HttpResponseRedirect(reverse('marc', args=[]))
-    
-    preamble = ('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
-                     '<collection '
-                     'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '
-                     'xmlns="http://www.loc.gov/MARC21/slim" '
-                     'xsi:schemaLocation="http://www.loc.gov/MARC21/slim '
-                     'http://www.loc.gov/standards/marcxml/schema/'
-                     'MARC21slim.xsd">')
-    outfile = HttpResponse('', content_type='application/marc')
-    outfile['Content-Disposition'] = 'attachment; filename='+ now().strftime('%Y%m%d%H%M%S') + '.' +  format
-
-    if format == 'xml':
-        outfile.write(preamble)
-    for record in selected_records:
-        if record.startswith('edition_'):
-            record_id = long(record[8:])
-            try:
-                edition = models.Edition.objects.get(pk=record_id)
-            except:
-                continue
-            record_id = marc.makestub(edition)
-        else:
-            record_id = long(record[7:])
-        if format == 'xml':
-            record_url = models.MARCRecord.objects.get(
-                            pk=record_id
-                         ).xml_record
-        elif format == 'mrc':
-            record_url = models.MARCRecord.objects.get(
-                            pk=record_id
-                         ).mrc_record
-        try:
-            record_file = default_storage.open(record_url).read()
-            outfile.write(record_file)
-        except IOError:
-            logger.warning('MARC record with id number %s has a problem with its URL' % record_id)
-    if format == 'xml':
-        outfile.write('</collection>')
-
-    return outfile
 
             

marc/forms.py

@@ -0,0 +1,22 @@
+from django import forms
+
+from selectable.forms import (
+    AutoCompleteSelectWidget,
+    AutoCompleteSelectField
+)
+
+from regluit.core.lookups import EditionLookup
+
+class MARCUploadForm(forms.Form):
+    edition = AutoCompleteSelectField(
+            EditionLookup,
+            label='Edition',
+            widget=AutoCompleteSelectWidget(EditionLookup),
+            required=True,
+            error_messages={'required': 'Please specify an edition.'},
+        )    
+    file = forms.FileField(label='Select a MARCXML file.')
+    source = forms.ChoiceField(label='This file is ...', choices=[
+       ( 'loc' , 'from Library of Congress (print)'),
+       ( 'raw' , 'prepared for Unglue.it'),
+    ])

marc/load.py

@@ -0,0 +1,291 @@
+"""
+This takes a MARCXML filename as an argument and converts it into
+MARC records for the unglued edition (in .xml and .mrc formats).
+Consider it a catalogolem: http://commons.wikimedia.org/wiki/File:Arcimboldo_Librarian_Stokholm.jpg
+Use the MARCXML file for the non-unglued edition from Library of Congress.
+"""
+
+import pymarc
+import logging
+from datetime import datetime
+from StringIO import StringIO
+
+from django.conf import settings
+from django.core.urlresolvers import reverse
+
+import regluit.core.cc as cc
+
+def stub(edition):
+    
+    record = pymarc.Record()
+
+    now = datetime.now()
+    
+    #mostly to identify this record as a stub
+    field001 = pymarc.Field(tag='001', data='stb'+now.strftime('%y%m%d%H%M%S'))
+    record.add_ordered_field(field001)
+    
+    add_stuff(record)
+    
+    # fun fun fun 008
+    new_field_value= now.strftime('%y%m%d')+'s'
+    if edition.publication_date and len(edition.publication_date)>3:
+        new_field_value += edition.publication_date[0:4]
+    else:
+        new_field_value += '||||'
+    new_field_value += '||||xx |||||o|||||||||||eng||'
+    field008 = pymarc.Field(tag='008', data=new_field_value)
+    record.add_ordered_field(field008)   
+        
+    # add IBSN for if available
+    if edition.isbn_13:
+        field020 = pymarc.Field(
+            tag='020',
+            indicators = [' ', ' '],
+            subfields = ['a', edition.isbn_13]
+        )
+        record.add_ordered_field(field020)
+    
+    # OCLC number
+    if edition.oclc:
+        field035 = pymarc.Field(
+            tag='035',
+            indicators = [' ', ' '],
+            subfields = ['a', '(OCoLC)' + edition.oclc]
+        )
+        record.add_ordered_field(field035)
+        
+    # author name
+    num_auths = len(edition.authnames())
+    if num_auths:
+        field100 = pymarc.Field(
+            tag='100',
+            indicators = ['1', ' '],
+            subfields = [
+                'a', edition.authnames()[0],
+            ]
+        )
+        record.add_ordered_field(field100)
+    if num_auths > 1:
+        for auth in edition.authnames()[1:]:
+            field = pymarc.Field(
+                tag='700',
+                indicators = ['1', ' '],
+                subfields = [
+                    'a', auth,
+                    'e', 'joint author.',
+                ]
+            )
+            record.add_ordered_field(field)
+
+    # add subfield to 245 indicating format
+    field245 = pymarc.Field(
+        tag='245',
+        indicators = ['1', '0'],
+        subfields = [
+            'a', edition.title,
+            'a', '[electronic resource]',
+        ]
+    )
+    record.add_ordered_field(field245)
+    
+    # publisher, date
+    if edition.publisher:
+        field260 = pymarc.Field(
+            tag='260',
+            indicators = [' ', ' '],
+            subfields = [
+                'b', edition.publisher,
+            ]
+        )
+        if edition.publication_date:
+            field260.add_subfield('c', unicode(edition.publication_date))
+        record.add_ordered_field(field260)
+        
+    if edition.description:
+        #add 520 field (description)
+        field520 =  pymarc.Field(
+            tag='520',
+            indicators = [' ', ' '],
+            subfields = [
+                'a', edition.description,
+            ]
+        )
+        record.add_ordered_field(field520)
+        
+    add_license(record, edition)
+    
+    return record
+
+#load a with minimal change
+def raw(marcfile,  edition):
+    record = pymarc.parse_xml_to_array(marcfile)[0]
+    for field in record:
+        if field.tag in ('001', '003', '005', '006', '007', '856') or int( field.tag ) > 900:
+            record.remove_field(field)
+    add_stuff(record)
+    return record
+    
+#load a record from library of Congress
+def from_lc(marcfile,  edition):
+    
+    # save lccn for later (if there is one) before deleting it
+    print_lccn = None
+    record = pymarc.parse_xml_to_array(marcfile)[0]
+    for lccn in record.get_fields('010'):
+        for validlccn in lccn.get_subfields('a'):
+            print_lccn = validlccn
+    for field in record:
+        if field.tag in ('001', '003', '005', '006', '007', '010', '040', '856') or int( field.tag ) > 900:
+            record.remove_field(field)
+
+    add_stuff(record)
+    
+    field008 = record.get_fields('008')[0]
+    record.remove_field(field008)
+    old_field_value = field008.value()
+    new_field_value = old_field_value[:23] + 'o' + old_field_value[24:]
+    field008 = pymarc.Field(tag='008', data=new_field_value)
+    record.add_ordered_field(field008)   
+        
+    # add IBSN for ebook where applicable; relegate print ISBN to $z
+    isbn = edition.isbn_13
+    try:
+        field020 = record.get_fields('020')[0]
+        print_isbn = field020.get_subfields('a')[0]
+        field020.delete_subfield('a')
+        if isbn:
+            field020.add_subfield('a', isbn)
+        field020.add_subfield('z', print_isbn)
+    except IndexError:
+        print_isbn = None
+
+    # change 050 and 082 indicators because LOC is no longer responsible for these
+    # no easy indicator change function, so we'll just reconstruct the fields
+    try:
+        field050 = record.get_fields('050')[0]
+        field050_new = field050
+        field050_new.indicators = [' ', '4']
+        record.remove_field(field050)
+        record.add_ordered_field(field050_new)
+    except:
+        pass # if no 050 field, don't need to change indicator
+    
+    try:
+        field082 = record.get_fields('082')[0]
+        field082_new = field082
+        field082_new.indicators = [' ', '4']
+        record.remove_field(field082)
+        record.add_ordered_field(field082_new)
+    except:
+        pass # if no 082 field, don't need to change indicator
+    
+    # add subfield to 245 indicating format
+    try:
+        field245 = record.get_fields('245')[0]
+        field245.add_subfield('a', '[electronic resource]')
+    except IndexError:
+        pass
+        
+    # modify 300 field (physical description)
+    try:
+        field300 = record.get_fields('300')[0]
+        subfield_a = field300.get_subfields('a')[0]
+        if (
+            subfield_a[-2:] == ' ;' or 
+            subfield_a[-2:] == ' :' or 
+            subfield_a[-2:] == ' +'
+        ):
+            subfield_a = subfield_a[:-2]
+        new300a = '1 online resource (' + subfield_a + ')'
+        if field300.get_subfields('b'):
+            new300a += ' :'
+        field300.delete_subfield('a')
+        field300.add_subfield('a', new300a)
+        field300.delete_subfield('c')
+    except:
+        pass
+    
+    add_license(record, edition)
+
+    # add 588 field (source of description) - credit where credit is due
+    if print_lccn:
+        field588 = pymarc.Field(
+            tag='588',
+            indicators = [' ', ' '],
+            subfields = [
+                'a', 'Description based on print version record from the Library of Congress.',
+            ]
+        )
+        record.add_ordered_field(field588)
+    
+    # add 776 field (related editions) - preserve pISBN, LCCN, OCLCnum
+    title = record.get_fields('245')[0].get_subfields('a')[0]
+    title = title.split('/')[0]
+    
+    subfields = ['i', 'Print version: ','t', title,]
+    
+    if print_isbn:
+        subfields.extend(['z', print_isbn])
+    elif isbn:
+        subfields.extend(['z', isbn])
+    if print_lccn:
+        subfields.extend(['w', '(DLC) ' + print_lccn, ])
+    if edition.oclc:
+        subfields.extend(['w', '(OCoLC) ' + edition.oclc,])
+
+    field776 = pymarc.Field(
+        tag='776',
+        indicators = ['0', '8'],
+        subfields = subfields
+    )
+    
+    record.add_ordered_field(field776)
+        
+    return record
+
+def add_license(record, edition):
+    if edition.license:
+        # add 536 field (funding information)
+        field536 = pymarc.Field(
+            tag='536',
+            indicators = [' ', ' '],
+            subfields = [
+                'a', edition.funding_info,
+            ]
+        )
+        record.add_ordered_field(field536)
+    
+        # add 540 field (terms governing use)
+        field540 = pymarc.Field(
+            tag='540',
+            indicators = [' ', ' '],
+            subfields = [
+                'a', dict(cc.CHOICES)[edition.license],
+                'u', dict(cc.GRANTS)[edition.license], 
+            ]
+        )
+        record.add_ordered_field(field540)
+
+def add_stuff(record):
+    # add field indicating record originator
+    field003 = pymarc.Field(tag='003', data='UnglueIt')
+    record.add_ordered_field(field003)
+    
+    # update timestamp of record
+    datestamp = datetime.now().strftime('%Y%m%d%H%M%S') + '.0'
+    field005 = pymarc.Field(tag='005', data=datestamp)
+    record.add_ordered_field(field005)
+
+    # change 006, 007, 008 because this is an online resource
+    field006 = pymarc.Field(
+        tag='006',
+        data='m     o  d        '
+    )
+    record.add_ordered_field(field006)
+    
+    field007 = pymarc.Field(
+        tag='007',
+        data='cr'
+    )
+    record.add_ordered_field(field007)

marc/management/commands/migrate_records.py

@@ -0,0 +1,31 @@
+from django.core.management.base import BaseCommand
+from django.core.files.storage import default_storage
+
+from regluit.marc.models import MARCRecord
+from regluit.core.models import MARCRecord as OldRecord
+
+class Command(BaseCommand):
+    help = "migrate records (id<stoprecord) from files"
+    args = "<stoprecord>"
+    
+    def handle(self, stoprecord=None, **options):
+        editions=[]
+        old_records=OldRecord.objects.all().order_by('-id')
+        if stoprecord:
+            old_records = old_records.filter(id__lt=int(stoprecord))
+        for old_record in old_records:
+            if old_record.edition.pk not in editions:
+                new_record, created = MARCRecord.objects.get_or_create(id=old_record.pk)
+                try:
+                    xml_file = default_storage.open(old_record.xml_record)
+                    xml_data = xml_file.read()
+                    new_record.guts = xml_data
+                    new_record.edition = old_record.edition
+                    editions.append(old_record.edition.pk)
+                    xml_file.close()
+                    new_record.save()
+                    print 'record %s updated' % new_record.id
+                except IOError:
+                    if created:
+                        new_record.delete()
+                    print 'failed opening %s' % old_record.xml_record

marc/migrations/0001_initial.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'MARCRecord'
+        db.create_table('marc_marcrecord', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('guts', self.gf('django.db.models.fields.TextField')()),
+            ('edition', self.gf('django.db.models.fields.related.ForeignKey')(related_name='MARCRecords', null=True, to=orm['core.Edition'])),
+        ))
+        db.send_create_signal('marc', ['MARCRecord'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'MARCRecord'
+        db.delete_table('marc_marcrecord')
+
+
+    models = {
+        'core.edition': {
+            'Meta': {'object_name': 'Edition'},
+            'cover_image': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'publication_date': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'publisher_name': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editions'", 'null': 'True', 'to': "orm['core.PublisherName']"}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '1000'}),
+            'unglued': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'work': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editions'", 'null': 'True', 'to': "orm['core.Work']"})
+        },
+        'core.publisher': {
+            'Meta': {'object_name': 'Publisher'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'logo_url': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'key_publisher'", 'to': "orm['core.PublisherName']"}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'})
+        },
+        'core.publishername': {
+            'Meta': {'object_name': 'PublisherName'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'publisher': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'alternate_names'", 'null': 'True', 'to': "orm['core.Publisher']"})
+        },
+        'core.work': {
+            'Meta': {'ordering': "['title']", 'object_name': 'Work'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}),
+            'earliest_publication': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True'}),
+            'featured': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '5', 'db_index': 'True'}),
+            'num_wishes': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}),
+            'openlibrary_lookup': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'selected_edition': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'selected_works'", 'null': 'True', 'to': "orm['core.Edition']"}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '1000'})
+        },
+        'marc.marcrecord': {
+            'Meta': {'object_name': 'MARCRecord'},
+            'edition': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'MARCRecords'", 'null': 'True', 'to': "orm['core.Edition']"}),
+            'guts': ('django.db.models.fields.TextField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        }
+    }
+
+    complete_apps = ['marc']

marc/migrations/0002_auto__add_field_marcrecord_user__add_field_marcrecord_created.py

@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'MARCRecord.user'
+        db.add_column('marc_marcrecord', 'user',
+                      self.gf('django.db.models.fields.related.ForeignKey')(related_name='MARCRecords', null=True, to=orm['auth.User']),
+                      keep_default=False)
+
+        # Adding field 'MARCRecord.created'
+        db.add_column('marc_marcrecord', 'created',
+                      self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, default=datetime.datetime(2014, 10, 27, 0, 0), blank=True),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'MARCRecord.user'
+        db.delete_column('marc_marcrecord', 'user_id')
+
+        # Deleting field 'MARCRecord.created'
+        db.delete_column('marc_marcrecord', 'created')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'core.edition': {
+            'Meta': {'object_name': 'Edition'},
+            'cover_image': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'publication_date': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'publisher_name': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editions'", 'null': 'True', 'to': "orm['core.PublisherName']"}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '1000'}),
+            'unglued': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'work': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editions'", 'null': 'True', 'to': "orm['core.Work']"})
+        },
+        'core.publisher': {
+            'Meta': {'object_name': 'Publisher'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'logo_url': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'key_publisher'", 'to': "orm['core.PublisherName']"}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'})
+        },
+        'core.publishername': {
+            'Meta': {'object_name': 'PublisherName'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'publisher': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'alternate_names'", 'null': 'True', 'to': "orm['core.Publisher']"})
+        },
+        'core.work': {
+            'Meta': {'ordering': "['title']", 'object_name': 'Work'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}),
+            'earliest_publication': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True'}),
+            'featured': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '5', 'db_index': 'True'}),
+            'num_wishes': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}),
+            'openlibrary_lookup': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'selected_edition': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'selected_works'", 'null': 'True', 'to': "orm['core.Edition']"}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '1000'})
+        },
+        'marc.marcrecord': {
+            'Meta': {'object_name': 'MARCRecord'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'edition': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'MARCRecords'", 'null': 'True', 'to': "orm['core.Edition']"}),
+            'guts': ('django.db.models.fields.TextField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'MARCRecords'", 'null': 'True', 'to': "orm['auth.User']"})
+        }
+    }
+
+    complete_apps = ['marc']

marc/models.py

@@ -0,0 +1,163 @@
+import pymarc
+import logging
+from datetime import datetime
+from StringIO import StringIO
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.db import models
+
+from . import load
+
+# weak coupling
+EDITION_MODEL = "core.Edition"
+
+class AbstractEdition:
+    # define the methods and attributes an edition should have
+    isbn_13 = ''
+    oclc = ''
+    license = None
+    funding_info = ''
+    description = ''
+    publisher = ''
+    title = ''
+    publication_date = ''
+    
+    # the edition should be able to report ebook downloads, with should have format and url attributes
+    def downloads(self):
+        return []
+
+    # the edition should be able to report an "ebook via" url
+    def download_via_url(self):
+        return []
+
+    # these should be last name first
+    def authnames(self):
+        return []
+        
+def _xml(record):
+    return pymarc.record_to_xml(record)
+
+def _mrc(record):
+    mrc_file = StringIO()
+    writer = pymarc.MARCWriter(mrc_file)
+    writer.write(record)
+    mrc_file.seek(0)
+    return mrc_file.read()
+
+class MARCRecord(models.Model):
+    # the record  goes here
+    guts = models.TextField()
+    
+    #storage for parsed guts
+    _the_record = None
+    
+    # note capitalization of related_name
+    edition = models.ForeignKey(EDITION_MODEL, related_name="MARCRecords", null=True)
+    
+    user =  models.ForeignKey(User, related_name="MARCRecords", null=True ) 
+    created =  models.DateTimeField(auto_now_add=True)  
+
+
+    def __init__(self, *args,  **kwargs):
+        super(MARCRecord, self).__init__( *args, **kwargs)
+        edition = kwargs.pop('edition', None)
+        guts = kwargs.pop('guts', None)
+        if edition and not guts:
+            #make a stub _the_record from the edition
+            self._the_record = load.stub(edition)
+
+    @property
+    def accession(self):
+        zeroes = 9 - len(str(self.id))
+        return 'ung' + zeroes*'0' + str(self.id)
+    
+    def save(self, *args, **kwargs):
+        if self.id == None and self._the_record:
+            # get the id first, add assession number
+            stash_guts = self.guts
+            self.guts = ''
+            super(MARCRecord, self).save(*args, **kwargs) 
+            self.guts = stash_guts
+            field001 = self._the_record.get_fields('001')
+            if field001:
+                self._the_record.remove_field(field)
+            field001 = pymarc.Field(tag='001', data=self.accession)
+            self._the_record.add_ordered_field(field001)
+        super(MARCRecord, self).save(*args, **kwargs)
+    
+    def load_from_file(self, source='raw'):
+        #parse guts
+        if isinstance(self.guts, str) or isinstance(self.guts, unicode):
+            marcfile = StringIO(self.guts)
+        else:
+            marcfile = self.guts
+        if source == 'loc':
+            self._the_record = load.from_lc(marcfile, self.edition)
+        else:
+            self._the_record = load.raw(marcfile, self.edition)
+        self.guts = pymarc.record_to_xml(self._the_record)
+        self.save()
+
+        
+    # the record without 856 
+    def _record(self):
+        if self._the_record:
+            the_record = self._the_record
+        else:
+            the_record = pymarc.parse_xml_to_array(StringIO(self.guts))[0]
+        for field in the_record.get_fields('856'):
+            the_record.remove_field(field)
+        self._the_record = the_record
+        return the_record
+        
+    def direct_record(self):
+        the_record = self._record()
+        for book in self.edition.downloads():
+            field856 = pymarc.Field(
+                tag='856',
+                indicators = ['4', '0'],
+                subfields = [
+                    '3', book.format + ' version',
+                    'q', settings.CONTENT_TYPES[book.format],
+                    'u', book.url,
+                ]
+            )
+            the_record.add_ordered_field(field856)
+        return the_record
+
+    def direct_record_xml(self):
+        return _xml(self.direct_record())
+
+    def direct_record_mrc(self):
+        return _mrc(self.direct_record())
+
+    def via_record(self):
+        the_record = self._record()
+        field856_via = pymarc.Field(
+            tag='856',
+            indicators = ['4', '0'],
+            subfields = [
+                'u', self.edition.download_via_url(),
+            ]
+        )
+        the_record.add_ordered_field(field856_via)
+        return the_record
+
+    def via_record_xml(self):
+        return _xml(self.via_record())
+
+    def via_record_mrc(self):
+        return _mrc(self.via_record())
+    
+    def record(self, link_target='via', format='xml'):
+        if format == 'xml':
+            if link_target == 'via':
+                return self.via_record_xml()
+            elif link_target == 'direct':
+                return self.direct_record_xml()
+        elif  format == 'mrc':
+            if link_target == 'via':
+                return self.via_record_mrc()
+            elif link_target == 'direct':
+                return self.direct_record_mrc()

marc/tests.py

@@ -0,0 +1,135 @@
+"""
+django imports
+"""
+from django.test import TestCase
+from django.test.client import Client
+from django.db.models import get_model
+
+"""
+regluit imports
+"""
+from . import models, load
+
+a_marc_record = '''<?xml version="1.0" encoding="UTF-8"?><record xmlns="http://www.loc.gov/MARC21/slim" xmlns:cinclude="http://apache.org/cocoon/include/1.0" xmlns:zs="http://www.loc.gov/zing/srw/">
+  <leader>01021cam a2200301 a 4500</leader>
+  <controlfield tag="001">3057297</controlfield>
+  <controlfield tag="005">19970108080513.5</controlfield>
+  <controlfield tag="008">960131r19761970ke b     b    000 0 eng  </controlfield>
+  <datafield tag="035" ind1=" " ind2=" ">
+    <subfield code="9">(DLC)   96109467</subfield>
+  </datafield>
+  <datafield tag="906" ind1=" " ind2=" ">
+    <subfield code="a">7</subfield>
+    <subfield code="b">cbc</subfield>
+    <subfield code="c">orignew</subfield>
+    <subfield code="d">u</subfield>
+    <subfield code="e">ncip</subfield>
+    <subfield code="f">19</subfield>
+    <subfield code="g">y-gencatlg</subfield>
+  </datafield>
+  <datafield tag="955" ind1=" " ind2=" ">
+    <subfield code="a">082 done aa11</subfield>
+  </datafield>
+  <datafield tag="010" ind1=" " ind2=" ">
+    <subfield code="a">   96109467 </subfield>
+  </datafield>
+  <datafield tag="020" ind1=" " ind2=" ">
+    <subfield code="a">0195724135</subfield>
+  </datafield>
+  <datafield tag="040" ind1=" " ind2=" ">
+    <subfield code="a">DLC</subfield>
+    <subfield code="c">DLC</subfield>
+    <subfield code="d">DLC</subfield>
+  </datafield>
+  <datafield tag="043" ind1=" " ind2=" ">
+    <subfield code="a">f------</subfield>
+  </datafield>
+  <datafield tag="050" ind1="0" ind2="0">
+    <subfield code="a">PL8010</subfield>
+    <subfield code="b">.F5 1976</subfield>
+  </datafield>
+  <datafield tag="082" ind1="0" ind2="0">
+    <subfield code="a">398.2/096</subfield>
+    <subfield code="2">20</subfield>
+  </datafield>
+  <datafield tag="100" ind1="1" ind2=" ">
+    <subfield code="a">Finnegan, Ruth H.</subfield>
+  </datafield>
+  <datafield tag="245" ind1="1" ind2="0">
+    <subfield code="a">Oral literature in Africa /</subfield>
+    <subfield code="c">Ruth Finnegan.</subfield>
+  </datafield>
+  <datafield tag="260" ind1=" " ind2=" ">
+    <subfield code="a">Nairobi :</subfield>
+    <subfield code="b">Oxford University Press,</subfield>
+    <subfield code="c">1976 (1994 printing).</subfield>
+  </datafield>
+  <datafield tag="300" ind1=" " ind2=" ">
+    <subfield code="a">xviii, 558 p. :</subfield>
+    <subfield code="b">map ;</subfield>
+    <subfield code="c">21 cm.</subfield>
+  </datafield>
+  <datafield tag="440" ind1=" " ind2="0">
+    <subfield code="a">Oxford library of African literature</subfield>
+  </datafield>
+  <datafield tag="500" ind1=" " ind2=" ">
+    <subfield code="a">Originally published: Oxford : Clarendon Press, 1970.</subfield>
+  </datafield>
+  <datafield tag="504" ind1=" " ind2=" ">
+    <subfield code="a">Includes index and bibliographical references (p. [522]-536).</subfield>
+  </datafield>
+  <datafield tag="650" ind1=" " ind2="0">
+    <subfield code="a">Folk literature, African</subfield>
+    <subfield code="x">History and criticism.</subfield>
+  </datafield>
+  <datafield tag="650" ind1=" " ind2="0">
+    <subfield code="a">Oral tradition</subfield>
+    <subfield code="z">Africa.</subfield>
+  </datafield>
+  <datafield tag="922" ind1=" " ind2=" ">
+    <subfield code="a">ap</subfield>
+  </datafield>
+  <datafield tag="991" ind1=" " ind2=" ">
+    <subfield code="b">c-GenColl</subfield>
+    <subfield code="h">PL8010</subfield>
+    <subfield code="i">.F5 1976</subfield>
+    <subfield code="t">Copy 1</subfield>
+    <subfield code="w">BOOKS</subfield>
+  </datafield>
+</record>'''
+
+class MarcTests(TestCase):
+    work_id=None
+    
+    def test_records(self):
+        w = get_model('core','Work').objects.create(title="Work 1")
+        e = get_model('core','Edition').objects.create(title=w.title,work=w) 
+        eb = get_model('core','Ebook').objects.create(url = "http://example.org",edition = e,format = 'epub', rights='CC BY')
+
+        mr = models.MARCRecord.objects.create(guts=a_marc_record, edition=e )
+
+        mr.direct_record_xml()
+        mr.direct_record_mrc()
+        mr.via_record_xml()
+        mr.via_record_mrc()
+        
+        load.stub(e)
+        
+        w.description='test'
+        e.set_publisher('test pub')
+        e.publication_date = '2000'
+        e.add_author('joe writer')
+        id = get_model('core','Identifier').objects.create(work=w,edition=e, type='isbn', value='0030839939') 
+        id = get_model('core','Identifier').objects.create(work=w,edition=e, type='oclc', value='0074009772') 
+        
+        load.stub(e)
+        
+        mr2 = models.MARCRecord.objects.create(guts=a_marc_record, edition=e )
+        mr2.load_from_file('loc')
+        mr3 = models.MARCRecord.objects.create(guts=a_marc_record, edition=e )
+        mr3.load_from_file('raw')
+
+        mr2.direct_record_xml()
+        mr2.direct_record_mrc()
+        mr2.via_record_xml()
+        mr2.via_record_mrc()

marc/urls.py

@@ -0,0 +1,11 @@
+from django.conf.urls.defaults import *
+from django.contrib.auth.decorators import login_required
+from . import views 
+
+
+urlpatterns = patterns(
+    "regluit.marc.views",
+    url(r"^marc/concatenate/$", "marc_records", name="marc_concatenate"),
+    url(r"^marc/all/$", "all_marc_records", name="marc_all"),
+    url(r"^marc/upload/$", login_required(views.MARCUpload.as_view()), name="upload_marc"),
+)

marc/views.py

@@ -0,0 +1,132 @@
+from datetime import datetime
+from xml.sax import SAXParseException
+
+from django.contrib import messages
+from django.core.urlresolvers import reverse, reverse_lazy
+from django.db.models import get_model
+from django.http import HttpResponseRedirect, HttpResponse, HttpResponseNotFound
+from django.views.generic.edit import FormView
+
+from . import models
+from . import forms
+
+PREAMBLE = ('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+                 '<collection '
+                 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '
+                 'xmlns="http://www.loc.gov/MARC21/slim" '
+                 'xsi:schemaLocation="http://www.loc.gov/MARC21/slim '
+                 'http://www.loc.gov/standards/marcxml/schema/'
+                 'MARC21slim.xsd">')
+
+Edition = get_model(*models.EDITION_MODEL.split('.'))
+
+
+def marc_records(request, selected_records=None):
+    # TODO convert to streaming when we move to Django 1.5+
+    if request.method == 'POST':
+        params=request.POST
+    elif request.method == 'GET':
+        params=request.GET
+    else:
+        return HttpResponseNotFound
+    format = params.get('format', 'xml')
+    link_target = params.get('link_target', 'via')
+    
+    if selected_records==None:
+        # extract the user-selected records from the params QueryDict
+        selected_records = list(
+            k for k in params if params[k] == u'on'
+        )
+    if not selected_records:
+        messages.error(
+            request,
+            "There were no records to download."
+        )
+        return HttpResponseRedirect(reverse('marc', args=[]))
+    
+    outfile = HttpResponse('', content_type='application/marc')
+    outfile['Content-Disposition'] = 'attachment; filename='+ datetime.now().strftime('%Y%m%d%H%M%S') + '.' +  format
+
+    if format == 'xml':
+        outfile.write(PREAMBLE)
+
+    for record_name in selected_records:
+        if isinstance(record_name, models.MARCRecord):
+            record = record_name
+        elif isinstance(record_name, Edition):
+            record = models.MARCRecord(edition=record_name)
+        elif hasattr(record_name, 'edition'):
+            record = models.MARCRecord(edition=record_name.edition)
+        elif record_name.startswith('edition_'):
+            record_id = long(record_name[8:])
+            try:
+                edition = Edition.objects.get(pk=record_id)
+            except Edition.DoesNotExist:
+                continue
+            record = models.MARCRecord(edition=edition)
+        elif record_name.startswith('record_'):
+            record_id = long(record_name[7:])
+            try:
+                record = models.MARCRecord.objects.get(id=record_id )
+            except models.MARCRecord.DoesNotExist:
+                continue
+        
+        outfile.write(record.record(format=format, link_target=link_target))
+
+    if format == 'xml':
+        outfile.write('</collection>')
+
+    return outfile
+
+def all_marc_records(request):
+    selected_records = list(
+        'record_'+ str(record.pk) for record in models.MARCRecord.objects.all()
+    )
+    return marc_records(request, selected_records=selected_records)
+
+def qs_marc_records(request, qs):
+    # translate a queryset into a record list.
+    selected_records = []
+    for obj in qs:
+        # each object must have a marc_records() attribute which returns a record list. 
+        # records can be strings: 'record_xxx', 'edition_xxx', MARCRecord objs or objs with edition attributes
+        selected_records.extend(obj.marc_records())
+    return marc_records(request, selected_records=selected_records)
+
+class MARCUpload(FormView):
+    template_name = 'marc/upload.html'
+    form_class = forms.MARCUploadForm
+    success_url = reverse_lazy('upload_marc')
+    
+    # allow a get param to specify the edition
+    def get_initial(self):
+        if self.request.method == 'GET':
+            edition = self.request.GET.get('edition',None)
+            if Edition.objects.filter(id=edition).count():
+                edition = Edition.objects.filter(id=edition)[0]
+                if edition.ebooks.count() or edition.ebook_files.count():
+                    return {'edition':edition.id}
+        return {}
+
+    def form_valid(self, form):
+        edition = form.cleaned_data['edition']
+        source = form.cleaned_data['source']
+        try:
+            marcfile=self.request.FILES['file']
+            new_record = models.MARCRecord(
+                guts=marcfile,
+                edition=edition,
+                user= self.request.user
+            )
+            new_record.load_from_file(source)
+            messages.success(
+                self.request,
+                "You have successfully added a MARC record. Hooray! Add another?"
+            )
+        except SAXParseException:
+            messages.error(
+                self.request,
+                "Sorry, couldn't parse that file."
+            )
+        return super(MARCUpload,self).form_valid(form)
+        

settings/common.py

@@ -132,6 +132,7 @@ INSTALLED_APPS = (
     'regluit.frontend',
     'regluit.api',
     'regluit.core',
+    'regluit.marc',
     'regluit.payment',
     'regluit.utils',
     'registration',

static/less/book_list.less

@@ -309,7 +309,7 @@ ul.navigation li a:hover, ul.navigation li.active a {
 }
 
 div#content-block-content {
-  padding-bottom: 100px;
+  padding-bottom: 10px;
 }
 
 .listview.panelback, .listview.panelback div {
@@ -328,3 +328,19 @@ div#content-block-content {
 .nobold {
 	font-weight: normal;
 }
+
+div#libtools {    
+    margin-left: 15px;
+    margin-bottom: 1em;
+    border:1px solid @blue-grey;
+    border-radius: 10px;
+    padding: 10px;
+    
+    p {
+        margin-top: 0px;
+    }
+    div {
+        margin-top: 0px;
+        margin-left: 2em ;
+    }
+}

static/less/campaign2.less

@@ -420,3 +420,24 @@ ul.support li {
 	font-weight: bold;
 	margin: 20px auto;
 }
+
+div#libtools {    
+	border:1px solid @blue-grey;
+	.one-border-radius(10px);
+	padding:10px;
+    margin-left: 0px;
+    margin-top: 1em;
+    padding: 10px;
+    p {
+        margin-top: 0px;
+        margin-bottom: 0px;
+    }
+    span {
+        margin-top: 0px;
+        margin-left: 0.5em ;
+        display: inline-block;
+    }
+    input[type="submit"]{
+        margin-left: 4em;
+    }
+}

urls.py

@@ -17,6 +17,7 @@ urlpatterns = patterns('',
     url(r'', include('regluit.frontend.urls')),
     url(r'', include('regluit.payment.urls')),
     url(r'', include('regluit.libraryauth.urls')),
+    url(r'', include('regluit.marc.urls')),
     url(r'^selectable/', include('selectable.urls')),
     url(r'^admin/', include(admin_site.urls)), 
     url(r'^comments/', include('django.contrib.comments.urls')),