EbookFoundation
/
regluit
mirror of https://github.com/EbookFoundation/regluit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #404 from Gluejar/marc_package 

Marc package
pull/1/head
Raymond Yee 2014-10-31 12:55:46 -07:00
parent 7d53b6a48a fafd37a973
commit a666427497
41 changed files with 1344 additions and 643 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -10,5 +10,4 @@ build
deploy/last-update
logs/*
celerybeat.pid
marc/*
.gitignore~

14
admin.py
View File

@ -35,6 +35,7 @@ regluit imports
"""
from regluit import payment
from regluit.core import models
from regluit.marc.models import MARCRecord
from regluit.core.lookups import (
PublisherNameLookup,
WorkLookup,
@ -207,12 +208,17 @@ class MARCRecordAdminForm(forms.ModelForm):
widget=AutoCompleteSelectWidget(EditionLookup),
required=True,
)
user = AutoCompleteSelectField(
OwnerLookup,
widget=AutoCompleteSelectWidget(OwnerLookup),
required=True,
)
class Meta(object):
model = models.MARCRecord
model = MARCRecord
class MARCRecordAdmin(ModelAdmin):
list_display = ('edition',)
list_display = ('edition', 'user')
date_hierarchy = 'created'
form = MARCRecordAdminForm
admin_site = RegluitAdmin("Admin")
@ -238,7 +244,7 @@ admin_site.register(models.Wishlist, WishlistAdmin)
admin_site.register(models.UserProfile, UserProfileAdmin)
admin_site.register(models.CeleryTask, CeleryTaskAdmin)
admin_site.register(models.Press, PressAdmin)
admin_site.register(models.MARCRecord, MARCRecordAdmin)
admin_site.register(MARCRecord, MARCRecordAdmin)
# payments

3
core/bookloader.py
View File

@ -202,8 +202,7 @@ def update_edition(edition):
models.Identifier.get_or_add(type='goog',value=googlebooks_id,edition=edition,work=edition.work)
for a in d.get('authors', []):
a, created = models.Author.objects.get_or_create(name=a)
a.editions.add(edition)
edition.add_author(a)
add_ebooks(item, edition)

367
core/marc.py
View File

@ -1,367 +0,0 @@
"""
This takes a MARCXML filename as an argument and converts it into
MARC records for the unglued edition (in .xml and .mrc formats).
Consider it a catalogolem: http://commons.wikimedia.org/wiki/File:Arcimboldo_Librarian_Stokholm.jpg
Use the MARCXML file for the non-unglued edition from Library of Congress.
"""
import pymarc
import logging
from copy import deepcopy
from datetime import datetime
from StringIO import StringIO
from django.conf import settings
from django.core.files.storage import default_storage
from django.core.urlresolvers import reverse
import regluit.core.cc as cc
from regluit.core import models
def makestub(edition):
return makemarc(None, edition)
def makemarc(marcfile, edition):
logger = logging.getLogger(__name__)
try:
license = edition.ebooks.all()[0].rights
ebf = None
except IndexError:
license = None
try:
ebf = edition.ebook_files.all()[0]
except IndexError:
# no record if no ebooks
return None
logger.info("Making MARC records for edition %s " % edition)
# save lccn for later (if there is one) before deleting it
print_lccn = None
if marcfile:
record = pymarc.parse_xml_to_array(marcfile)[0]
for lccn in record.get_fields('010'):
for validlccn in lccn.get_subfields('a'):
print_lccn = validlccn
fields_to_delete = []
fields_to_delete += record.get_fields('001')
fields_to_delete += record.get_fields('003')
fields_to_delete += record.get_fields('005')
fields_to_delete += record.get_fields('006')
fields_to_delete += record.get_fields('007')
fields_to_delete += record.get_fields('010')
fields_to_delete += record.get_fields('040')
for field in fields_to_delete:
record.remove_field(field)
else:
record = pymarc.Record()
# create accession number and write 001 field
# (control field syntax is special)
if ebf:
(marc_record, created) = models.MARCRecord.objects.get_or_create(edition=edition,link_target='B2U')
else:
(marc_record, created) = models.MARCRecord.objects.get_or_create(edition=edition,link_target='UNGLUE')
field001 = pymarc.Field(tag='001', data=marc_record.accession)
record.add_ordered_field(field001)
# add field indicating record originator
field003 = pymarc.Field(tag='003', data='UnglueIt')
record.add_ordered_field(field003)
# update timestamp of record
now = datetime.now()
datestamp = now.strftime('%Y%m%d%H%M%S') + '.0'
field005 = pymarc.Field(tag='005', data=datestamp)
record.add_ordered_field(field005)
# change 006, 007, 008 because this is an online resource
field006 = pymarc.Field(
tag='006',
data='m o d '
)
record.add_ordered_field(field006)
field007 = pymarc.Field(
tag='007',
data='cr'
)
record.add_ordered_field(field007)
try:
field008 = record.get_fields('008')[0]
record.remove_field(field008)
old_field_value = field008.value()
new_field_value = old_field_value[:23] + 'o' + old_field_value[24:]
except IndexError:
# fun fun fun
new_field_value= now.strftime('%y%m%d')+'s'
if edition.publication_date and len(edition.publication_date)>3:
new_field_value += edition.publication_date[0:4]
else:
new_field_value += '||||'
new_field_value += '||||xx |||||o|||||||||||eng||'
field008 = pymarc.Field(tag='008', data=new_field_value)
record.add_ordered_field(field008)
# add IBSN for ebook where applicable; relegate print ISBN to $z
isbn = ''
try:
isbn = edition.identifiers.filter(type='isbn')[0].value
except IndexError:
pass
try:
field020 = record.get_fields('020')[0]
print_isbn = field020.get_subfields('a')[0]
field020.delete_subfield('a')
if isbn:
field020.add_subfield('a', isbn)
field020.add_subfield('z', print_isbn)
except IndexError:
print_isbn = None
# change 050 and 082 indicators because LOC is no longer responsible for these
# no easy indicator change function, so we'll just reconstruct the fields
try:
field050 = record.get_fields('050')[0]
field050_new = field050
field050_new.indicators = [' ', '4']
record.remove_field(field050)
record.add_ordered_field(field050_new)
except:
pass # if no 050 field, don't need to change indicator
try:
field082 = record.get_fields('082')[0]
field082_new = field082
field082_new.indicators = [' ', '4']
record.remove_field(field082)
record.add_ordered_field(field082_new)
except:
pass # if no 082 field, don't need to change indicator
# author name
try:
field100 = record.get_fields('100')[0]
except IndexError:
num_auths = edition.authors.count()
if num_auths:
field100 = pymarc.Field(
tag='100',
indicators = ['1', ' '],
subfields = [
'a', edition.authors.all()[0].last_name_first,
]
)
record.add_ordered_field(field100)
if num_auths > 1:
for auth in edition.authors.all()[1:]:
field = pymarc.Field(
tag='700',
indicators = ['1', ' '],
subfields = [
'a', auth.last_name_first,
'e', 'joint author.',
]
)
record.add_ordered_field(field)
# add subfield to 245 indicating format
try:
field245 = record.get_fields('245')[0]
except IndexError:
field245 = pymarc.Field(
tag='245',
indicators = ['1', '0'],
subfields = [
'a', edition.title,
]
)
record.add_ordered_field(field245)
field245.add_subfield('a', '[electronic resource]')
# publisher, date
try:
field260 = record.get_fields('260')[0]
except IndexError:
field260 = pymarc.Field(
tag='260',
indicators = [' ', ' '],
subfields = [
'b', edition.publisher_name.name if edition.publisher_name is not None else "",
'c', unicode(edition.publication_date),
]
)
record.add_ordered_field(field260)
# modify 300 field (physical description)
try:
field300 = record.get_fields('300')[0]
subfield_a = field300.get_subfields('a')[0]
if (
subfield_a[-2:] == ' ;' or
subfield_a[-2:] == ' :' or
subfield_a[-2:] == ' +'
):
subfield_a = subfield_a[:-2]
new300a = '1 online resource (' + subfield_a + ')'
if field300.get_subfields('b'):
new300a += ' :'
field300.delete_subfield('a')
field300.add_subfield('a', new300a)
field300.delete_subfield('c')
except:
pass
if license:
# add 536 field (funding information)
if edition.unglued:
funding_info = 'The book is available as a free download thanks to the generous support of interested readers and organizations, who made donations using the crowd-funding website Unglue.it.'
else:
if edition.ebooks.all()[0].rights in cc.LICENSE_LIST:
funding_info = 'The book is available as a free download thanks to a Creative Commons license.'
else:
funding_info = 'The book is available as a free download because it is in the Public Domain.'
field536 = pymarc.Field(
tag='536',
indicators = [' ', ' '],
subfields = [
'a', funding_info,
]
)
record.add_ordered_field(field536)
# add 540 field (terms governing use)
field540 = pymarc.Field(
tag='540',
indicators = [' ', ' '],
subfields = [
'a', dict(cc.CHOICES)[license],
'u', dict(cc.GRANTS)[license],
]
)
record.add_ordered_field(field540)
# add 588 field (source of description) - credit where credit is due
if print_lccn:
field588 = pymarc.Field(
tag='588',
indicators = [' ', ' '],
subfields = [
'a', 'Description based on print version record from the Library of Congress.',
]
)
record.add_ordered_field(field588)
# add 776 field (related editions) - preserve pISBN, LCCN, OCLCnum
title = record.get_fields('245')[0].get_subfields('a')[0]
title = title.split('/')[0]
try:
oclcnum = edition.identifiers.filter(type='oclc')[0].value
except IndexError:
oclcnum = None
subfields = ['i', 'Print version: ','t', title,]
if print_isbn:
subfields.extend(['z', print_isbn])
elif isbn:
subfields.extend(['z', isbn])
if print_lccn:
subfields.extend(['w', '(DLC) ' + print_lccn, ])
if oclcnum:
subfields.extend(['w', '(OCoLC) ' + oclcnum,])
field776 = pymarc.Field(
tag='776',
indicators = ['0', '8'],
subfields = subfields
)
record.add_ordered_field(field776)
"""
add 776 fields
indicators: 0 8
'$i Print version: '
$t Title. <--note space
$d is optional
$z pISBN goes here
harvest from 020 (was moved from $a to $z)
$w (DLC) LCCN_goes_here
harvest from 010 field before deletion
$w (OCoLC) OCLCnum_goes_here
harvest from identifiers db
"""
# strip any 9XX fields (they're for local use)
for i in range(900, 1000):
fields = record.get_fields(str(i))
for field in fields:
record.remove_field(field)
# add 856 fields with links for each available file
# doing this out of order as it's the only thing that differs
# between direct-link and via-unglue.it versions
if not ebf:
# need deepcopy() because omg referential transparency!
record_direct = deepcopy(record) # 2 records for unglued stuff
for format_tuple in settings.FORMATS:
format = format_tuple[0]
ebooks = edition.ebooks.filter(format=format)
if ebooks:
for book in ebooks:
field856 = pymarc.Field(
tag='856',
indicators = ['4', '0'],
subfields = [
'3', format + ' version',
'q', settings.CONTENT_TYPES[format],
'u', book.url,
]
)
record_direct.add_ordered_field(field856)
unglued_url = settings.BASE_URL_SECURE + reverse('download', args=[edition.work.id])
field856_via = pymarc.Field(
tag='856',
indicators = ['4', '0'],
subfields = [
'u', unglued_url,
]
)
record.add_ordered_field(field856_via)
if not ebf:
# this via_unglueit record needs its own accession number
field001 = record_direct.get_fields('001')[0]
record_direct.remove_field(field001)
(marc_record_direct, created) = models.MARCRecord.objects.get_or_create(edition=edition,link_target='DIRECT')
field001 = pymarc.Field(tag='001', data=marc_record_direct.accession)
record_direct.add_ordered_field(field001)
# write the unglued MARCxml records
xmlrecord = pymarc.record_to_xml(record_direct)
xml_file = default_storage.open(marc_record_direct.xml_record, 'w')
xml_file.write(xmlrecord)
xml_file.close()
# write the unglued .mrc records, then save to s3
mrc_file = default_storage.open(marc_record_direct.mrc_record, 'w')
writer = pymarc.MARCWriter(mrc_file)
writer.write(record_direct)
mrc_file.close()
xmlrecord = pymarc.record_to_xml(record)
xml_file = default_storage.open(marc_record.xml_record, 'w')
xml_file.write(xmlrecord)
xml_file.close()
mrc_file = default_storage.open(marc_record.mrc_record, 'w')
writer = pymarc.MARCWriter(mrc_file)
writer.write(record)
mrc_file.close()
return marc_record.pk

62
core/models.py
View File

@ -37,7 +37,7 @@ import regluit.core.isbn
import regluit.core.cc as cc
from regluit.core.epub import personalize, ungluify, test_epub, ask_epub
from regluit.core.pdf import ask_pdf, pdf_append
from regluit.marc.models import MARCRecord as NewMARC
from regluit.core.signals import (
successful_campaign,
unsuccessful_campaign,
@ -1008,6 +1008,10 @@ class Campaign(models.Model):
def user_to_pay(self):
return self.rh.owner
### for compatibility with MARC output
def marc_records(self):
return self.work.marc_records()
class Identifier(models.Model):
# olib, ltwk, goog, gdrd, thng, isbn, oclc, olwk, olib, gute, glue
type = models.CharField(max_length=4, null=False)
@ -1518,6 +1522,20 @@ class Work(models.Model):
# assume it's several users
return self.user_license(self.acqs.filter(user__in=user))
### for compatibility with MARC output
def marc_records(self):
record_list = []
record_list.extend(NewMARC.objects.filter(edition__work=self))
for obj in record_list:
break
else:
for ebook in self.ebooks():
record_list.append(ebook.edition)
break
return record_list
class Author(models.Model):
created = models.DateTimeField(auto_now_add=True)
name = models.CharField(max_length=500)
@ -1640,6 +1658,14 @@ class Edition(models.Model):
except Identifier.DoesNotExist:
return None
def add_author(self, author_name):
if author_name:
try:
author= Author.objects.get(name=author_name)
except Author.DoesNotExist:
author= Author.objects.create(name=author_name)
author.editions.add(self)
def set_publisher(self,publisher_name):
if publisher_name and publisher_name != '':
try:
@ -1652,6 +1678,40 @@ class Edition(models.Model):
self.publisher_name = pub_name
self.save()
#### following methods for compatibility with marc outputter
def downloads(self):
return self.ebooks.all()
def download_via_url(self):
return settings.BASE_URL_SECURE + reverse('download', args=[self.work.id])
def authnames(self):
return [auth.last_name_first for auth in self.authors.all()]
@property
def license(self):
try:
return self.ebooks.all()[0].rights
except:
return None
@property
def funding_info(self):
if self.ebooks.all().count()==0:
return ''
if self.unglued:
return 'The book is available as a free download thanks to the generous support of interested readers and organizations, who made donations using the crowd-funding website Unglue.it.'
else:
if self.ebooks.all()[0].rights in cc.LICENSE_LIST:
return 'The book is available as a free download thanks to a Creative Commons license.'
else:
return 'The book is available as a free download because it is in the Public Domain.'
@property
def description(self):
return self.work.description
class Publisher(models.Model):
created = models.DateTimeField(auto_now_add=True)
name = models.ForeignKey('PublisherName', related_name='key_publisher')

13
frontend/forms.py
View File

@ -770,17 +770,8 @@ class PressForm(forms.ModelForm):
class KindleEmailForm(forms.Form):
kindle_email = forms.EmailField()
class MARCUngluifyForm(forms.Form):
edition = AutoCompleteSelectField(
EditionLookup,
label='Edition',
widget=AutoCompleteSelectWidget(EditionLookup),
required=True,
error_messages={'required': 'Please specify an edition.'},
)
file = forms.FileField(label='Download a MARCXML file from Library of Congress; then upload it here.')
class MARCFormatForm(forms.ModelForm):
class LibModeForm(forms.ModelForm):
class Meta:
model = Libpref
fields = ('marc_link_target',)
fields = ()

7
frontend/templates/bypub_list.html
View File

@ -20,3 +20,10 @@
{% endblock %}
{% block add_more %}{% endblock %}
{% block marcform %}
<form method="GET" action="{% url bypubname_list_marc facet pubname.id %}">
{% include 'marc_form.html' %}
<input type="submit" name="submit" value="download MARC" id="submit">
</form>
{% endblock %}

11
frontend/templates/campaign_list.html
View File

@ -102,6 +102,17 @@ location.hash = "#2";
{% endifequal %}
</div>
</div>
{% if request.user.libpref %}
<div id="libtools">
<p>for libraries...</p>
{% block marcform %}
<form method="GET" action="{% url campaign_list_marc facet %}">
{% include 'marc_form.html' %}
<input type="submit" name="submit" value="download MARC" id="submit">
</form>
{% endblock %}
</div>
{% endif %}
</div>
</div>
</div>

9
frontend/templates/cc_list.html
View File

@ -94,6 +94,15 @@
</div>
</div>
</div>
{% if request.user.libpref %}
<div id="libtools">
<p>for libraries...</p>
<form method="GET" action="{% url cc_list_marc facet %}">
{% include 'marc_form.html' %}
<input type="submit" name="submit" value="download MARC" id="submit">
</form>
</div>
{% endif %}
</div>
</div>
</div>

13
frontend/templates/edition_display.html
View File

@ -43,17 +43,8 @@
See <a href="https://encrypted.google.com/books?id={{ edition.googlebooks_id }}">this edition on Google Books</a><br />
{% endif %}
{% endif %}
{% if edition.MARCrecords.all %}
{% for record in edition.MARCrecords.all %}
Download {{record.link_target}} MARC record for this edition: (<a href="{% url marc_concatenate %}?record_{{ record.id }}=on&amp;format=xml">XML</a>) (<a href="{% url marc_concatenate %}?record_{{ record.id }}=on&amp;format=mrc">mrc</a>)<br />
{% endfor %}
{% else %}
{% if edition.ebooks.all %}
Download {{record.link_target}} MARC record for this edition: (<a href="{% url marc_concatenate %}?edition_{{ edition.id }}=on&amp;format=xml">XML</a>) (<a href="{% url marc_concatenate %}?edition_{{ edition.id }}=on&amp;format=mrc">mrc</a>)<br />
{% endif %}
{% endif %}
{% if user.is_staff %}{% if edition.ebooks.0 or edition.ebook_files.0 %}
<a href="{% url MARCUngluify %}?edition={{ edition.id }}">Upload</a> a MARC record for this edition. <br />
{% if user.libpref %}{% if edition.ebooks.all or edition.ebook_files.all %}
<a href="{% url upload_marc %}?edition={{ edition.id }}">Upload</a> a MARC record for this edition. <br />
{% endif %} {% endif %}
</div>

8
frontend/templates/libraries.html
View File

@ -31,6 +31,8 @@ We’ve added "Buy-to-Unglue" campaigns to so that libraries can lend ebooks on
<dl>
<dt><a href="{% url registration_register %}?next={{ request.get_full_path|urlencode }}">Sign up.</a> It's Free!</dt>
<dd>Starting an account, for yourself or your library, is free. Use it to help us distribute unglued and ungluing ebooks. </dd>
<dt>Add free ebooks to your collection with our <a href="{% url marc %}">MARC records</a>!</dt>
<dd>We're building a comprehensive database of freely licensed ebooks. We provide <a href="{% url marc %}">MARC records for these ebooks</a> wherever we can. They can live in the catalog in your ILS; they link to an epub file in the Internet Archive. Of course, because of unglued ebooks' Creative Commons licenses, you're free to shift that epub to other formats, host the file on your own servers, and edit the MARC record accordingly. And if you want, help us <a href="{% url marc %}">improve</a> our records.</dd>
<dt>Become an Unglue.it participating library</dt>
<dd><ol><li><a href="{% url library_create %}">Make your library page</a> on Unglue.it. It takes just a few minutes. It's the first step towards becoming an Unglue.it participating library.</li>
<li>Review our <a href="https://www.docracy.com/0_uyw26qv9c/unglue-it-library-license-agreement">LIBRARY LICENSE AGREEMENT</a>. </li>
@ -42,10 +44,6 @@ The library license gives download access to one library member at a time for 14
</dd>
<dt>Stay in touch.</dt>
<dd>You can follow us on Twitter (<a href="http://twitter.com/unglueit">@unglueit</a>), <a href="http://facebook/com/unglueit">Facebook</a>, and our <a href="http://blog.unglue.it">blog</a>, and <a href="http://eepurl.com/fKLfI">subscribe to our newsletter</a> (1-2 emails per month).</dd>
<dt>Add unglued ebooks to your collection.</dt>
<dd>We provide <a href="{% url marc %}">MARC records for unglued ebooks</a> wherever we can. We also strive to have MARC records available for all unglued ebooks. They can live in the catalog in your ILS; they link to an epub file in the Internet Archive. Of course, because of unglued ebooks' Creative Commons licenses, you're free to shift that epub to other formats, host the file on your own servers, and edit the MARC record accordingly.</dd>
<dt>Join our <a href="http://goo.gl/lCTLI">catalogers list</a>.</dt>
<dd>We're able to get MARC records into OCLC and SkyRiver because of the support of catalogers like you. Want to help? <a href="http://goo.gl/lCTLI">Add yourself to the list.</a> Usually all you'll need to do is modify an existing record for an earlier edition and upload the record. We'll supply you with the link and the unglued edition. We'll only contact you when there's an unglued book to be cataloged.</dd>
<dt>Spread the word.</dt>
<dd>There are social media sharing links on most pages on the site. There are some right here!
<div id="widgetcode">Copy/paste this into your site:<br /><textarea rows="7" cols="22">&lt;iframe src="https://{{request.META.HTTP_HOST}}/api/widget/{{work.first_isbn_13}}/" width="152" height="325" frameborder="0"&gt;&lt;/iframe&gt;</textarea></div>
@ -59,7 +57,7 @@ The library license gives download access to one library member at a time for 14
<dt>Educate yourself and your patrons about ebook issues and Creative Commons licenses.</dt>
<dd>Checkout limits, publishers who won't sell ebooks to libraries, DRM, companies tracking readers' behavior, library prices far in excess of consumer costs, incompatible technologies and formats, cataloging silos...you know why it's hard for you to deliver a seamless ereading experience to your patrons. Make sure they know, too. And make sure everyone knows how solutions, like <a href="http://creativecommons.org">Creative Commons</a> licenses, can help libraries and readers while respecting copyright and protecting creators' rights.</dd>
<dt>Support <a href="{% url campaign_list 'ending' %}">our active campaigns</a>.</dt>
<dd>Ultimately ebooks can't be unglued unless authors and publishers are paid for their work. Many of our staunchest supporters are librarians. There are also several libraries which have supported campaigns, including Leddy Library (University of Windsor, Ontario); the University of Alberta library ; and the Z. Smith Reynolds library (Wake Forest University.</dd>
<dd>Ultimately ebooks can't be unglued unless authors and publishers are paid for their work. Many of our staunchest supporters are librarians. There are also several libraries which have supported campaigns, including Leddy Library (University of Windsor, Ontario); the University of Alberta library ; and the Z. Smith Reynolds library (Wake Forest University).</dd>
<dt>Give feedback and ask questions.</dt>
<dd>Want to know more? Need help? Have ideas for how we could improve the site or make it more library-friendly? Contact us at: <a href="mailto:libraries@gluejar.com">libraries@gluejar.com</a>.</dd>
</dl>

69
frontend/templates/marc.html
View File

@ -44,8 +44,8 @@ ul.local li {
<p>Go ahead: add unglued ebooks to your library catalog!</p>
{% if request.user.is_staff %}
<p>Hi, {{ request.user.username }}. Unglue.it staffers can also <a href="{% url MARCUngluify %}">add new records</a>.</p>
{% if request.user.libpref %}
<p>Hi, {{ request.user.username }}. Librarians can also <a href="{% url upload_marc %}">add new records</a>.</p>
{% endif %}
{% if messages %}
@ -55,53 +55,38 @@ ul.local li {
{% endfor %}
</ul>
{% endif %}
{% if userlist %}
<h2>Records for <a href="{% url supporter userlist %}">{{ userlist }}</a></h2>
{% else %}
{% if request.user.is_authenticated %}
<p> To restrict this list to works on your list <a href="{% url user_marc request.user.username %}">click here</a> </p>
{% endif %}
{% endif %}
{% if records %}
<div class="marc">
<form method="POST" id="record_form" action="{% url marc_concatenate %}">
{% csrf_token %}
Record format:
<select name="format">
<option value="xml">xml</option>
<option value="mrc">mrc</option>
</select>
<br /><br />
{% for record in records %}
<input type="checkbox" name="record_{{ record.id }}" id="record_{{ record.id }}" />
<label for="record_{{ record.id }}">
Record for <a href="{% url work record.edition.work.id %}" class="title clearfix">{{ record.edition.work.title }}{% if record.edition.isbn_13 %} (ISBN {{ record.edition.isbn_13 }}){% endif %}</a>
</label>
<br />
{% endfor %}
<input type="submit" name="submit" value="Download" id="submit">
<form method="POST" id="record_form" action="{% url marc_all %}">
{% include 'marc_form.html' %}
<input type="submit" name="submit" value="Download All Records" id="submit">
<span id="unsubmit"><img src="/static/images/loading.gif" alt="spinning wheel" /> Fetching record(s)...</span>
</form>
</div>
{% else %}
<p>Sorry; we don't have any records meeting your needs at this time. </p>
<br />
{% if request.user.is_authenticated %}
<p>...or</p>
<div class="marc">
<form method="POST" id="record_form" action="{% url user_marc request.user.username %}">
{% include 'marc_form.html' %}
<input type="submit" name="submit" value="Download {{request.user.username}}'s Records" id="submit">
<span id="unsubmit"><img src="/static/images/loading.gif" alt="spinning wheel" /> Fetching record(s)...</span>
</form>
<br /><hr>
{% endif %}
<p>The 856 link options are:</p>
<ul class="local">
<li> Download page link. MARC records link through Unglue.it download page ( library user authentication, context sensitive help for getting files onto user's device)</li>
<li> Links to each file type if available. MARC records link directly to files ( less help, not available for "Buy-to-unglue" titles) </li>
</ul>
</div>
<hr>
{% if request.user.is_authenticated %}
<div>
<p>Your preference for the links in the 856 field is:</p>
<ul class="local">
<li>{{ libpref.get_marc_link_target_display }}</li>
</ul>
<p>The options are:</p>
<ul class="local">
<li> Raw link if available. MARC records link direct to provider (one click, no help, not available for "Buy-to-unglue" titles) </li>
<li> Unglue.it link. MARC records link through Unglue.it download page (extra click, library user authentication, context sensitive help for getting files onto user's device)</li>
</ul>
<p>You can <a href="{% url marc_config %}">change your preferences here</a>.</p>
{% if request.user.libpref %}
<p>You have enabled librarian tools. You can <a href="{% url marc_config %}">change your librarian status here</a>.</p>
{% else %}
<p>You have not enabled librarian tools. You can <a href="{% url marc_config %}">change your librarian status here</a>.</p>
{% endif %}
</div>
{% endif %}
{% endblock %}

62
frontend/templates/marc/upload.html Normal file
View File

@ -0,0 +1,62 @@
{% extends "basedocumentation.html" %}
{% block title %}Add new MARC records{% endblock %}
{% block extra_extra_head %}
{{ block.super }}
<link rel="stylesheet" href="https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.13/themes/ui-lightness/jquery-ui.css" type="text/css" media="screen">
<link href="/static/selectable/css/dj.selectable.css" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="{{ jquery_ui_home }}"></script>
<script type="text/javascript" src="/static/selectable/js/jquery.dj.selectable.js"></script>
{% endblock %}
{% block doccontent %}
{% if not request.user.libpref %}
<p><i>If you want to load MARC records, you probably want to <a href="{% url marc_config %}">enable Unglue.it's librarian tools</a> first!</i></p>
{% endif %}
<h2>Adding MARC records to Unglue.it</h2>
{% if messages %}
<ul class="messages">
{% for message in messages %}
<li{% if message.tags %} class="{{ message.tags }}"{% endif %}>{{ message }}</li>
{% endfor %}
</ul>
{% endif %}
{% if not form.initial.edition %}
<h3>Selecting an edition</h3>
<p>
MARC records are associated with unglue.it edition records. Look for links in the "more" tab of any unglue.it work page that has a "Download" button on it.
{% endif %}
<h3>Adapting records from LoC</h3>
<p>
For ebooks which have existing print editions cataloged in the Library of Congress, we can automatically convert those to Unglue.it ebook MARC records, which will then be automatically affixed to the ebook edition in our database and <a href="{% url marc %}">provided to ungluers</a>.
</p>
<p>
To get the XML record, search <a href="http://catalog.loc.gov/">http://catalog.loc.gov/</a>, select the print edition, and click on the permalink (will look something like <a href="http://lccn.loc.gov/63008219">http://lccn.loc.gov/63008219</a>). This page has a MARCXML link.
</p>
<p>
The record loader will automatically add links to all the unglued ebook files known to the Unglue.it database. <I>Make sure those links are in the database before adding the record.</i> </p>
<h3>Editing stub records from LoC</h3>
{% if form.initial.edition %}
The current records for this edition are here. They may be auto-generated stubs:
<div id="libtools">
<form method="POST" id="record_form" action="{% url marc_concatenate %}">
{% include 'marc_form.html' %}
<input type="hidden" name="edition_{{form.initial.edition}}" value="on">
<input type="submit" name="submit" value="Download MARC" id="submit">
</form>
</div>
{% endif %}
<p>
Edit the record, then upload it as a "record prepared for Unglue.it". The record loader strips 001, 003, 005, 006, 007, 856 and >900 fields, and adds them back in on download.
</p>
<h3>Load a record</h3>
<form action="" enctype="multipart/form-data" method="post">{% csrf_token %}
{{ form.as_p }}
<input type="submit" value="Add record" />
</form>
{% endblock %}

33
frontend/templates/marc_config.html
View File

@ -10,7 +10,7 @@
</style>
{% endblock %}
{% block title %}Change your MARC record preferences{% endblock %}
{% block title %}Change your librarian preferences{% endblock %}
{% block ce_content %}
{% if messages %}
@ -20,14 +20,33 @@
{% endfor %}
</ul>
{% endif %}
Back to <a href="{% url marc %}">record download page</a><br /><br />
<h3>Librarian Tools</h3>
If you enable Librarian tools, you'll find links to download MARC record(s) at the bottom of many Unglue.it pages.
<ul>
<li> work pages</li>
<li> lists of works</li>
<li> supporter pages </li>
</ul>
Your current MARC record link target preference is:<br />
{{ libpref.get_marc_link_target_display }}
{% if request.user.is_authenticated %}
<form method="post" action="{% url marc_config %}">
{% csrf_token %}
{{ form }}
<br />
<input type="submit" value="Change" />
<div>
{% if request.user.libpref %}
<p>You have enabled librarian tools. </p>
{{ form }}
<input type="submit" value="Disable Librarian Tools" name="disable" />
{% else %}
<p>You have not enabled librarian tools.
{{ form }}
<br />
<input type="submit" value="Enable Librarian Tools" name="enable" />
{% endif %}
</div>
</form>
<p>
Back to <a href="{% url marc %}">record download page</a>
</p>
{% endif %}
{% endblock %}

12
frontend/templates/marc_form.html Normal file
View File

@ -0,0 +1,12 @@
{% csrf_token %}
<span style="padding: 5px;">record format:
<select name="format">
<option value="xml" selected>xml</option>
<option value="mrc">mrc</option>
</select></span>&nbsp;&nbsp;
<span style="padding: 5px;">856 links:
<select name="link_target">
<option value="via" selected>to download page</option>
<option value="direct">to each file type</option>
</select>
</span>

42
frontend/templates/marcungluify.html
View File

@ -1,42 +0,0 @@
{% extends "basedocumentation.html" %}
{% block title %}Add new MARC records{% endblock %}
{% block extra_extra_head %}
{{ block.super }}
<link rel="stylesheet" href="https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.13/themes/ui-lightness/jquery-ui.css" type="text/css" media="screen">
<link href="/static/selectable/css/dj.selectable.css" type="text/css" media="all" rel="stylesheet" />
<script type="text/javascript" src="{{ jquery_ui_home }}"></script>
<script type="text/javascript" src="/static/selectable/js/jquery.dj.selectable.js"></script>
{% endblock %}
{% block doccontent %}
<h2>Make your unglued MARC records here</h2>
{% if messages %}
<ul class="messages">
{% for message in messages %}
<li{% if message.tags %} class="{{ message.tags }}"{% endif %}>{{ message }}</li>
{% endfor %}
</ul>
{% endif %}
<p>
For unglued ebooks which have existing print editions cataloged in the Library of Congress, we can automatically convert those to unglued ebook MARC records, which will then be automatically affixed to the ebook edition in our database and <a href="{% url marc %}">provided to ungluers</a>.
</p>
<p>
To get the XML record, search <a href="http://catalog.loc.gov/">http://catalog.loc.gov/</a>, select the print edition, and click on the permalink (will look something like <a href="http://lccn.loc.gov/63008219">http://lccn.loc.gov/63008219</a>). This page has a MARCXML link.
</p>
<p>
The robot cataloger will automatically add links to all the unglued ebook files known to the Unglue.it database. <I>Make sure you have added those links to the database before creating the record.</i> Likewise, it will autofill the ISBN, and you should have already provided that.
</p>
<p>
If you want to add a MARC record for a non-unglued but publicly available book, create a MARCRecord instance in the admin interface and populate its fields there. (This assumes that the MARC record is for the freely available ebook version and exists in a linkable form. If it does not, upload it to S3 first, by analogy with the <a href="http://django-storages.readthedocs.org/en/latest/backends/amazon-S3.html#tests">django-storages documentation</a>, and use that link.)
</p>
<form action="" enctype="multipart/form-data" method="post">{% csrf_token %}
{{ form.as_p }}
<input type="submit" value="Add record" />
</form>
{% endblock %}

11
frontend/templates/supporter.html
View File

@ -389,6 +389,17 @@ function highlightTarget(targetdiv) {
{% endifequal %}
</div>
</div>
{% if request.user.libpref %}
<div id="libtools">
<p>for libraries...</p>
{% block marcform %}
<form method="GET" action="{% url user_marc supporter.username %}">
{% include 'marc_form.html' %}
<input type="submit" name="submit" value="download MARC for {{supporter.username}} " id="submit">
</form>
{% endblock %}
</div>
{% endif %}
</div>
</div>
</div>

11
frontend/templates/unglued_list.html
View File

@ -105,6 +105,17 @@
</div>
</div>
</div>
{% if request.user.libpref %}
<div id="libtools">
<p>for libraries...</p>
{% block marcform %}
<form method="GET" action="{% url unglued_list_marc facet %}">
{% include 'marc_form.html' %}
<input type="submit" name="submit" value="download MARC" id="submit">
</form>
{% endblock %}
</div>
{% endif %}
</div>
</div>
</div>

9
frontend/templates/work.html
View File

@ -711,6 +711,15 @@
</div>
{% endif %}
</div>
{% if request.user.libpref and work.first_ebook %}
<div id="libtools">
<p>for libraries...</p>
<form method="POST" id="record_form" action="{% url work_marc work.id %}">
{% include 'marc_form.html' %}
<input type="submit" name="submit" value="Download MARC" id="submit">
</form>
</div>
{% endif %}
</div>
</div>
</div>

11
frontend/templates/work_list.html
View File

@ -131,6 +131,17 @@
{% endifequal %}
</div>
</div>
{% if request.user.libpref %}
<div id="libtools">
<p>for libraries...</p>
{% block marcform %}
<form method="GET" action="{% url work_list_marc facet %}">
{% include 'marc_form.html' %}
<input type="submit" name="submit" value="download MARC" id="submit">
</form>
{% endblock %}
</div>
{% endif %}
</div>
</div>
</div>

17
frontend/urls.py
View File

@ -36,8 +36,7 @@ from regluit.frontend.views import (
ByPubView,
kindle_config,
send_to_kindle,
MARCUngluifyView,
MARCConfigView,
LibModeView,
DownloadView,
)
@ -47,7 +46,7 @@ urlpatterns = patterns(
url(r"^landing/$", "home", {'landing': True}, name="landing"),
url(r"^next/$", "next", name="next"),
url(r"^supporter/(?P<supporter_username>[^/]+)/$", "supporter", {'template_name': 'supporter.html'}, name="supporter"),
url(r"^supporter/(?P<userlist>[^/]+)/marc/$", "marc_admin", name="user_marc"),
url(r"^supporter/(?P<userlist>[^/]+)/marc/$", "userlist_marc", name="user_marc"),
url(r"^library/(?P<library_name>[^/]+)/$", "library", name="library"),
url(r"^accounts/manage/$", login_required(ManageAccount.as_view()), name="manage_account"),
url(r"^search/$", "search", name="search"),
@ -71,14 +70,19 @@ urlpatterns = patterns(
url(r"^wishlist/$", "wishlist", name="wishlist"),
url(r"^msg/$", "msg", name="msg"),
url(r"^campaigns/(?P<facet>\w*)$", CampaignListView.as_view(), name='campaign_list'),
url(r"^campaigns/(?P<facet>\w*)/marc/$", CampaignListView.as_view(send_marc=True), name='campaign_list_marc'),
url(r"^lists/(?P<facet>\w*)$", WorkListView.as_view(), name='work_list'),
url(r"^lists/(?P<facet>\w*)/marc/$", WorkListView.as_view(send_marc=True), name='work_list_marc'),
url(r"^pid/all/(?P<pubname>\d+)$", ByPubView.as_view(), name='bypubname_list'),
url(r"^pid/(?P<facet>\w*)/(?P<pubname>\d+)$", ByPubView.as_view(), name='bypubname_list'),
url(r"^pid/(?P<facet>\w*)/(?P<pubname>\d+)/marc/$", ByPubView.as_view(send_marc=True), name='bypubname_list_marc'),
url(r"^bypub/all/(?P<pubname>.*)$", ByPubListView.as_view(), name='bypub_list'),
url(r"^bypub/(?P<facet>\w*)/(?P<pubname>.*)$", ByPubListView.as_view(), name='bypub_list'),
url(r"^unglued/(?P<facet>\w*)$", UngluedListView.as_view(), name='unglued_list'),
url(r"^unglued/(?P<facet>\w*)/marc/$", UngluedListView.as_view(send_marc=True), name='unglued_list_marc'),
url(r"^creativecommons/$", CCListView.as_view(), name='cc_list'),
url(r"^creativecommons/(?P<facet>[\w\-]*)$", CCListView.as_view(), name='cc_list_detail'),
url(r"^creativecommons/(?P<facet>[\w\-]*)/marc/$", CCListView.as_view(send_marc=True), name='cc_list_marc'),
url(r"^goodreads/auth/$", "goodreads_auth", name="goodreads_auth"),
url(r"^goodreads/auth_cb/$", "goodreads_cb", name="goodreads_cb"),
url(r"^goodreads/flush/$","goodreads_flush_assoc", name="goodreads_flush_assoc"),
@ -94,6 +98,7 @@ urlpatterns = patterns(
url(r"^work/(?P<work_id>\d+)/download/$", DownloadView.as_view(), name="thank"),
url(r"^work/(?P<work_id>\d+)/unglued/(?P<format>\w+)/$", "download_campaign", name="download_campaign"),
url(r"^work/(?P<work_id>\d+)/borrow/$", "borrow", name="borrow"),
url(r"^work/(?P<work_id>\d+)/marc/$", "work_marc", name="work_marc"),
url(r"^work/(?P<work_id>\d+)/reserve/$", "reserve", name="reserve"),
url(r"^work/(?P<work_id>\d+)/feature/$", "feature", name="feature"),
url(r"^work/(?P<work_id>\d+)/merge/$", login_required(MergeView.as_view()), name="merge"),
@ -150,10 +155,8 @@ urlpatterns = patterns(
url(r"^accounts/edit/kindle_config/$", "kindle_config", name="kindle_config"),
url(r"^accounts/edit/kindle_config/(?P<work_id>\d+)/$", "kindle_config", name="kindle_config_download"),
url(r"^send_to_kindle/(?P<work_id>\d+)/(?P<javascript>\d)/$", "send_to_kindle", name="send_to_kindle"),
url(r"^marc/$", "marc_admin", name="marc"),
url(r"^marc/ungluify/$", staff_member_required(MARCUngluifyView.as_view()), name="MARCUngluify"),
url(r"^marc/concatenate/$", "marc_concatenate", name="marc_concatenate"),
url(r"^accounts/edit/marc_config/$", login_required(MARCConfigView.as_view()), name="marc_config"),
url(r"^marc/$", direct_to_template, {'template': 'marc.html'}, name="marc"),
url(r"^accounts/edit/marc_config/$", login_required(LibModeView.as_view()), name="marc_config"),
)
if settings.DEBUG:

179
frontend/views.py
View File

@ -17,7 +17,6 @@ from notification import models as notification
from random import randint
from re import sub
from xml.etree import ElementTree as ET
from xml.sax import SAXParseException
from tastypie.models import ApiKey
'''
@ -72,7 +71,6 @@ from regluit.core import (
librarything,
userlists,
goodreads,
marc
)
import regluit.core.cc as cc
from regluit.core.bookloader import merge_works, detach_edition
@ -116,8 +114,7 @@ from regluit.frontend.forms import (
MsgForm,
PressForm,
KindleEmailForm,
MARCUngluifyForm,
MARCFormatForm,
LibModeForm,
DateCalculatorForm
)
@ -145,6 +142,7 @@ from regluit.booxtream.exceptions import BooXtreamError
from regluit.pyepub import InvalidEpub
from regluit.libraryauth.views import Authenticator
from regluit.libraryauth.models import Library
from regluit.marc.views import qs_marc_records
logger = logging.getLogger(__name__)
@ -559,10 +557,10 @@ def new_edition(request, work_id, edition_id, by=None):
elif request.POST.has_key('add_subject_submit') and admin:
new_subject = request.POST['add_subject'].strip()
try:
author= models.Subject.objects.get(name=new_subject)
subject= models.Subject.objects.get(name=new_subject)
except models.Subject.DoesNotExist:
author=models.Subject.objects.create(name=new_subject)
edition.new_subjects.append(new_subject)
subject=models.Subject.objects.create(name=new_subject)
edition.new_subjects.append(subject)
form = EditionForm(instance=edition, data=request.POST, files=request.FILES)
edition.ebook_form = EbookForm( instance= models.Ebook(user = request.user, edition = edition, provider = 'x' ), prefix = 'ebook_%d'%edition.id)
@ -605,11 +603,7 @@ def new_edition(request, work_id, edition_id, by=None):
else:
models.Identifier.set(type=id_type, value=id_val, edition=edition, work=work)
for author_name in edition.new_author_names:
try:
author= models.Author.objects.get(name=author_name)
except models.Author.DoesNotExist:
author=models.Author.objects.create(name=author_name)
author.editions.add(edition)
edition.add_author(author_name)
for subject_name in edition.new_subjects:
try:
subject= models.Subject.objects.get(name=subject_name)
@ -783,6 +777,7 @@ def subjects(request):
return render(request, 'subjects.html', {'subjects': subjects})
class FilterableListView(ListView):
send_marc = False
def get_queryset(self):
if self.request.GET.has_key('pub_lang'):
if self.model is models.Campaign:
@ -802,6 +797,12 @@ class FilterableListView(ListView):
context['WISHED_LANGS']=settings.WISHED_LANGS
return context
def render_to_response(self, context, **response_kwargs):
if self.send_marc:
return qs_marc_records(self.request, qs=self.object_list)
else:
return super(FilterableListView,self).render_to_response(context, **response_kwargs)
recommended_user = User.objects.filter( username=settings.UNGLUEIT_RECOMMENDED_USERNAME)
class WorkListView(FilterableListView):
@ -881,6 +882,8 @@ class ByPubView(WorkListView):
context = super(ByPubView, self).get_context_data(**kwargs)
context['pubname'] = self.publisher_name
context['publisher'] = self.publisher
context['facet'] = self.kwargs.get('facet','all')
return context
class ByPubListView(ByPubView):
@ -913,7 +916,7 @@ class CCListView(FilterableListView):
def get_context_data(self, **kwargs):
context = super(CCListView, self).get_context_data(**kwargs)
facet = self.kwargs.get('facet','')
facet = self.kwargs.get('facet','all')
qs=self.get_queryset()
context['ungluers'] = userlists.work_list_users(qs,5)
context['activetab'] = "#1"
@ -3091,144 +3094,44 @@ def send_to_kindle(request, work_id, javascript='0'):
return local_response(request, javascript, context, 2)
def marc_admin(request, userlist=None):
link_target = 'UNGLUE'
libpref = {'marc_link_target': settings.MARC_CHOICES[1]}
try:
libpref = request.user.libpref
link_target = libpref.marc_link_target
except:
if not request.user.is_anonymous():
libpref = models.Libpref(user=request.user)
unwanted = 'UNGLUE' if link_target == 'DIRECT' else 'DIRECT'
def userlist_marc(request, userlist=None):
if userlist:
records = []
user = get_object_or_404(User,username=userlist)
for work in user.wishlist.works.all():
records.extend(models.MARCRecord.objects.filter(edition__work=work,link_target=link_target))
records.extend(models.MARCRecord.objects.filter(edition__work=work,link_target='B2U'))
return qs_marc_records(request, qs=user.wishlist.works.all())
else:
records = models.MARCRecord.objects.exclude(link_target=unwanted)
return render(
request,
'marc.html',
{'records': records, 'libpref' : libpref , 'userlist' : userlist}
)
return qs_marc_records(request, qs=request.user.wishlist.works.all())
class MARCUngluifyView(FormView):
template_name = 'marcungluify.html'
form_class = MARCUngluifyForm
success_url = reverse_lazy('MARCUngluify')
return render( request,'marc.html',{'userlist' : [] })
# allow a get param to specify the edition
def get_initial(self):
if self.request.method == 'GET':
edition = self.request.GET.get('edition',None)
if models.Edition.objects.filter(id=edition).count():
edition = models.Edition.objects.filter(id=edition)[0]
if edition.ebooks.count() or edition.ebook_files.count():
return {'edition':edition.id}
return {}
def work_marc(request, work_id):
work = safe_get_work(work_id)
return qs_marc_records(request, qs=[ work ])
def form_valid(self, form):
edition = form.cleaned_data['edition']
try:
marc.makemarc(
marcfile=self.request.FILES['file'],
edition=edition
)
messages.success(
self.request,
"You have successfully added a MARC record. Hooray! Add another?"
)
except SAXParseException:
messages.error(
self.request,
"Sorry, couldn't parse that file."
)
return super(MARCUngluifyView,self).form_valid(form)
class MARCConfigView(FormView):
class LibModeView(FormView):
template_name = 'marc_config.html'
form_class = MARCFormatForm
form_class = LibModeForm
success_url = reverse_lazy('marc')
def form_valid(self, form):
marc_link_target = form.cleaned_data['marc_link_target']
try:
libpref = self.request.user.libpref
except:
libpref = models.Libpref(user=self.request.user)
libpref.marc_link_target = marc_link_target
libpref.save()
messages.success(
self.request,
"Your preferences have been changed."
)
enable= form.data.has_key('enable')
if enable:
try:
libpref = self.request.user.libpref
except:
libpref = models.Libpref(user=self.request.user)
libpref.save()
messages.success(self.request,"Tools are enabled.")
else:
try:
self.request.user.libpref.delete()
except:
pass
messages.success(self.request,"Tools are disabled." )
if reverse('marc_config', args=[]) in self.request.META['HTTP_REFERER']:
return HttpResponseRedirect(reverse('marc_config', args=[]))
else:
return super(MARCConfigView, self).form_valid(form)
def marc_concatenate(request):
if request.method == 'POST':
params=request.POST
elif request.method == 'GET':
params=request.GET
else:
return HttpResponseNotFound
format = params['format']
# extract the user-selected records from the params QueryDict
selected_records = list(
k for k in params if params[k] == u'on'
)
if not selected_records:
messages.error(
request,
"Please select at least one record to download."
)
return HttpResponseRedirect(reverse('marc', args=[]))
preamble = ('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
'<collection '
'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '
'xmlns="http://www.loc.gov/MARC21/slim" '
'xsi:schemaLocation="http://www.loc.gov/MARC21/slim '
'http://www.loc.gov/standards/marcxml/schema/'
'MARC21slim.xsd">')
outfile = HttpResponse('', content_type='application/marc')
outfile['Content-Disposition'] = 'attachment; filename='+ now().strftime('%Y%m%d%H%M%S') + '.' + format
if format == 'xml':
outfile.write(preamble)
for record in selected_records:
if record.startswith('edition_'):
record_id = long(record[8:])
try:
edition = models.Edition.objects.get(pk=record_id)
except:
continue
record_id = marc.makestub(edition)
else:
record_id = long(record[7:])
if format == 'xml':
record_url = models.MARCRecord.objects.get(
pk=record_id
).xml_record
elif format == 'mrc':
record_url = models.MARCRecord.objects.get(
pk=record_id
).mrc_record
try:
record_file = default_storage.open(record_url).read()
outfile.write(record_file)
except IOError:
logger.warning('MARC record with id number %s has a problem with its URL' % record_id)
if format == 'xml':
outfile.write('</collection>')
return outfile
return super(LibModeView, self).form_valid(form)

0
marc/__init__.py Executable file
View File

22
marc/forms.py Normal file
View File

@ -0,0 +1,22 @@
from django import forms
from selectable.forms import (
AutoCompleteSelectWidget,
AutoCompleteSelectField
)
from regluit.core.lookups import EditionLookup
class MARCUploadForm(forms.Form):
edition = AutoCompleteSelectField(
EditionLookup,
label='Edition',
widget=AutoCompleteSelectWidget(EditionLookup),
required=True,
error_messages={'required': 'Please specify an edition.'},
)
file = forms.FileField(label='Select a MARCXML file.')
source = forms.ChoiceField(label='This file is ...', choices=[
( 'loc' , 'from Library of Congress (print)'),
( 'raw' , 'prepared for Unglue.it'),
])

291
marc/load.py Normal file
View File

@ -0,0 +1,291 @@
"""
This takes a MARCXML filename as an argument and converts it into
MARC records for the unglued edition (in .xml and .mrc formats).
Consider it a catalogolem: http://commons.wikimedia.org/wiki/File:Arcimboldo_Librarian_Stokholm.jpg
Use the MARCXML file for the non-unglued edition from Library of Congress.
"""
import pymarc
import logging
from datetime import datetime
from StringIO import StringIO
from django.conf import settings
from django.core.urlresolvers import reverse
import regluit.core.cc as cc
def stub(edition):
record = pymarc.Record()
now = datetime.now()
#mostly to identify this record as a stub
field001 = pymarc.Field(tag='001', data='stb'+now.strftime('%y%m%d%H%M%S'))
record.add_ordered_field(field001)
add_stuff(record)
# fun fun fun 008
new_field_value= now.strftime('%y%m%d')+'s'
if edition.publication_date and len(edition.publication_date)>3:
new_field_value += edition.publication_date[0:4]
else:
new_field_value += '||||'
new_field_value += '||||xx |||||o|||||||||||eng||'
field008 = pymarc.Field(tag='008', data=new_field_value)
record.add_ordered_field(field008)
# add IBSN for if available
if edition.isbn_13:
field020 = pymarc.Field(
tag='020',
indicators = [' ', ' '],
subfields = ['a', edition.isbn_13]
)
record.add_ordered_field(field020)
# OCLC number
if edition.oclc:
field035 = pymarc.Field(
tag='035',
indicators = [' ', ' '],
subfields = ['a', '(OCoLC)' + edition.oclc]
)
record.add_ordered_field(field035)
# author name
num_auths = len(edition.authnames())
if num_auths:
field100 = pymarc.Field(
tag='100',
indicators = ['1', ' '],
subfields = [
'a', edition.authnames()[0],
]
)
record.add_ordered_field(field100)
if num_auths > 1:
for auth in edition.authnames()[1:]:
field = pymarc.Field(
tag='700',
indicators = ['1', ' '],
subfields = [
'a', auth,
'e', 'joint author.',
]
)
record.add_ordered_field(field)
# add subfield to 245 indicating format
field245 = pymarc.Field(
tag='245',
indicators = ['1', '0'],
subfields = [
'a', edition.title,
'a', '[electronic resource]',
]
)
record.add_ordered_field(field245)
# publisher, date
if edition.publisher:
field260 = pymarc.Field(
tag='260',
indicators = [' ', ' '],
subfields = [
'b', edition.publisher,
]
)
if edition.publication_date:
field260.add_subfield('c', unicode(edition.publication_date))
record.add_ordered_field(field260)
if edition.description:
#add 520 field (description)
field520 = pymarc.Field(
tag='520',
indicators = [' ', ' '],
subfields = [
'a', edition.description,
]
)
record.add_ordered_field(field520)
add_license(record, edition)
return record
#load a with minimal change
def raw(marcfile, edition):
record = pymarc.parse_xml_to_array(marcfile)[0]
for field in record:
if field.tag in ('001', '003', '005', '006', '007', '856') or int( field.tag ) > 900:
record.remove_field(field)
add_stuff(record)
return record
#load a record from library of Congress
def from_lc(marcfile, edition):
# save lccn for later (if there is one) before deleting it
print_lccn = None
record = pymarc.parse_xml_to_array(marcfile)[0]
for lccn in record.get_fields('010'):
for validlccn in lccn.get_subfields('a'):
print_lccn = validlccn
for field in record:
if field.tag in ('001', '003', '005', '006', '007', '010', '040', '856') or int( field.tag ) > 900:
record.remove_field(field)
add_stuff(record)
field008 = record.get_fields('008')[0]
record.remove_field(field008)
old_field_value = field008.value()
new_field_value = old_field_value[:23] + 'o' + old_field_value[24:]
field008 = pymarc.Field(tag='008', data=new_field_value)
record.add_ordered_field(field008)
# add IBSN for ebook where applicable; relegate print ISBN to $z
isbn = edition.isbn_13
try:
field020 = record.get_fields('020')[0]
print_isbn = field020.get_subfields('a')[0]
field020.delete_subfield('a')
if isbn:
field020.add_subfield('a', isbn)
field020.add_subfield('z', print_isbn)
except IndexError:
print_isbn = None
# change 050 and 082 indicators because LOC is no longer responsible for these
# no easy indicator change function, so we'll just reconstruct the fields
try:
field050 = record.get_fields('050')[0]
field050_new = field050
field050_new.indicators = [' ', '4']
record.remove_field(field050)
record.add_ordered_field(field050_new)
except:
pass # if no 050 field, don't need to change indicator
try:
field082 = record.get_fields('082')[0]
field082_new = field082
field082_new.indicators = [' ', '4']
record.remove_field(field082)
record.add_ordered_field(field082_new)
except:
pass # if no 082 field, don't need to change indicator
# add subfield to 245 indicating format
try:
field245 = record.get_fields('245')[0]
field245.add_subfield('a', '[electronic resource]')
except IndexError:
pass
# modify 300 field (physical description)
try:
field300 = record.get_fields('300')[0]
subfield_a = field300.get_subfields('a')[0]
if (
subfield_a[-2:] == ' ;' or
subfield_a[-2:] == ' :' or
subfield_a[-2:] == ' +'
):
subfield_a = subfield_a[:-2]
new300a = '1 online resource (' + subfield_a + ')'
if field300.get_subfields('b'):
new300a += ' :'
field300.delete_subfield('a')
field300.add_subfield('a', new300a)
field300.delete_subfield('c')
except:
pass
add_license(record, edition)
# add 588 field (source of description) - credit where credit is due
if print_lccn:
field588 = pymarc.Field(
tag='588',
indicators = [' ', ' '],
subfields = [
'a', 'Description based on print version record from the Library of Congress.',
]
)
record.add_ordered_field(field588)
# add 776 field (related editions) - preserve pISBN, LCCN, OCLCnum
title = record.get_fields('245')[0].get_subfields('a')[0]
title = title.split('/')[0]
subfields = ['i', 'Print version: ','t', title,]
if print_isbn:
subfields.extend(['z', print_isbn])
elif isbn:
subfields.extend(['z', isbn])
if print_lccn:
subfields.extend(['w', '(DLC) ' + print_lccn, ])
if edition.oclc:
subfields.extend(['w', '(OCoLC) ' + edition.oclc,])
field776 = pymarc.Field(
tag='776',
indicators = ['0', '8'],
subfields = subfields
)
record.add_ordered_field(field776)
return record
def add_license(record, edition):
if edition.license:
# add 536 field (funding information)
field536 = pymarc.Field(
tag='536',
indicators = [' ', ' '],
subfields = [
'a', edition.funding_info,
]
)
record.add_ordered_field(field536)
# add 540 field (terms governing use)
field540 = pymarc.Field(
tag='540',
indicators = [' ', ' '],
subfields = [
'a', dict(cc.CHOICES)[edition.license],
'u', dict(cc.GRANTS)[edition.license],
]
)
record.add_ordered_field(field540)
def add_stuff(record):
# add field indicating record originator
field003 = pymarc.Field(tag='003', data='UnglueIt')
record.add_ordered_field(field003)
# update timestamp of record
datestamp = datetime.now().strftime('%Y%m%d%H%M%S') + '.0'
field005 = pymarc.Field(tag='005', data=datestamp)
record.add_ordered_field(field005)
# change 006, 007, 008 because this is an online resource
field006 = pymarc.Field(
tag='006',
data='m o d '
)
record.add_ordered_field(field006)
field007 = pymarc.Field(
tag='007',
data='cr'
)
record.add_ordered_field(field007)

0
marc/management/__init__.py Normal file
View File

0
marc/management/commands/__init__.py Normal file
View File

31
marc/management/commands/migrate_records.py Normal file
View File

@ -0,0 +1,31 @@
from django.core.management.base import BaseCommand
from django.core.files.storage import default_storage
from regluit.marc.models import MARCRecord
from regluit.core.models import MARCRecord as OldRecord
class Command(BaseCommand):
help = "migrate records (id<stoprecord) from files"
args = "<stoprecord>"
def handle(self, stoprecord=None, **options):
editions=[]
old_records=OldRecord.objects.all().order_by('-id')
if stoprecord:
old_records = old_records.filter(id__lt=int(stoprecord))
for old_record in old_records:
if old_record.edition.pk not in editions:
new_record, created = MARCRecord.objects.get_or_create(id=old_record.pk)
try:
xml_file = default_storage.open(old_record.xml_record)
xml_data = xml_file.read()
new_record.guts = xml_data
new_record.edition = old_record.edition
editions.append(old_record.edition.pk)
xml_file.close()
new_record.save()
print 'record %s updated' % new_record.id
except IOError:
if created:
new_record.delete()
print 'failed opening %s' % old_record.xml_record

73
marc/migrations/0001_initial.py Normal file
View File

@ -0,0 +1,73 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'MARCRecord'
db.create_table('marc_marcrecord', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('guts', self.gf('django.db.models.fields.TextField')()),
('edition', self.gf('django.db.models.fields.related.ForeignKey')(related_name='MARCRecords', null=True, to=orm['core.Edition'])),
))
db.send_create_signal('marc', ['MARCRecord'])
def backwards(self, orm):
# Deleting model 'MARCRecord'
db.delete_table('marc_marcrecord')
models = {
'core.edition': {
'Meta': {'object_name': 'Edition'},
'cover_image': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'publication_date': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'publisher_name': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editions'", 'null': 'True', 'to': "orm['core.PublisherName']"}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '1000'}),
'unglued': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'work': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editions'", 'null': 'True', 'to': "orm['core.Work']"})
},
'core.publisher': {
'Meta': {'object_name': 'Publisher'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'logo_url': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
'name': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'key_publisher'", 'to': "orm['core.PublisherName']"}),
'url': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'})
},
'core.publishername': {
'Meta': {'object_name': 'PublisherName'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'publisher': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'alternate_names'", 'null': 'True', 'to': "orm['core.Publisher']"})
},
'core.work': {
'Meta': {'ordering': "['title']", 'object_name': 'Work'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}),
'earliest_publication': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True'}),
'featured': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '5', 'db_index': 'True'}),
'num_wishes': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}),
'openlibrary_lookup': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
'selected_edition': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'selected_works'", 'null': 'True', 'to': "orm['core.Edition']"}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '1000'})
},
'marc.marcrecord': {
'Meta': {'object_name': 'MARCRecord'},
'edition': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'MARCRecords'", 'null': 'True', 'to': "orm['core.Edition']"}),
'guts': ('django.db.models.fields.TextField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
}
}
complete_apps = ['marc']

116
marc/migrations/0002_auto__add_field_marcrecord_user__add_field_marcrecord_created.py Normal file
View File

@ -0,0 +1,116 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'MARCRecord.user'
db.add_column('marc_marcrecord', 'user',
self.gf('django.db.models.fields.related.ForeignKey')(related_name='MARCRecords', null=True, to=orm['auth.User']),
keep_default=False)
# Adding field 'MARCRecord.created'
db.add_column('marc_marcrecord', 'created',
self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, default=datetime.datetime(2014, 10, 27, 0, 0), blank=True),
keep_default=False)
def backwards(self, orm):
# Deleting field 'MARCRecord.user'
db.delete_column('marc_marcrecord', 'user_id')
# Deleting field 'MARCRecord.created'
db.delete_column('marc_marcrecord', 'created')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'core.edition': {
'Meta': {'object_name': 'Edition'},
'cover_image': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'publication_date': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'publisher_name': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editions'", 'null': 'True', 'to': "orm['core.PublisherName']"}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '1000'}),
'unglued': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'work': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editions'", 'null': 'True', 'to': "orm['core.Work']"})
},
'core.publisher': {
'Meta': {'object_name': 'Publisher'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'logo_url': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
'name': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'key_publisher'", 'to': "orm['core.PublisherName']"}),
'url': ('django.db.models.fields.URLField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'})
},
'core.publishername': {
'Meta': {'object_name': 'PublisherName'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'publisher': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'alternate_names'", 'null': 'True', 'to': "orm['core.Publisher']"})
},
'core.work': {
'Meta': {'ordering': "['title']", 'object_name': 'Work'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True', 'blank': 'True'}),
'earliest_publication': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True'}),
'featured': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '5', 'db_index': 'True'}),
'num_wishes': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}),
'openlibrary_lookup': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
'selected_edition': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'selected_works'", 'null': 'True', 'to': "orm['core.Edition']"}),
'title': ('django.db.models.fields.CharField', [], {'max_length': '1000'})
},
'marc.marcrecord': {
'Meta': {'object_name': 'MARCRecord'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'edition': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'MARCRecords'", 'null': 'True', 'to': "orm['core.Edition']"}),
'guts': ('django.db.models.fields.TextField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'MARCRecords'", 'null': 'True', 'to': "orm['auth.User']"})
}
}
complete_apps = ['marc']

0
marc/migrations/__init__.py Normal file
View File

163
marc/models.py Normal file
View File

@ -0,0 +1,163 @@
import pymarc
import logging
from datetime import datetime
from StringIO import StringIO
from django.conf import settings
from django.contrib.auth.models import User
from django.db import models
from . import load
# weak coupling
EDITION_MODEL = "core.Edition"
class AbstractEdition:
# define the methods and attributes an edition should have
isbn_13 = ''
oclc = ''
license = None
funding_info = ''
description = ''
publisher = ''
title = ''
publication_date = ''
# the edition should be able to report ebook downloads, with should have format and url attributes
def downloads(self):
return []
# the edition should be able to report an "ebook via" url
def download_via_url(self):
return []
# these should be last name first
def authnames(self):
return []
def _xml(record):
return pymarc.record_to_xml(record)
def _mrc(record):
mrc_file = StringIO()
writer = pymarc.MARCWriter(mrc_file)
writer.write(record)
mrc_file.seek(0)
return mrc_file.read()
class MARCRecord(models.Model):
# the record goes here
guts = models.TextField()
#storage for parsed guts
_the_record = None
# note capitalization of related_name
edition = models.ForeignKey(EDITION_MODEL, related_name="MARCRecords", null=True)
user = models.ForeignKey(User, related_name="MARCRecords", null=True )
created = models.DateTimeField(auto_now_add=True)
def __init__(self, *args, **kwargs):
super(MARCRecord, self).__init__( *args, **kwargs)
edition = kwargs.pop('edition', None)
guts = kwargs.pop('guts', None)
if edition and not guts:
#make a stub _the_record from the edition
self._the_record = load.stub(edition)
@property
def accession(self):
zeroes = 9 - len(str(self.id))
return 'ung' + zeroes*'0' + str(self.id)
def save(self, *args, **kwargs):
if self.id == None and self._the_record:
# get the id first, add assession number
stash_guts = self.guts
self.guts = ''
super(MARCRecord, self).save(*args, **kwargs)
self.guts = stash_guts
field001 = self._the_record.get_fields('001')
if field001:
self._the_record.remove_field(field)
field001 = pymarc.Field(tag='001', data=self.accession)
self._the_record.add_ordered_field(field001)
super(MARCRecord, self).save(*args, **kwargs)
def load_from_file(self, source='raw'):
#parse guts
if isinstance(self.guts, str) or isinstance(self.guts, unicode):
marcfile = StringIO(self.guts)
else:
marcfile = self.guts
if source == 'loc':
self._the_record = load.from_lc(marcfile, self.edition)
else:
self._the_record = load.raw(marcfile, self.edition)
self.guts = pymarc.record_to_xml(self._the_record)
self.save()
# the record without 856
def _record(self):
if self._the_record:
the_record = self._the_record
else:
the_record = pymarc.parse_xml_to_array(StringIO(self.guts))[0]
for field in the_record.get_fields('856'):
the_record.remove_field(field)
self._the_record = the_record
return the_record
def direct_record(self):
the_record = self._record()
for book in self.edition.downloads():
field856 = pymarc.Field(
tag='856',
indicators = ['4', '0'],
subfields = [
'3', book.format + ' version',
'q', settings.CONTENT_TYPES[book.format],
'u', book.url,
]
)
the_record.add_ordered_field(field856)
return the_record
def direct_record_xml(self):
return _xml(self.direct_record())
def direct_record_mrc(self):
return _mrc(self.direct_record())
def via_record(self):
the_record = self._record()
field856_via = pymarc.Field(
tag='856',
indicators = ['4', '0'],
subfields = [
'u', self.edition.download_via_url(),
]
)
the_record.add_ordered_field(field856_via)
return the_record
def via_record_xml(self):
return _xml(self.via_record())
def via_record_mrc(self):
return _mrc(self.via_record())
def record(self, link_target='via', format='xml'):
if format == 'xml':
if link_target == 'via':
return self.via_record_xml()
elif link_target == 'direct':
return self.direct_record_xml()
elif format == 'mrc':
if link_target == 'via':
return self.via_record_mrc()
elif link_target == 'direct':
return self.direct_record_mrc()

135
marc/tests.py Normal file
View File

@ -0,0 +1,135 @@
"""
django imports
"""
from django.test import TestCase
from django.test.client import Client
from django.db.models import get_model
"""
regluit imports
"""
from . import models, load
a_marc_record = '''<?xml version="1.0" encoding="UTF-8"?><record xmlns="http://www.loc.gov/MARC21/slim" xmlns:cinclude="http://apache.org/cocoon/include/1.0" xmlns:zs="http://www.loc.gov/zing/srw/">
<leader>01021cam a2200301 a 4500</leader>
<controlfield tag="001">3057297</controlfield>
<controlfield tag="005">19970108080513.5</controlfield>
<controlfield tag="008">960131r19761970ke b b 000 0 eng </controlfield>
<datafield tag="035" ind1=" " ind2=" ">
<subfield code="9">(DLC) 96109467</subfield>
</datafield>
<datafield tag="906" ind1=" " ind2=" ">
<subfield code="a">7</subfield>
<subfield code="b">cbc</subfield>
<subfield code="c">orignew</subfield>
<subfield code="d">u</subfield>
<subfield code="e">ncip</subfield>
<subfield code="f">19</subfield>
<subfield code="g">y-gencatlg</subfield>
</datafield>
<datafield tag="955" ind1=" " ind2=" ">
<subfield code="a">082 done aa11</subfield>
</datafield>
<datafield tag="010" ind1=" " ind2=" ">
<subfield code="a"> 96109467 </subfield>
</datafield>
<datafield tag="020" ind1=" " ind2=" ">
<subfield code="a">0195724135</subfield>
</datafield>
<datafield tag="040" ind1=" " ind2=" ">
<subfield code="a">DLC</subfield>
<subfield code="c">DLC</subfield>
<subfield code="d">DLC</subfield>
</datafield>
<datafield tag="043" ind1=" " ind2=" ">
<subfield code="a">f------</subfield>
</datafield>
<datafield tag="050" ind1="0" ind2="0">
<subfield code="a">PL8010</subfield>
<subfield code="b">.F5 1976</subfield>
</datafield>
<datafield tag="082" ind1="0" ind2="0">
<subfield code="a">398.2/096</subfield>
<subfield code="2">20</subfield>
</datafield>
<datafield tag="100" ind1="1" ind2=" ">
<subfield code="a">Finnegan, Ruth H.</subfield>
</datafield>
<datafield tag="245" ind1="1" ind2="0">
<subfield code="a">Oral literature in Africa /</subfield>
<subfield code="c">Ruth Finnegan.</subfield>
</datafield>
<datafield tag="260" ind1=" " ind2=" ">
<subfield code="a">Nairobi :</subfield>
<subfield code="b">Oxford University Press,</subfield>
<subfield code="c">1976 (1994 printing).</subfield>
</datafield>
<datafield tag="300" ind1=" " ind2=" ">
<subfield code="a">xviii, 558 p. :</subfield>
<subfield code="b">map ;</subfield>
<subfield code="c">21 cm.</subfield>
</datafield>
<datafield tag="440" ind1=" " ind2="0">
<subfield code="a">Oxford library of African literature</subfield>
</datafield>
<datafield tag="500" ind1=" " ind2=" ">
<subfield code="a">Originally published: Oxford : Clarendon Press, 1970.</subfield>
</datafield>
<datafield tag="504" ind1=" " ind2=" ">
<subfield code="a">Includes index and bibliographical references (p. [522]-536).</subfield>
</datafield>
<datafield tag="650" ind1=" " ind2="0">
<subfield code="a">Folk literature, African</subfield>
<subfield code="x">History and criticism.</subfield>
</datafield>
<datafield tag="650" ind1=" " ind2="0">
<subfield code="a">Oral tradition</subfield>
<subfield code="z">Africa.</subfield>
</datafield>
<datafield tag="922" ind1=" " ind2=" ">
<subfield code="a">ap</subfield>
</datafield>
<datafield tag="991" ind1=" " ind2=" ">
<subfield code="b">c-GenColl</subfield>
<subfield code="h">PL8010</subfield>
<subfield code="i">.F5 1976</subfield>
<subfield code="t">Copy 1</subfield>
<subfield code="w">BOOKS</subfield>
</datafield>
</record>'''
class MarcTests(TestCase):
work_id=None
def test_records(self):
w = get_model('core','Work').objects.create(title="Work 1")
e = get_model('core','Edition').objects.create(title=w.title,work=w)
eb = get_model('core','Ebook').objects.create(url = "http://example.org",edition = e,format = 'epub', rights='CC BY')
mr = models.MARCRecord.objects.create(guts=a_marc_record, edition=e )
mr.direct_record_xml()
mr.direct_record_mrc()
mr.via_record_xml()
mr.via_record_mrc()
load.stub(e)
w.description='test'
e.set_publisher('test pub')
e.publication_date = '2000'
e.add_author('joe writer')
id = get_model('core','Identifier').objects.create(work=w,edition=e, type='isbn', value='0030839939')
id = get_model('core','Identifier').objects.create(work=w,edition=e, type='oclc', value='0074009772')
load.stub(e)
mr2 = models.MARCRecord.objects.create(guts=a_marc_record, edition=e )
mr2.load_from_file('loc')
mr3 = models.MARCRecord.objects.create(guts=a_marc_record, edition=e )
mr3.load_from_file('raw')
mr2.direct_record_xml()
mr2.direct_record_mrc()
mr2.via_record_xml()
mr2.via_record_mrc()

11
marc/urls.py Normal file
View File

@ -0,0 +1,11 @@
from django.conf.urls.defaults import *
from django.contrib.auth.decorators import login_required
from . import views
urlpatterns = patterns(
"regluit.marc.views",
url(r"^marc/concatenate/$", "marc_records", name="marc_concatenate"),
url(r"^marc/all/$", "all_marc_records", name="marc_all"),
url(r"^marc/upload/$", login_required(views.MARCUpload.as_view()), name="upload_marc"),
)

132
marc/views.py Normal file
View File

@ -0,0 +1,132 @@
from datetime import datetime
from xml.sax import SAXParseException
from django.contrib import messages
from django.core.urlresolvers import reverse, reverse_lazy
from django.db.models import get_model
from django.http import HttpResponseRedirect, HttpResponse, HttpResponseNotFound
from django.views.generic.edit import FormView
from . import models
from . import forms
PREAMBLE = ('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
'<collection '
'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '
'xmlns="http://www.loc.gov/MARC21/slim" '
'xsi:schemaLocation="http://www.loc.gov/MARC21/slim '
'http://www.loc.gov/standards/marcxml/schema/'
'MARC21slim.xsd">')
Edition = get_model(*models.EDITION_MODEL.split('.'))
def marc_records(request, selected_records=None):
# TODO convert to streaming when we move to Django 1.5+
if request.method == 'POST':
params=request.POST
elif request.method == 'GET':
params=request.GET
else:
return HttpResponseNotFound
format = params.get('format', 'xml')
link_target = params.get('link_target', 'via')
if selected_records==None:
# extract the user-selected records from the params QueryDict
selected_records = list(
k for k in params if params[k] == u'on'
)
if not selected_records:
messages.error(
request,
"There were no records to download."
)
return HttpResponseRedirect(reverse('marc', args=[]))
outfile = HttpResponse('', content_type='application/marc')
outfile['Content-Disposition'] = 'attachment; filename='+ datetime.now().strftime('%Y%m%d%H%M%S') + '.' + format
if format == 'xml':
outfile.write(PREAMBLE)
for record_name in selected_records:
if isinstance(record_name, models.MARCRecord):
record = record_name
elif isinstance(record_name, Edition):
record = models.MARCRecord(edition=record_name)
elif hasattr(record_name, 'edition'):
record = models.MARCRecord(edition=record_name.edition)
elif record_name.startswith('edition_'):
record_id = long(record_name[8:])
try:
edition = Edition.objects.get(pk=record_id)
except Edition.DoesNotExist:
continue
record = models.MARCRecord(edition=edition)
elif record_name.startswith('record_'):
record_id = long(record_name[7:])
try:
record = models.MARCRecord.objects.get(id=record_id )
except models.MARCRecord.DoesNotExist:
continue
outfile.write(record.record(format=format, link_target=link_target))
if format == 'xml':
outfile.write('</collection>')
return outfile
def all_marc_records(request):
selected_records = list(
'record_'+ str(record.pk) for record in models.MARCRecord.objects.all()
)
return marc_records(request, selected_records=selected_records)
def qs_marc_records(request, qs):
# translate a queryset into a record list.
selected_records = []
for obj in qs:
# each object must have a marc_records() attribute which returns a record list.
# records can be strings: 'record_xxx', 'edition_xxx', MARCRecord objs or objs with edition attributes
selected_records.extend(obj.marc_records())
return marc_records(request, selected_records=selected_records)
class MARCUpload(FormView):
template_name = 'marc/upload.html'
form_class = forms.MARCUploadForm
success_url = reverse_lazy('upload_marc')
# allow a get param to specify the edition
def get_initial(self):
if self.request.method == 'GET':
edition = self.request.GET.get('edition',None)
if Edition.objects.filter(id=edition).count():
edition = Edition.objects.filter(id=edition)[0]
if edition.ebooks.count() or edition.ebook_files.count():
return {'edition':edition.id}
return {}
def form_valid(self, form):
edition = form.cleaned_data['edition']
source = form.cleaned_data['source']
try:
marcfile=self.request.FILES['file']
new_record = models.MARCRecord(
guts=marcfile,
edition=edition,
user= self.request.user
)
new_record.load_from_file(source)
messages.success(
self.request,
"You have successfully added a MARC record. Hooray! Add another?"
)
except SAXParseException:
messages.error(
self.request,
"Sorry, couldn't parse that file."
)
return super(MARCUpload,self).form_valid(form)

1
settings/common.py
View File

@ -132,6 +132,7 @@ INSTALLED_APPS = (
'regluit.frontend',
'regluit.api',
'regluit.core',
'regluit.marc',
'regluit.payment',
'regluit.utils',
'registration',

2
static/css/book_list.css
View File

File diff suppressed because one or more lines are too long

2
static/css/campaign2.css
View File

File diff suppressed because one or more lines are too long

18
static/less/book_list.less
View File

@ -309,7 +309,7 @@ ul.navigation li a:hover, ul.navigation li.active a {
}
div#content-block-content {
padding-bottom: 100px;
padding-bottom: 10px;
}
.listview.panelback, .listview.panelback div {
@ -328,3 +328,19 @@ div#content-block-content {
.nobold {
font-weight: normal;
}
div#libtools {
margin-left: 15px;
margin-bottom: 1em;
border:1px solid @blue-grey;
border-radius: 10px;
padding: 10px;
p {
margin-top: 0px;
}
div {
margin-top: 0px;
margin-left: 2em ;
}
}

21
static/less/campaign2.less
View File

@ -420,3 +420,24 @@ ul.support li {
font-weight: bold;
margin: 20px auto;
}
div#libtools {
border:1px solid @blue-grey;
.one-border-radius(10px);
padding:10px;
margin-left: 0px;
margin-top: 1em;
padding: 10px;
p {
margin-top: 0px;
margin-bottom: 0px;
}
span {
margin-top: 0px;
margin-left: 0.5em ;
display: inline-block;
}
input[type="submit"]{
margin-left: 4em;
}
}

1
urls.py
View File

@ -17,6 +17,7 @@ urlpatterns = patterns('',
url(r'', include('regluit.frontend.urls')),
url(r'', include('regluit.payment.urls')),
url(r'', include('regluit.libraryauth.urls')),
url(r'', include('regluit.marc.urls')),
url(r'^selectable/', include('selectable.urls')),
url(r'^admin/', include(admin_site.urls)),
url(r'^comments/', include('django.contrib.comments.urls')),