regluit/core/validation.py

# encoding: utf-8
'''
methods to validate and clean identifiers
'''
import re
import datetime

from dateutil.parser import parse
from PyPDF2 import PdfFileReader

from django.forms import ValidationError
from django.utils.translation import ugettext_lazy as _

from regluit.pyepub import EPUB
from regluit.mobi import Mobi
from .isbn import ISBN

ID_VALIDATION = {
    'http': (re.compile(r"(https?|ftp)://(-\.)?([^\s/?\.#]+\.?)+(/[^\s]*)?$",
                        flags=re.IGNORECASE|re.S),
             "The Web Address must be a valid http(s) URL."),
    'isbn':  (r'^([\dxX\-–— ]+|delete)$',
              "The ISBN must be a valid ISBN-13."),
    'doab': (r'^(\d{1,6}|delete)$',
             "The value must be 1-6 digits."),
    'gtbg': (r'^(\d{1,6}|delete)$',
             "The Gutenberg number must be 1-6 digits."),
    'doi': (r'^(https?://dx\.doi\.org/|https?://doi\.org/)?(10\.\d+/\S+|delete)$',
            "The DOI value must be a valid DOI."),
    'oclc': (r'^(\d{8,12}|delete)$',
             "The OCLCnum must be 8 or more digits."),
    'goog': (r'^([a-zA-Z0-9\-_]{12}|delete)$',
             "The Google id must be 12 alphanumeric characters, dash or underscore."),
    'gdrd': (r'^(\d{1,8}|delete)$',
             "The Goodreads ID must be 1-8 digits."),
    'thng': (r'(^\d{1,8}|delete)$',
             "The LibraryThing ID must be 1-8 digits."),
    'olwk': (r'^(/works/\)?OLd{1,8}W|delete)$',
             "The Open Library Work ID looks like 'OL####W'."),
    'glue': (r'^(\d{1,6}|delete)$',
             "The Unglue.it ID must be 1-6 digits."),
    'ltwk': (r'^(\d{1,8}|delete)$',
             "The LibraryThing work ID must be 1-8 digits."),
}

def isbn_cleaner(value):
    if value == 'delete':
        return value
    if not value:
        raise ValidationError('no identifier value found')
    elif value == 'delete':
        return value
    isbn = ISBN(value)
    if isbn.error:
        raise ValidationError(isbn.error)
    isbn.validate()
    return isbn.to_string()

def olwk_cleaner(value):
    if not value == 'delete' and value.startswith('/works/'):
        value = '/works/{}'.format(value)
    return value

doi_match = re.compile(r'10\.\d+/\S+')

def doi_cleaner(value):
    if not value == 'delete' and not value.startswith('10.'):
        try:
            return doi_match.search(value).group(0)
        except AttributeError:
            return ''
    return value

ID_MORE_VALIDATION = {
    'isbn': isbn_cleaner,
    'olwk': olwk_cleaner,
    'doi': doi_cleaner,
}

def identifier_cleaner(id_type, quiet=False):
    if ID_VALIDATION.has_key(id_type):
        (regex, err_msg) = ID_VALIDATION[id_type]
        extra = ID_MORE_VALIDATION.get(id_type, None)
        if isinstance(regex, (str, unicode)):
            regex = re.compile(regex)
        def cleaner(value):
            if not value:
                return None
            try:
                if regex.match(value):
                    if extra:
                        value = extra(value)
                    return value
                else:
                    raise ValidationError(err_msg)
            except ValidationError as ve:
                if quiet:
                    return None
                else:
                    raise ve
        return cleaner
    return lambda value: value

def test_file(the_file, fformat):
    if the_file and the_file.name:
        if fformat == 'epub':
            try:
                book = EPUB(the_file.file)
            except Exception as e:
                raise ValidationError(_('Are you sure this is an EPUB file?: %s' % e))
        elif fformat == 'mobi':
            try:
                book = Mobi(the_file.file)
                book.parse()
            except Exception as e:
                raise ValidationError(_('Are you sure this is a MOBI file?: %s' % e))
        elif fformat == 'pdf':
            try:
                PdfFileReader(the_file.file)
            except Exception, e:
                raise ValidationError(_('%s is not a valid PDF file' % the_file.name))
    return True

def valid_xml_char_ordinal(c):
    codepoint = ord(c)
    # conditions ordered by presumed frequency
    return (
        0x20 <= codepoint <= 0xD7FF or
        codepoint in (0x9, 0xA, 0xD) or
        0xE000 <= codepoint <= 0xFFFD or
        0x10000 <= codepoint <= 0x10FFFF
        )

def valid_subject(subject_name):
    num_commas = 0
    for c in subject_name:
        if not valid_xml_char_ordinal(c):
            return False
        if c == ',':
            num_commas += 1
            if num_commas > 2:
                return False
    return True

reverse_name_comma = re.compile(r',(?! *Jr[\., ])')

def unreverse_name(name):
    name = name.strip('.')
    if not reverse_name_comma.search(name):
        return name
    (last, rest) = name.split(',', 1)
    if not ',' in rest:
        return '%s %s' % (rest.strip(), last.strip())
    (first, rest) = rest.split(',', 1)
    return '%s %s, %s' % (first.strip(), last.strip(), rest.strip())

def authlist_cleaner(authlist):
    ''' given a author string or list of author strings, checks that the author string
        is not a list of author names and that no author is repeated'''
    if isinstance(authlist, str):
        authlist = [authlist]
    cleaned = []
    for auth in authlist:
        for cleaned_auth in auth_cleaner(auth):
            if cleaned_auth not in cleaned:
                cleaned.append(cleaned_auth)
    return cleaned

# Match comma but not ", Jr"
comma_list_delim = re.compile(r',(?! *Jr[\., ])')
spaces = re.compile(r'\s+')
_and_ = re.compile(r',? (and|\&) ')
semicolon_list_delim = re.compile(r'[\;|\&]')
reversed_name = re.compile(r'(de |la |los |von |van )*\w+, \w+.?( \w+.?)?(, Jr\.?)?')

def auth_cleaner(auth):
    ''' given a author string checks that the author string
        is not a list of author names'''
    cleaned = []
    if ';' in auth or reversed_name.match(auth):
        authlist = semicolon_list_delim.split(auth)
        authlist = [unreverse_name(name) for name in authlist]
    else:
        auth = _and_.sub(',', auth)
        authlist = comma_list_delim.split(auth)
    for auth in authlist:
        cleaned.append(spaces.sub(' ', auth.strip()))
    return cleaned

MATCHYEAR = re.compile(r'(1|2)\d\d\d')
MATCHYMD = re.compile(r'(1|2)\d\d\d-\d\d-\d\d')

def validate_date(date_string):
    ymd = MATCHYMD.search(date_string)
    if ymd:
        return ymd.group(0)
    try:
        date = parse(date_string.strip(), default=datetime.date(999, 1, 1))
        if date.year != 999:
            return date.strftime('%Y')
    except ValueError:
        year = MATCHYEAR.search(date_string)
        if year:
            return year.group(0)
        return ''