From 17a5f33f0a619be302fb31054b927b9a12d7e592 Mon Sep 17 00:00:00 2001 From: Mike Benowitz Date: Tue, 9 Jul 2019 17:32:22 -0400 Subject: [PATCH 1/5] Initial commit of testing configuration using `pytest` --- .coveragerc | 7 +++ dev-requirements.txt | 4 ++ esIndexer.py | 47 ++++++++------- helpers/config.py | 11 ++++ main.py | 46 ++++++--------- model/elastic.py | 6 +- tests/__init__.py | 0 tests/__pycache__/tmp71s1x3zr | 0 tests/test_es_indexer.py | 107 ++++++++++++++++++++++++++++++++++ tests/test_main_ingest.py | 81 +++++++++++++++++++++++++ 10 files changed, 254 insertions(+), 55 deletions(-) create mode 100644 .coveragerc create mode 100644 dev-requirements.txt create mode 100644 helpers/config.py create mode 100644 tests/__init__.py create mode 100644 tests/__pycache__/tmp71s1x3zr create mode 100644 tests/test_es_indexer.py create mode 100644 tests/test_main_ingest.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..cd0a4a9 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +[run] +omit = tests/* + +[report] + +exclude_lines = + if __name__ == '__main__': \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..6dd0228 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,4 @@ +flake8 +pytest +pytest-cov +pytest-mock \ No newline at end of file diff --git a/esIndexer.py b/esIndexer.py index 5631ca4..1f1fdcd 100644 --- a/esIndexer.py +++ b/esIndexer.py @@ -1,12 +1,8 @@ import os from datetime import datetime -from elasticsearch.helpers import bulk, BulkIndexError, streaming_bulk +from elasticsearch.helpers import BulkIndexError, streaming_bulk from elasticsearch import Elasticsearch -from elasticsearch.exceptions import ( - ConnectionError, - TransportError, - ConflictError -) +from elasticsearch.exceptions import ConnectionError from elasticsearch_dsl import connections from elasticsearch_dsl.wrappers import Range @@ -16,7 +12,7 @@ from sqlalchemy.dialects import postgresql from model.cce import CCE as dbCCE from model.renewal import Renewal as dbRenewal -from model.registration import Registration as dbRegistration +from model.registration import Registration as dbRegistration # noqa: F401 from model.elastic import ( CCE, Registration, @@ -31,7 +27,10 @@ class ESIndexer(): self.ccr_index = os.environ['ES_CCR_INDEX'] self.client = None self.session = manager.session - self.loadFromTime = loadFromTime if loadFromTime else datetime.strptime('1970-01-01', '%Y-%m-%d') + if loadFromTime: + self.loadFromTime = loadFromTime + else: + self.loadFromTime = datetime.strptime('1970-01-01', '%Y-%m-%d') self.createElasticConnection() self.createIndex() @@ -57,7 +56,7 @@ class ESIndexer(): CCE.init() if self.client.indices.exists(index=self.ccr_index) is False: Renewal.init() - + def indexRecords(self, recType='cce'): """Process the current batch of updating records. This utilizes the elasticsearch-py bulk helper to import records in chunks of the @@ -68,14 +67,17 @@ class ESIndexer(): success, failure = 0, 0 errors = [] try: - for status, work in streaming_bulk(self.client, self.process(recType)): + for status, work in streaming_bulk( + self.client, + self.process(recType) + ): print(status, work) if not status: errors.append(work) failure += 1 else: success += 1 - + print('Success {} | Failure: {}'.format(success, failure)) except BulkIndexError as err: print('One or more records in the chunk failed to import') @@ -91,7 +93,8 @@ class ESIndexer(): for ccr in self.retrieveRenewals(): esRen = ESRen(ccr) esRen.indexRen() - if esRen.renewal.rennum == '': continue + if esRen.renewal.rennum == '': + continue yield esRen.renewal.to_dict(True) def retrieveEntries(self): @@ -99,7 +102,7 @@ class ESIndexer(): .filter(dbCCE.date_modified > self.loadFromTime) for cce in retQuery.all(): yield cce - + def retrieveRenewals(self): renQuery = self.session.query(dbRenewal)\ .filter(dbRenewal.date_modified > self.loadFromTime) @@ -110,10 +113,10 @@ class ESIndexer(): class ESDoc(): def __init__(self, cce): self.dbRec = cce - self.entry = None - + self.entry = None + self.initEntry() - + def initEntry(self): print('Creating ES record for {}'.format(self.dbRec)) @@ -122,9 +125,9 @@ class ESDoc(): def indexEntry(self): self.entry.uuid = self.dbRec.uuid self.entry.title = self.dbRec.title - self.entry.authors = [ a.name for a in self.dbRec.authors ] - self.entry.publishers = [ p.name for p in self.dbRec.publishers ] - self.entry.lccns = [ l.lccn for l in self.dbRec.lccns ] + self.entry.authors = [a.name for a in self.dbRec.authors] + self.entry.publishers = [p.name for p in self.dbRec.publishers] + self.entry.lccns = [l.lccn for l in self.dbRec.lccns] self.entry.registrations = [ Registration(regnum=r.regnum, regdate=r.reg_date) for r in self.dbRec.registrations @@ -134,10 +137,10 @@ class ESDoc(): class ESRen(): def __init__(self, ccr): self.dbRen = ccr - self.renewal = None - + self.renewal = None + self.initRenewal() - + def initRenewal(self): print('Creating ES record for {}'.format(self.dbRen)) diff --git a/helpers/config.py b/helpers/config.py new file mode 100644 index 0000000..7f58beb --- /dev/null +++ b/helpers/config.py @@ -0,0 +1,11 @@ +import os +import yaml + + +def loadConfig(): + with open('config.yaml', 'r') as yamlFile: + config = yaml.safe_load(yamlFile) + for section in config: + sectionDict = config[section] + for key, value in sectionDict.items(): + os.environ[key] = value diff --git a/main.py b/main.py index 733c4b8..0f608f2 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,11 @@ import argparse from datetime import datetime, timedelta -import os -import yaml + +from sessionManager import SessionManager +from builder import CCEReader +from renBuilder import CCRReader +from esIndexer import ESIndexer +from helpers.config import loadConfig def main(secondsAgo=None, year=None, exclude=None, reinit=False): @@ -19,26 +23,27 @@ def main(secondsAgo=None, year=None, exclude=None, reinit=False): loadCCE(manager, loadFromTime, year) if exclude != 'ccr': loadCCR(manager, loadFromTime, year) - + indexUpdates(manager, loadFromTime) - + manager.closeConnection() - + def loadCCE(manager, loadFromTime, selectedYear): cceReader = CCEReader(manager) cceReader.loadYears(selectedYear) cceReader.getYearFiles(loadFromTime) cceReader.importYearData() - + def loadCCR(manager, loadFromTime, selectedYear): ccrReader = CCRReader(manager) ccrReader.loadYears(selectedYear, loadFromTime) ccrReader.importYears() + def indexUpdates(manager, loadFromTime): - esIndexer = ESIndexer(manager, None) + esIndexer = ESIndexer(manager, loadFromTime) esIndexer.indexRecords(recType='cce') esIndexer.indexRecords(recType='ccr') @@ -48,28 +53,16 @@ def parseArgs(): description='Load CCE XML and CCR TSV into PostgresQL' ) parser.add_argument('-t', '--time', type=int, required=False, - help='Time ago in seconds to check for file updates' - ) + help='Time ago in seconds to check for file updates') parser.add_argument('-y', '--year', type=str, required=False, - help='Specific year to load CCE entries and/or renewals from' - ) + help='Specific year to load CCE entries and/or renewals from') parser.add_argument('-x', '--exclude', type=str, required=False, - choices=['cce', 'ccr'], - help='Specify to exclude either entries or renewals from this run' - ) + choices=['cce', 'ccr'], + help='Specify to exclude either entries or renewals from this run') parser.add_argument('--REINITIALIZE', action='store_true') return parser.parse_args() -def loadConfig(): - with open('config.yaml', 'r') as yamlFile: - config = yaml.safe_load(yamlFile) - for section in config: - sectionDict = config[section] - for key, value in sectionDict.items(): - os.environ[key] = value - - if __name__ == '__main__': args = parseArgs() try: @@ -77,14 +70,9 @@ if __name__ == '__main__': except FileNotFoundError: pass - from sessionManager import SessionManager - from builder import CCEReader, CCEFile - from renBuilder import CCRReader, CCRFile - from esIndexer import ESIndexer - main( secondsAgo=args.time, year=args.year, exclude=args.exclude, reinit=args.REINITIALIZE - ) \ No newline at end of file + ) diff --git a/model/elastic.py b/model/elastic.py index 559f9f0..b458433 100644 --- a/model/elastic.py +++ b/model/elastic.py @@ -1,7 +1,5 @@ import os -import yaml from elasticsearch_dsl import ( - Index, Document, Keyword, Text, @@ -45,7 +43,7 @@ class Renewal(BaseDoc): claimants = Nested(Claimant) class Index: - name = os.environ['ES_CCR_INDEX'] + name = os.environ.get('ES_CCR_INDEX', None) class CCE(BaseDoc): @@ -58,4 +56,4 @@ class CCE(BaseDoc): registrations = Nested(Registration) class Index: - name = os.environ['ES_CCE_INDEX'] + name = os.environ.get('ES_CCE_INDEX', None) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__pycache__/tmp71s1x3zr b/tests/__pycache__/tmp71s1x3zr new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_es_indexer.py b/tests/test_es_indexer.py new file mode 100644 index 0000000..957d16d --- /dev/null +++ b/tests/test_es_indexer.py @@ -0,0 +1,107 @@ +from datetime import datetime +from unittest.mock import MagicMock, call +import pytest +from elasticsearch.exceptions import ConnectionError + +from esIndexer import ESIndexer, ESDoc, ESRen + + +class TestIndexer(object): + @pytest.fixture + def setEnvVars(self, mocker): + mocker.patch.dict('os.environ', { + 'ES_CCE_INDEX': 'test_cce', + 'ES_CCR_INDEX': 'test_ccr', + 'ES_HOST': 'test', + 'ES_PORT': '9999', + 'ES_TIMEOUT': '0' + }) + + def test_indexerInit(self, mocker, setEnvVars): + mockConfig = mocker.patch('esIndexer.configure_mappers') + mockConn = mocker.patch('esIndexer.ESIndexer.createElasticConnection') + mockCreate = mocker.patch('esIndexer.ESIndexer.createIndex') + mockManager = MagicMock() + mockManager.session = 'session' + + testIndexer = ESIndexer(mockManager, 10) + + assert testIndexer.cce_index == 'test_cce' + assert testIndexer.ccr_index == 'test_ccr' + assert testIndexer.session == 'session' + assert mockConn.called + assert mockCreate.called + assert mockConfig.called + + def test_indexerInit_no_time(self, mocker, setEnvVars): + mockConfig = mocker.patch('esIndexer.configure_mappers') + mockConn = mocker.patch('esIndexer.ESIndexer.createElasticConnection') + mockCreate = mocker.patch('esIndexer.ESIndexer.createIndex') + mockManager = MagicMock() + mockManager.session = 'session' + + testIndexer = ESIndexer(mockManager, None) + + assert testIndexer.cce_index == 'test_cce' + assert testIndexer.ccr_index == 'test_ccr' + assert testIndexer.session == 'session' + assert testIndexer.loadFromTime == datetime(1970, 1, 1) + assert mockConn.called + assert mockCreate.called + assert mockConfig.called + + def test_elastic_connection(self, mocker, setEnvVars): + mockConfig = mocker.patch('esIndexer.configure_mappers') + mockCreate = mocker.patch('esIndexer.ESIndexer.createIndex') + mockManager = MagicMock() + mockManager.session = 'session' + + mockElastic = mocker.patch('esIndexer.Elasticsearch') + mockElastic.return_value = 'test_client' + mocker.patch('esIndexer.connections') + + testIndexer = ESIndexer(mockManager, 10) + + assert testIndexer.cce_index == 'test_cce' + assert testIndexer.ccr_index == 'test_ccr' + assert testIndexer.session == 'session' + assert testIndexer.client == 'test_client' + assert mockCreate.called + assert mockConfig.called + assert mockElastic.called_once_with( + hosts=[{'host': 'test', 'port': '9999'}], + timeout='0' + ) + + def test_elastic_conn_err(self, mocker, setEnvVars): + mocker.patch('esIndexer.configure_mappers') + mocker.patch('esIndexer.ESIndexer.createIndex') + mockManager = MagicMock() + mockManager.session = 'session' + + mockElastic = mocker.patch('esIndexer.Elasticsearch') + mockElastic.side_effect = ConnectionError + mocker.patch('esIndexer.connections') + with pytest.raises(ConnectionError): + ESIndexer(mockManager, 10) + + #def test_index_create(self, mocker, monkeypatch, setEnvVars): + # mockConfig = mocker.patch('esIndexer.configure_mappers') + # mockConn = mocker.patch('esIndexer.ESIndexer.createElasticConnection') + # mockCCE = mocker.patch('esIndexer.CCE') + # mockCCR = mocker.patch('esIndexer.Renewal') + # mockManager = MagicMock() + # mockManager.session = 'session' + + # mockClient = MagicMock() + # mockClient.indices.exists.side_effect = [False, False] + + # testIndexer = ESIndexer(mockManager, 10) + + # assert testIndexer.cce_index == 'test_cce' + # assert testIndexer.ccr_index == 'test_ccr' + # assert testIndexer.session == 'session' + # assert mockConn.called + # assert mockConfig.called + # assert mockCCE.init.called + # assert mockCCR.init.called diff --git a/tests/test_main_ingest.py b/tests/test_main_ingest.py new file mode 100644 index 0000000..49af6e9 --- /dev/null +++ b/tests/test_main_ingest.py @@ -0,0 +1,81 @@ +import sys +from unittest.mock import MagicMock, call +from main import main, loadCCE, loadCCR, indexUpdates, parseArgs + + +class TestHandler(object): + def test_main_plain(self, mocker): + mockSession = mocker.patch('main.SessionManager') + mockLoadCCE = mocker.patch('main.loadCCE') + mockLoadCCR = mocker.patch('main.loadCCR') + mockIndex = mocker.patch('main.indexUpdates') + + main() + + assert mockSession.called + assert mockLoadCCE.called + assert mockLoadCCR.called + assert mockIndex.called + + def test_main_args(self, mocker): + mockSession = mocker.patch('main.SessionManager') + mockLoadCCE = mocker.patch('main.loadCCE') + mockLoadCCR = mocker.patch('main.loadCCR') + mockIndex = mocker.patch('main.indexUpdates') + + main( + secondsAgo=10, + year=1900, + exclude='ccr', + reinit=True + ) + + assert mockSession.called + assert mockLoadCCE.called + assert mockLoadCCR.not_called + assert mockIndex.called + + def test_cce_load(self, mocker): + mockReader = mocker.patch('main.CCEReader') + mockCCE = MagicMock() + mockReader.return_value = mockCCE + + loadCCE('manager', 10, None) + + assert mockReader.called_once_with('manager') + assert mockCCE.loadYears.called_once_with(None) + assert mockCCE.getYearFiles.called_once_with(10) + assert mockCCE.importYearData.called + + def test_ccr_load(self, mocker): + mockReader = mocker.patch('main.CCRReader') + mockCCR = MagicMock() + mockReader.return_value = mockCCR + + loadCCR('manager', 10, None) + + assert mockReader.called_once_with('manager') + assert mockCCR.loadYears.called_once_with(None, 10) + assert mockCCR.importYears.called + + def test_indexer(self, mocker): + mockIndexer = mocker.patch('main.ESIndexer') + mockInd = MagicMock() + mockIndexer.return_value = mockInd + + indexUpdates('manager', 10) + + assert mockIndexer.called_once_with('manager', 10) + assert mockInd.indexRecords.mock_calls == \ + [call(recType='cce'), call(recType='ccr')] + + def test_parseArgs_success(self, mocker): + args = ['main', '--time', '10', '--year', '1900', '--exclude', 'ccr'] + mocker.patch.object(sys, 'argv', args) + + args = parseArgs() + + assert int(args.time) == 10 + assert int(args.year) == 1900 + assert args.exclude == 'ccr' + assert args.REINITIALIZE is False From 0f8d50ad4d537e220dde558c735d651f97f79790 Mon Sep 17 00:00:00 2001 From: Mike Benowitz Date: Wed, 10 Jul 2019 17:27:42 -0400 Subject: [PATCH 2/5] Further test additions and linting fixes --- .coveragerc | 2 +- esIndexer.py | 24 ++--- model/author.py | 9 +- model/cce.py | 63 ++++++----- tests/test_es_indexer.py | 199 ++++++++++++++++++++++++++++++----- tests/test_helpers_config.py | 20 ++++ tests/test_helpers_errors.py | 8 ++ tests/test_model_author.py | 10 ++ tests/test_model_cce.py | 58 ++++++++++ 9 files changed, 316 insertions(+), 77 deletions(-) create mode 100644 tests/test_helpers_config.py create mode 100644 tests/test_helpers_errors.py create mode 100644 tests/test_model_author.py create mode 100644 tests/test_model_cce.py diff --git a/.coveragerc b/.coveragerc index cd0a4a9..08f0252 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,5 @@ [run] -omit = tests/* +omit = tests/*, */__init__.py [report] diff --git a/esIndexer.py b/esIndexer.py index 1f1fdcd..8ec24e9 100644 --- a/esIndexer.py +++ b/esIndexer.py @@ -25,14 +25,13 @@ class ESIndexer(): def __init__(self, manager, loadFromTime): self.cce_index = os.environ['ES_CCE_INDEX'] self.ccr_index = os.environ['ES_CCR_INDEX'] - self.client = None + self.client = self.createElasticConnection() self.session = manager.session if loadFromTime: self.loadFromTime = loadFromTime else: self.loadFromTime = datetime.strptime('1970-01-01', '%Y-%m-%d') - self.createElasticConnection() self.createIndex() configure_mappers() @@ -42,14 +41,16 @@ class ESIndexer(): port = os.environ['ES_PORT'] timeout = int(os.environ['ES_TIMEOUT']) try: - self.client = Elasticsearch( + client = Elasticsearch( hosts=[{'host': host, 'port': port}], timeout=timeout ) except ConnectionError as err: print('Failed to connect to ElasticSearch instance') raise err - connections.connections._conns['default'] = self.client + connections.connections._conns['default'] = client + + return client def createIndex(self): if self.client.indices.exists(index=self.cce_index) is False: @@ -71,7 +72,6 @@ class ESIndexer(): self.client, self.process(recType) ): - print(status, work) if not status: errors.append(work) failure += 1 @@ -113,14 +113,11 @@ class ESIndexer(): class ESDoc(): def __init__(self, cce): self.dbRec = cce - self.entry = None - - self.initEntry() + self.entry = self.initEntry() def initEntry(self): print('Creating ES record for {}'.format(self.dbRec)) - - self.entry = CCE(meta={'id': self.dbRec.uuid}) + return CCE(meta={'id': self.dbRec.uuid}) def indexEntry(self): self.entry.uuid = self.dbRec.uuid @@ -137,14 +134,11 @@ class ESDoc(): class ESRen(): def __init__(self, ccr): self.dbRen = ccr - self.renewal = None - - self.initRenewal() + self.renewal = self.initRenewal() def initRenewal(self): print('Creating ES record for {}'.format(self.dbRen)) - - self.renewal = Renewal(meta={'id': self.dbRen.renewal_num}) + return Renewal(meta={'id': self.dbRen.renewal_num}) def indexRen(self): self.renewal.uuid = self.dbRen.uuid diff --git a/model/author.py b/model/author.py index 3d5dedf..8f6daca 100644 --- a/model/author.py +++ b/model/author.py @@ -1,18 +1,11 @@ from sqlalchemy import ( Column, - Date, ForeignKey, Integer, - String, Unicode, Boolean ) -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import relationship, backref -from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.orm.exc import NoResultFound - from model.core import Base, Core @@ -25,4 +18,4 @@ class Author(Core, Base): cce_id = Column(Integer, ForeignKey('cce.id'), index=True) def __repr__(self): - return ''.format(self.name, self.primary) \ No newline at end of file + return ''.format(self.name, self.primary) diff --git a/model/cce.py b/model/cce.py index ca36304..1bd3397 100644 --- a/model/cce.py +++ b/model/cce.py @@ -1,19 +1,15 @@ from lxml import etree -import uuid from sqlalchemy import ( Column, Date, ForeignKey, Integer, - String, Unicode, Boolean ) from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import relationship, backref -from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.orm import relationship from model.core import Base, Core @@ -45,22 +41,29 @@ class CCE(Core, Base): registrations = relationship('Registration', backref='cce') lccns = relationship('LCCN', backref='cce', cascade='all, delete-orphan') - authors = relationship('Author', backref='cce', cascade='all, delete-orphan') - publishers = relationship('Publisher', backref='cce', cascade='all, delete-orphan') + authors = relationship('Author', + backref='cce', cascade='all, delete-orphan') + publishers = relationship('Publisher', + backref='cce', cascade='all, delete-orphan') def __repr__(self): - return ''.format(self.registrations, self.uuid, self.title) + return ''.format( + self.registrations, + self.uuid, + self.title + ) - def addRelationships(self, volume, xml, lccn=[], authors=[], publishers=[], registrations=[]): + def addRelationships(self, volume, xml, + lccn=[], authors=[], publishers=[], registrations=[]): self.volume = volume self.addLCCN(lccn) self.addAuthor(authors) self.addPublisher(publishers) self.addRegistration(registrations) self.addXML(xml) - + def addLCCN(self, lccns): - self.lccns = [ LCCN(lccn=lccn) for lccn in lccns ] + self.lccns = [LCCN(lccn=lccn) for lccn in lccns] def addXML(self, xml): xmlString = etree.tostring(xml, encoding='utf-8').decode() @@ -72,7 +75,7 @@ class CCE(Core, Base): print('No author name! for {}'.format(self.uuid)) continue self.authors.append(Author(name=auth[0], primary=auth[1])) - + def addPublisher(self, publishers): for pub in publishers: if pub[0] is None: @@ -80,7 +83,7 @@ class CCE(Core, Base): continue claimant = True if pub[1] == 'yes' else False self.publishers.append(Publisher(name=pub[0], claimant=claimant)) - + def addRegistration(self, registrations): self.registrations = [ Registration( @@ -91,16 +94,18 @@ class CCE(Core, Base): ) for reg in registrations ] - - def updateRelationships(self, xml, lccn=[], authors=[], publishers=[], registrations=[]): + + def updateRelationships(self, xml, + lccn=[], authors=[], + publishers=[], registrations=[]): self.addXML(xml) self.updateLCCN(lccn) self.updateAuthors(authors) self.updatePublishers(publishers) self.updateRegistrations(registrations) - + def updateLCCN(self, lccns): - currentLCCNs = [ l.lccn for l in self.lccns ] + currentLCCNs = [l.lccn for l in self.lccns] if lccns != currentLCCNs: self.lccns = [ l for l in self.lccns @@ -108,9 +113,9 @@ class CCE(Core, Base): ] for new in list(set(lccns) - set(currentLCCNs)): self.lccns.append(LCCN(lccn=new)) - + def updateAuthors(self, authors): - currentAuthors = [ (a.name, a.primary) for a in self.authors ] + currentAuthors = [(a.name, a.primary) for a in self.authors] newAuthors = filter(lambda x: x[0] is None, authors) if newAuthors != currentAuthors: self.authors = [ @@ -119,9 +124,9 @@ class CCE(Core, Base): ] for new in list(set(newAuthors) - set(currentAuthors)): self.authors.append(Author(name=new[0], primary=new[1])) - + def updatePublishers(self, publishers): - currentPublishers = [ (a.name, a.claimant) for a in self.publishers ] + currentPublishers = [(a.name, a.claimant) for a in self.publishers] newPublishers = [ (p[0], True if p[1] == 'yes' else False) for p in filter(lambda x: x[0] is None, publishers) @@ -133,15 +138,15 @@ class CCE(Core, Base): ] for new in list(set(newPublishers) - set(currentPublishers)): self.publishers.append(Publisher(name=new[0], claimant=new[1])) - + def updateRegistrations(self, registrations): existingRegs = [ self.updateReg(r, registrations) for r in self.registrations - if r.regnum in [ n['regnum'] for n in registrations ] + if r.regnum in [n['regnum'] for n in registrations] ] newRegs = [ r for r in registrations - if r['regnum'] not in [ n.regnum for n in existingRegs ] + if r['regnum'] not in [n.regnum for n in existingRegs] ] self.registrations = existingRegs + [ Registration( @@ -152,16 +157,18 @@ class CCE(Core, Base): ) for reg in newRegs ] - + def updateReg(self, reg, registrations): newReg = CCE.getReg(reg.regnum, registrations) - if newReg: reg.update(newReg) + if newReg: + reg.update(newReg) return reg - + def setParentCCE(self, parentID): self.parent_cce_id = parentID @staticmethod def getReg(regnum, newRegs): for new in newRegs: - if regnum == new['regnum']: return new + if regnum == new['regnum']: + return new diff --git a/tests/test_es_indexer.py b/tests/test_es_indexer.py index 957d16d..67c47db 100644 --- a/tests/test_es_indexer.py +++ b/tests/test_es_indexer.py @@ -1,6 +1,7 @@ from datetime import datetime from unittest.mock import MagicMock, call import pytest +from elasticsearch.helpers import BulkIndexError from elasticsearch.exceptions import ConnectionError from esIndexer import ESIndexer, ESDoc, ESRen @@ -17,21 +18,20 @@ class TestIndexer(object): 'ES_TIMEOUT': '0' }) - def test_indexerInit(self, mocker, setEnvVars): - mockConfig = mocker.patch('esIndexer.configure_mappers') - mockConn = mocker.patch('esIndexer.ESIndexer.createElasticConnection') - mockCreate = mocker.patch('esIndexer.ESIndexer.createIndex') + @pytest.fixture + def testIndexer(self, mocker, setEnvVars): + mocker.patch('esIndexer.configure_mappers') + mocker.patch('esIndexer.ESIndexer.createElasticConnection') + mocker.patch('esIndexer.ESIndexer.createIndex') mockManager = MagicMock() mockManager.session = 'session' - testIndexer = ESIndexer(mockManager, 10) + return ESIndexer(mockManager, 10) + def test_indexerInit(self, mocker, testIndexer): assert testIndexer.cce_index == 'test_cce' assert testIndexer.ccr_index == 'test_ccr' assert testIndexer.session == 'session' - assert mockConn.called - assert mockCreate.called - assert mockConfig.called def test_indexerInit_no_time(self, mocker, setEnvVars): mockConfig = mocker.patch('esIndexer.configure_mappers') @@ -85,23 +85,172 @@ class TestIndexer(object): with pytest.raises(ConnectionError): ESIndexer(mockManager, 10) - #def test_index_create(self, mocker, monkeypatch, setEnvVars): - # mockConfig = mocker.patch('esIndexer.configure_mappers') - # mockConn = mocker.patch('esIndexer.ESIndexer.createElasticConnection') - # mockCCE = mocker.patch('esIndexer.CCE') - # mockCCR = mocker.patch('esIndexer.Renewal') - # mockManager = MagicMock() - # mockManager.session = 'session' + def test_index_create(self, mocker, setEnvVars): + mockConfig = mocker.patch('esIndexer.configure_mappers') + mockClient = MagicMock() + mockClient.indices.exists.side_effect = [False, False] + mockConn = mocker.patch('esIndexer.ESIndexer.createElasticConnection') + mockConn.return_value = mockClient + mockCCE = mocker.patch('esIndexer.CCE') + mockCCR = mocker.patch('esIndexer.Renewal') + mockManager = MagicMock() + mockManager.session = 'session' - # mockClient = MagicMock() - # mockClient.indices.exists.side_effect = [False, False] + testIndexer = ESIndexer(mockManager, 10) - # testIndexer = ESIndexer(mockManager, 10) + assert testIndexer.cce_index == 'test_cce' + assert testIndexer.ccr_index == 'test_ccr' + assert testIndexer.session == 'session' + assert mockConn.called + assert mockConfig.called + assert mockCCE.init.called + assert mockCCR.init.called - # assert testIndexer.cce_index == 'test_cce' - # assert testIndexer.ccr_index == 'test_ccr' - # assert testIndexer.session == 'session' - # assert mockConn.called - # assert mockConfig.called - # assert mockCCE.init.called - # assert mockCCR.init.called + def test_bulk_index_success(self, mocker, testIndexer): + mockProcess = mocker.patch('esIndexer.ESIndexer.process') + mockProcess.return_value = ['test1', 'test2', 'test3'] + mockStreaming = mocker.patch('esIndexer.streaming_bulk') + mockStreaming.return_value = [ + (True, 'test1'), + (False, 'test2'), + (True, 'test3') + ] + + testIndexer.indexRecords() + + assert mockProcess.called + assert mockStreaming.called + + def test_bulk_index_failure(self, mocker, testIndexer): + mockProcess = mocker.patch('esIndexer.ESIndexer.process') + mockProcess.return_value = ['test1', 'test2', 'test3'] + mockStreaming = mocker.patch('esIndexer.streaming_bulk') + mockStreaming.side_effect = BulkIndexError + + with pytest.raises(BulkIndexError): + testIndexer.indexRecords() + + def test_process_cce(self, mocker, testIndexer): + mockRetrieve = mocker.patch('esIndexer.ESIndexer.retrieveEntries') + mockRetrieve.return_value = ['test1', 'test2', 'test3'] + mockDoc = mocker.patch('esIndexer.ESDoc') + mockDoc().entry.to_dict.side_effect = ['test1', 'test2', 'test3'] + + processed = [p for p in testIndexer.process('cce')] + assert processed[0] == 'test1' + + def test_process_ccr(self, mocker, testIndexer): + mockRens = [] + for i in range(1, 4): + tmpRen = MagicMock() + tmpRen.renewal.rennum = '' if i == 1 else i + tmpRen.renewal.to_dict.return_value = 'test{}'.format(str(i)) + mockRens.append(tmpRen) + + mockRetrieve = mocker.patch('esIndexer.ESIndexer.retrieveRenewals') + mockRetrieve.return_value = ['test1', 'test2', 'test3'] + mockDoc = mocker.patch('esIndexer.ESRen') + mockDoc.side_effect = mockRens + + processed = [p for p in testIndexer.process('ccr')] + assert processed[0] == 'test2' + + def test_retrieveEntries(self, mocker, testIndexer): + mockSession = MagicMock() + mockAll = MagicMock() + mockAll.all.return_value = ['cce1', 'cce2', 'cce3'] + mockSession.query().filter.return_value = mockAll + testIndexer.session = mockSession + + entries = [e for e in testIndexer.retrieveEntries()] + + assert entries[1] == 'cce2' + + def test_retrieveRenewals(self, mocker, testIndexer): + mockSession = MagicMock() + mockAll = MagicMock() + mockAll.all.return_value = ['ccr1', 'ccr2', 'ccr3'] + mockSession.query().filter.return_value = mockAll + testIndexer.session = mockSession + + renewals = [r for r in testIndexer.retrieveRenewals()] + + assert renewals[2] == 'ccr3' + + +class TestESDoc(object): + def test_ESDocInit(self, mocker): + mockInit = mocker.patch('esIndexer.ESDoc.initEntry') + mockInit.return_value = 'testCCE' + + testDoc = ESDoc('testRec') + + assert testDoc.dbRec == 'testRec' + assert testDoc.entry == 'testCCE' + + def test_ESDocCreateEntry(self): + mockRec = MagicMock() + mockRec.uuid = 'testUUID' + + testDoc = ESDoc(mockRec) + + assert testDoc.entry.meta.id == 'testUUID' + + def test_esDoc_index(self, mocker): + mockEntry = MagicMock() + mockInit = mocker.patch('esIndexer.ESDoc.initEntry') + mockInit.return_value = mockEntry + + mockDB = MagicMock() + mockDB.uuid = 'testUUID' + mockDB.title = 'Test Title' + + mockReg = MagicMock() + mockReg.regnum = 'T0000' + mockReg.reg_date = '1999-12-31' + mockDB.registrations = [mockReg] + + testDoc = ESDoc(mockDB) + testDoc.indexEntry() + + assert testDoc.entry.uuid == 'testUUID' + assert testDoc.entry.registrations[0].regnum == 'T0000' + + +class TestESDen(object): + def test_ESRenInit(self, mocker): + mockInit = mocker.patch('esIndexer.ESRen.initRenewal') + mockInit.return_value = 'testCCR' + + testDoc = ESRen('testRen') + + assert testDoc.dbRen == 'testRen' + assert testDoc.renewal == 'testCCR' + + def test_ESRenCreateRenewal(self): + mockRec = MagicMock() + mockRec.renewal_num = 'testRennum' + + testRen = ESRen(mockRec) + + assert testRen.renewal.meta.id == 'testRennum' + + def test_esRen_index(self, mocker): + mockRenewal = MagicMock() + mockInit = mocker.patch('esIndexer.ESRen.initRenewal') + mockInit.return_value = mockRenewal + + mockDB = MagicMock() + mockDB.uuid = 'testUUID' + mockDB.renewal_num = 'R0000' + + mockCla = MagicMock() + mockCla.name = 'Test Claimant' + mockCla.claimant_type = 'T' + mockDB.claimants = [mockCla] + + testRen = ESRen(mockDB) + testRen.indexRen() + + assert testRen.renewal.uuid == 'testUUID' + assert testRen.renewal.claimants[0].claim_type == 'T' diff --git a/tests/test_helpers_config.py b/tests/test_helpers_config.py new file mode 100644 index 0000000..05dce29 --- /dev/null +++ b/tests/test_helpers_config.py @@ -0,0 +1,20 @@ +import os +from unittest.mock import mock_open, patch +from helpers.config import loadConfig + + +class TestConfigHelpers(object): + def test_config_loader(self): + testYAMLText = """ + TESTING: + FIRST: STRING1 + SECOND: '10' + + EXTRA: + VALUE: SOMETHING""" + mockOpen = mock_open(read_data=testYAMLText) + with patch('helpers.config.open', mockOpen): + loadConfig() + assert os.environ['FIRST'] == 'STRING1' + assert os.environ['SECOND'] == '10' + assert os.environ['VALUE'] == 'SOMETHING' diff --git a/tests/test_helpers_errors.py b/tests/test_helpers_errors.py new file mode 100644 index 0000000..05f9e74 --- /dev/null +++ b/tests/test_helpers_errors.py @@ -0,0 +1,8 @@ +from helpers.errors import DataError + + +class TestErrorHelpers(object): + def test_create_DataError(self): + newDataErr = DataError('testing', source='pytest') + assert newDataErr.message == 'testing' + assert newDataErr.source == 'pytest' diff --git a/tests/test_model_author.py b/tests/test_model_author.py new file mode 100644 index 0000000..ab3e3fa --- /dev/null +++ b/tests/test_model_author.py @@ -0,0 +1,10 @@ +from model.author import Author + + +class TestModelAuthor(object): + def test_authorCreate(self): + testAuthor = Author() + testAuthor.name = 'Tester' + testAuthor.primary = True + + assert str(testAuthor) == '' diff --git a/tests/test_model_cce.py b/tests/test_model_cce.py new file mode 100644 index 0000000..506be9d --- /dev/null +++ b/tests/test_model_cce.py @@ -0,0 +1,58 @@ +from unittest.mock import MagicMock, DEFAULT +import pytest + +from model.cce import CCE + + +class TestModelCCE(object): + @pytest.fixture + def mockCCE(self): + return CCE() + + def test_cceCreate(self, mockCCE): + mockCCE.uuid = 'testUUID' + mockCCE.title = 'Testing' + mockReg = MagicMock() + mockCCE.registrations = [mockReg] + + assert str(mockCCE) == ''.format( + str(mockReg), 'testUUID', 'Testing' + ) + + def test_addRelationships(self, mocker, mockCCE): + addMocks = mocker.patch.multiple('model.cce.CCE', addLCCN=DEFAULT, + addAuthor=DEFAULT, + addPublisher=DEFAULT, + addRegistration=DEFAULT, + addXML=DEFAULT) + + mockCCE = CCE() + mockVol = MagicMock() + mockVol.name = 'testVol' + mockCCE.addRelationships( + mockVol, + '', + lccn=[1, 2, 3], + authors=['author1'], + publishers=['pub1'], + registrations=['reg1'] + ) + + assert mockCCE.volume.name == 'testVol' + assert addMocks['addAuthor'].called_once_with(['author1']) + assert addMocks['addPublisher'].called_once_with(['pub1']) + assert addMocks['addRegistration'].called_once_with(['reg1']) + assert addMocks['addLCCN'].called_once_with([1, 2, 3]) + + def test_addLCCN(self, mocker, mockCCE): + mockLCCN = mocker.patch('model.cce.LCCN') + mockLCs = [] + for i in range(1, 3): + lcMock = MagicMock() + lcMock.name = 'lccn{}'.format(i) + mockLCs.append(lcMock) + mockLCCN.side_effect = mockLCs + + mockCCE.addLCCN([1, 2]) + + assert mockCCE.lccns[1].name == 'lccn2' From 4b1c0b5f1dbe96926d4177476d013e50de330e7c Mon Sep 17 00:00:00 2001 From: Mike Benowitz Date: Mon, 5 Aug 2019 14:40:32 -0400 Subject: [PATCH 3/5] Fix missing index issue for indexing updates --- esIndexer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esIndexer.py b/esIndexer.py index 8ec24e9..78c8ba1 100644 --- a/esIndexer.py +++ b/esIndexer.py @@ -117,7 +117,7 @@ class ESDoc(): def initEntry(self): print('Creating ES record for {}'.format(self.dbRec)) - return CCE(meta={'id': self.dbRec.uuid}) + return CCE(meta={'id': self.dbRec.uuid, 'index': 'cce'}) def indexEntry(self): self.entry.uuid = self.dbRec.uuid @@ -138,7 +138,7 @@ class ESRen(): def initRenewal(self): print('Creating ES record for {}'.format(self.dbRen)) - return Renewal(meta={'id': self.dbRen.renewal_num}) + return Renewal(meta={'id': self.dbRen.renewal_num, 'index': 'ccr'}) def indexRen(self): self.renewal.uuid = self.dbRen.uuid From 0a8fb1cde4ceb33ecdbf4a2dc2eb1fc0e42e9497 Mon Sep 17 00:00:00 2001 From: Mike Benowitz Date: Mon, 5 Aug 2019 14:41:27 -0400 Subject: [PATCH 4/5] Fix issues with delete-cascade The important tables `xml` and `registration` were not properly set for their `CASCADE` behavior, in addiiton `XML` needed to have the `single_parent` option enabled to allow for cascading-deletes (since otherwise a single entry could be referenced by an entry and a error. --- model/cce.py | 6 +++++- model/xml.py | 15 +++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/model/cce.py b/model/cce.py index 1bd3397..639087f 100644 --- a/model/cce.py +++ b/model/cce.py @@ -39,7 +39,11 @@ class CCE(Core, Base): volume_id = Column(Integer, ForeignKey('volume.id')) - registrations = relationship('Registration', backref='cce') + registrations = relationship( + 'Registration', + backref='cce', + cascade='all, delete-orphan' + ) lccns = relationship('LCCN', backref='cce', cascade='all, delete-orphan') authors = relationship('Author', backref='cce', cascade='all, delete-orphan') diff --git a/model/xml.py b/model/xml.py index 8697ac7..fc2a44b 100644 --- a/model/xml.py +++ b/model/xml.py @@ -14,9 +14,11 @@ from sqlalchemy import ( from model.core import Base, Core + @compiles(String, 'postgresql') def compile_xml(type_, compiler, **kw): - return "XML" + return 'XML' + ENTRY_XML = Table( 'entry_xml', @@ -32,19 +34,24 @@ ERROR_XML = Table( Column('xml_id', Integer, ForeignKey('xml.id'), index=True) ) + class XML(Core, Base): __tablename__ = 'xml' id = Column(Integer, primary_key=True) xml_source = Column(String) - + entry = relationship( 'CCE', secondary=ENTRY_XML, - backref='xml_sources' + backref='xml_sources', + single_parent=True, + cascade='all, delete-orphan' ) error_entry = relationship( 'ErrorCCE', secondary=ERROR_XML, - backref='xml_sources' + backref='xml_sources', + single_parent=True, + cascade='all, delete-orphan' ) From 73834ff63f6f68ee7fc00060d5b26faf8ed421a2 Mon Sep 17 00:00:00 2001 From: Mike Benowitz Date: Mon, 5 Aug 2019 15:20:27 -0400 Subject: [PATCH 5/5] Add error handling for 404 and 500 results from PostgreSQL --- api/prints/swagger/swag.py | 585 ++++++++++++++++++++----------------- api/prints/uuid.py | 55 ++-- helpers/errors.py | 9 +- 3 files changed, 354 insertions(+), 295 deletions(-) diff --git a/api/prints/swagger/swag.py b/api/prints/swagger/swag.py index d044e2c..87318b3 100644 --- a/api/prints/swagger/swag.py +++ b/api/prints/swagger/swag.py @@ -5,405 +5,440 @@ class SwaggerDoc(): def getDocs(self): return { - "swagger": "2.0", - "info": { - "title": "CCE Search", - "description": "API for searching Copyright Registrations and Renewals", - "contact": { - "responsibleOrganization": "NYPL", - "responsibleDeveloper": "Michael Benowitz", - "email": "michaelbenowitz@nypl.org", - "url": "www.nypl.org", + 'swagger': '2.0', + 'info': { + 'title': 'CCE Search', + 'description': 'API for searching Copyright Registrations and Renewals', + 'contact': { + 'responsibleOrganization': 'NYPL', + 'responsibleDeveloper': 'Michael Benowitz', + 'email': 'michaelbenowitz@nypl.org', + 'url': 'www.nypl.org', }, - "version": "v0.1" + 'version': 'v0.1' }, - "basePath": "/", # base bash for blueprint registration - "schemes": [ - "http", - "https" + 'basePath': '/', # base bash for blueprint registration + 'schemes': [ + 'http', + 'https' ], - "paths": { - "/search/fulltext": { - "get": { - "tags": ["Search"], - "summary": "Returns a set of registration and renewal objects", - "description": "Accepts a search_query string with full boolean logic to fuzzy search across both registration and renewal records", - "parameters": [ + 'paths': { + '/search/fulltext': { + 'get': { + 'tags': ['Search'], + 'summary': 'Returns a set of registration and renewal objects', + 'description': 'Accepts a search_query string with full boolean logic to fuzzy search across both registration and renewal records', + 'parameters': [ { - "name": "query", - "in": "query", - "type": "string", - "required": True, - "default": "*" + 'name': 'query', + 'in': 'query', + 'type': 'string', + 'required': True, + 'default': '*' },{ - "name": "source", - "in": "query", - "type": "boolean", - "required": False, - "default": False, - "description": "Return source XML/CSV data" + 'name': 'source', + 'in': 'query', + 'type': 'boolean', + 'required': False, + 'default': False, + 'description': 'Return source XML/CSV data' },{ - "name": "page", - "in": "query", - "type": "number", - "required": False, - "default": 0 + 'name': 'page', + 'in': 'query', + 'type': 'number', + 'required': False, + 'default': 0 },{ - "name": "per_page", - "in": "query", - "type": "number", - "required": False, - "default": 10 + 'name': 'per_page', + 'in': 'query', + 'type': 'number', + 'required': False, + 'default': 10 } ], - "responses": { + 'responses': { 200: { - "description": "A list of copyright registrations and renewals", - "schema": { - "$ref": "#/definitions/MultiResponse" + 'description': 'A list of copyright registrations and renewals', + 'schema': { + '$ref': '#/definitions/MultiResponse' } } } } }, - "/search/registration/{regnum}": { - "get": { - "tags": ["Search"], - "summary": "Returns a set of registration and renewal objects", - "description": "Accepts a copyright registration number and returns all matching records", - "parameters": [ + '/search/registration/{regnum}': { + 'get': { + 'tags': ['Search'], + 'summary': 'Returns a set of registration and renewal objects', + 'description': 'Accepts a copyright registration number and returns all matching records', + 'parameters': [ { - "name": "regnum", - "in": "path", - "required": True, - "schema": { - "type": "string" + 'name': 'regnum', + 'in': 'path', + 'required': True, + 'schema': { + 'type': 'string' }, - "description": "Standard copyright registration number" + 'description': 'Standard copyright registration number' },{ - "name": "source", - "in": "query", - "type": "boolean", - "required": False, - "default": False, - "description": "Return source XML/CSV data" + 'name': 'source', + 'in': 'query', + 'type': 'boolean', + 'required': False, + 'default': False, + 'description': 'Return source XML/CSV data' },{ - "name": "page", - "in": "query", - "type": "number", - "required": False, - "default": 0 + 'name': 'page', + 'in': 'query', + 'type': 'number', + 'required': False, + 'default': 0 },{ - "name": "per_page", - "in": "query", - "type": "number", - "required": False, - "default": 10 + 'name': 'per_page', + 'in': 'query', + 'type': 'number', + 'required': False, + 'default': 10 } ], - "responses": { + 'responses': { 200: { - "description": "A list of copyright registrations and renewals", - "schema": { - "$ref": "#/definitions/MultiResponse" + 'description': 'A list of copyright registrations and renewals', + 'schema': { + '$ref': '#/definitions/MultiResponse' } } } } }, - "/search/renewal/{rennum}": { - "get": { - "tags": ["Search"], - "summary": "Returns a set of registration and renewal objects", - "description": "Accepts a copyright renewal number and returns all matching records", - "parameters": [ + '/search/renewal/{rennum}': { + 'get': { + 'tags': ['Search'], + 'summary': 'Returns a set of registration and renewal objects', + 'description': 'Accepts a copyright renewal number and returns all matching records', + 'parameters': [ { - "name": "rennum", - "in": "path", - "required": True, - "schema": { - "type": "string" + 'name': 'rennum', + 'in': 'path', + 'required': True, + 'schema': { + 'type': 'string' }, - "description": "Standard copyright renewal number" + 'description': 'Standard copyright renewal number' },{ - "name": "source", - "in": "query", - "type": "boolean", - "required": False, - "default": False, - "description": "Return source XML/CSV data" + 'name': 'source', + 'in': 'query', + 'type': 'boolean', + 'required': False, + 'default': False, + 'description': 'Return source XML/CSV data' },{ - "name": "page", - "in": "query", - "type": "number", - "required": False, - "default": 0 + 'name': 'page', + 'in': 'query', + 'type': 'number', + 'required': False, + 'default': 0 },{ - "name": "per_page", - "in": "query", - "type": "number", - "required": False, - "default": 10 + 'name': 'per_page', + 'in': 'query', + 'type': 'number', + 'required': False, + 'default': 10 } ], - "responses": { + 'responses': { 200: { - "description": "A list of copyright registrations and renewals", - "schema": { - "$ref": "#/definitions/MultiResponse" + 'description': 'A list of copyright registrations and renewals', + 'schema': { + '$ref': '#/definitions/MultiResponse' } } } } }, - "/registration/{uuid}": { - "get": { - "tags": ["Lookup"], - "summary": "Return a specific Registration record by UUID", - "description": "Accepts a UUID and returns a registration record", - "parameters": [{ - "name": "uuid", - "in": "path", - "required": True, - "schema": { - "type": "string" + '/registration/{uuid}': { + 'get': { + 'tags': ['Lookup'], + 'summary': 'Return a specific Registration record by UUID', + 'description': 'Accepts a UUID and returns a registration record', + 'parameters': [{ + 'name': 'uuid', + 'in': 'path', + 'required': True, + 'schema': { + 'type': 'string' }, - "description": "Standard UUID" + 'description': 'Standard UUID' }], - "responses": { + 'responses': { 200: { - "description": "A single Registration record", - "schema": { - "$ref": "#/definitions/SingleResponse" + 'description': 'A single Registration record', + 'schema': { + '$ref': '#/definitions/SingleResponse' + } + }, + 404: { + 'description': 'A message noting that the UUID could not be found', + 'schema': { + '$ref': '#/definitions/ErrorResponse' + } + }, + 500: { + 'description': 'Generic internal error message', + 'schema': { + '$ref': '#/definitions/ErrorResponse' } } } } }, - "/renewal/{uuid}": { - "get": { - "tags": ["Lookup"], - "summary": "Return a specific Renewal record by UUID", - "description": "Accepts a UUID and returns either an orphan renewal record or the parent registration with associated renewals", - "parameters": [{ - "name": "uuid", - "in": "path", - "required": True, - "schema": { - "type": "string" + '/renewal/{uuid}': { + 'get': { + 'tags': ['Lookup'], + 'summary': 'Return a specific Renewal record by UUID', + 'description': 'Accepts a UUID and returns either an orphan renewal record or the parent registration with associated renewals', + 'parameters': [{ + 'name': 'uuid', + 'in': 'path', + 'required': True, + 'schema': { + 'type': 'string' }, - "description": "Standard UUID" + 'description': 'Standard UUID' }], - "responses": { + 'responses': { 200: { - "description": "A single Renewal or Registration record", - "schema": { - "$ref": "#/definitions/SingleResponse" + 'description': 'A single Renewal or Registration record', + 'schema': { + '$ref': '#/definitions/SingleResponse' + } + }, + 404: { + 'description': 'A message noting that the UUID could not be found', + 'schema': { + '$ref': '#/definitions/ErrorResponse' + } + }, + 500: { + 'description': 'Generic internal error message', + 'schema': { + '$ref': '#/definitions/ErrorResponse' } } } } } }, - "definitions": { - "SingleResponse": { - "type": "object", - "properties": { - "status": { - "type": "integer" + 'definitions': { + 'ErrorResponse': { + 'type': 'object', + 'properties': { + 'status': { + 'type': 'integer' }, - "data": { - "type": "object", - "anyOf": [ - {"$ref": "#/definitions/Registration"}, - {"$ref": "#/definitions/Renewal"} + 'message': { + 'type': 'string' + } + } + }, + 'SingleResponse': { + 'type': 'object', + 'properties': { + 'status': { + 'type': 'integer' + }, + 'data': { + 'type': 'object', + 'anyOf': [ + {'$ref': '#/definitions/Registration'}, + {'$ref': '#/definitions/Renewal'} ] } } }, - "MultiResponse": { - "type": "object", - "properties": { - "total": { - "type": "integer", + 'MultiResponse': { + 'type': 'object', + 'properties': { + 'total': { + 'type': 'integer', }, - "query": { - "type": "object", - "$ref": "#/definitions/Query" + 'query': { + 'type': 'object', + '$ref': '#/definitions/Query' }, - "paging": { - "type": "object", - "$ref": "#/definitions/Paging" + 'paging': { + 'type': 'object', + '$ref': '#/definitions/Paging' }, - "results": { - "type": "array", - "items": { - "anyOf": [ - {"$ref": "#/definitions/Registration"}, - {"$ref": "#/definitions/Renewal"} + 'results': { + 'type': 'array', + 'items': { + 'anyOf': [ + {'$ref': '#/definitions/Registration'}, + {'$ref': '#/definitions/Renewal'} ] } } } }, - "Query": { - "type": "object", - "properties": { - "endpoint": { - "type": "string" + 'Query': { + 'type': 'object', + 'properties': { + 'endpoint': { + 'type': 'string' }, - "term": { - "type": "string" + 'term': { + 'type': 'string' } } }, - "Paging": { - "type": "object", - "properties": { - "first": { - "type": "string" + 'Paging': { + 'type': 'object', + 'properties': { + 'first': { + 'type': 'string' }, - "previous": { - "type": "string" + 'previous': { + 'type': 'string' }, - "next": { - "type": "string" + 'next': { + 'type': 'string' }, - "last": { - "type": "string" + 'last': { + 'type': 'string' } } }, - "Registration": { - "type": "object", - "properties": { - "title": { - "type": "string" + 'Registration': { + 'type': 'object', + 'properties': { + 'title': { + 'type': 'string' }, - "copies": { - "type": "string" + 'copies': { + 'type': 'string' }, - "copy_date": { - "type": "string" + 'copy_date': { + 'type': 'string' }, - "description": { - "type": "string" + 'description': { + 'type': 'string' }, - "authors": { - "type": "array", - "items": { - "$ref": "#/definitions/Agent" + 'authors': { + 'type': 'array', + 'items': { + '$ref': '#/definitions/Agent' } }, - "publishers": { - "type": "array", - "items": { - "$ref": "#/definitions/Agent" + 'publishers': { + 'type': 'array', + 'items': { + '$ref': '#/definitions/Agent' } }, - "registrations": { - "type": "array", - "items": { - "$ref": "#/definitions/RegRegistration" + 'registrations': { + 'type': 'array', + 'items': { + '$ref': '#/definitions/RegRegistration' } }, - "renewals":{ - "type": "array", - "items": { - "$ref": "#/definitions/Renewal" + 'renewals':{ + 'type': 'array', + 'items': { + '$ref': '#/definitions/Renewal' } }, - "source": { - "type": "object", - "properties": { - "page": { - "type": "integer" + 'source': { + 'type': 'object', + 'properties': { + 'page': { + 'type': 'integer' }, - "page_position": { - "type": "integer" + 'page_position': { + 'type': 'integer' }, - "part": { - "type": "string" + 'part': { + 'type': 'string' }, - "series": { - "type": "string" + 'series': { + 'type': 'string' }, - "url": { - "type": "string" + 'url': { + 'type': 'string' }, - "year": { - "type": "integer" + 'year': { + 'type': 'integer' } } } } }, - "Agent": { - "type": "string" + 'Agent': { + 'type': 'string' }, - "RegRegistration": { - "type": "object", - "properties": { - "number": { - "type": "string" + 'RegRegistration': { + 'type': 'object', + 'properties': { + 'number': { + 'type': 'string' }, - "date": { - "type": "string" + 'date': { + 'type': 'string' } } }, - "Renewal": { - "type": "object", - "properties": { - "type": { - "type": "string" + 'Renewal': { + 'type': 'object', + 'properties': { + 'type': { + 'type': 'string' }, - "title": { - "type": "string" + 'title': { + 'type': 'string' }, - "author": { - "type": "string" + 'author': { + 'type': 'string' }, - "new_matter": { - "type": "string" + 'new_matter': { + 'type': 'string' }, - "renewal_num": { - "type": "string" + 'renewal_num': { + 'type': 'string' }, - "renewal_date": { - "type": "string" + 'renewal_date': { + 'type': 'string' }, - "notes": { - "type": "string" + 'notes': { + 'type': 'string' }, - "volume": { - "type": "string" + 'volume': { + 'type': 'string' }, - "part": { - "type": "string" + 'part': { + 'type': 'string' }, - "number": { - "type": "string" + 'number': { + 'type': 'string' }, - "page": { - "type": "string" + 'page': { + 'type': 'string' }, - "claimants": { - "type": "array", - "items": { - "$ref": "#/definitions/Claimant" + 'claimants': { + 'type': 'array', + 'items': { + '$ref': '#/definitions/Claimant' } } } }, - "Claimant": { - "type": "object", - "properties": { - "name": { - "type": "string" + 'Claimant': { + 'type': 'object', + 'properties': { + 'name': { + 'type': 'string' }, - "type": { - "type": "string" + 'type': { + 'type': 'string' } } } diff --git a/api/prints/uuid.py b/api/prints/uuid.py index 15bab27..8f6a0cf 100644 --- a/api/prints/uuid.py +++ b/api/prints/uuid.py @@ -1,40 +1,57 @@ -from flask import ( - Blueprint, request, session, url_for, redirect, current_app, jsonify -) +from flask import Blueprint, request, jsonify +from sqlalchemy.exc import DataError +from sqlalchemy.orm.exc import NoResultFound from api.db import db -from api.elastic import elastic from api.response import SingleResponse from model.cce import CCE from model.registration import Registration from model.renewal import Renewal, RENEWAL_REG -from model.volume import Volume +from helpers.errors import LookupError uuid = Blueprint('uuid', __name__, url_prefix='/') @uuid.route('/registration/', methods=['GET']) def regQuery(uuid): - dbEntry = db.session.query(CCE)\ - .outerjoin(Registration, RENEWAL_REG, Renewal)\ - .filter(CCE.uuid == uuid).one() - + err = None regRecord = SingleResponse('uuid', request.base_url) - regRecord.result = SingleResponse.parseEntry(dbEntry, xml=True) - regRecord.createDataBlock() - return jsonify(regRecord.createResponse(200)) + try: + dbEntry = db.session.query(CCE)\ + .outerjoin(Registration, RENEWAL_REG, Renewal)\ + .filter(CCE.uuid == uuid).one() + + regRecord.result = SingleResponse.parseEntry(dbEntry, xml=True) + regRecord.createDataBlock() + status = 200 + except NoResultFound: + status = 404 + err = LookupError('Unable to locate UUID {} in database'.format(uuid)) + except DataError: + status = 500 + err = LookupError('Malformed UUID {} received'.format(uuid)) + return jsonify(regRecord.createResponse(status, err=err)) @uuid.route('/renewal/', methods=['GET']) def renQuery(uuid): - dbRenewal = db.session.query(Renewal)\ - .outerjoin(RENEWAL_REG, Registration, CCE)\ - .filter(Renewal.uuid == uuid).one() - + err = None renRecord = SingleResponse('uuid', request.base_url) - renRecord.result = parseRetRenewal(dbRenewal) - renRecord.createDataBlock() - return jsonify(renRecord.createResponse(200)) + try: + dbRenewal = db.session.query(Renewal)\ + .outerjoin(RENEWAL_REG, Registration, CCE)\ + .filter(Renewal.uuid == uuid).one() + + renRecord.result = parseRetRenewal(dbRenewal) + renRecord.createDataBlock() + status = 200 + except NoResultFound: + status = 404 + err = LookupError('Unable to locate UUID {} in database'.format(uuid)) + except DataError: + status = 500 + err = LookupError('Malformed UUID {} received'.format(uuid)) + return jsonify(renRecord.createResponse(status, err=err)) def parseRetRenewal(dbRenewal): diff --git a/helpers/errors.py b/helpers/errors.py index 8b7a1d6..0247bd8 100644 --- a/helpers/errors.py +++ b/helpers/errors.py @@ -4,4 +4,11 @@ class DataError(Exception): self.message = message for key, value in kwargs.items(): - setattr(self, key, value) \ No newline at end of file + setattr(self, key, value) + +class LookupError(Exception): + def __init__(self, message, **kwargs): + self.message = message + + for key, value in kwargs.items(): + setattr(self, key, value)