From e9c4703c4868083104dc241ac7102bbda6d3a06b Mon Sep 17 00:00:00 2001 From: Greg Newby Date: Mon, 8 Jul 2019 09:44:07 -0400 Subject: [PATCH 1/8] Update autocat3.service --- autocat3.service | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autocat3.service b/autocat3.service index 3789912..4eba36a 100644 --- a/autocat3.service +++ b/autocat3.service @@ -5,10 +5,11 @@ After=network.target [Service] User=autocat Type=simple +LimitNOFILE=4096 RuntimeDirectory=autocat WorkingDirectory=/var/lib/autocat/autocat3 ExecStartPre=-/usr/bin/mkdir /var/run/autocat ExecStart=/var/lib/autocat/.local/bin/pipenv run python CherryPyApp.py [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target From ef3153a819e1f1b79981d615d38225afaac6e1f9 Mon Sep 17 00:00:00 2001 From: Greg Newby Date: Sat, 20 Jul 2019 10:40:26 -0400 Subject: [PATCH 2/8] More detail on restarting It's still not restarting cleanly after a system boot. We'll try to dig into this. For now, I wanted to track the commands that were helpful in restarting by hand, after a system boot. --- configuring.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/configuring.txt b/configuring.txt index b529550..ac9b095 100644 --- a/configuring.txt +++ b/configuring.txt @@ -70,6 +70,20 @@ sudo chown -R autocat:pgweb /var/run/autocat sudo systemctl enable /var/lib/autocat/autocat3/autocat3.service sudo systemctl start autocat3 +# If there are problems with `sudo systemctl enable /var/lib/autocat/autocat3/autocat3.service` +# it might be due to monit (the monitoring system) fighting to restart. Or it might be that the +# service is in an uncertain state, not stopped. Try: +sudo service autocat3 stop +systemctl disable autocat3.service +systemctl enable /var/lib/autocat/autocat3/autocat3.service +systemctl restart autocat3.service +systemctl status autocat3.service + +# `sudo rm /etc/systemd/system/autocat3.service` is probably not needed. As of July 20 2019, +# we are having problems with monit trying to restart. It might be necessary to also stop +# monit. Further diagnosis is forthcoming. + + ## updates su - autocat From 2fe006388defdc891e6836d106b992d286cb96bd Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 26 Jul 2019 11:54:24 -0400 Subject: [PATCH 3/8] stop using buitins as global --- BaseSearcher.py | 3 +++ BibrecPage.py | 2 ++ CherryPyApp.py | 8 -------- CloudStorage.py | 1 + DublinCoreI18n.py | 2 +- Formatters.py | 1 + HTMLFormatter.py | 1 + Page.py | 1 + SearchPage.py | 2 ++ StartPage.py | 1 + templates/results.opds | 2 +- 11 files changed, 14 insertions(+), 10 deletions(-) diff --git a/BaseSearcher.py b/BaseSearcher.py index 312f4c2..8bc6f99 100644 --- a/BaseSearcher.py +++ b/BaseSearcher.py @@ -33,6 +33,9 @@ from libgutenberg import DublinCore from libgutenberg import GutenbergDatabaseDublinCore from libgutenberg import GutenbergGlobals as gg +from i18n_tool import ugettext as _ +from i18n_tool import ungettext as __ + import DublinCoreI18n from SupportedLocales import FB_LANGS, TWITTER_LANGS, GOOGLE_LANGS, PAYPAL_LANGS, FLATTR_LANGS diff --git a/BibrecPage.py b/BibrecPage.py index 6d0ede2..1ed1a47 100644 --- a/BibrecPage.py +++ b/BibrecPage.py @@ -17,6 +17,8 @@ from __future__ import unicode_literals import cherrypy from libgutenberg import GutenbergGlobals as gg +from i18n_tool import ugettext as _ +from i18n_tool import ungettext as __ import BaseSearcher import Page diff --git a/CherryPyApp.py b/CherryPyApp.py index 9ab743c..65eefbc 100644 --- a/CherryPyApp.py +++ b/CherryPyApp.py @@ -24,17 +24,9 @@ import traceback import cherrypy from cherrypy.process import plugins -import six -from six.moves import builtins from libgutenberg import GutenbergDatabase -import i18n_tool -# Make translator functions available everywhere. Do this early, at -# least before Genshi starts loading templates. -builtins._ = i18n_tool.ugettext -builtins.__ = i18n_tool.ungettext - # this import causes ConnectionPool.ConnectionPool to become the cherrypy connection pool import ConnectionPool diff --git a/CloudStorage.py b/CloudStorage.py index 4b3679b..0d0d1f2 100644 --- a/CloudStorage.py +++ b/CloudStorage.py @@ -28,6 +28,7 @@ import requests_oauthlib from requests import RequestException from oauthlib.oauth2.rfc6749.errors import OAuth2Error +from i18n_tool import ugettext as _ import BaseSearcher # pylint: disable=R0921 diff --git a/DublinCoreI18n.py b/DublinCoreI18n.py index 6ac5a8c..1bb4999 100644 --- a/DublinCoreI18n.py +++ b/DublinCoreI18n.py @@ -17,7 +17,7 @@ from __future__ import unicode_literals import cherrypy import babel - +from i18n_tool import ugettext as _ class DublinCoreI18nMixin (object): """ Translator Mixin for GutenbergDatabaseDublinCore class. """ diff --git a/Formatters.py b/Formatters.py index 793d4c5..02cc4e6 100644 --- a/Formatters.py +++ b/Formatters.py @@ -23,6 +23,7 @@ import genshi.filters import cherrypy +from i18n_tool import ugettext as _ import HTMLFormatter import OPDSFormatter import JSONFormatter diff --git a/HTMLFormatter.py b/HTMLFormatter.py index 2730a7d..81bacef 100644 --- a/HTMLFormatter.py +++ b/HTMLFormatter.py @@ -26,6 +26,7 @@ from libgutenberg.MediaTypes import mediatypes as mt import BaseSearcher import BaseFormatter +from i18n_tool import ugettext as _ # filetypes ignored on desktop site NO_DESKTOP_FILETYPES = 'plucker qioo rdf rst rst.gen rst.master tei cover.medium cover.small'.split () diff --git a/Page.py b/Page.py index ca6d5f3..ff8a24f 100644 --- a/Page.py +++ b/Page.py @@ -23,6 +23,7 @@ from libgutenberg.GutenbergDatabase import DatabaseError import BaseSearcher import Formatters +from i18n_tool import ugettext as _ class Page (object): """ Base for all pages. """ diff --git a/SearchPage.py b/SearchPage.py index 000338d..7642d31 100644 --- a/SearchPage.py +++ b/SearchPage.py @@ -21,6 +21,8 @@ from libgutenberg.DublinCore import DublinCore import BaseSearcher from Page import SearchPage +from i18n_tool import ugettext as _ +from i18n_tool import ungettext as __ class BookSearchPage (SearchPage): diff --git a/StartPage.py b/StartPage.py index ab8dd9b..8ec4997 100644 --- a/StartPage.py +++ b/StartPage.py @@ -16,6 +16,7 @@ from __future__ import unicode_literals import BaseSearcher import Page +from i18n_tool import ugettext as _ class Start (Page.Page): """ The start page. """ diff --git a/templates/results.opds b/templates/results.opds index 0e78c1b..8fcb2e0 100644 --- a/templates/results.opds +++ b/templates/results.opds @@ -14,7 +14,7 @@ which contains *all* Project Gutenberg metadata in one RDF/XML file. Date: Fri, 26 Jul 2019 12:30:32 -0400 Subject: [PATCH 4/8] revert --- autocat3.service | 1 - 1 file changed, 1 deletion(-) diff --git a/autocat3.service b/autocat3.service index 4eba36a..a652abc 100644 --- a/autocat3.service +++ b/autocat3.service @@ -5,7 +5,6 @@ After=network.target [Service] User=autocat Type=simple -LimitNOFILE=4096 RuntimeDirectory=autocat WorkingDirectory=/var/lib/autocat/autocat3 ExecStartPre=-/usr/bin/mkdir /var/run/autocat From bb4a416d1432bc298a3b613e4833971387067b9d Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 26 Jul 2019 14:22:09 -0400 Subject: [PATCH 5/8] still need an import --- CherryPyApp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/CherryPyApp.py b/CherryPyApp.py index 65eefbc..690bbeb 100644 --- a/CherryPyApp.py +++ b/CherryPyApp.py @@ -26,6 +26,7 @@ import cherrypy from cherrypy.process import plugins from libgutenberg import GutenbergDatabase +import i18n_tool # this import causes ConnectionPool.ConnectionPool to become the cherrypy connection pool import ConnectionPool From f8c1099cab42f2dcedb975ff31f1fd470a8f1cb9 Mon Sep 17 00:00:00 2001 From: eric Date: Mon, 29 Jul 2019 14:31:45 -0400 Subject: [PATCH 6/8] add diagnostics page --- CherryPyApp.py | 4 ++++ diagnostics.py | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100755 diagnostics.py diff --git a/CherryPyApp.py b/CherryPyApp.py index 65eefbc..ab0e7d9 100644 --- a/CherryPyApp.py +++ b/CherryPyApp.py @@ -38,6 +38,7 @@ from SearchPage import BookSearchPage, AuthorSearchPage, SubjectSearchPage, Book from BibrecPage import BibrecPage import CoverPages import QRCodePage +import diagnostics import Sitemap import Formatters @@ -255,6 +256,9 @@ def main(): d.connect('iplimit', r'/iplimit/', controller=Page.NullPage()) + d.connect('diagnostics', r'/diagnostics/', + controller=diagnostics.DiagnosticsPage()) + d.connect('stats', r'/stats/', controller=Page.NullPage(), _static=True) diff --git a/diagnostics.py b/diagnostics.py new file mode 100755 index 0000000..cae3c42 --- /dev/null +++ b/diagnostics.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*- +""" +diagnostics.py + +Copyright 2019 by Eric Hellman + +Distributable under the GNU General Public License Version 3 or newer. + +""" +import json +import resource +import sys +from Page import Page +from cherrypy.lib.sessions import RamSession + +class DiagnosticsPage (Page): + """ Python health. """ + + def index (self, **dummy_kwargs): + """ return stats. """ + stats = {} + stats['sessions'] = len(RamSession.cache) + stats['allocated_blocks'] = sys.getallocatedblocks() + stats['rusage_self'] = resource.rusage(resource.RUSAGE_SELF) + stats['rusage_children'] = resource.rusage(resource.RUSAGE_CHILDREN) + return json.dumps(stats) From 41aedb60102a9ea8a5df5fa20effe855405d7f5b Mon Sep 17 00:00:00 2001 From: eric Date: Mon, 29 Jul 2019 17:45:12 -0400 Subject: [PATCH 7/8] also check size of session cache --- diagnostics.py | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/diagnostics.py b/diagnostics.py index cae3c42..72770ad 100755 --- a/diagnostics.py +++ b/diagnostics.py @@ -11,9 +11,45 @@ Distributable under the GNU General Public License Version 3 or newer. import json import resource import sys +from collections import Mapping, Container +from sys import getsizeof from Page import Page from cherrypy.lib.sessions import RamSession +def deep_getsizeof(o, ids): + """Find the memory footprint of a Python object + + This is a recursive function that rills down a Python object graph + like a dictionary holding nested ditionaries with lists of lists + and tuples and sets. + + The sys.getsizeof function does a shallow size of only. It counts each + object inside a container as pointer only regardless of how big it + really is. + + :param o: the object + :param ids: + :return: + https://github.com/the-gigi/deep/blob/master/deeper.py + """ + d = deep_getsizeof + if id(o) in ids: + return 0 + + r = getsizeof(o) + ids.add(id(o)) + + if isinstance(o, str): + return r + + if isinstance(o, Mapping): + return r + sum(d(k, ids) + d(v, ids) for k, v in o.items()) + + if isinstance(o, Container): + return r + sum(d(x, ids) for x in o) + + return r + class DiagnosticsPage (Page): """ Python health. """ @@ -21,7 +57,9 @@ class DiagnosticsPage (Page): """ return stats. """ stats = {} stats['sessions'] = len(RamSession.cache) + stats['sessions_storage'] = deep_getsizeof(RamSession.cache, set()) stats['allocated_blocks'] = sys.getallocatedblocks() - stats['rusage_self'] = resource.rusage(resource.RUSAGE_SELF) - stats['rusage_children'] = resource.rusage(resource.RUSAGE_CHILDREN) + stats['rusage_self'] = resource.getrusage(resource.RUSAGE_SELF) + stats['rusage_children'] = resource.getrusage(resource.RUSAGE_CHILDREN) return json.dumps(stats) + From b699f5ddfb3f1120c4555ddb1b24ca0c0f73cc83 Mon Sep 17 00:00:00 2001 From: eric Date: Mon, 29 Jul 2019 17:52:17 -0400 Subject: [PATCH 8/8] set correct content-type header --- diagnostics.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/diagnostics.py b/diagnostics.py index 72770ad..e5acd62 100755 --- a/diagnostics.py +++ b/diagnostics.py @@ -13,8 +13,9 @@ import resource import sys from collections import Mapping, Container from sys import getsizeof -from Page import Page +import cherrypy from cherrypy.lib.sessions import RamSession +from Page import Page def deep_getsizeof(o, ids): """Find the memory footprint of a Python object @@ -52,7 +53,8 @@ def deep_getsizeof(o, ids): class DiagnosticsPage (Page): """ Python health. """ - + + @cherrypy.tools.json_out() def index (self, **dummy_kwargs): """ return stats. """ stats = {} @@ -61,5 +63,5 @@ class DiagnosticsPage (Page): stats['allocated_blocks'] = sys.getallocatedblocks() stats['rusage_self'] = resource.getrusage(resource.RUSAGE_SELF) stats['rusage_children'] = resource.getrusage(resource.RUSAGE_CHILDREN) - return json.dumps(stats) + return stats