Merge branch 'master' into gutenberg2

BaseSearcher.py

@@ -33,6 +33,9 @@ from libgutenberg import DublinCore
 from libgutenberg import GutenbergDatabaseDublinCore
 from libgutenberg import GutenbergGlobals as gg
 
+from i18n_tool import ugettext as _
+from i18n_tool import ungettext as __
+
 import DublinCoreI18n
 from SupportedLocales import FB_LANGS, TWITTER_LANGS, GOOGLE_LANGS, PAYPAL_LANGS, FLATTR_LANGS
 

BibrecPage.py

@@ -17,6 +17,8 @@ from __future__ import unicode_literals
 import cherrypy
 
 from libgutenberg import GutenbergGlobals as gg
+from i18n_tool import ugettext as _
+from i18n_tool import ungettext as __
 
 import BaseSearcher
 import Page

CherryPyApp.py

@@ -24,16 +24,9 @@ import traceback
 
 import cherrypy
 from cherrypy.process import plugins
-import six
-from six.moves import builtins
 
 from libgutenberg import GutenbergDatabase
-
 import i18n_tool
-# Make translator functions available everywhere. Do this early, at
-# least before Genshi starts loading templates.
-builtins._ = i18n_tool.ugettext
-builtins.__ = i18n_tool.ungettext
 
 # this import causes ConnectionPool.ConnectionPool to become the cherrypy connection pool
 import ConnectionPool
@@ -46,6 +39,7 @@ from SearchPage import BookSearchPage, AuthorSearchPage, SubjectSearchPage, Book
 from BibrecPage import BibrecPage
 import CoverPages
 import QRCodePage
+import diagnostics
 import Sitemap
 import Formatters
 
@@ -263,6 +257,9 @@ def main():
     d.connect('iplimit', r'/iplimit/',
                controller=Page.NullPage())
 
+    d.connect('diagnostics', r'/diagnostics/',
+               controller=diagnostics.DiagnosticsPage())
+
     d.connect('stats', r'/stats/',
                controller=Page.NullPage(), _static=True)
 

CloudStorage.py

@@ -28,6 +28,7 @@ import requests_oauthlib
 from requests import RequestException
 from oauthlib.oauth2.rfc6749.errors import OAuth2Error
 
+from i18n_tool import ugettext as _
 import BaseSearcher
 
 # pylint: disable=R0921

DublinCoreI18n.py

@@ -17,7 +17,7 @@ from __future__ import unicode_literals
 
 import cherrypy
 import babel
-
+from i18n_tool import ugettext as _
 
 class DublinCoreI18nMixin (object):
     """ Translator Mixin for GutenbergDatabaseDublinCore class. """

Formatters.py

@@ -23,6 +23,7 @@ import genshi.filters
 
 import cherrypy
 
+from i18n_tool import ugettext as _
 import HTMLFormatter
 import OPDSFormatter
 import JSONFormatter

HTMLFormatter.py

@@ -26,6 +26,7 @@ from libgutenberg.MediaTypes import mediatypes as mt
 
 import BaseSearcher
 import BaseFormatter
+from i18n_tool import ugettext as _
 
 # filetypes ignored on desktop site
 NO_DESKTOP_FILETYPES = 'plucker qioo rdf rst rst.gen rst.master tei cover.medium cover.small'.split ()

Page.py

@@ -23,6 +23,7 @@ from libgutenberg.GutenbergDatabase import DatabaseError
 
 import BaseSearcher
 import Formatters
+from i18n_tool import ugettext as _
 
 class Page (object):
     """ Base for all pages. """

SearchPage.py

@@ -21,6 +21,8 @@ from libgutenberg.DublinCore import DublinCore
 
 import BaseSearcher
 from Page import SearchPage
+from i18n_tool import ugettext as _
+from i18n_tool import ungettext as __
 
 
 class BookSearchPage (SearchPage):

StartPage.py

@@ -16,6 +16,7 @@ from __future__ import unicode_literals
 
 import BaseSearcher
 import Page
+from i18n_tool import ugettext as _
 
 class Start (Page.Page):
     """ The start page. """

autocat3.service

@@ -11,4 +11,4 @@ ExecStartPre=-/usr/bin/mkdir /var/run/autocat
 ExecStart=/usr/local/bin/pipenv run python CherryPyApp.py
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target

configuring.txt

@@ -70,6 +70,20 @@ sudo chown -R autocat:pgweb /var/run/autocat
 sudo systemctl enable /var/lib/autocat/autocat3/autocat3.service
 sudo systemctl start autocat3
 
+# If there are problems with `sudo systemctl enable /var/lib/autocat/autocat3/autocat3.service`
+# it might be due to monit (the monitoring system) fighting to restart. Or it might be that the
+# service is in an uncertain state, not stopped. Try:
+sudo service autocat3 stop
+systemctl disable autocat3.service
+systemctl enable /var/lib/autocat/autocat3/autocat3.service
+systemctl restart autocat3.service
+systemctl status autocat3.service
+
+# `sudo rm /etc/systemd/system/autocat3.service` is probably not needed. As of July 20 2019,
+# we are having problems with monit trying to restart. It might be necessary to also stop
+# monit. Further diagnosis is forthcoming.
+
+
 ## updates
 
 su - autocat

diagnostics.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+#  -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-
+"""
+diagnostics.py
+
+Copyright 2019 by Eric Hellman
+
+Distributable under the GNU General Public License Version 3 or newer.
+
+"""
+import json
+import resource
+import sys
+from collections import Mapping, Container
+from sys import getsizeof
+import cherrypy
+from cherrypy.lib.sessions import RamSession
+from Page import Page
+
+def deep_getsizeof(o, ids):
+    """Find the memory footprint of a Python object
+
+    This is a recursive function that rills down a Python object graph
+    like a dictionary holding nested ditionaries with lists of lists
+    and tuples and sets.
+
+    The sys.getsizeof function does a shallow size of only. It counts each
+    object inside a container as pointer only regardless of how big it
+    really is.
+
+    :param o: the object
+    :param ids:
+    :return:
+    https://github.com/the-gigi/deep/blob/master/deeper.py
+    """
+    d = deep_getsizeof
+    if id(o) in ids:
+        return 0
+
+    r = getsizeof(o)
+    ids.add(id(o))
+
+    if isinstance(o, str):
+        return r
+
+    if isinstance(o, Mapping):
+        return r + sum(d(k, ids) + d(v, ids) for k, v in o.items())
+
+    if isinstance(o, Container):
+        return r + sum(d(x, ids) for x in o)
+
+    return r
+
+class DiagnosticsPage (Page):
+    """ Python health. """
+
+    @cherrypy.tools.json_out()
+    def index (self, **dummy_kwargs):
+        """ return stats. """
+        stats = {}
+        stats['sessions'] = len(RamSession.cache)
+        stats['sessions_storage'] = deep_getsizeof(RamSession.cache, set())
+        stats['allocated_blocks'] = sys.getallocatedblocks()
+        stats['rusage_self'] = resource.getrusage(resource.RUSAGE_SELF)
+        stats['rusage_children'] = resource.getrusage(resource.RUSAGE_CHILDREN)
+        return stats
+

templates/results.opds

@@ -14,7 +14,7 @@ which contains *all* Project Gutenberg metadata in one RDF/XML file.
 <?python
   import re
   from libgutenberg import GutenbergGlobals as gg
-
+  from i18n_tool import ugettext as _
   if os.format == 'stanza':
      os.type_opds = "application/atom+xml"
      opds_relations = {