Merge pull request #4 from gutenbergtools/onedrive

Fix Onedrive, reorganize config

CherryPy.conf

@@ -0,0 +1,90 @@
+[global]
+# CherryPyApp.conf
+# These parameters get over-written by values in ~/.autocat3 or /etc/autocat3.conf
+# any sensetive parameters such as keys should be stored there
+environment: 'production'
+
+pidfile:  '/var/run/autocat/autocat3.pid'
+
+server.socket_host: '::'
+server.socket_port: 8000
+server.socket_queue_size: 10
+server.thread_pool: 20
+server.thread_pool_max: 20
+
+# change host and postgres params in .autocat3 or /etc/autocat3.conf files
+pghost:     'localhost'
+pgport:     5432
+pgdatabase: 'gutenberg'
+pguser:     'postgres'
+
+host:        'www.gutenberg.org'
+host_mobile: 'm.gutenberg.org'
+host_https:  1
+file_host:   'www.gutenberg.org'
+
+
+sqlalchemy.pool_size: 20
+sqlalchemy.max_overflow: 0
+sqlalchemy.timeout: 3
+
+facebook_app_id:      '115319388529183'
+
+dropbox_client_id:     '6s833cia5ndi4b5'
+dropbox_client_secret: 'add secret in .autocat3 or /etc/autocat3.conf files'
+
+gdrive_client_id:      '586299000268.apps.googleusercontent.com'
+gdrive_client_secret:  'add secret in .autocat3 or /etc/autocat3.conf files'
+
+msdrive_client_id:     '6902b111-a9d6-461f-bd8a-83dafee3da66'
+msdrive_client_secret: 'add secret in .autocat3 or /etc/autocat3.conf files'
+
+recaptcha_public_key:  '6Lf0cpkUAAAAAIHB6OW_6H0SIqt5O2xmgsnF4Is8'
+recaptcha_private_key: 'add secret in .autocat or /etc/autocat3.conf files'
+
+
+log.screen: False
+log.error_file:  CherryPyApp.install_dir + '/log/error.log'
+log.access_file: CherryPyApp.install_dir + '/log/access.log'
+log.rot_max_bytes: 104857600
+log.rot_backup_count: 2
+tools.log_headers.on: False
+tools.log_tracebacks.on: True
+
+document_root:    'https://www.gutenberg.org'
+
+[/]
+tools.proxy.on: True
+tools.proxy.local: 'X-Forwarded-Host'
+
+
+tools.encode.on: True
+tools.encode.encoding: 'utf-8'
+
+tools.gzip.on: True
+tools.gzip.mime_types: ['text/html', 'application/xhtml+xml', 'application/atom+xml']
+
+tools.I18nTool.on: True
+tools.I18nTool.default: 'en_US'
+tools.I18nTool.mo_dir: CherryPyApp.install_dir + '/i18n'
+tools.I18nTool.domain: 'messages'
+
+tools.sessions.on: True
+#tools.sessions.storage_type = "postgres"
+tools.sessions.table_name = "cherrypy.sessions"
+tools.sessions.timeout: 30
+tools.sessions.path: '/'
+tools.sessions.domain: '.gutenberg.org'
+#tools.sessions.domain: 'localhost'
+
+tools.expires.on: True
+tools.expires.secs: 0
+tools.expires.force: True
+
+[/index.html]
+tools.staticfile.on: True
+tools.staticfile.filename: CherryPyApp.install_dir + '/test/index.html'
+
+[/test.pdf]
+tools.staticfile.on: True
+tools.staticfile.filename: CherryPyApp.install_dir + '/test/test.pdf'

CherryPyApp.py

@@ -51,12 +51,10 @@ import Formatters
 import Timer
 
 plugins.Timer = Timer.TimerPlugin
+install_dir = os.path.dirname (os.path.abspath (__file__))
 
-if six.PY3:
-    CHERRYPY_CONFIG = (os.path.expanduser ('~/.autocat3'), '/etc/autocat3.conf')
-    # CCHERRYPY_CONFIG = ('/etc/autocat3.conf')
-else:
-    CHERRYPY_CONFIG = ('/etc/autocat.conf', os.path.expanduser ('~/.autocat'))
+CHERRYPY_CONFIG = os.path.join(install_dir, 'CherryPy.conf')
+LOCAL_CONFIG = [os.path.expanduser('~/.autocat3'), '/etc/autocat3.conf']
 
 class MyRoutesDispatcher (cherrypy.dispatch.RoutesDispatcher):
     """ Dispatcher that tells us the matched route.
@@ -81,8 +79,7 @@ def main ():
         'uid': 0,
         'gid': 0,
         'server_name': 'localhost',
-        'genshi.template_dir': os.path.join (
-            os.path.dirname (os.path.abspath (__file__)), 'templates'),
+        'genshi.template_dir': os.path.join (install_dir, 'templates'),
         'daemonize': False,
         'pidfile': None,
         'host': 'localhost',
@@ -91,13 +88,7 @@ def main ():
         })
 
     config_filename = None
-    for config_filename in CHERRYPY_CONFIG:
-        try:
-            cherrypy.config.update (config_filename)
-            break
-        except IOError:
-            pass
-
+    cherrypy.config.update (CHERRYPY_CONFIG)
     # Rotating Logs
     #
 
@@ -121,6 +112,8 @@ def main ():
     h.setFormatter (cherrypy._cplogging.logfmt)
     cherrypy.log.access_log.addHandler (h)
     
+
+
     if not cherrypy.config['daemonize']:
         ch = logging.StreamHandler ()
         ch.setLevel (logging.DEBUG)
@@ -131,8 +124,15 @@ def main ():
     #
 
     cherrypy.log ('*' * 80, context = 'ENGINE', severity = logging.INFO)
-    cherrypy.log ("Using config file '%s'." % config_filename,
+    cherrypy.log ("Using config file '%s'." % CHERRYPY_CONFIG,
                   context = 'ENGINE', severity = logging.INFO)
+    for config_filename in LOCAL_CONFIG:
+        try:
+            cherrypy.config.update (config_filename)
+            cherrypy.log ('loaded %s' % config_filename, context = 'ENGINE', severity = logging.INFO)
+            break
+        except IOError:
+            pass
 
     # after cherrypy.config is parsed
     Formatters.init ()
@@ -307,7 +307,7 @@ def main ():
 
     cherrypy.log ("Mounting root", context = 'ENGINE', severity = logging.INFO)
 
-    app = cherrypy.tree.mount (root = None, config = config_filename)
+    app = cherrypy.tree.mount (root = None, config = CHERRYPY_CONFIG)
 
     app.merge ({'/': {'request.dispatch': d}})
     return app

CloudStorage.py

@@ -65,9 +65,10 @@ class CloudOAuth2Session (requests_oauthlib.OAuth2Session): # pylint: disable=R0
         prefix = self.name_prefix
 
         host = config['file_host']
+        host_https = config['host_https']
         urlgen = routes.URLGenerator (cherrypy.routes_mapper, {
             'HTTP_HOST': host,
-            'HTTPS': 1
+            'HTTPS': host_https
         })
 
         client_id     = config[prefix + '_client_id']
@@ -134,7 +135,7 @@ class CloudStorage (object):
     session_class   = CloudOAuth2Session
     user_agent      = None
     upload_endpoint = None
-    re_filename     = re.compile (r'[/\<>:"|?*]')
+    re_filename     = re.compile (r'[/\<>:"|?* ]')
 
 
     def __init__ (self):
@@ -297,7 +298,11 @@ class EbookMetaData (object):
 
     def get_source_url (self):
         """ Return the url of the ebook file on gutenberg.org. """
-
+        protocol = 'https://' if cherrypy.config['host_https'] else 'http://'
+        if self.id == 99999:
+            # test filename
             return urllib.parse.urljoin (
-            'https://' + cherrypy.config['file_host'],
+                protocol + str(cherrypy.config['file_host']) , 'test.pdf')
+        return urllib.parse.urljoin (
+            protocol + cherrypy.config['file_host'],
             'ebooks/%d.%s' % (self.id, self.filetype))

MSDrive.py

@@ -4,11 +4,10 @@
 """
 MSDrive.py
 
-Copyright 2014,15 by Marcello Perathoner
-
 Distributable under the GNU General Public License Version 3 or newer.
 
-The send-to-microsoft-drive pages.
+The send-to-onedrive pages using the Graph API
+https://docs.microsoft.com/en-us/graph/api/driveitem-createuploadsession?view=graph-rest-1.0
 
 """
 
@@ -18,8 +17,7 @@ from contextlib import closing
 
 import CloudStorage
 
-
-class MSDriveSession (CloudStorage.CloudOAuth2Session):
+class MSDriveSession(CloudStorage.CloudOAuth2Session):
     """ Hold parameters for OAuth. """
 
     #
@@ -29,27 +27,54 @@ class MSDriveSession (CloudStorage.CloudOAuth2Session):
     #
 
     name_prefix = 'msdrive'
-    oauth2_auth_endpoint  = 'https://login.live.com/oauth20_authorize.srf'
-    oauth2_token_endpoint = 'https://login.live.com/oauth20_token.srf'
-    oauth2_scope          = 'wl.signin wl.basic wl.skydrive wl.skydrive_update'
+    oauth2_auth_endpoint = 'https://login.microsoftonline.com/common/oauth2/v2.0/authorize'
+    oauth2_token_endpoint = 'https://login.microsoftonline.com/common/oauth2/v2.0/token'
+    oauth2_scope = 'Files.ReadWrite'
 
-
-class MSDrive (CloudStorage.CloudStorage):
-    """ Send files to Microsoft Drive. """
+class MSDrive(CloudStorage.CloudStorage):
+    """ Send files to Microsoft OneDrive. """
 
     name = 'OneDrive'
     session_class = MSDriveSession
-    user_agent            = 'PG2MSDrive/0.2'
-    upload_endpoint       = 'https://apis.live.net/v5.0/me/skydrive/files/'
+    user_agent = 'PG2OneDrive/2019.0'
+    #upload_endpoint = 'https://apis.live.net/v5.0/me/skydrive/files/'
+    upload_endpoint = 'https://graph.microsoft.com/v1.0/me/drive/items/root:/Documents/Gutenberg/{filename}:/createUploadSession'
     
 
-    def upload_file (self, session, request):
-        """ Upload a file to microsoft drive. """
 
-        url = self.upload_endpoint + self.fix_filename (session.ebook.get_filename ())
+    def upload_file(self, session, response):
+        """ Upload a file to microsoft onedrive. """
+        filename = self.fix_filename(session.ebook.get_filename())
+        item_data = {
+            'name': filename,
+            'description': 'A Project Gutenberg Ebook',
+            "@microsoft.graph.conflictBehavior": "rename", 
+        }
+        filesize = int(response.headers['Content-Length'])
+        url = self.upload_endpoint.format(filename=filename)
+        chunk_size = 327680 # weird onedrive thing related to FAT tables
+        upload_data = session.post(url, json={'item': item_data}).json()
 
-        # MSDrive does not like such never-heard-of-before
-        # content-types like 'epub', so we just send it without
-        # content-type.
-        with closing (session.put (url, data = request.iter_content (1024 * 1024))) as r:
-            r.raise_for_status ()
+        def headers(start, end, filesize):
+            return {
+                'Content-Length': str(end - start + 1),
+                'Content-Range': 'bytes {}-{}/{}'.format(start, end, filesize)
+            }
+
+        if 'uploadUrl' in upload_data:
+            session_uri = upload_data['uploadUrl']
+            start = 0
+            end = min(chunk_size - 1, filesize - 1)
+
+            for chunk in response.iter_content(chunk_size):
+                r = session.put(
+                    session_uri,
+                    data=chunk,
+                    headers=headers(start, end, filesize),
+                )
+                start = start + chunk_size
+                end = min(end + chunk_size, filesize - 1)
+                r.raise_for_status()
+        else:
+            CloudStorage.log('no uploadUrl in %s' % upload_data)
+        session.close()

README.md

@@ -1,5 +1,5 @@
 # autocat3_original
-## Intruduction
+## Introduction
 
 **autocat3** is a python-based application used for supporting [Project Gutenberg](gutenberg.org). 
 CherryPy is used as the web framwork which is easy to develop.

Test.py

@@ -9,7 +9,10 @@ import CherryPyApp
 
 class TestInstantiation(unittest.TestCase):
     def setUp(self):
-        CherryPyApp.CHERRYPY_CONFIG = [os.path.join(os.path.dirname(__file__), 'test.conf')]
+        CherryPyApp.CHERRYPY_CONFIG = os.path.join(
+            os.path.dirname(os.path.abspath (__file__)),
+            'test.conf'
+        )
 
     def test_main(self):
         CherryPyApp.main()

configuring.txt

@@ -45,7 +45,12 @@ pipenv --three
 git checkout remotes/origin/master
 pipenv install
 
-# add conf file
+# add local conf file. keep secrets here!
+# for production .autocat3 should set values for these parameters:
+#   pghost, pguser, 
+#   dropbox_client_secret, gdrive_client_secret, msdrive_client_secret, recaptcha_private_key
+#   log.error_file, log.access_file
+
 #!! from local
 # scp [local:]~/autocat3.conf [autocat@host:]~/.autocat3
 

test/index.html

@@ -0,0 +1,19 @@
+<html>
+<head>
+<link rel="stylesheet" type="text/css" href="https://www.gutenberg.org/css/pg-desktop-one.css" />
+</head>
+<body>
+<div class="body">
+<p><a href="/test.pdf">Download Test</a></p>
+<p>
+<a href="/ebooks/send/dropbox/99999.pdf" title="Send to Dropbox." rel="nofollow"><span class="icon icon_dropbox">Dropbox</span></a>
+</p>
+<p>
+<a href="/ebooks/send/gdrive/99999.pdf" title="Send to Google Drive." rel="nofollow"><span class="icon icon_gdrive">Google Drive</span></a>
+</p>
+<p>
+<a href="/ebooks/send/msdrive/99999.pdf" title="Send to OneDrive." rel="nofollow"><span class="icon icon_msdrive">OneDrive</span></a>
+</p>
+</div>
+</body>
+</html>