Improve the flexibility and ease-of-use for docker-compose deployment (#560)

* docker-compose improvements
    * Use gevent gunicorn workers
    * Makes logs easier to access
* Customization of the logs location
* Improve secret key generation & only generate secret keys if one isn't defined (Closes #123)
* Install requirements required by plugins

CTFd/config.py

@@ -2,13 +2,26 @@ import os
 
 ''' GENERATE SECRET KEY '''
 
-with open('.ctfd_secret_key', 'a+b') as secret:
+if not os.environ.get('SECRET_KEY'):
-    secret.seek(0)  # Seek to beginning of file since a+ mode leaves you at the end and w+ deletes the file
+    # Attempt to read the secret from the secret file
+    # This will fail if the secret has not been written
+    try:
+        with open('.ctfd_secret_key', 'rb') as secret:
             key = secret.read()
+    except (OSError, IOError):
+        key = None
+
     if not key:
         key = os.urandom(64)
+        # Attempt to write the secret file
+        # This will fail if the filesystem is read-only
+        try:
+            with open('.ctfd_secret_key', 'wb') as secret:
                 secret.write(key)
                 secret.flush()
+        except (OSError, IOError):
+            pass
+
 
 ''' SERVER SETTINGS '''
 
@@ -72,6 +85,13 @@ class Config(object):
     '''
     MAILFROM_ADDR = "noreply@ctfd.io"
 
+    '''
+    LOG_FOLDER is the location where logs are written
+    These are the logs for CTFd key submissions, registrations, and logins
+    The default location is the CTFd/logs folder
+    '''
+    LOG_FOLDER = os.environ.get('LOG_FOLDER') or os.path.join(os.path.dirname(os.path.abspath(__file__)), 'logs')
+
     '''
     UPLOAD_FOLDER is the location where files are uploaded.
     The default destination is the CTFd/uploads folder. If you need Amazon S3 files

CTFd/utils.py

@@ -95,28 +95,23 @@ def init_logs(app):
     logger_logins.setLevel(logging.INFO)
     logger_regs.setLevel(logging.INFO)
 
-    try:
+    log_dir = app.config['LOG_FOLDER']
-        parent = os.path.dirname(__file__)
-    except:
-        parent = os.path.dirname(os.path.realpath(sys.argv[0]))
-
-    log_dir = os.path.join(parent, 'logs')
     if not os.path.exists(log_dir):
         os.makedirs(log_dir)
 
-    logs = [
+    logs = {
-        os.path.join(parent, 'logs', 'keys.log'),
+        'keys': os.path.join(log_dir, 'keys.log'),
-        os.path.join(parent, 'logs', 'logins.log'),
+        'logins': os.path.join(log_dir, 'logins.log'),
-        os.path.join(parent, 'logs', 'registers.log')
+        'registers': os.path.join(log_dir, 'registers.log')
-    ]
+    }
 
-    for log in logs:
+    for log in logs.values():
         if not os.path.exists(log):
             open(log, 'a').close()
 
-    key_log = logging.handlers.RotatingFileHandler(os.path.join(parent, 'logs', 'keys.log'), maxBytes=10000)
+    key_log = logging.handlers.RotatingFileHandler(logs['keys'], maxBytes=10000)
-    login_log = logging.handlers.RotatingFileHandler(os.path.join(parent, 'logs', 'logins.log'), maxBytes=10000)
+    login_log = logging.handlers.RotatingFileHandler(logs['logins'], maxBytes=10000)
-    register_log = logging.handlers.RotatingFileHandler(os.path.join(parent, 'logs', 'registers.log'), maxBytes=10000)
+    register_log = logging.handlers.RotatingFileHandler(logs['registers'], maxBytes=10000)
 
     logger_keys.addHandler(key_log)
     logger_logins.addHandler(login_log)

Dockerfile

@@ -8,6 +8,11 @@ WORKDIR /opt/CTFd
 VOLUME ["/opt/CTFd"]
 
 RUN pip install -r requirements.txt
+RUN for d in CTFd/plugins/*; do \
+      if [ -f "$d/requirements.txt" ]; then \
+        pip install -r $d/requirements.txt; \
+      fi; \
+    done;
 
 RUN chmod +x /opt/CTFd/docker-entrypoint.sh
 

docker-compose.yml

@@ -7,20 +7,34 @@ services:
     ports:
       - "8000:8000"
     environment:
+      - UPLOAD_FOLDER=/var/uploads
+      - LOG_FOLDER=/var/log/CTFd
       - DATABASE_URL=mysql+pymysql://root:ctfd@db/ctfd
     volumes:
-      - .data/CTFd/logs:/opt/CTFd/CTFd/logs
+      - .data/CTFd/logs:/var/log/CTFd
-      - .data/CTFd/uploads:/opt/CTFd/CTFd/uploads
+      - .data/CTFd/uploads:/var/uploads
+      - .:/opt/CTFd:ro
     depends_on:
       - db
+    networks:
+        default:
+        internal:
 
   db:
     image: mariadb:10.2
-    # This command is required to set important mariadb defaults
+    restart: always
-    command: [mysqld, --character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci, --wait_timeout=28800]
     environment:
       - MYSQL_ROOT_PASSWORD=ctfd
       - MYSQL_USER=ctfd
       - MYSQL_PASSWORD=ctfd
     volumes:
       - .data/mysql:/var/lib/mysql
+    networks:
+        internal:
+    # This command is required to set important mariadb defaults
+    command: [mysqld, --character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci, --wait_timeout=28800]
+
+networks:
+    default:
+    internal:
+        internal: true

docker-entrypoint.sh

@@ -15,4 +15,9 @@ if [ -n "$DATABASE_URL" ]
 fi
 
 echo "Starting CTFd"
-gunicorn --bind 0.0.0.0:8000 -w 1 'CTFd:create_app()' --access-logfile '/opt/CTFd/CTFd/logs/access.log' --error-logfile '/opt/CTFd/CTFd/logs/error.log'
+gunicorn 'CTFd:create_app()' \
+    --bind '0.0.0.0:8000' \
+    --workers 1 \
+    --worker-class 'gevent' \
+    --access-logfile "${LOG_FOLDER:-/opt/CTFd/CTFd/logs}/access.log" \
+    --error-logfile "${LOG_FOLDER:-/opt/CTFd/CTFd/logs}/error.log"

requirements.txt

@@ -18,3 +18,4 @@ mistune==0.8.3
 netaddr==0.7.19
 redis==2.10.6
 datafreeze==0.1.0
+gevent==1.2.2