Fix imports/exports and update Flask-SQLAlchemy to reduce warnings (#974)

* Fix imports/exports to accept JSON 
    * MariaDB does not properly understand JSON so it must accept strings instead of dicts
    * MariaDB outputs strings instead of JSON for its JSON type so the export serializer will attempt to cast output JSON strings to JSON objects
* Update Makefile to not lint any uploads
* Update Flask-SQLAlchemy to 2.4.0 to remove a large amount of warnings

CTFd/utils/exports/__init__.py

@@ -7,7 +7,7 @@ from datafreeze.format import SERIALIZERS
 from flask import current_app as app
 from flask_migrate import upgrade, stamp
 from datafreeze.format.fjson import JSONSerializer, JSONEncoder
-from sqlalchemy.exc import OperationalError
+from sqlalchemy.exc import OperationalError, ProgrammingError
 from alembic.util import CommandError
 import dataset
 import datafreeze
@@ -35,9 +35,23 @@ class CTFdSerializer(JSONSerializer):
             else:
                 fh = self.fileobj
 
-            data = json.dumps(result,
-                              cls=JSONEncoder,
-                              indent=self.export.get_int('indent'))
+            # Certain databases (MariaDB) store JSON as LONGTEXT.
+            # Before emitting a file we should standardize to valid JSON (i.e. a dict)
+            # See Issue #973
+            for i, r in enumerate(result['results']):
+                data = r.get('requirements')
+                if data:
+                    try:
+                        if isinstance(data, six.string_types):
+                            result['results'][i]['requirements'] = json.loads(data)
+                    except ValueError:
+                        pass
+
+            data = json.dumps(
+                result,
+                cls=JSONEncoder,
+                indent=self.export.get_int('indent')
+            )
 
             callback = self.export.get('callback')
             if callback:
@@ -208,7 +222,18 @@ def import_ctf(backup, erase=True):
                         requirements = entry.get('requirements')
                         if requirements and isinstance(requirements, six.string_types):
                             entry['requirements'] = json.loads(requirements)
-                    table.insert(entry)
+
+                    try:
+                        table.insert(entry)
+                    except ProgrammingError:
+                        # MariaDB does not like JSON objects and prefers strings because it internally
+                        # represents JSON with LONGTEXT.
+                        # See Issue #973
+                        requirements = entry.get('requirements')
+                        if requirements and isinstance(requirements, dict):
+                            entry['requirements'] = json.dumps(requirements)
+                        table.insert(entry)
+
                     db.session.commit()
                 if postgres:
                     # This command is to set the next primary key ID for the re-inserted tables in Postgres. However,

Makefile

@@ -1,5 +1,5 @@
 lint:
-	flake8 --ignore=E402,E501,E712 CTFd/ tests/
+	flake8 --ignore=E402,E501,E712 --exclude=CTFd/uploads CTFd/ tests/
 
 test:
 	pytest --cov=CTFd --disable-warnings -n auto

requirements.txt

@@ -1,6 +1,6 @@
 Flask==1.0.2
 Werkzeug==0.15.2
-Flask-SQLAlchemy==2.3.2
+Flask-SQLAlchemy==2.4.0
 Flask-Session==0.3.1
 Flask-Caching==1.4.0
 Flask-Migrate==2.3.0

tests/utils/test_exports.py

@@ -1,17 +1,20 @@
 # -*- coding: utf-8 -*-
-from tests.helpers import (create_ctfd,
-                           destroy_ctfd,
-                           register_user,
-                           login_as_user,
-                           gen_challenge,
-                           gen_flag,
-                           gen_user,
-                           gen_hint)
+from tests.helpers import (
+    create_ctfd,
+    destroy_ctfd,
+    register_user,
+    login_as_user,
+    gen_challenge,
+    gen_flag,
+    gen_user,
+    gen_hint
+)
 from CTFd.models import Challenges, Flags, Users
 from CTFd.utils import text_type
 from CTFd.utils.exports import import_ctf, export_ctf
 import json
 import os
+import zipfile
 
 
 def test_export_ctf():
@@ -20,8 +23,11 @@ def test_export_ctf():
     if not app.config.get('SQLALCHEMY_DATABASE_URI').startswith('sqlite'):
         with app.app_context():
             register_user(app)
-            chal = gen_challenge(app.db, name=text_type('🐺'))
-            chal_id = chal.id
+            chal1 = gen_challenge(app.db, name=text_type('🐺'))
+            gen_challenge(app.db, name=text_type('🐺'), requirements={
+                "prerequisites": [1]
+            })
+            chal_id = chal1.id
             gen_hint(app.db, chal_id)
 
             client = login_as_user(app)
@@ -38,6 +44,11 @@ def test_export_ctf():
 
             with open('export.test_export_ctf.zip', 'wb') as f:
                 f.write(backup.read())
+
+            export = zipfile.ZipFile('export.test_export_ctf.zip', 'r')
+            data = json.loads(export.read('db/challenges.json'))
+            assert data['results'][1]['requirements'] == {"prerequisites": [1]}
+
             os.remove('export.test_export_ctf.zip')
     destroy_ctfd(app)
 
@@ -53,10 +64,13 @@ def test_import_ctf():
                 user_email = user + "@ctfd.io"
                 gen_user(app.db, name=user, email=user_email)
 
-            for x in range(10):
+            for x in range(9):
                 chal = gen_challenge(app.db, name='chal_name{}'.format(x))
                 gen_flag(app.db, challenge_id=chal.id, content='flag')
 
+            chal = gen_challenge(app.db, name='chal_name10', requirements={"prerequisites": [1]})
+            gen_flag(app.db, challenge_id=chal.id, content='flag')
+
             app.db.session.commit()
 
             backup = export_ctf()
@@ -76,4 +90,7 @@ def test_import_ctf():
                 assert Users.query.count() == 11
                 assert Challenges.query.count() == 10
                 assert Flags.query.count() == 10
+
+                chal = Challenges.query.filter_by(name='chal_name10').first()
+                assert chal.requirements == {"prerequisites": [1]}
     destroy_ctfd(app)