From 4ec45dc4bc4084b04a451efcab4e23fef5860f04 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Tue, 30 Apr 2019 20:36:25 -0400 Subject: [PATCH] Fix imports/exports and update Flask-SQLAlchemy to reduce warnings (#974) * Fix imports/exports to accept JSON * MariaDB does not properly understand JSON so it must accept strings instead of dicts * MariaDB outputs strings instead of JSON for its JSON type so the export serializer will attempt to cast output JSON strings to JSON objects * Update Makefile to not lint any uploads * Update Flask-SQLAlchemy to 2.4.0 to remove a large amount of warnings --- CTFd/utils/exports/__init__.py | 35 +++++++++++++++++++++++++----- Makefile | 2 +- requirements.txt | 2 +- tests/utils/test_exports.py | 39 ++++++++++++++++++++++++---------- 4 files changed, 60 insertions(+), 18 deletions(-) diff --git a/CTFd/utils/exports/__init__.py b/CTFd/utils/exports/__init__.py index 26ed0fe..083c9d3 100644 --- a/CTFd/utils/exports/__init__.py +++ b/CTFd/utils/exports/__init__.py @@ -7,7 +7,7 @@ from datafreeze.format import SERIALIZERS from flask import current_app as app from flask_migrate import upgrade, stamp from datafreeze.format.fjson import JSONSerializer, JSONEncoder -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import OperationalError, ProgrammingError from alembic.util import CommandError import dataset import datafreeze @@ -35,9 +35,23 @@ class CTFdSerializer(JSONSerializer): else: fh = self.fileobj - data = json.dumps(result, - cls=JSONEncoder, - indent=self.export.get_int('indent')) + # Certain databases (MariaDB) store JSON as LONGTEXT. + # Before emitting a file we should standardize to valid JSON (i.e. a dict) + # See Issue #973 + for i, r in enumerate(result['results']): + data = r.get('requirements') + if data: + try: + if isinstance(data, six.string_types): + result['results'][i]['requirements'] = json.loads(data) + except ValueError: + pass + + data = json.dumps( + result, + cls=JSONEncoder, + indent=self.export.get_int('indent') + ) callback = self.export.get('callback') if callback: @@ -208,7 +222,18 @@ def import_ctf(backup, erase=True): requirements = entry.get('requirements') if requirements and isinstance(requirements, six.string_types): entry['requirements'] = json.loads(requirements) - table.insert(entry) + + try: + table.insert(entry) + except ProgrammingError: + # MariaDB does not like JSON objects and prefers strings because it internally + # represents JSON with LONGTEXT. + # See Issue #973 + requirements = entry.get('requirements') + if requirements and isinstance(requirements, dict): + entry['requirements'] = json.dumps(requirements) + table.insert(entry) + db.session.commit() if postgres: # This command is to set the next primary key ID for the re-inserted tables in Postgres. However, diff --git a/Makefile b/Makefile index 44b96e2..e35c87a 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ lint: - flake8 --ignore=E402,E501,E712 CTFd/ tests/ + flake8 --ignore=E402,E501,E712 --exclude=CTFd/uploads CTFd/ tests/ test: pytest --cov=CTFd --disable-warnings -n auto diff --git a/requirements.txt b/requirements.txt index bb35b58..9be8c83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ Flask==1.0.2 Werkzeug==0.15.2 -Flask-SQLAlchemy==2.3.2 +Flask-SQLAlchemy==2.4.0 Flask-Session==0.3.1 Flask-Caching==1.4.0 Flask-Migrate==2.3.0 diff --git a/tests/utils/test_exports.py b/tests/utils/test_exports.py index a937a6a..acf9bcb 100644 --- a/tests/utils/test_exports.py +++ b/tests/utils/test_exports.py @@ -1,17 +1,20 @@ # -*- coding: utf-8 -*- -from tests.helpers import (create_ctfd, - destroy_ctfd, - register_user, - login_as_user, - gen_challenge, - gen_flag, - gen_user, - gen_hint) +from tests.helpers import ( + create_ctfd, + destroy_ctfd, + register_user, + login_as_user, + gen_challenge, + gen_flag, + gen_user, + gen_hint +) from CTFd.models import Challenges, Flags, Users from CTFd.utils import text_type from CTFd.utils.exports import import_ctf, export_ctf import json import os +import zipfile def test_export_ctf(): @@ -20,8 +23,11 @@ def test_export_ctf(): if not app.config.get('SQLALCHEMY_DATABASE_URI').startswith('sqlite'): with app.app_context(): register_user(app) - chal = gen_challenge(app.db, name=text_type('🐺')) - chal_id = chal.id + chal1 = gen_challenge(app.db, name=text_type('🐺')) + gen_challenge(app.db, name=text_type('🐺'), requirements={ + "prerequisites": [1] + }) + chal_id = chal1.id gen_hint(app.db, chal_id) client = login_as_user(app) @@ -38,6 +44,11 @@ def test_export_ctf(): with open('export.test_export_ctf.zip', 'wb') as f: f.write(backup.read()) + + export = zipfile.ZipFile('export.test_export_ctf.zip', 'r') + data = json.loads(export.read('db/challenges.json')) + assert data['results'][1]['requirements'] == {"prerequisites": [1]} + os.remove('export.test_export_ctf.zip') destroy_ctfd(app) @@ -53,10 +64,13 @@ def test_import_ctf(): user_email = user + "@ctfd.io" gen_user(app.db, name=user, email=user_email) - for x in range(10): + for x in range(9): chal = gen_challenge(app.db, name='chal_name{}'.format(x)) gen_flag(app.db, challenge_id=chal.id, content='flag') + chal = gen_challenge(app.db, name='chal_name10', requirements={"prerequisites": [1]}) + gen_flag(app.db, challenge_id=chal.id, content='flag') + app.db.session.commit() backup = export_ctf() @@ -76,4 +90,7 @@ def test_import_ctf(): assert Users.query.count() == 11 assert Challenges.query.count() == 10 assert Flags.query.count() == 10 + + chal = Challenges.query.filter_by(name='chal_name10').first() + assert chal.requirements == {"prerequisites": [1]} destroy_ctfd(app)