Improve caching (#1014)

* Cache get place code for users and teams. 
    * Fix score changing test helpers to clear standings cache when generating a score changing row
    * `utils._get_config` will now return `KeyError` instead of None. 
    * Separate `/api/v1/[users,teams]/[me,id]/[solves,fails,awards]` into seperate API endpoints
* Install `Flask-DebugToolbar` in development

Main goals covered in #1012
selenium-screenshot-testing
Kevin Chung 2019-06-15 02:07:24 -04:00 committed by GitHub
parent e978867a2f
commit e627391b12
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 336 additions and 225 deletions

View File

@ -5,9 +5,11 @@ from CTFd.schemas.teams import TeamSchema
from CTFd.schemas.submissions import SubmissionSchema
from CTFd.schemas.awards import AwardSchema
from CTFd.cache import clear_standings
from CTFd.utils.decorators.visibility import check_account_visibility
from CTFd.utils.config.visibility import accounts_visible, scores_visible
from CTFd.utils.user import get_current_team, is_admin, authed
from CTFd.utils.decorators.visibility import (
check_account_visibility,
check_score_visibility,
)
from CTFd.utils.user import get_current_team, is_admin
from CTFd.utils.decorators import authed_only, admins_only
import copy
@ -221,23 +223,74 @@ class TeamMembers(Resource):
return {"success": True, "data": members}
@teams_namespace.route("/<team_id>/solves")
@teams_namespace.param("team_id", "Team ID or 'me'")
class TeamSolves(Resource):
def get(self, team_id):
if team_id == "me":
if not authed():
abort(403)
team = get_current_team()
solves = team.get_solves(admin=True)
else:
if accounts_visible() is False or scores_visible() is False:
abort(404)
team = Teams.query.filter_by(id=team_id).first_or_404()
@teams_namespace.route("/me/solves")
class TeamPrivateSolves(Resource):
@authed_only
def get(self):
team = get_current_team()
solves = team.get_solves(admin=True)
if (team.banned or team.hidden) and is_admin() is False:
abort(404)
solves = team.get_solves(admin=is_admin())
view = "admin" if is_admin() else "user"
schema = SubmissionSchema(view=view, many=True)
response = schema.dump(solves)
if response.errors:
return {"success": False, "errors": response.errors}, 400
return {"success": True, "data": response.data}
@teams_namespace.route("/me/fails")
class TeamPrivateFails(Resource):
@authed_only
def get(self):
team = get_current_team()
fails = team.get_fails(admin=True)
view = "admin" if is_admin() else "user"
schema = SubmissionSchema(view=view, many=True)
response = schema.dump(fails)
if response.errors:
return {"success": False, "errors": response.errors}, 400
if is_admin():
data = response.data
else:
data = []
count = len(response.data)
return {"success": True, "data": data, "meta": {"count": count}}
@teams_namespace.route("/me/awards")
class TeamPrivateAwards(Resource):
@authed_only
def get(self):
team = get_current_team()
awards = team.get_awards(admin=True)
schema = AwardSchema(many=True)
response = schema.dump(awards)
if response.errors:
return {"success": False, "errors": response.errors}, 400
return {"success": True, "data": response.data}
@teams_namespace.route("/<team_id>/solves")
@teams_namespace.param("team_id", "Team ID")
class TeamPublicSolves(Resource):
@check_account_visibility
@check_score_visibility
def get(self, team_id):
team = Teams.query.filter_by(id=team_id).first_or_404()
if (team.banned or team.hidden) and is_admin() is False:
abort(404)
solves = team.get_solves(admin=is_admin())
view = "admin" if is_admin() else "user"
schema = SubmissionSchema(view=view, many=True)
@ -250,22 +303,16 @@ class TeamSolves(Resource):
@teams_namespace.route("/<team_id>/fails")
@teams_namespace.param("team_id", "Team ID or 'me'")
class TeamFails(Resource):
@teams_namespace.param("team_id", "Team ID")
class TeamPublicFails(Resource):
@check_account_visibility
@check_score_visibility
def get(self, team_id):
if team_id == "me":
if not authed():
abort(403)
team = get_current_team()
fails = team.get_fails(admin=True)
else:
if accounts_visible() is False or scores_visible() is False:
abort(404)
team = Teams.query.filter_by(id=team_id).first_or_404()
team = Teams.query.filter_by(id=team_id).first_or_404()
if (team.banned or team.hidden) and is_admin() is False:
abort(404)
fails = team.get_fails(admin=is_admin())
if (team.banned or team.hidden) and is_admin() is False:
abort(404)
fails = team.get_fails(admin=is_admin())
view = "admin" if is_admin() else "user"
@ -285,22 +332,16 @@ class TeamFails(Resource):
@teams_namespace.route("/<team_id>/awards")
@teams_namespace.param("team_id", "Team ID or 'me'")
class TeamAwards(Resource):
@teams_namespace.param("team_id", "Team ID")
class TeamPublicAwards(Resource):
@check_account_visibility
@check_score_visibility
def get(self, team_id):
if team_id == "me":
if not authed():
abort(403)
team = get_current_team()
awards = team.get_awards(admin=True)
else:
if accounts_visible() is False or scores_visible() is False:
abort(404)
team = Teams.query.filter_by(id=team_id).first_or_404()
team = Teams.query.filter_by(id=team_id).first_or_404()
if (team.banned or team.hidden) and is_admin() is False:
abort(404)
awards = team.get_awards(admin=is_admin())
if (team.banned or team.hidden) and is_admin() is False:
abort(404)
awards = team.get_awards(admin=is_admin())
schema = AwardSchema(many=True)
response = schema.dump(awards)

View File

@ -1,5 +1,6 @@
from flask import request
from flask_restplus import Namespace, Resource
from CTFd.cache import clear_standings
from CTFd.models import db, get_class_by_tablename, Unlocks
from CTFd.utils.user import get_current_user
from CTFd.schemas.unlocks import UnlockSchema
@ -72,6 +73,7 @@ class UnlockList(Resource):
award = award_schema.load(award)
db.session.add(award.data)
db.session.commit()
clear_standings()
response = schema.dump(response.data)

View File

@ -10,14 +10,15 @@ from CTFd.models import (
Submissions,
Notifications,
)
from CTFd.utils.decorators import authed_only, admins_only, authed, ratelimit
from CTFd.utils.decorators import authed_only, admins_only, ratelimit
from CTFd.cache import clear_standings
from CTFd.utils.config import get_mail_provider
from CTFd.utils.email import sendmail, user_created_notification
from CTFd.utils.user import get_current_user, is_admin
from CTFd.utils.decorators.visibility import check_account_visibility
from CTFd.utils.config.visibility import accounts_visible, scores_visible
from CTFd.utils.decorators.visibility import (
check_account_visibility,
check_score_visibility,
)
from CTFd.schemas.submissions import SubmissionSchema
from CTFd.schemas.awards import AwardSchema
@ -156,23 +157,72 @@ class UserPrivate(Resource):
return {"success": True, "data": response.data}
@users_namespace.route("/<user_id>/solves")
@users_namespace.param("user_id", "User ID or 'me'")
class UserSolves(Resource):
def get(self, user_id):
if user_id == "me":
if not authed():
abort(403)
user = get_current_user()
solves = user.get_solves(admin=True)
else:
if accounts_visible() is False or scores_visible() is False:
abort(404)
user = Users.query.filter_by(id=user_id).first_or_404()
@users_namespace.route("/me/solves")
class UserPrivateSolves(Resource):
@authed_only
def get(self):
user = get_current_user()
solves = user.get_solves(admin=True)
if (user.banned or user.hidden) and is_admin() is False:
abort(404)
solves = user.get_solves(admin=is_admin())
view = "user" if not is_admin() else "admin"
response = SubmissionSchema(view=view, many=True).dump(solves)
if response.errors:
return {"success": False, "errors": response.errors}, 400
return {"success": True, "data": response.data}
@users_namespace.route("/me/fails")
class UserPrivateFails(Resource):
@authed_only
def get(self):
user = get_current_user()
fails = user.get_fails(admin=True)
view = "user" if not is_admin() else "admin"
response = SubmissionSchema(view=view, many=True).dump(fails)
if response.errors:
return {"success": False, "errors": response.errors}, 400
if is_admin():
data = response.data
else:
data = []
count = len(response.data)
return {"success": True, "data": data, "meta": {"count": count}}
@users_namespace.route("/me/awards")
@users_namespace.param("user_id", "User ID")
class UserPrivateAwards(Resource):
@authed_only
def get(self):
user = get_current_user()
awards = user.get_awards(admin=True)
view = "user" if not is_admin() else "admin"
response = AwardSchema(view=view, many=True).dump(awards)
if response.errors:
return {"success": False, "errors": response.errors}, 400
return {"success": True, "data": response.data}
@users_namespace.route("/<user_id>/solves")
@users_namespace.param("user_id", "User ID")
class UserPublicSolves(Resource):
@check_account_visibility
@check_score_visibility
def get(self, user_id):
user = Users.query.filter_by(id=user_id).first_or_404()
if (user.banned or user.hidden) and is_admin() is False:
abort(404)
solves = user.get_solves(admin=is_admin())
view = "user" if not is_admin() else "admin"
response = SubmissionSchema(view=view, many=True).dump(solves)
@ -184,22 +234,16 @@ class UserSolves(Resource):
@users_namespace.route("/<user_id>/fails")
@users_namespace.param("user_id", "User ID or 'me'")
class UserFails(Resource):
@users_namespace.param("user_id", "User ID")
class UserPublicFails(Resource):
@check_account_visibility
@check_score_visibility
def get(self, user_id):
if user_id == "me":
if not authed():
abort(403)
user = get_current_user()
fails = user.get_fails(admin=True)
else:
if accounts_visible() is False or scores_visible() is False:
abort(404)
user = Users.query.filter_by(id=user_id).first_or_404()
user = Users.query.filter_by(id=user_id).first_or_404()
if (user.banned or user.hidden) and is_admin() is False:
abort(404)
fails = user.get_fails(admin=is_admin())
if (user.banned or user.hidden) and is_admin() is False:
abort(404)
fails = user.get_fails(admin=is_admin())
view = "user" if not is_admin() else "admin"
response = SubmissionSchema(view=view, many=True).dump(fails)
@ -217,21 +261,15 @@ class UserFails(Resource):
@users_namespace.route("/<user_id>/awards")
@users_namespace.param("user_id", "User ID or 'me'")
class UserAwards(Resource):
class UserPublicAwards(Resource):
@check_account_visibility
@check_score_visibility
def get(self, user_id):
if user_id == "me":
if not authed():
abort(403)
user = get_current_user()
awards = user.get_awards(admin=True)
else:
if accounts_visible() is False or scores_visible() is False:
abort(404)
user = Users.query.filter_by(id=user_id).first_or_404()
user = Users.query.filter_by(id=user_id).first_or_404()
if (user.banned or user.hidden) and is_admin() is False:
abort(404)
awards = user.get_awards(admin=is_admin())
if (user.banned or user.hidden) and is_admin() is False:
abort(404)
awards = user.get_awards(admin=is_admin())
view = "user" if not is_admin() else "admin"
response = AwardSchema(view=view, many=True).dump(awards)

View File

@ -26,11 +26,13 @@ def clear_config():
def clear_standings():
from CTFd.utils.scores import get_standings
from CTFd.utils.scores import get_standings, get_team_standings, get_user_standings
from CTFd.api.v1.scoreboard import ScoreboardDetail, ScoreboardList
from CTFd.api import api
cache.delete_memoized(get_standings)
cache.delete_memoized(get_team_standings)
cache.delete_memoized(get_user_standings)
cache.delete(make_cache_key(path=api.name + "." + ScoreboardList.endpoint))
cache.delete(make_cache_key(path=api.name + "." + ScoreboardDetail.endpoint))
cache.delete_memoized(ScoreboardList.get)

View File

@ -1,6 +1,5 @@
from flask_sqlalchemy import SQLAlchemy
from flask_marshmallow import Marshmallow
from sqlalchemy.sql.expression import union_all
from sqlalchemy.orm import validates, column_property
from sqlalchemy.ext.hybrid import hybrid_property
from CTFd.utils.crypto import hash_password
@ -346,65 +345,9 @@ class Users(db.Model):
to no imports within the CTFd application as importing from the
application itself will result in a circular import.
"""
scores = (
db.session.query(
Solves.user_id.label("user_id"),
db.func.sum(Challenges.value).label("score"),
db.func.max(Solves.id).label("id"),
db.func.max(Solves.date).label("date"),
)
.join(Challenges)
.filter(Challenges.value != 0)
.group_by(Solves.user_id)
)
from CTFd.utils.scores import get_user_standings
awards = (
db.session.query(
Awards.user_id.label("user_id"),
db.func.sum(Awards.value).label("score"),
db.func.max(Awards.id).label("id"),
db.func.max(Awards.date).label("date"),
)
.filter(Awards.value != 0)
.group_by(Awards.user_id)
)
if not admin:
freeze = Configs.query.filter_by(key="freeze").first()
if freeze and freeze.value:
freeze = int(freeze.value)
freeze = datetime.datetime.utcfromtimestamp(freeze)
scores = scores.filter(Solves.date < freeze)
awards = awards.filter(Awards.date < freeze)
results = union_all(scores, awards).alias("results")
sumscores = (
db.session.query(
results.columns.user_id,
db.func.sum(results.columns.score).label("score"),
db.func.max(results.columns.id).label("id"),
db.func.max(results.columns.date).label("date"),
)
.group_by(results.columns.user_id)
.subquery()
)
if admin:
standings_query = (
db.session.query(Users.id.label("user_id"))
.join(sumscores, Users.id == sumscores.columns.user_id)
.order_by(sumscores.columns.score.desc(), sumscores.columns.id)
)
else:
standings_query = (
db.session.query(Users.id.label("user_id"))
.join(sumscores, Users.id == sumscores.columns.user_id)
.filter(Users.banned == False, Users.hidden == False)
.order_by(sumscores.columns.score.desc(), sumscores.columns.id)
)
standings = standings_query.all()
standings = get_user_standings(admin=admin)
# http://codegolf.stackexchange.com/a/4712
try:
@ -533,65 +476,9 @@ class Teams(db.Model):
to no imports within the CTFd application as importing from the
application itself will result in a circular import.
"""
scores = (
db.session.query(
Solves.team_id.label("team_id"),
db.func.sum(Challenges.value).label("score"),
db.func.max(Solves.id).label("id"),
db.func.max(Solves.date).label("date"),
)
.join(Challenges)
.filter(Challenges.value != 0)
.group_by(Solves.team_id)
)
from CTFd.utils.scores import get_team_standings
awards = (
db.session.query(
Awards.team_id.label("team_id"),
db.func.sum(Awards.value).label("score"),
db.func.max(Awards.id).label("id"),
db.func.max(Awards.date).label("date"),
)
.filter(Awards.value != 0)
.group_by(Awards.team_id)
)
if not admin:
freeze = Configs.query.filter_by(key="freeze").first()
if freeze and freeze.value:
freeze = int(freeze.value)
freeze = datetime.datetime.utcfromtimestamp(freeze)
scores = scores.filter(Solves.date < freeze)
awards = awards.filter(Awards.date < freeze)
results = union_all(scores, awards).alias("results")
sumscores = (
db.session.query(
results.columns.team_id,
db.func.sum(results.columns.score).label("score"),
db.func.max(results.columns.id).label("id"),
db.func.max(results.columns.date).label("date"),
)
.group_by(results.columns.team_id)
.subquery()
)
if admin:
standings_query = (
db.session.query(Teams.id.label("team_id"))
.join(sumscores, Teams.id == sumscores.columns.team_id)
.order_by(sumscores.columns.score.desc(), sumscores.columns.id)
)
else:
standings_query = (
db.session.query(Teams.id.label("team_id"))
.join(sumscores, Teams.id == sumscores.columns.team_id)
.filter(Teams.banned == False)
.order_by(sumscores.columns.score.desc(), sumscores.columns.id)
)
standings = standings_query.all()
standings = get_team_standings(admin=admin)
# http://codegolf.stackexchange.com/a/4712
try:

View File

@ -34,11 +34,14 @@ def _get_config(key):
return False
else:
return value
# Flask-Caching is unable to roundtrip a value of None.
# Return an exception so that we can still cache and avoid the db hit
return KeyError
def get_config(key, default=None):
value = _get_config(key)
if value is None:
if value is KeyError:
return default
else:
return value

View File

@ -2,7 +2,7 @@ from CTFd.cache import cache
from CTFd.models import Pages
# @cache.memoize()
@cache.memoize()
def get_pages():
db_pages = Pages.query.filter(
Pages.route != "index", Pages.draft.isnot(True), Pages.hidden.isnot(True)

View File

@ -1,7 +1,7 @@
from sqlalchemy.sql.expression import union_all
from CTFd.cache import cache
from CTFd.models import db, Solves, Awards, Challenges
from CTFd.models import db, Teams, Users, Solves, Awards, Challenges
from CTFd.utils.dates import unix_time_to_utc
from CTFd.utils import get_config
from CTFd.utils.modes import get_model
@ -111,5 +111,134 @@ def get_standings(count=None, admin=False):
else:
standings = standings_query.limit(count).all()
db.session.close()
return standings
@cache.memoize(timeout=60)
def get_team_standings(count=None, admin=False):
scores = (
db.session.query(
Solves.team_id.label("team_id"),
db.func.sum(Challenges.value).label("score"),
db.func.max(Solves.id).label("id"),
db.func.max(Solves.date).label("date"),
)
.join(Challenges)
.filter(Challenges.value != 0)
.group_by(Solves.team_id)
)
awards = (
db.session.query(
Awards.team_id.label("team_id"),
db.func.sum(Awards.value).label("score"),
db.func.max(Awards.id).label("id"),
db.func.max(Awards.date).label("date"),
)
.filter(Awards.value != 0)
.group_by(Awards.team_id)
)
freeze = get_config("freeze")
if not admin and freeze:
scores = scores.filter(Solves.date < unix_time_to_utc(freeze))
awards = awards.filter(Awards.date < unix_time_to_utc(freeze))
results = union_all(scores, awards).alias("results")
sumscores = (
db.session.query(
results.columns.team_id,
db.func.sum(results.columns.score).label("score"),
db.func.max(results.columns.id).label("id"),
db.func.max(results.columns.date).label("date"),
)
.group_by(results.columns.team_id)
.subquery()
)
if admin:
standings_query = (
db.session.query(Teams.id.label("team_id"))
.join(sumscores, Teams.id == sumscores.columns.team_id)
.order_by(sumscores.columns.score.desc(), sumscores.columns.id)
)
else:
standings_query = (
db.session.query(Teams.id.label("team_id"))
.join(sumscores, Teams.id == sumscores.columns.team_id)
.filter(Teams.banned == False)
.order_by(sumscores.columns.score.desc(), sumscores.columns.id)
)
if count is None:
standings = standings_query.all()
else:
standings = standings_query.limit(count).all()
return standings
@cache.memoize(timeout=60)
def get_user_standings(count=None, admin=False):
scores = (
db.session.query(
Solves.user_id.label("user_id"),
db.func.sum(Challenges.value).label("score"),
db.func.max(Solves.id).label("id"),
db.func.max(Solves.date).label("date"),
)
.join(Challenges)
.filter(Challenges.value != 0)
.group_by(Solves.user_id)
)
awards = (
db.session.query(
Awards.user_id.label("user_id"),
db.func.sum(Awards.value).label("score"),
db.func.max(Awards.id).label("id"),
db.func.max(Awards.date).label("date"),
)
.filter(Awards.value != 0)
.group_by(Awards.user_id)
)
freeze = get_config("freeze")
if not admin and freeze:
scores = scores.filter(Solves.date < unix_time_to_utc(freeze))
awards = awards.filter(Awards.date < unix_time_to_utc(freeze))
results = union_all(scores, awards).alias("results")
sumscores = (
db.session.query(
results.columns.user_id,
db.func.sum(results.columns.score).label("score"),
db.func.max(results.columns.id).label("id"),
db.func.max(results.columns.date).label("date"),
)
.group_by(results.columns.user_id)
.subquery()
)
if admin:
standings_query = (
db.session.query(Users.id.label("user_id"))
.join(sumscores, Users.id == sumscores.columns.user_id)
.order_by(sumscores.columns.score.desc(), sumscores.columns.id)
)
else:
standings_query = (
db.session.query(Users.id.label("user_id"))
.join(sumscores, Users.id == sumscores.columns.user_id)
.filter(Users.banned == False, Users.hidden == False)
.order_by(sumscores.columns.score.desc(), sumscores.columns.id)
)
if count is None:
standings = standings_query.all()
else:
standings = standings_query.limit(count).all()
return standings

View File

@ -10,7 +10,8 @@ psycopg2-binary==2.7.5
codecov==2.0.15
moto==1.3.7
bandit==1.5.1
flask_profiler==1.8.1
flask_profiler==1.7
pytest-xdist==1.28.0
pytest-cov==2.6.1
sphinx_rtd_theme==0.4.3
flask-debugtoolbar==0.10.1

View File

@ -1,5 +1,5 @@
Flask==1.0.2
Werkzeug==0.15.2
Werkzeug==0.15.3
Flask-SQLAlchemy==2.4.0
Flask-Session==0.3.1
Flask-Caching==1.4.0

View File

@ -8,6 +8,7 @@ args = parser.parse_args()
app = create_app()
if args.profile:
from flask_debugtoolbar import DebugToolbarExtension
import flask_profiler
app.config["flask_profiler"] = {
"enabled": app.config["DEBUG"],
@ -19,6 +20,11 @@ if args.profile:
},
}
flask_profiler.init_app(app)
app.config['DEBUG_TB_PROFILER_ENABLED'] = True
app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False
toolbar = DebugToolbarExtension()
toolbar.init_app(app)
print(" * Flask profiling running at http://127.0.0.1:4000/flask-profiler/")
app.run(debug=True, threaded=True, host="127.0.0.1", port=4000)

View File

@ -394,7 +394,7 @@ def test_api_team_get_me_solves_not_logged_in():
app = create_ctfd(user_mode="teams")
with app.app_context():
with app.test_client() as client:
r = client.get("/api/v1/teams/me/solves")
r = client.get("/api/v1/teams/me/solves", json="")
assert r.status_code == 403
destroy_ctfd(app)
@ -474,7 +474,7 @@ def test_api_team_get_me_fails_not_logged_in():
app = create_ctfd(user_mode="teams")
with app.app_context():
with app.test_client() as client:
r = client.get("/api/v1/teams/me/fails")
r = client.get("/api/v1/teams/me/fails", json="")
assert r.status_code == 403
destroy_ctfd(app)
@ -551,7 +551,7 @@ def test_api_team_get_me_awards_not_logged_in():
app = create_ctfd(user_mode="teams")
with app.app_context():
with app.test_client() as client:
r = client.get("/api/v1/teams/me/awards")
r = client.get("/api/v1/teams/me/awards", json="")
assert r.status_code == 403
destroy_ctfd(app)

View File

@ -494,7 +494,7 @@ def test_api_user_get_me_solves_not_logged_in():
app = create_ctfd()
with app.app_context():
with app.test_client() as client:
r = client.get("/api/v1/users/me/solves")
r = client.get("/api/v1/users/me/solves", json="")
assert r.status_code == 403
destroy_ctfd(app)
@ -569,7 +569,7 @@ def test_api_user_get_me_fails_not_logged_in():
app = create_ctfd()
with app.app_context():
with app.test_client() as client:
r = client.get("/api/v1/users/me/fails")
r = client.get("/api/v1/users/me/fails", json="")
assert r.status_code == 403
destroy_ctfd(app)
@ -641,7 +641,7 @@ def test_api_user_get_me_awards_not_logged_in():
app = create_ctfd()
with app.app_context():
with app.test_client() as client:
r = client.get("/api/v1/users/me/awards")
r = client.get("/api/v1/users/me/awards", json="")
assert r.status_code == 403
destroy_ctfd(app)

View File

@ -20,7 +20,7 @@ from CTFd.models import (
Unlocks,
Users,
)
from CTFd.cache import cache
from CTFd.cache import cache, clear_standings
from sqlalchemy_utils import drop_database
from collections import namedtuple
from mock import Mock, patch
@ -268,6 +268,7 @@ def gen_award(db, user_id, team_id=None, name="award_name", value=100):
award.date = datetime.datetime.utcnow()
db.session.add(award)
db.session.commit()
clear_standings()
return award
@ -364,6 +365,7 @@ def gen_solve(
solve.date = datetime.datetime.utcnow()
db.session.add(solve)
db.session.commit()
clear_standings()
return solve