EbookFoundation
/
readthedocs.org
mirror of https://github.com/EbookFoundation/readthedocs.org.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add private symlink structure. (#2121) 

This builds out a separate private_* symlink structure for all private projects. This runs both Public & Private symlinks for each project, as it might have versions that have both privacy levels.
break-out-core-urls-views
Eric Holscher 2016-04-11 13:10:02 -07:00 committed by Anthony
parent bd3e0b9f57
commit 712e75d30a
6 changed files with 248 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
readthedocs/core/symlink.py
View File

@ -60,8 +60,8 @@ from collections import OrderedDict
from django.conf import settings
from readthedocs.projects.constants import LOG_TEMPLATE
from readthedocs.projects.models import Domain
from readthedocs.projects import constants
from readthedocs.projects.models import Domain, Project
from readthedocs.projects.utils import run
log = logging.getLogger(__name__)
@ -70,10 +70,6 @@ log = logging.getLogger(__name__)
class Symlink(object):
"""Base class for symlinking of projects."""
CNAME_ROOT = os.path.join(settings.SITE_ROOT, 'public_cname_root')
WEB_ROOT = os.path.join(settings.SITE_ROOT, 'public_web_root')
PROJECT_CNAME_ROOT = os.path.join(settings.SITE_ROOT, 'public_cname_project')
def __init__(self, project):
self.project = project
self.project_root = os.path.join(
@ -86,7 +82,7 @@ class Symlink(object):
def _log(self, msg, level='info'):
logger = getattr(log, level)
logger(LOG_TEMPLATE
logger(constants.LOG_TEMPLATE
.format(project=self.project.slug,
version='',
msg=msg)
@ -98,6 +94,8 @@ class Symlink(object):
This will leave it in the proper state for the single_project setting.
"""
if not self.run_sanity_check():
return
if os.path.islink(self.project_root) and not self.project.single_version:
self._log("Removing single version symlink")
os.unlink(self.project_root)
@ -161,7 +159,7 @@ class Symlink(object):
run('ln -nsf %s %s' % (self.project.doc_path, project_cname_symlink))
def remove_symlink_cname(self, domain):
"""Remove single_version symlink"""
"""Remove CNAME symlink"""
self._log(u"Removing symlink for CNAME {0}".format(domain.domain))
symlink = os.path.join(self.CNAME_ROOT, domain.domain)
os.unlink(symlink)
@ -173,7 +171,7 @@ class Symlink(object):
$WEB_ROOT/<project>
"""
subprojects = set()
rels = self.project.subprojects.all()
rels = self.get_subprojects()
if rels.count():
# Don't creat the `projects/` directory unless subprojects exist.
if not os.path.exists(self.subproject_root):
@ -211,7 +209,7 @@ class Symlink(object):
"""
translations = {}
for trans in self.project.translations.all():
for trans in self.get_translations():
translations[trans.language] = trans.slug
# Make sure the language directory is a directory
@ -243,7 +241,10 @@ class Symlink(object):
Link from $WEB_ROOT/<project> ->
HOME/user_builds/<project>/rtd-builds/latest/
"""
default_version = self.project.get_default_version()
default_version = self.get_default_version()
if default_version is None:
return
self._log("Symlinking single_version")
symlink = self.project_root
@ -266,8 +267,7 @@ class Symlink(object):
version_dir = os.path.join(self.WEB_ROOT, self.project.slug, self.project.language)
# Include active public versions,
# as well as public verisons that are built but not active, for archived versions
version_queryset = (self.project.versions.protected(only_active=False).filter(built=True) |
self.project.versions.protected(only_active=True))
version_queryset = self.get_version_queryset()
if version_queryset.count():
if not os.path.exists(version_dir):
os.makedirs(version_dir)
@ -283,3 +283,54 @@ class Symlink(object):
for old_ver in os.listdir(version_dir):
if old_ver not in versions:
os.unlink(os.path.join(version_dir, old_ver))
class PublicSymlink(Symlink):
CNAME_ROOT = os.path.join(settings.SITE_ROOT, 'public_cname_root')
WEB_ROOT = os.path.join(settings.SITE_ROOT, 'public_web_root')
PROJECT_CNAME_ROOT = os.path.join(settings.SITE_ROOT, 'public_cname_project')
def get_version_queryset(self):
return (self.project.versions.protected(only_active=False).filter(built=True) |
self.project.versions.protected(only_active=True))
def get_subprojects(self):
return self.project.subprojects.protected()
def get_translations(self):
return self.project.translations.protected()
def get_default_version(self):
default_version = self.project.get_default_version()
if self.project.versions.protected().filter(slug=default_version).exists():
return default_version
return None
def run_sanity_check(self):
return self.project.privacy_level in [constants.PUBLIC, constants.PROTECTED]
class PrivateSymlink(Symlink):
CNAME_ROOT = os.path.join(settings.SITE_ROOT, 'private_cname_root')
WEB_ROOT = os.path.join(settings.SITE_ROOT, 'private_web_root')
PROJECT_CNAME_ROOT = os.path.join(settings.SITE_ROOT, 'private_cname_project')
def run_sanity_check(self):
return self.project.privacy_level == constants.PRIVATE
def get_version_queryset(self):
return (self.project.versions.private(only_active=False).filter(built=True) |
self.project.versions.private(only_active=True))
def get_subprojects(self):
return self.project.subprojects.private()
def get_translations(self):
return self.project.translations.private()
def get_default_version(self):
default_version = self.project.get_default_version()
version_qs = self.project.versions.private().filter(slug=default_version)
if version_qs.exists():
return default_version
return None

47
readthedocs/privacy/backend.py
View File

@ -57,6 +57,13 @@ class ProjectManager(models.Manager):
else:
return queryset
def private(self, user=None):
queryset = self.filter(privacy_level=constants.PRIVATE)
if user:
return self._add_user_repos(queryset, user)
else:
return queryset
# Aliases
def dashboard(self, user=None):
@ -176,8 +183,8 @@ class RelatedProjectManager(models.Manager):
This shouldn't be used as a subclass.
"""
use_for_related_fields = True
project_field = 'project'
def _add_user_repos(self, queryset, user=None):
# Hack around get_objects_for_user not supporting global perms
@ -187,11 +194,35 @@ class RelatedProjectManager(models.Manager):
# Add in possible user-specific views
project_qs = get_objects_for_user(user, 'projects.view_project')
pks = [p.pk for p in project_qs]
queryset = self.get_queryset().filter(project__pk__in=pks) | queryset
kwargs = {'%s__pk__in' % self.project_field: pks}
queryset = self.get_queryset().filter(**kwargs) | queryset
return queryset.distinct()
def public(self, user=None, project=None):
queryset = self.filter(project__privacy_level=constants.PUBLIC)
kwargs = {'%s__privacy_level' % self.project_field: constants.PUBLIC}
queryset = self.filter(**kwargs)
if user:
queryset = self._add_user_repos(queryset, user)
if project:
queryset = queryset.filter(project=project)
return queryset
def protected(self, user=None, project=None):
kwargs = {
'%s__privacy_level__in' % self.project_field: [constants.PUBLIC, constants.PROTECTED]
}
queryset = self.filter(**kwargs)
if user:
queryset = self._add_user_repos(queryset, user)
if project:
queryset = queryset.filter(project=project)
return queryset
def private(self, user=None, project=None):
kwargs = {
'%s__privacy_level' % self.project_field: constants.PRIVATE,
}
queryset = self.filter(**kwargs)
if user:
queryset = self._add_user_repos(queryset, user)
if project:
@ -202,6 +233,16 @@ class RelatedProjectManager(models.Manager):
return self.public(user)
class ParentRelatedProjectManager(RelatedProjectManager):
project_field = 'parent'
use_for_related_fields = True
class ChildRelatedProjectManager(RelatedProjectManager):
project_field = 'child'
use_for_related_fields = True
class RelatedBuildManager(models.Manager):
'''For models with association to a project through :py:cls:`Build`'''

3
readthedocs/privacy/loader.py
View File

@ -21,6 +21,9 @@ RelatedBuildManager = import_by_path(
RelatedUserManager = import_by_path(
getattr(settings, 'RELATED_USER_MANAGER',
'readthedocs.privacy.backend.RelatedUserManager'))
ChildRelatedProjectManager = import_by_path(
getattr(settings, 'CHILD_RELATED_PROJECT_MANAGER',
'readthedocs.privacy.backend.ChildRelatedProjectManager'))
# Permissions
AdminPermission = import_by_path(

6
readthedocs/projects/models.py
View File

@ -18,10 +18,10 @@ from taggit.managers import TaggableManager
from readthedocs.api.client import api
from readthedocs.core.utils import broadcast
from readthedocs.core.resolver import resolve_domain
from readthedocs.restapi.client import api as apiv2
from readthedocs.builds.constants import LATEST, LATEST_VERBOSE_NAME, STABLE
from readthedocs.privacy.loader import RelatedProjectManager, ProjectManager
from readthedocs.privacy.loader import (RelatedProjectManager, ProjectManager,
ChildRelatedProjectManager)
from readthedocs.projects import constants
from readthedocs.projects.exceptions import ProjectImportError
from readthedocs.projects.templatetags.projects_tags import sort_version_aware
@ -58,6 +58,8 @@ class ProjectRelationship(models.Model):
related_name='superprojects')
alias = models.CharField(_('Alias'), max_length=255, null=True, blank=True)
objects = ChildRelatedProjectManager()
def __unicode__(self):
return "%s -> %s" % (self.parent, self.child)

23
readthedocs/projects/tasks.py
View File

@ -26,7 +26,7 @@ from readthedocs.builds.constants import (LATEST,
BUILD_STATE_FINISHED)
from readthedocs.builds.models import Build, Version
from readthedocs.core.utils import send_email, run_on_app_servers, broadcast
from readthedocs.core.symlink import Symlink
from readthedocs.core.symlink import PublicSymlink, PrivateSymlink
from readthedocs.cdn.purge import purge
from readthedocs.doc_builder.loader import get_builder_class
from readthedocs.doc_builder.config import load_yaml_config
@ -627,26 +627,29 @@ def update_search(version_pk, commit, delete_non_commit_files=True):
@task(queue='web')
def symlink_project(project_pk):
project = Project.objects.get(pk=project_pk)
sym = Symlink(project=project)
sym.run()
for symlink in [PublicSymlink, PrivateSymlink]:
sym = symlink(project=project)
sym.run()
@task(queue='web')
def symlink_domain(project_pk, domain_pk, delete=False):
project = Project.objects.get(pk=project_pk)
domain = Domain.objects.get(pk=domain_pk)
sym = Symlink(project=project)
if delete:
sym.remove_symlink_cname(domain)
else:
sym.symlink_cnames(domain)
for symlink in [PublicSymlink, PrivateSymlink]:
sym = symlink(project=project)
if delete:
sym.remove_symlink_cname(domain)
else:
sym.symlink_cnames(domain)
@task(queue='web')
def symlink_subproject(project_pk):
project = Project.objects.get(pk=project_pk)
sym = Symlink(project=project)
sym.symlink_subprojects()
for symlink in [PublicSymlink, PrivateSymlink]:
sym = symlink(project=project)
sym.symlink_subprojects()
@task(queue='web')

180
readthedocs/rtd_tests/tests/test_project_symlinks.py
View File

@ -9,7 +9,7 @@ from django_dynamic_fixture import get
from readthedocs.builds.models import Version
from readthedocs.projects.models import Project, Domain
from readthedocs.core.symlink import Symlink
from readthedocs.core.symlink import PublicSymlink, PrivateSymlink
def patched(fn):
@ -26,18 +26,33 @@ def patched(fn):
return wrapper
class TestSubprojects(TestCase):
class TestSymlinkCnames(TestCase):
def setUp(self):
self.project = get(Project, slug='kong')
self.subproject = get(Project, slug='sub')
self.symlink = Symlink(self.project)
self.version = get(Version, verbose_name='latest', active=True, project=self.project)
self.symlink = PublicSymlink(self.project)
self.args = {
'web_root': self.symlink.WEB_ROOT,
'subproject_root': self.symlink.subproject_root,
'cname_root': self.symlink.CNAME_ROOT,
'project_root': self.symlink.project_root,
}
self.commands = []
@patched
def test_symlink_cname(self):
self.cname = get(Domain, project=self.project, url='http://woot.com', cname=True)
self.symlink.symlink_cnames()
self.args['cname'] = self.cname.domain
commands = [
'ln -nsf {project_root} {cname_root}/{cname}',
]
for index, command in enumerate(commands):
self.assertEqual(self.commands[index], command.format(**self.args))
class BaseSubprojects(object):
@patched
def test_subproject_normal(self):
self.project.add_subproject(self.subproject)
@ -76,53 +91,32 @@ class TestSubprojects(TestCase):
self.assertTrue(not os.path.lexists(subproject_link))
class TestSymlinkCnames(TestCase):
class TestPublicSubprojects(BaseSubprojects, TestCase):
def setUp(self):
self.project = get(Project, slug='kong')
self.version = get(Version, verbose_name='latest', active=True, project=self.project)
self.symlink = Symlink(self.project)
self.subproject = get(Project, slug='sub')
self.symlink = PublicSymlink(self.project)
self.args = {
'cname_root': self.symlink.CNAME_ROOT,
'project_root': self.symlink.project_root,
'web_root': self.symlink.WEB_ROOT,
'subproject_root': self.symlink.subproject_root,
}
self.commands = []
@patched
def test_symlink_cname(self):
self.cname = get(Domain, project=self.project, url='http://woot.com', cname=True)
self.symlink.symlink_cnames()
self.args['cname'] = self.cname.domain
commands = [
'ln -nsf {project_root} {cname_root}/{cname}',
]
for index, command in enumerate(commands):
self.assertEqual(self.commands[index], command.format(**self.args))
class TestSymlinkTranslations(TestCase):
commands = []
class TestPrivateSubprojects(BaseSubprojects, TestCase):
def setUp(self):
self.project = get(Project, slug='kong')
self.translation = get(Project, slug='pip')
self.translation.language = 'de'
self.translation.main_lanuage_project = self.project
self.project.translations.add(self.translation)
self.translation.save()
self.project.save()
self.symlink = Symlink(self.project)
get(Version, verbose_name='master', active=True, project=self.project)
get(Version, verbose_name='master', active=True, project=self.translation)
self.project = get(Project, slug='kong', privacy_level='private')
self.subproject = get(Project, slug='sub', privacy_level='private')
self.symlink = PrivateSymlink(self.project)
self.args = {
'project_root': self.symlink.project_root,
'translation_root': os.path.join(self.symlink.WEB_ROOT, self.translation.slug),
'web_root': self.symlink.WEB_ROOT,
'subproject_root': self.symlink.subproject_root,
}
self.assertIn(self.translation, self.project.translations.all())
self.commands = []
class BaseSymlinkTranslations(object):
@patched
def test_symlink_basic(self):
'''Test basic scenario, language english, translation german'''
@ -184,25 +178,67 @@ class TestSymlinkTranslations(TestCase):
self.commands.index(command.format(**self.args))
))
def test_remove_language(self):
self.symlink.symlink_translations()
trans_link = os.path.join(
self.symlink.project_root, self.translation.language
)
self.assertTrue(os.path.lexists(trans_link))
trans = self.project.translations.first()
self.project.translations.remove(trans)
self.symlink.symlink_translations()
self.assertTrue(not os.path.lexists(trans_link))
class TestSymlinkSingleVersion(TestCase):
class TestPublicSymlinkTranslations(BaseSymlinkTranslations, TestCase):
def setUp(self):
self.project = get(Project, slug='kong')
self.translation = get(Project, slug='pip')
self.translation.language = 'de'
self.translation.main_lanuage_project = self.project
self.project.translations.add(self.translation)
self.translation.save()
self.project.save()
self.symlink = PublicSymlink(self.project)
get(Version, verbose_name='master', active=True, project=self.project)
get(Version, verbose_name='master', active=True, project=self.translation)
self.args = {
'project_root': self.symlink.project_root,
'translation_root': os.path.join(self.symlink.WEB_ROOT, self.translation.slug),
}
self.assertIn(self.translation, self.project.translations.all())
self.commands = []
class TestPrivateSymlinkTranslations(BaseSymlinkTranslations, TestCase):
def setUp(self):
self.project = get(Project, slug='kong', privacy_level='private')
self.translation = get(Project, slug='pip', privacy_level='private')
self.translation.language = 'de'
self.translation.main_lanuage_project = self.project
self.project.translations.add(self.translation)
self.translation.save()
self.project.save()
self.symlink = PrivateSymlink(self.project)
get(Version, verbose_name='master', active=True, project=self.project)
get(Version, verbose_name='master', active=True, project=self.translation)
self.args = {
'project_root': self.symlink.project_root,
'translation_root': os.path.join(self.symlink.WEB_ROOT, self.translation.slug),
}
self.assertIn(self.translation, self.project.translations.all())
self.commands = []
class TestPublicSymlinkSingleVersion(TestCase):
def setUp(self):
self.project = get(Project, slug='kong')
self.version = get(Version, verbose_name='latest', active=True, project=self.project)
self.symlink = Symlink(self.project)
self.symlink = PublicSymlink(self.project)
self.args = {
'project_root': self.symlink.project_root,
'doc_path': self.project.rtd_build_path(),
@ -220,18 +256,7 @@ class TestSymlinkSingleVersion(TestCase):
self.assertEqual(self.commands[index], command.format(**self.args))
class TestSymlinkVersions(TestCase):
def setUp(self):
self.project = get(Project, slug='kong')
self.stable = get(Version, slug='stable', verbose_name='stable', active=True, project=self.project)
self.symlink = Symlink(self.project)
self.args = {
'project_root': self.symlink.project_root,
'latest_path': self.project.rtd_build_path('latest'),
'stable_path': self.project.rtd_build_path('stable'),
}
self.commands = []
class BaseSymlinkVersions(object):
@patched
def test_symlink_versions(self):
@ -244,6 +269,21 @@ class TestSymlinkVersions(TestCase):
for index, command in enumerate(commands):
self.assertEqual(self.commands[index], command.format(**self.args))
class TestPublicSymlinkVersions(BaseSymlinkVersions, TestCase):
def setUp(self):
self.project = get(Project, slug='kong')
self.stable = get(
Version, slug='stable', verbose_name='stable', active=True, project=self.project)
self.symlink = PublicSymlink(self.project)
self.args = {
'project_root': self.symlink.project_root,
'latest_path': self.project.rtd_build_path('latest'),
'stable_path': self.project.rtd_build_path('stable'),
}
self.commands = []
@patched
def test_no_symlink_private_versions(self):
self.stable.privacy_level = 'private'
@ -268,12 +308,32 @@ class TestSymlinkVersions(TestCase):
self.assertTrue(not os.path.lexists(version_link))
class TestSymlinkUnicode(TestCase):
class TestPrivateSymlinkVersions(BaseSymlinkVersions, TestCase):
def setUp(self):
self.project = get(Project, slug='kong', privacy_level='private')
self.stable = get(
Version, slug='stable', verbose_name='stable',
active=True, project=self.project, privacy_level='private')
self.project.versions.filter(slug='latest').update(privacy_level='private')
self.symlink = PrivateSymlink(self.project)
self.args = {
'project_root': self.symlink.project_root,
'latest_path': self.project.rtd_build_path('latest'),
'stable_path': self.project.rtd_build_path('stable'),
}
self.commands = []
# Unicode
class TestPublicSymlinkUnicode(TestCase):
def setUp(self):
self.project = get(Project, slug='kong', name=u'foo-∫')
self.stable = get(Version, slug='stable', verbose_name=u'foo-∂', active=True, project=self.project)
self.symlink = Symlink(self.project)
self.stable = get(
Version, slug='stable', verbose_name=u'foo-∂', active=True, project=self.project)
self.symlink = PublicSymlink(self.project)
self.args = {
'project_root': self.symlink.project_root,
'latest_path': self.project.rtd_build_path('latest'),