Merge pull request #4899 from rtfd/humitos/admin/crud-env-variables

CRUD for EnvironmentVariables from Project's admin

docs/builds.rst

@@ -225,3 +225,8 @@ The *Sphinx* and *Mkdocs* builders set the following RTD-specific environment va
 +-------------------------+--------------------------------------------------+----------------------+
 | ``READTHEDOCS_PROJECT`` | The RTD name of the project which is being built | ``myexampleproject`` |
 +-------------------------+--------------------------------------------------+----------------------+
+
+.. tip::
+
+   In case extra environment variables are needed to the build process (like secrets, tokens, etc),
+   you can add them going to **Admin > Environment Variables** in your project. See :doc:`guides/environment-variables`.

docs/guides/environment-variables.rst

@@ -0,0 +1,37 @@
+I Need Secrets (or Environment Variables) in my Build
+=====================================================
+
+It may happen that your documentation depends on an authenticated service to be built properly.
+In this case, you will require some secrets to access these services.
+
+Read the Docs provides a way to define environment variables for your project to be used in the build process.
+All these variables will be exposed to all the commands executed when building your documentation.
+
+To define an environment variable, you need to
+
+#. Go to your project **Admin > Environment Variables**
+#. Click on "Add Environment Variable" button
+#. Input a ``Name`` and ``Value`` (your secret needed here)
+#. Click "Save" button
+
+.. note::
+
+   Values will never be exposed to users, even to owners of the project. Once you create an environment variable you won't be able to see its value anymore because of security purposes.
+
+After adding an environment variable from your project's admin, you can access it from your build process using Python, for example:
+
+.. code-block:: python
+
+   # conf.py
+   import os
+   import requests
+
+   # Access to our custom environment variables
+   username = os.environ.get('USERNAME')
+   password = os.environ.get('PASSWORD')
+
+   # Request a username/password protected URL
+   response = requests.get(
+       'https://httpbin.org/basic-auth/username/password',
+       auth=(username, password),
+   )

docs/guides/specifying-dependencies.rst

@@ -35,7 +35,7 @@ Using the project admin dashboard
 Once the requirements file has been created;
 
 - Login to Read the Docs and go to the project admin dashboard.
-- Go to ``Admin > Advanced Settings > Requirements file``.
+- Go to **Admin > Advanced Settings > Requirements file**.
 - Specify the path of the requirements file you just created. The path should be relative to the root directory of the documentation project.
 
 Using a conda environment file

readthedocs/projects/forms.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+
 """Project forms."""
 
 from __future__ import (
@@ -8,6 +9,18 @@ from __future__ import (
     unicode_literals,
 )
 
+try:
+    # TODO: remove this when we deprecate Python2
+    # re.fullmatch is >= Py3.4 only
+    from re import fullmatch
+except ImportError:
+    # https://stackoverflow.com/questions/30212413/backport-python-3-4s-regular-expression-fullmatch-to-python-2
+    import re
+
+    def fullmatch(regex, string, flags=0):
+        """Emulate python-3.4 re.fullmatch()."""  # noqa
+        return re.match("(?:" + regex + r")\Z", string, flags=flags)
+
 from random import choice
 
 from builtins import object
@@ -27,11 +40,11 @@ from readthedocs.core.utils.extend import SettingsOverrideObject
 from readthedocs.integrations.models import Integration
 from readthedocs.oauth.models import RemoteRepository
 from readthedocs.projects import constants
-from readthedocs.projects.constants import PUBLIC
 from readthedocs.projects.exceptions import ProjectSpamError
 from readthedocs.projects.models import (
     Domain,
     EmailHook,
+    EnvironmentVariable,
     Feature,
     Project,
     ProjectRelationship,
@@ -767,3 +780,49 @@ class FeatureForm(forms.ModelForm):
     def __init__(self, *args, **kwargs):
         super(FeatureForm, self).__init__(*args, **kwargs)
         self.fields['feature_id'].choices = Feature.FEATURES
+
+
+class EnvironmentVariableForm(forms.ModelForm):
+
+    """
+    Form to add an EnvironmentVariable to a Project.
+
+    This limits the name of the variable.
+    """
+
+    project = forms.CharField(widget=forms.HiddenInput(), required=False)
+
+    class Meta(object):
+        model = EnvironmentVariable
+        fields = ('name', 'value', 'project')
+
+    def __init__(self, *args, **kwargs):
+        self.project = kwargs.pop('project', None)
+        super(EnvironmentVariableForm, self).__init__(*args, **kwargs)
+
+    def clean_project(self):
+        return self.project
+
+    def clean_name(self):
+        name = self.cleaned_data['name']
+        if name.startswith('__'):
+            raise forms.ValidationError(
+                _("Variable name can't start with __ (double underscore)"),
+            )
+        elif name.startswith('READTHEDOCS'):
+            raise forms.ValidationError(
+                _("Variable name can't start with READTHEDOCS"),
+            )
+        elif self.project.environmentvariable_set.filter(name=name).exists():
+            raise forms.ValidationError(
+                _('There is already a variable with this name for this project'),
+            )
+        elif ' ' in name:
+            raise forms.ValidationError(
+                _("Variable name can't contain spaces"),
+            )
+        elif not fullmatch('[a-zA-Z0-9_]+', name):
+            raise forms.ValidationError(
+                _('Only letters, numbers and underscore are allowed'),
+            )
+        return name

readthedocs/projects/models.py

@@ -8,6 +8,7 @@ import fnmatch
 import logging
 import os
 from builtins import object  # pylint: disable=redefined-builtin
+from six.moves import shlex_quote
 
 from django.conf import settings
 from django.contrib.auth.models import User
@@ -1062,6 +1063,7 @@ class Feature(models.Model):
         return dict(self.FEATURES).get(self.feature_id, self.feature_id)
 
 
+@python_2_unicode_compatible
 class EnvironmentVariable(TimeStampedModel, models.Model):
     name = models.CharField(
         max_length=128,
@@ -1076,3 +1078,10 @@ class EnvironmentVariable(TimeStampedModel, models.Model):
         on_delete=models.CASCADE,
         help_text=_('Project where this variable will be used'),
     )
+
+    def __str__(self):
+        return self.name
+
+    def save(self, *args, **kwargs):  # pylint: disable=arguments-differ
+        self.value = shlex_quote(self.value)
+        return super(EnvironmentVariable, self).save(*args, **kwargs)

readthedocs/projects/urls/private.py

@@ -1,18 +1,38 @@
-"""Project URLs for authenticated users"""
+"""Project URLs for authenticated users."""
+
+from __future__ import (
+    absolute_import,
+    division,
+    print_function,
+    unicode_literals,
+)
 
-from __future__ import absolute_import
 from django.conf.urls import url
 
 from readthedocs.constants import pattern_opts
+from readthedocs.projects.backends.views import ImportDemoView, ImportWizardView
 from readthedocs.projects.views import private
 from readthedocs.projects.views.private import (
-    ProjectDashboard, ImportView,
-    ProjectUpdate, ProjectAdvancedUpdate,
-    DomainList, DomainCreate, DomainDelete, DomainUpdate,
-    IntegrationList, IntegrationCreate, IntegrationDetail, IntegrationDelete,
-    IntegrationExchangeDetail, IntegrationWebhookSync, ProjectAdvertisingUpdate)
-from readthedocs.projects.backends.views import ImportWizardView, ImportDemoView
-
+    DomainCreate,
+    DomainDelete,
+    DomainList,
+    DomainUpdate,
+    EnvironmentVariableCreate,
+    EnvironmentVariableDelete,
+    EnvironmentVariableList,
+    EnvironmentVariableDetail,
+    ImportView,
+    IntegrationCreate,
+    IntegrationDelete,
+    IntegrationDetail,
+    IntegrationExchangeDetail,
+    IntegrationList,
+    IntegrationWebhookSync,
+    ProjectAdvancedUpdate,
+    ProjectAdvertisingUpdate,
+    ProjectDashboard,
+    ProjectUpdate,
+)
 
 urlpatterns = [
     url(r'^$',
@@ -171,3 +191,20 @@ subproject_urls = [
 ]
 
 urlpatterns += subproject_urls
+
+environmentvariable_urls = [
+    url(r'^(?P<project_slug>[-\w]+)/environmentvariables/$',
+        EnvironmentVariableList.as_view(),
+        name='projects_environmentvariables'),
+    url(r'^(?P<project_slug>[-\w]+)/environmentvariables/create/$',
+        EnvironmentVariableCreate.as_view(),
+        name='projects_environmentvariables_create'),
+    url(r'^(?P<project_slug>[-\w]+)/environmentvariables/(?P<environmentvariable_pk>[-\w]+)/$',
+        EnvironmentVariableDetail.as_view(),
+        name='projects_environmentvariables_detail'),
+    url(r'^(?P<project_slug>[-\w]+)/environmentvariables/(?P<environmentvariable_pk>[-\w]+)/delete/$',
+        EnvironmentVariableDelete.as_view(),
+        name='projects_environmentvariables_delete'),
+]
+
+urlpatterns += environmentvariable_urls

readthedocs/projects/views/private.py

@@ -43,6 +43,7 @@ from readthedocs.projects import tasks
 from readthedocs.projects.forms import (
     DomainForm,
     EmailHookForm,
+    EnvironmentVariableForm,
     IntegrationForm,
     ProjectAdvancedForm,
     ProjectAdvertisingForm,
@@ -59,6 +60,7 @@ from readthedocs.projects.forms import (
 from readthedocs.projects.models import (
     Domain,
     EmailHook,
+    EnvironmentVariable,
     Project,
     ProjectRelationship,
     WebHook,
@@ -887,3 +889,37 @@ class ProjectAdvertisingUpdate(PrivateViewMixin, UpdateView):
 
     def get_success_url(self):
         return reverse('projects_advertising', args=[self.object.slug])
+
+
+class EnvironmentVariableMixin(ProjectAdminMixin, PrivateViewMixin):
+
+    """Environment Variables to be added when building the Project."""
+
+    model = EnvironmentVariable
+    form_class = EnvironmentVariableForm
+    lookup_url_kwarg = 'environmentvariable_pk'
+
+    def get_success_url(self):
+        return reverse(
+            'projects_environmentvariables',
+            args=[self.get_project().slug],
+        )
+
+
+class EnvironmentVariableList(EnvironmentVariableMixin, ListView):
+    pass
+
+
+class EnvironmentVariableCreate(EnvironmentVariableMixin, CreateView):
+    pass
+
+
+class EnvironmentVariableDetail(EnvironmentVariableMixin, DetailView):
+    pass
+
+
+class EnvironmentVariableDelete(EnvironmentVariableMixin, DeleteView):
+
+    # This removes the delete confirmation
+    def get(self, request, *args, **kwargs):
+        return self.http_method_not_allowed(request, *args, **kwargs)

readthedocs/restapi/serializers.py

@@ -43,6 +43,12 @@ class ProjectAdminSerializer(ProjectSerializer):
         slug_field='feature_id',
     )
 
+    def get_environment_variables(self, obj):
+        return {
+            variable.name: variable.value
+            for variable in obj.environmentvariable_set.all()
+        }
+
     class Meta(ProjectSerializer.Meta):
         fields = ProjectSerializer.Meta.fields + (
             'enable_epub_build',

readthedocs/rtd_tests/tests/test_privacy_urls.py

@@ -14,7 +14,7 @@ from taggit.models import Tag
 from readthedocs.builds.models import Build, BuildCommandResult
 from readthedocs.core.utils.tasks import TaskNoPermission
 from readthedocs.integrations.models import HttpExchange, Integration
-from readthedocs.projects.models import Project, Domain
+from readthedocs.projects.models import Project, Domain, EnvironmentVariable
 from readthedocs.oauth.models import RemoteRepository, RemoteOrganization
 from readthedocs.rtd_tests.utils import create_user
 
@@ -150,6 +150,7 @@ class ProjectMixin(URLAccessMixin):
             status_code=200,
         )
         self.domain = get(Domain, url='http://docs.foobar.com', project=self.pip)
+        self.environment_variable = get(EnvironmentVariable, project=self.pip)
         self.default_kwargs = {
             'project_slug': self.pip.slug,
             'subproject_slug': self.subproject.slug,
@@ -162,6 +163,7 @@ class ProjectMixin(URLAccessMixin):
             'domain_pk': self.domain.pk,
             'integration_pk': self.integration.pk,
             'exchange_pk': self.webhook_exchange.pk,
+            'environmentvariable_pk': self.environment_variable.pk,
         }
 
 
@@ -241,11 +243,13 @@ class PrivateProjectAdminAccessTest(PrivateProjectMixin, TestCase):
         '/dashboard/pip/integrations/sync/': {'status_code': 405},
         '/dashboard/pip/integrations/{integration_id}/sync/': {'status_code': 405},
         '/dashboard/pip/integrations/{integration_id}/delete/': {'status_code': 405},
+        '/dashboard/pip/environmentvariables/{environmentvariable_id}/delete/': {'status_code': 405},
     }
 
     def get_url_path_ctx(self):
         return {
             'integration_id': self.integration.id,
+            'environmentvariable_id': self.environment_variable.id,
         }
 
     def login(self):
@@ -275,6 +279,7 @@ class PrivateProjectUserAccessTest(PrivateProjectMixin, TestCase):
         '/dashboard/pip/integrations/sync/': {'status_code': 405},
         '/dashboard/pip/integrations/{integration_id}/sync/': {'status_code': 405},
         '/dashboard/pip/integrations/{integration_id}/delete/': {'status_code': 405},
+        '/dashboard/pip/environmentvariables/{environmentvariable_id}/delete/': {'status_code': 405},
     }
 
     # Filtered out by queryset on projects that we don't own.
@@ -283,6 +288,7 @@ class PrivateProjectUserAccessTest(PrivateProjectMixin, TestCase):
     def get_url_path_ctx(self):
         return {
             'integration_id': self.integration.id,
+            'environmentvariable_id': self.environment_variable.id,
         }
 
     def login(self):

readthedocs/rtd_tests/tests/test_project_forms.py

@@ -25,13 +25,14 @@ from readthedocs.projects.constants import (
 )
 from readthedocs.projects.exceptions import ProjectSpamError
 from readthedocs.projects.forms import (
+    EnvironmentVariableForm,
     ProjectAdvancedForm,
     ProjectBasicsForm,
     ProjectExtraForm,
     TranslationForm,
     UpdateProjectForm,
 )
-from readthedocs.projects.models import Project
+from readthedocs.projects.models import Project, EnvironmentVariable
 
 
 class TestProjectForms(TestCase):
@@ -511,3 +512,89 @@ class TestTranslationForms(TestCase):
             instance=self.project_b_en
         )
         self.assertTrue(form.is_valid())
+
+
+class TestProjectEnvironmentVariablesForm(TestCase):
+
+    def setUp(self):
+        self.project = get(Project)
+
+    def test_use_invalid_names(self):
+        data = {
+            'name': 'VARIABLE WITH SPACES',
+            'value': 'string here',
+        }
+        form = EnvironmentVariableForm(data, project=self.project)
+        self.assertFalse(form.is_valid())
+        self.assertIn(
+            "Variable name can't contain spaces",
+            form.errors['name'],
+        )
+
+        data = {
+            'name': 'READTHEDOCS__INVALID',
+            'value': 'string here',
+        }
+        form = EnvironmentVariableForm(data, project=self.project)
+        self.assertFalse(form.is_valid())
+        self.assertIn(
+            "Variable name can't start with READTHEDOCS",
+            form.errors['name'],
+        )
+
+        data = {
+            'name': 'INVALID_CHAR*',
+            'value': 'string here',
+        }
+        form = EnvironmentVariableForm(data, project=self.project)
+        self.assertFalse(form.is_valid())
+        self.assertIn(
+            'Only letters, numbers and underscore are allowed',
+            form.errors['name'],
+        )
+
+        data = {
+            'name': '__INVALID',
+            'value': 'string here',
+        }
+        form = EnvironmentVariableForm(data, project=self.project)
+        self.assertFalse(form.is_valid())
+        self.assertIn(
+            "Variable name can't start with __ (double underscore)",
+            form.errors['name'],
+        )
+
+        get(EnvironmentVariable, name='EXISTENT_VAR', project=self.project)
+        data = {
+            'name': 'EXISTENT_VAR',
+            'value': 'string here',
+        }
+        form = EnvironmentVariableForm(data, project=self.project)
+        self.assertFalse(form.is_valid())
+        self.assertIn(
+            'There is already a variable with this name for this project',
+            form.errors['name'],
+        )
+
+    def test_create(self):
+        data = {
+            'name': 'MYTOKEN',
+            'value': 'string here',
+        }
+        form = EnvironmentVariableForm(data, project=self.project)
+        form.save()
+
+        self.assertEqual(EnvironmentVariable.objects.count(), 1)
+        self.assertEqual(EnvironmentVariable.objects.first().name, 'MYTOKEN')
+        self.assertEqual(EnvironmentVariable.objects.first().value, "'string here'")
+
+        data = {
+            'name': 'ESCAPED',
+            'value': r'string escaped here: #$\1[]{}\|',
+        }
+        form = EnvironmentVariableForm(data, project=self.project)
+        form.save()
+
+        self.assertEqual(EnvironmentVariable.objects.count(), 2)
+        self.assertEqual(EnvironmentVariable.objects.first().name, 'ESCAPED')
+        self.assertEqual(EnvironmentVariable.objects.first().value, r"'string escaped here: #$\1[]{}\|'")

readthedocs/templates/projects/environmentvariable_detail.html

@@ -0,0 +1,30 @@
+{% extends "projects/project_edit_base.html" %}
+
+{% load i18n %}
+
+{% block title %}{% trans "Environment Variables" %}{% endblock %}
+
+{% block nav-dashboard %} class="active"{% endblock %}
+
+{% block editing-option-edit-environment-variables %}class="active"{% endblock %}
+
+{% block project-environment-variables-active %}active{% endblock %}
+{% block project_edit_content_header %}
+  {% blocktrans trimmed with name=environmentvariable.name %}
+    Environment Variable: {{ name }}
+  {% endblocktrans %}
+{% endblock %}
+
+{% block project_edit_content %}
+
+  <p>
+    {% blocktrans trimmed %}
+      The value of the environment variable is not shown here for sercurity purposes.
+    {% endblocktrans %}
+  </p>
+
+  <form method="post" action="{% url 'projects_environmentvariables_delete' project_slug=project.slug environmentvariable_pk=environmentvariable.pk %}">
+    {% csrf_token %}
+    <input type="submit" value="{% trans "Delete" %}">
+  </form>
+{% endblock %}

readthedocs/templates/projects/environmentvariable_form.html

@@ -0,0 +1,22 @@
+{% extends "projects/project_edit_base.html" %}
+
+{% load i18n %}
+
+{% block title %}{% trans "Environment Variables" %}{% endblock %}
+
+{% block nav-dashboard %} class="active"{% endblock %}
+
+{% block editing-option-edit-environment-variables %}class="active"{% endblock %}
+
+{% block project-environment-variables-active %}active{% endblock %}
+{% block project_edit_content_header %}{% trans "Environment Variables" %}{% endblock %}
+
+{% block project_edit_content %}
+  <form
+      method="post"
+      action="{% url 'projects_environmentvariables_create' project_slug=project.slug %}">
+    {% csrf_token %}
+    {{ form.as_p }}
+    <input type="submit" value="{% trans "Save" %}">
+  </form>
+{% endblock %}

readthedocs/templates/projects/environmentvariable_list.html

@@ -0,0 +1,47 @@
+{% extends "projects/project_edit_base.html" %}
+
+{% load i18n %}
+
+{% block title %}{% trans "Environment Variables" %}{% endblock %}
+
+{% block nav-dashboard %} class="active"{% endblock %}
+
+{% block editing-option-edit-environment-variables %}class="active"{% endblock %}
+
+{% block project-environment-variables-active %}active{% endblock %}
+{% block project_edit_content_header %}{% trans "Environment Variables" %}{% endblock %}
+
+{% block project_edit_content %}
+  <p>Environment variables allow you to change the way that your build behaves. Take into account that these environment variables are available to all build steps.</p>
+
+  <div class="button-bar">
+    <ul>
+      <li>
+        <a class="button"
+           href="{% url 'projects_environmentvariables_create' project_slug=project.slug %}">
+          {% trans "Add Environment Variable" %}
+        </a>
+      </li>
+    </ul>
+  </div>
+
+  <div class="module-list">
+    <div class="module-list-wrapper">
+      <ul>
+        {% for environmentvariable in object_list %}
+          <li class="module-item">
+            <a href="{% url 'projects_environmentvariables_detail' project_slug=project.slug environmentvariable_pk=environmentvariable.pk %}">
+              {{ environmentvariable.name }}
+            </a>
+          </li>
+        {% empty %}
+          <li class="module-item">
+            <p class="quiet">
+              {% trans 'No environment variables are currently configured.' %}
+            </p>
+          </li>
+        {% endfor %}
+      </ul>
+    </div>
+  </div>
+{% endblock %}

readthedocs/templates/projects/project_edit_base.html

@@ -23,6 +23,7 @@
         <li class="{% block project-translations-active %}{% endblock %}"><a href="{% url "projects_translations" project.slug %}">{% trans "Translations" %}</a></li>
         <li class="{% block project-subprojects-active %}{% endblock %}"><a href="{% url "projects_subprojects" project.slug %}">{% trans "Subprojects" %}</a></li>
         <li class="{% block project-integrations-active %}{% endblock %}"><a href="{% url "projects_integrations" project.slug %}">{% trans "Integrations" %}</a></li>
+        <li class="{% block project-environment-variables-active %}{% endblock %}"><a href="{% url "projects_environmentvariables" project.slug %}">{% trans "Environment Variables" %}</a></li>
         <li class="{% block project-notifications-active %}{% endblock %}"><a href="{% url "projects_notifications" project.slug %}">{% trans "Notifications" %}</a></li>
         {% if USE_PROMOS %}
           <li class="{% block project-ads-active %}{% endblock %}"><a href="{% url "projects_advertising" project.slug %}">{% trans "Advertising" %} </a></li>