Add modeling for webhooks, wrap webhook creation some with model creation (#2726)

* Add modeling for http transactions and wrappers around webhook views

* Transaction -> Exchange

* Move request payload normalization to util function

media/css/core.css

@@ -1029,3 +1029,105 @@ select.dropdown { display: none; }
 
 .domain-machine { color: #999; }
 .domain-canonical { font-weight: bold; }
+
+/* Integrations */
+div.module-list-wrapper.httptransactions li span.status {
+    padding: .2em .4em;
+    margin-right: .3em;
+    border-radius: .3em;
+    color: #fff;
+}
+div.module-list-wrapper.httptransactions li span.status.status-pass {
+    background: #5a5;
+}
+div.module-list-wrapper.httptransactions li span.status.status-fail {
+    background: #a55;
+}
+
+div.httptransaction dl dt {
+  display: inline-block;
+  font-weight: bold;
+  font-family: 'inconsolata', 'bitstream vera sans mono', 'andale mono', 'lucida console', monospace;
+  font-size: .9em;
+}
+
+div.httptransaction dl dd {
+  display: inline;
+  font-family: 'inconsolata', 'bitstream vera sans mono', 'andale mono', 'lucida console', monospace;
+  font-size: .9em;
+}
+div.httptransaction dl dd:after {
+  display: block;
+  content: '';
+}
+
+div.httptransaction div.highlight pre {
+  padding: 1em;
+  background: #f4f4f4;
+  border: 1px solid #ccc;
+  font-size: .9em;
+}
+
+/* Pygments */
+div.highlight pre .hll { background-color: #ffffcc }
+div.highlight pre .c { color: #60a0b0; font-style: italic } /* Comment */
+div.highlight pre .err { border: 1px solid #FF0000 } /* Error */
+div.highlight pre .k { color: #007020; font-weight: bold } /* Keyword */
+div.highlight pre .o { color: #666666 } /* Operator */
+div.highlight pre .cm { color: #60a0b0; font-style: italic } /* Comment.Multiline */
+div.highlight pre .cp { color: #007020 } /* Comment.Preproc */
+div.highlight pre .c1 { color: #60a0b0; font-style: italic } /* Comment.Single */
+div.highlight pre .cs { color: #60a0b0; background-color: #fff0f0 } /* Comment.Special */
+div.highlight pre .gd { color: #A00000 } /* Generic.Deleted */
+div.highlight pre .ge { font-style: italic } /* Generic.Emph */
+div.highlight pre .gr { color: #FF0000 } /* Generic.Error */
+div.highlight pre .gh { color: #000080; font-weight: bold } /* Generic.Heading */
+div.highlight pre .gi { color: #00A000 } /* Generic.Inserted */
+div.highlight pre .go { color: #888888 } /* Generic.Output */
+div.highlight pre .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */
+div.highlight pre .gs { font-weight: bold } /* Generic.Strong */
+div.highlight pre .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
+div.highlight pre .gt { color: #0044DD } /* Generic.Traceback */
+div.highlight pre .kc { color: #007020; font-weight: bold } /* Keyword.Constant */
+div.highlight pre .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */
+div.highlight pre .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */
+div.highlight pre .kp { color: #007020 } /* Keyword.Pseudo */
+div.highlight pre .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */
+div.highlight pre .kt { color: #902000 } /* Keyword.Type */
+div.highlight pre .m { color: #40a070 } /* Literal.Number */
+div.highlight pre .s { color: #4070a0 } /* Literal.String */
+div.highlight pre .na { color: #4070a0 } /* Name.Attribute */
+div.highlight pre .nb { color: #007020 } /* Name.Builtin */
+div.highlight pre .nc { color: #0e84b5; font-weight: bold } /* Name.Class */
+div.highlight pre .no { color: #60add5 } /* Name.Constant */
+div.highlight pre .nd { color: #555555; font-weight: bold } /* Name.Decorator */
+div.highlight pre .ni { color: #d55537; font-weight: bold } /* Name.Entity */
+div.highlight pre .ne { color: #007020 } /* Name.Exception */
+div.highlight pre .nf { color: #06287e } /* Name.Function */
+div.highlight pre .nl { color: #002070; font-weight: bold } /* Name.Label */
+div.highlight pre .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */
+div.highlight pre .nt { color: #062873; font-weight: bold } /* Name.Tag */
+div.highlight pre .nv { color: #bb60d5 } /* Name.Variable */
+div.highlight pre .ow { color: #007020; font-weight: bold } /* Operator.Word */
+div.highlight pre .w { color: #bbbbbb } /* Text.Whitespace */
+div.highlight pre .mb { color: #40a070 } /* Literal.Number.Bin */
+div.highlight pre .mf { color: #40a070 } /* Literal.Number.Float */
+div.highlight pre .mh { color: #40a070 } /* Literal.Number.Hex */
+div.highlight pre .mi { color: #40a070 } /* Literal.Number.Integer */
+div.highlight pre .mo { color: #40a070 } /* Literal.Number.Oct */
+div.highlight pre .sb { color: #4070a0 } /* Literal.String.Backtick */
+div.highlight pre .sc { color: #4070a0 } /* Literal.String.Char */
+div.highlight pre .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */
+div.highlight pre .s2 { color: #4070a0 } /* Literal.String.Double */
+div.highlight pre .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */
+div.highlight pre .sh { color: #4070a0 } /* Literal.String.Heredoc */
+div.highlight pre .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */
+div.highlight pre .sx { color: #c65d09 } /* Literal.String.Other */
+div.highlight pre .sr { color: #235388 } /* Literal.String.Regex */
+div.highlight pre .s1 { color: #4070a0 } /* Literal.String.Single */
+div.highlight pre .ss { color: #517918 } /* Literal.String.Symbol */
+div.highlight pre .bp { color: #007020 } /* Name.Builtin.Pseudo */
+div.highlight pre .vc { color: #bb60d5 } /* Name.Variable.Class */
+div.highlight pre .vg { color: #bb60d5 } /* Name.Variable.Global */
+div.highlight pre .vi { color: #bb60d5 } /* Name.Variable.Instance */
+div.highlight pre .il { color: #40a070 } /* Literal.Number.Integer.Long */

readthedocs/core/views/hooks.py

@@ -139,6 +139,10 @@ def _build_url(url, projects, branches):
 def github_build(request):
     """GitHub webhook consumer
 
+    .. warning:: **DEPRECATED**
+        Use :py:cls:`readthedocs.restapi.views.intergrations.GitHubWebhookView`
+        instead of this view function
+
     This will search for projects matching either a stripped down HTTP or SSH
     URL. The search is error prone, use the API v2 webhook for new webhooks.
 
@@ -189,6 +193,10 @@ def github_build(request):
 def gitlab_build(request):
     """GitLab webhook consumer
 
+    .. warning:: **DEPRECATED**
+        Use :py:cls:`readthedocs.restapi.views.intergrations.GitLabWebhookView`
+        instead of this view function
+
     Search project repository URLs using the site URL from GitLab webhook payload.
     This search is error-prone, use the API v2 webhook view for new webhooks.
     """
@@ -220,6 +228,10 @@ def gitlab_build(request):
 def bitbucket_build(request):
     """Consume webhooks from multiple versions of Bitbucket's API
 
+    .. warning:: **DEPRECATED**
+        Use :py:cls:`readthedocs.restapi.views.intergrations.BitbucketWebhookView`
+        instead of this view function
+
     New webhooks are set up with v2, but v1 webhooks will still point to this
     endpoint. There are also "services" that point here and submit
     ``application/x-www-form-urlencoded`` data.
@@ -277,6 +289,12 @@ def bitbucket_build(request):
 
 @csrf_exempt
 def generic_build(request, project_id_or_slug=None):
+    """Generic webhook build endpoint
+
+    .. warning:: **DEPRECATED**
+        Use :py:cls:`readthedocs.restapi.views.intergrations.GenericWebhookView`
+        instead of this view function
+    """
     try:
         project = Project.objects.get(pk=project_id_or_slug)
     # Allow slugs too

readthedocs/integrations/migrations/0001_add_http_exchange.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.12 on 2017-03-16 18:30
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import jsonfield.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='HttpExchange',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('object_id', models.PositiveIntegerField()),
+                ('date', models.DateTimeField(auto_now_add=True, verbose_name='Date')),
+                ('request_headers', jsonfield.fields.JSONField(verbose_name='Request headers')),
+                ('request_body', models.TextField(verbose_name='Request body')),
+                ('response_headers', jsonfield.fields.JSONField(verbose_name='Request headers')),
+                ('response_body', models.TextField(verbose_name='Response body')),
+                ('status_code', models.IntegerField(default=200, verbose_name='Status code')),
+                ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
+            ],
+            options={
+                'ordering': ['-date'],
+            },
+        ),
+    ]

readthedocs/integrations/models.py

@@ -0,0 +1,141 @@
+"""Integration models for external services"""
+
+import json
+import uuid
+
+from django.db import models, transaction
+from django.utils.translation import ugettext_lazy as _
+from django.contrib.contenttypes.fields import GenericForeignKey
+from django.contrib.contenttypes.models import ContentType
+from django.utils.safestring import mark_safe
+from rest_framework import status
+from jsonfield import JSONField
+from pygments import highlight
+from pygments.lexers import JsonLexer
+from pygments.formatters import HtmlFormatter
+
+from .utils import normalize_request_payload
+
+
+class HttpExchangeManager(models.Manager):
+
+    """HTTP exchange manager methods"""
+
+    @transaction.atomic
+    def from_exchange(self, req, resp, related_object, payload=None):
+        """Create object from Django request and response objects
+
+        If an explicit Request ``payload`` is not specified, the payload will be
+        determined directly from the Request object. This makes a good effort to
+        normalize the data, however we don't enforce that the payload is JSON
+
+        :param req: Request object to store
+        :type req: HttpRequest
+        :param resp: Response object to store
+        :type resp: HttpResponse
+        :param related_object: Object to use for generic relation
+        :param payload: Alternate payload object to store
+        :type payload: dict
+        """
+        request_payload = payload
+        if request_payload is None:
+            request_payload = normalize_request_payload(req)
+        try:
+            request_body = json.dumps(request_payload, sort_keys=True)
+        except TypeError:
+            request_body = str(request_payload)
+        # This is the rawest form of request header we have, the WSGI
+        # headers. HTTP headers are prefixed with `HTTP_`, which we remove,
+        # and because the keys are all uppercase, we'll normalize them to
+        # title case-y hyphen separated values.
+        request_headers = dict(
+            (key[5:].title().replace('_', '-'), str(val))
+            for (key, val) in req.META.items()
+            if key.startswith('HTTP_')
+        )
+        request_headers['Content-Type'] = req.content_type
+
+        response_payload = resp.data if hasattr(resp, 'data') else resp.content
+        try:
+            response_body = json.dumps(response_payload, sort_keys=True)
+        except TypeError:
+            response_body = str(response_payload)
+        response_headers = dict(resp.items())
+
+        fields = {
+            'status_code': resp.status_code,
+            'request_headers': request_headers,
+            'request_body': request_body,
+            'response_body': response_body,
+            'response_headers': response_headers,
+        }
+        fields['related_object'] = related_object
+        obj = self.create(**fields)
+        self.delete_limit(related_object)
+        return obj
+
+    def delete_limit(self, related_object, limit=10):
+        queryset = self.filter(
+            content_type=ContentType.objects.get(
+                app_label=related_object._meta.app_label,
+                model=related_object._meta.model_name,
+            ),
+            object_id=related_object.pk
+        )
+        for exchange in queryset[limit:]:
+            exchange.delete()
+
+
+class HttpExchange(models.Model):
+
+    """HTTP request/response exchange"""
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+
+    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
+    object_id = models.PositiveIntegerField()
+    related_object = GenericForeignKey('content_type', 'object_id')
+
+    date = models.DateTimeField(_('Date'), auto_now_add=True)
+
+    request_headers = JSONField(_('Request headers'))
+    request_body = models.TextField(_('Request body'))
+
+    response_headers = JSONField(_('Request headers'))
+    response_body = models.TextField(_('Response body'))
+
+    status_code = models.IntegerField(
+        _('Status code'), default=status.HTTP_200_OK
+    )
+
+    objects = HttpExchangeManager()
+
+    class Meta:
+        ordering = ['-date']
+
+    def __unicode__(self):
+        return _('Exchange {0}').format(self.pk)
+
+    @property
+    def failed(self):
+        # Assume anything that isn't 2xx level status code is an error
+        return int(self.status_code / 100) != 2
+
+    def formatted_json(self, field):
+        """Try to return pretty printed and Pygment highlighted code"""
+        value = getattr(self, field) or ''
+        try:
+            json_value = json.dumps(json.loads(value), sort_keys=True, indent=2)
+            formatter = HtmlFormatter()
+            html = highlight(json_value, JsonLexer(), formatter)
+            return mark_safe(html)
+        except (ValueError, TypeError):
+            return value
+
+    @property
+    def formatted_request_body(self):
+        return self.formatted_json('request_body')
+
+    @property
+    def formatted_response_body(self):
+        return self.formatted_json('response_body')

readthedocs/integrations/utils.py

@@ -0,0 +1,22 @@
+"""Integration utility functions"""
+
+
+def normalize_request_payload(request):
+    """Normalize the request body, hopefully to JSON
+
+    This will attempt to return a JSON body, backing down to a string body next.
+
+    :param request: HTTP request object
+    :type request: django.http.HttpRequest
+    :returns: The request body as a string
+    :rtype: str
+    """
+    request_payload = getattr(request, 'data', {})
+    if request.content_type != 'application/json':
+        # Here, request_body can be a dict or a MergeDict. Probably best to
+        # normalize everything first
+        try:
+            request_payload = dict(request_payload.items())
+        except AttributeError:
+            pass
+    return request_payload

readthedocs/oauth/utils.py

@@ -16,6 +16,11 @@ def attach_webhook(project, request=None):
             service = service_cls
             break
     else:
+        messages.error(
+            request,
+            _('Webhook activation failed. '
+              'There are no connected services for this project.')
+        )
         return None
 
     user_accounts = service.for_user(request.user)
@@ -25,8 +30,8 @@ def attach_webhook(project, request=None):
             messages.success(request, _('Webhook activated'))
             project.has_valid_webhook = True
             project.save()
-            break
+            return True
-    else:
+    # No valid account found
     if user_accounts:
         messages.error(
             request,
@@ -40,3 +45,4 @@ def attach_webhook(project, request=None):
                     network=service.adapter().get_provider().name
                 ))
         )
+    return False

readthedocs/projects/urls/private.py

@@ -7,6 +7,7 @@ from readthedocs.projects.views.private import (
     ProjectDashboard, ImportView,
     ProjectUpdate, ProjectAdvancedUpdate,
     DomainList, DomainCreate, DomainDelete, DomainUpdate,
+    IntegrationList, IntegrationExchangeDetail, IntegrationWebhookSync,
     ProjectAdvertisingUpdate)
 from readthedocs.projects.backends.views import ImportWizardView, ImportDemoView
 
@@ -105,10 +106,6 @@ urlpatterns = [
         private.project_redirects_delete,
         name='projects_redirects_delete'),
 
-    url(r'^(?P<project_slug>[-\w]+)/resync_webhook/$',
-        private.project_resync_webhook,
-        name='projects_resync_webhook'),
-
     url(r'^(?P<project_slug>[-\w]+)/advertising/$',
         ProjectAdvertisingUpdate.as_view(),
         name='projects_advertising'),
@@ -130,3 +127,17 @@ domain_urls = [
 ]
 
 urlpatterns += domain_urls
+
+integration_urls = [
+    url(r'^(?P<project_slug>[-\w]+)/integrations/$',
+        IntegrationList.as_view(),
+        name='projects_integrations'),
+    url(r'^(?P<project_slug>[-\w]+)/integrations/exchange/(?P<exchange_pk>[-\w]+)/$',
+        IntegrationExchangeDetail.as_view(),
+        name='projects_integrations_exchange_detail'),
+    url(r'^(?P<project_slug>[-\w]+)/integrations/sync/$',
+        IntegrationWebhookSync.as_view(),
+        name='projects_integrations_sync'),
+]
+
+urlpatterns += integration_urls

readthedocs/projects/views/base.py

@@ -79,9 +79,6 @@ class ProjectAdminMixin(object):
         kwargs['project'] = self.get_project()
         return self.form_class(data, files, **kwargs)
 
-    def get_success_url(self, **kwargs):
-        return reverse('projects_domains', args=[self.get_project().slug])
-
 
 class ProjectSpamMixin(object):
 

readthedocs/projects/views/private.py

@@ -5,8 +5,8 @@ import logging
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.models import User
 from django.contrib import messages
+from django.contrib.contenttypes.models import ContentType
 from django.core.urlresolvers import reverse
-from django.conf import settings
 from django.http import (HttpResponseRedirect, HttpResponseNotAllowed,
                          Http404, HttpResponseBadRequest)
 from django.shortcuts import get_object_or_404, render_to_response, render
@@ -18,7 +18,7 @@ from django.middleware.csrf import get_token
 from formtools.wizard.views import SessionWizardView
 from allauth.socialaccount.models import SocialAccount
 
-from vanilla import CreateView, DeleteView, UpdateView
+from vanilla import CreateView, DeleteView, UpdateView, DetailView, GenericView
 
 from readthedocs.bookmarks.models import Bookmark
 from readthedocs.builds.models import Version
@@ -27,6 +27,7 @@ from readthedocs.builds.filters import VersionFilter
 from readthedocs.builds.models import VersionAlias
 from readthedocs.core.utils import trigger_build, broadcast
 from readthedocs.core.mixins import ListViewWithForm
+from readthedocs.integrations.models import HttpExchange
 from readthedocs.projects.forms import (
     ProjectBasicsForm, ProjectExtraForm,
     ProjectAdvancedForm, UpdateProjectForm, SubprojectForm,
@@ -656,6 +657,9 @@ class DomainMixin(ProjectAdminMixin, PrivateViewMixin):
     form_class = DomainForm
     lookup_url_kwarg = 'domain_pk'
 
+    def get_success_url(self):
+        return reverse('projects_domains', args=[self.get_project().slug])
+
 
 class DomainList(DomainMixin, ListViewWithForm):
     pass
@@ -673,25 +677,63 @@ class DomainDelete(DomainMixin, DeleteView):
     pass
 
 
-@login_required
+class IntegrationMixin(object):
-def project_resync_webhook(request, project_slug):
+
+    """Project external service mixin for listing webhook objects
+
+    This mixin will be used more once we have modeling around webhooks and
+    external integrations.
     """
-    Resync a project webhook.
+
+    def get_success_url(self):
+        return reverse('projects_integrations', args=[self.get_project().slug])
+
+    def get_template_names(self):
+        if self.template_name:
+            return self.template_name
+        return 'projects/integration{0}.html'.format(self.template_name_suffix)
+
+
+class IntegrationExchangeMixin(ProjectAdminMixin, PrivateViewMixin):
+
+    """Project webhook exchange mixin for listing exchange objects"""
+
+    model = HttpExchange
+    lookup_url_kwarg = 'exchange_pk'
+
+    def get_queryset(self):
+        self.project = self.get_project()
+        return self.model.objects.filter(
+            content_type=ContentType.objects.filter(
+                app_label='projects',
+                model='project'
+            ),
+            object_id=self.project.pk
+        )
+
+
+class IntegrationList(IntegrationMixin, IntegrationExchangeMixin, ListView):
+    pass
+
+
+class IntegrationExchangeDetail(IntegrationMixin, IntegrationExchangeMixin, DetailView):
+    template_name = 'projects/integration_exchange_detail.html'
+
+
+class IntegrationWebhookSync(PrivateViewMixin, ProjectAdminMixin, GenericView):
+
+    """Resync a project webhook
 
     The signal will add a success/failure message on the request.
     """
-    project = get_object_or_404(Project.objects.for_admin_user(request.user),
-                                slug=project_slug)
-    if request.method == 'POST':
-        attach_webhook(project=project, request=request)
-        return HttpResponseRedirect(reverse('projects_detail',
-                                            args=[project.slug]))
 
-    return render_to_response(
+    def post(self, request, *args, **kwargs):
-        'projects/project_resync_webhook.html',
+        # pylint: disable=unused-argument
-        {'project': project},
+        attach_webhook(project=self.get_project(), request=request)
-        context_instance=RequestContext(request)
+        return HttpResponseRedirect(self.get_success_url())
-    )
+
+    def get_success_url(self):
+        return reverse('projects_integrations', args=[self.get_project().slug])
 
 
 class ProjectAdvertisingUpdate(PrivateViewMixin, UpdateView):

readthedocs/restapi/urls.py

@@ -71,6 +71,9 @@ integration_urls = [
     url(r'webhook/bitbucket/(?P<project_slug>{project_slug})/'.format(**pattern_opts),
         integrations.BitbucketWebhookView.as_view(),
         name='api_webhook_bitbucket'),
+    url(r'webhook/generic/(?P<project_slug>{project_slug})/'.format(**pattern_opts),
+        integrations.GenericWebhookView.as_view(),
+        name='api_webhook_generic'),
 ]
 
 urlpatterns += function_urls

readthedocs/restapi/views/integrations.py

@@ -1,10 +1,11 @@
+import json
 import logging
 
 from rest_framework import permissions
 from rest_framework.views import APIView
 from rest_framework.renderers import JSONRenderer
 from rest_framework.response import Response
-from rest_framework.exceptions import ParseError
+from rest_framework.exceptions import APIException, ParseError, NotFound
 
 from django.shortcuts import get_object_or_404
 from django.http import Http404
@@ -12,6 +13,8 @@ from django.http import Http404
 from readthedocs.core.views.hooks import build_branches
 from readthedocs.core.signals import (webhook_github, webhook_bitbucket,
                                       webhook_gitlab)
+from readthedocs.integrations.models import HttpExchange
+from readthedocs.integrations.utils import normalize_request_payload
 from readthedocs.projects.models import Project
 
 
@@ -28,20 +31,44 @@ class WebhookMixin(object):
     renderer_classes = (JSONRenderer,)
 
     def post(self, request, project_slug, format=None):
+        """Set up webhook post view with request and project objects"""
+        self.request = request
+        self.project = None
         try:
-            project = Project.objects.get(slug=project_slug)
+            self.project = Project.objects.get(slug=project_slug)
-            resp = self.handle_webhook(request, project, request.data)
+            resp = self.handle_webhook()
             if resp is None:
                 log.info('Unhandled webhook event')
                 resp = {'detail': 'Unhandled webhook event'}
+            resp = Response(resp)
         except Project.DoesNotExist:
-            raise Http404('Project does not exist')
+            raise NotFound('Project not found')
-        return Response(resp)
+        return resp
 
-    def handle_webhook(self, request, project, data=None):
+    def finalize_response(self, req, *args, **kwargs):
+        """If the project was set on POST, store an HTTP exchange"""
+        resp = super(WebhookMixin, self).finalize_response(req, *args, **kwargs)
+        if hasattr(self, 'project') and self.project:
+            HttpExchange.objects.from_exchange(
+                req,
+                resp,
+                related_object=self.project,
+                payload=self.get_payload(),
+            )
+        return resp
+
+    def handle_webhook(self):
         """Handle webhook payload"""
         raise NotImplementedError
 
+    def get_payload(self):
+        """Don't specify any special handling of the payload data
+
+        The exchange will record ``request.data`` instead of assume any
+        special handling of the payload data
+        """
+        return None
+
     def get_response_push(self, project, branches):
         """Build branches on push events and return API response
 
@@ -65,7 +92,7 @@ class WebhookMixin(object):
         triggered = True if to_build else False
         return {'build_triggered': triggered,
                 'project': project.slug,
-                'versions': to_build}
+                'versions': list(to_build)}
 
 
 class GitHubWebhookView(WebhookMixin, APIView):
@@ -84,15 +111,25 @@ class GitHubWebhookView(WebhookMixin, APIView):
         }
     """
 
-    def handle_webhook(self, request, project, data=None):
+    def get_payload(self):
+        if self.request.content_type == 'application/x-www-form-urlencoded':
+            try:
+                return json.loads(self.request.data['payload'])
+            except ValueError:
+                pass
+        return normalize_request_payload(self.request)
+
+    def handle_webhook(self):
+        data = self.get_payload()
         # Get event and trigger other webhook events
-        event = request.META.get('HTTP_X_GITHUB_EVENT', 'push')
+        event = self.request.META.get('HTTP_X_GITHUB_EVENT', 'push')
-        webhook_github.send(Project, project=project, data=data, event=event)
+        webhook_github.send(Project, project=self.project,
+                            data=data, event=event)
         # Handle push events and trigger builds
         if event == GITHUB_PUSH:
             try:
-                branches = [request.data['ref'].replace('refs/heads/', '')]
+                branches = [data['ref'].replace('refs/heads/', '')]
-                return self.get_response_push(project, branches)
+                return self.get_response_push(self.project, branches)
             except KeyError:
                 raise ParseError('Parameter "ref" is required')
 
@@ -112,15 +149,16 @@ class GitLabWebhookView(WebhookMixin, APIView):
         }
     """
 
-    def handle_webhook(self, request, project, data=None):
+    def handle_webhook(self):
         # Get event and trigger other webhook events
-        event = data.get('object_kind', GITLAB_PUSH)
+        event = self.request.data.get('object_kind', GITLAB_PUSH)
-        webhook_gitlab.send(Project, project=project, data=data, event=event)
+        webhook_gitlab.send(Project, project=self.project,
+                            data=self.request.data, event=event)
         # Handle push events and trigger builds
         if event == GITLAB_PUSH:
             try:
-                branches = [request.data['ref'].replace('refs/heads/', '')]
+                branches = [self.request.data['ref'].replace('refs/heads/', '')]
-                return self.get_response_push(project, branches)
+                return self.get_response_push(self.project, branches)
             except KeyError:
                 raise ParseError('Parameter "ref" is required')
 
@@ -148,16 +186,39 @@ class BitbucketWebhookView(WebhookMixin, APIView):
         }
     """
 
-    def handle_webhook(self, request, project, data=None):
+    def handle_webhook(self):
         # Get event and trigger other webhook events
-        event = request.META.get('HTTP_X_EVENT_KEY', BITBUCKET_PUSH)
+        event = self.request.META.get('HTTP_X_EVENT_KEY', BITBUCKET_PUSH)
-        webhook_bitbucket.send(Project, project=project, data=data, event=event)
+        webhook_bitbucket.send(Project, project=self.project,
+                               data=self.request.data, event=event)
         # Handle push events and trigger builds
         if event == BITBUCKET_PUSH:
             try:
-                changes = data['push']['changes']
+                changes = self.request.data['push']['changes']
                 branches = [change['new']['name']
                             for change in changes]
-                return self.get_response_push(project, branches)
+                return self.get_response_push(self.project, branches)
             except KeyError:
                 raise ParseError('Invalid request')
+
+
+class GenericWebhookView(WebhookMixin, APIView):
+
+    """Generic webhook consumer
+
+    Expects the following JSON::
+
+        {
+            "branches": ["master"]
+        }
+    """
+
+    def handle_webhook(self):
+        try:
+            branches = list(self.request.data.get(
+                'branches',
+                [self.project.get_default_branch()]
+            ))
+            return self.get_response_push(self.project, branches)
+        except TypeError:
+            raise ParseError('Invalid request')

readthedocs/rtd_tests/tests/test_integrations.py

@@ -0,0 +1,141 @@
+import django_dynamic_fixture as fixture
+from django.test import TestCase, RequestFactory
+from django.contrib.contenttypes.models import ContentType
+from rest_framework.test import APIClient
+from rest_framework.test import APIRequestFactory
+from rest_framework.response import Response
+
+from readthedocs.integrations.models import HttpExchange
+from readthedocs.projects.models import Project
+
+
+class HttpExchangeTests(TestCase):
+
+    """Test HttpExchange model by using existing views
+
+    This doesn't mock out a req/resp cycle, as manually creating these outside
+    views misses a number of attributes on the request object.
+    """
+
+    def test_exchange_json_request_body(self):
+        client = APIClient()
+        client.login(username='super', password='test')
+        project = fixture.get(Project, main_language_project=None)
+        resp = client.post(
+            '/api/v2/webhook/github/{0}/'.format(project.slug),
+            {'ref': 'exchange_json'},
+            format='json'
+        )
+        exchange = HttpExchange.objects.get(
+            content_type=ContentType.objects.filter(
+                app_label='projects',
+                model='project'
+            ),
+            object_id=project.pk
+        )
+        self.assertEqual(
+            exchange.request_body,
+            '{"ref": "exchange_json"}'
+        )
+        self.assertEqual(
+            exchange.request_headers,
+            {u'Content-Type': u'application/json; charset=None',
+             u'Cookie': u''}
+        )
+        self.assertEqual(
+            exchange.response_body,
+            ('{{"build_triggered": false, "project": "{0}", "versions": []}}'
+             .format(project.slug)),
+        )
+        self.assertEqual(
+            exchange.response_headers,
+            {u'Allow': u'POST, OPTIONS',
+             u'Content-Type': u'text/html; charset=utf-8'}
+        )
+
+    def test_exchange_form_request_body(self):
+        client = APIClient()
+        client.login(username='super', password='test')
+        project = fixture.get(Project, main_language_project=None)
+        resp = client.post(
+            '/api/v2/webhook/github/{0}/'.format(project.slug),
+            'payload=%7B%22ref%22%3A+%22exchange_form%22%7D',
+            content_type='application/x-www-form-urlencoded',
+        )
+        exchange = HttpExchange.objects.get(
+            content_type=ContentType.objects.filter(
+                app_label='projects',
+                model='project'
+            ),
+            object_id=project.pk
+        )
+        self.assertEqual(
+            exchange.request_body,
+            '{"ref": "exchange_form"}'
+        )
+        self.assertEqual(
+            exchange.request_headers,
+            {u'Content-Type': u'application/x-www-form-urlencoded',
+             u'Cookie': u''}
+        )
+        self.assertEqual(
+            exchange.response_body,
+            ('{{"build_triggered": false, "project": "{0}", "versions": []}}'
+             .format(project.slug)),
+        )
+        self.assertEqual(
+            exchange.response_headers,
+            {u'Allow': u'POST, OPTIONS',
+             u'Content-Type': u'text/html; charset=utf-8'}
+        )
+
+    def test_extraneous_exchanges_deleted_in_correct_order(self):
+        client = APIClient()
+        client.login(username='super', password='test')
+        project = fixture.get(Project, main_language_project=None)
+
+        self.assertEqual(
+            HttpExchange.objects.filter(
+                content_type=ContentType.objects.get(
+                    app_label='projects',
+                    model='project',
+                ),
+                object_id=project.pk
+            ).count(),
+            0
+        )
+
+        for _ in range(10):
+            resp = client.post(
+                '/api/v2/webhook/github/{0}/'.format(project.slug),
+                {'ref': 'deleted'},
+                format='json'
+            )
+        for _ in range(10):
+            resp = client.post(
+                '/api/v2/webhook/github/{0}/'.format(project.slug),
+                {'ref': 'preserved'},
+                format='json'
+            )
+
+        self.assertEqual(
+            HttpExchange.objects.filter(
+                content_type=ContentType.objects.get(
+                    app_label='projects',
+                    model='project',
+                ),
+                object_id=project.pk
+            ).count(),
+            10
+        )
+        self.assertEqual(
+            HttpExchange.objects.filter(
+                content_type=ContentType.objects.get(
+                    app_label='projects',
+                    model='project',
+                ),
+                object_id=project.pk,
+                request_body='{"ref": "preserved"}',
+            ).count(),
+            10
+        )

readthedocs/rtd_tests/tests/test_privacy_urls.py

@@ -6,6 +6,7 @@ from django.core.urlresolvers import reverse
 
 from readthedocs.builds.models import Build, VersionAlias, BuildCommandResult
 from readthedocs.comments.models import DocumentComment, NodeSnapshot
+from readthedocs.integrations.models import HttpExchange
 from readthedocs.projects.models import Project, Domain
 from readthedocs.oauth.models import RemoteRepository, RemoteOrganization
 from readthedocs.rtd_tests.utils import create_user
@@ -59,7 +60,15 @@ class URLAccessMixin(object):
         if self.context_data and getattr(response, 'context'):
             self._test_context(response)
         for (key, val) in response_attrs.items():
-            self.assertEqual(getattr(response, key), val)
+            resp_val = getattr(response, key)
+            self.assertEqual(
+                resp_val,
+                val,
+                ('Attribute mismatch for view {view} ({path}): '
+                 '{key} != {expected} (got {value})'
+                 .format(view=name, path=path, key=key, expected=val,
+                         value=resp_val))
+            )
         return response
 
     def _test_context(self, response):
@@ -95,17 +104,17 @@ class URLAccessMixin(object):
                     raise Exception('URL argument not in test kwargs. Please add `%s`' % key)
                 added_kwargs[key] = self.default_kwargs[key]
             path = reverse(name, kwargs=added_kwargs)
-            print "Tested %s (%s)" % (name, path)
             self.assertResponse(path=path, name=name)
-            print "Passed %s (%s)" % (name, path)
             added_kwargs = {}
 
     def setUp(self):
         # Previous Fixtures
         self.owner = create_user(username='owner', password='test')
         self.tester = create_user(username='tester', password='test')
-        self.pip = get(Project, slug='pip', users=[self.owner], privacy_level='public')
+        self.pip = get(Project, slug='pip', users=[self.owner],
-        self.private = get(Project, slug='private', privacy_level='private')
+                       privacy_level='public', main_language_project=None)
+        self.private = get(Project, slug='private', privacy_level='private',
+                           main_language_project=None)
 
 
 class ProjectMixin(URLAccessMixin):
@@ -115,9 +124,17 @@ class ProjectMixin(URLAccessMixin):
         self.build = get(Build, project=self.pip)
         self.tag = get(Tag, slug='coolness')
         self.alias = get(VersionAlias, slug='that_alias', project=self.pip)
-        self.subproject = get(Project, slug='sub', language='ja', users=[self.owner])
+        self.subproject = get(Project, slug='sub', language='ja',
+                              users=[self.owner], main_language_project=None)
         self.pip.add_subproject(self.subproject)
         self.pip.translations.add(self.subproject)
+        # For whatever reason, fixtures hates JSONField
+        self.webhook_exchange = HttpExchange.objects.create(
+            related_object=self.pip,
+            request_headers='{"foo": "bar"}',
+            response_headers='{"foo": "bar"}',
+            status_code=200,
+        )
         self.domain = get(Domain, url='http://docs.foobar.com', project=self.pip)
         self.default_kwargs = {
             'project_slug': self.pip.slug,
@@ -129,6 +146,7 @@ class ProjectMixin(URLAccessMixin):
             'child_slug': self.subproject.slug,
             'build_pk': self.build.pk,
             'domain_pk': self.domain.pk,
+            'exchange_pk': self.webhook_exchange.pk,
         }
 
 
@@ -204,6 +222,7 @@ class PrivateProjectAdminAccessTest(PrivateProjectMixin, TestCase):
         '/dashboard/pip/users/delete/': {'status_code': 405},
         '/dashboard/pip/notifications/delete/': {'status_code': 405},
         '/dashboard/pip/redirects/delete/': {'status_code': 405},
+        '/dashboard/pip/integrations/sync/': {'status_code': 405},
     }
 
     def login(self):
@@ -229,6 +248,7 @@ class PrivateProjectUserAccessTest(PrivateProjectMixin, TestCase):
         '/dashboard/pip/users/delete/': {'status_code': 405},
         '/dashboard/pip/notifications/delete/': {'status_code': 405},
         '/dashboard/pip/redirects/delete/': {'status_code': 405},
+        '/dashboard/pip/integrations/sync/': {'status_code': 405},
     }
 
     # Filtered out by queryset on projects that we don't own.
@@ -298,6 +318,7 @@ class APIMixin(URLAccessMixin):
             'api_webhook_github': {'status_code': 405},
             'api_webhook_gitlab': {'status_code': 405},
             'api_webhook_bitbucket': {'status_code': 405},
+            'api_webhook_generic': {'status_code': 405},
             'remoteorganization-detail': {'status_code': 404},
             'remoterepository-detail': {'status_code': 404},
         }

readthedocs/settings/base.py

@@ -104,6 +104,7 @@ class CommunityBaseSettings(Settings):
             'readthedocs.donate',
             'readthedocs.payments',
             'readthedocs.notifications',
+            'readthedocs.integrations',
 
             # allauth
             'allauth',

readthedocs/templates/core/project_bar_base.html

@@ -28,15 +28,14 @@
 
     {% if not project.has_valid_webhook and request.user|is_admin:project %}
       <p class="build-failure">
-        {% url "projects_resync_webhook" project.slug as resync_url %}
+        {% url "projects_integrations_sync" project.slug as sync_url %}
         {% blocktrans %}
-        This repository doesn't have a valid webhook set up.
+        This repository doesn't have a valid webhook set up,
-        That means it won't be rebuilt on commits to the repository.
+        commits won't trigger new builds for this project.
         <br>
-        You can <a href='{{ resync_url }}'>resync your webhook</a> to fix this.
+        You can <a href='{{ sync_url }}'>sync your webhook</a> to fix this.
         {% endblocktrans %}
       </p>
-      <br>
     {% endif %}
 
     {% if project.skip %}

readthedocs/templates/projects/integration_exchange_detail.html

@@ -0,0 +1,41 @@
+{% extends "projects/project_edit_base.html" %}
+
+{% load i18n %}
+
+{% block title %}{% trans "Integrations" %}{% endblock %}
+
+{% block nav-dashboard %} class="active"{% endblock %}
+
+{% block editing-option-edit-integrations %}class="active"{% endblock %}
+
+{% block project-integrations-active %}active{% endblock %}
+{% block project_edit_content_header %}{% trans "Exchange" %}{% endblock %}
+
+{% block project_edit_content %}
+  <h4>Response</h4>
+
+  <div class="httpexchange">
+    <dl class="term-list-flat http-header">
+      <dt>Status:</dt>
+      <dd>{{ httpexchange.status_code }}</dd>
+
+      {% for header, value in httpexchange.response_headers.items %}
+        <dt>{{ header }}:</dt>
+        <dd>{{ value }}</dd>
+      {% endfor %}
+    </dl>
+    {{ httpexchange.formatted_response_body }}
+  </div>
+
+  <h4>Request</h4>
+
+  <div class="httpexchange">
+    <dl class="term-list-flat http-header">
+      {% for header, value in httpexchange.request_headers.items %}
+        <dt>{{ header }}:</dt>
+        <dd>{{ value }}</dd>
+      {% endfor %}
+    </dl>
+    {{ httpexchange.formatted_request_body }}
+  </div>
+{% endblock %}

readthedocs/templates/projects/integration_list.html

@@ -0,0 +1,46 @@
+{% extends "projects/project_edit_base.html" %}
+
+{% load i18n %}
+
+{% block title %}{% trans "Integrations" %}{% endblock %}
+
+{% block nav-dashboard %} class="active"{% endblock %}
+
+{% block editing-option-edit-integrations %}class="active"{% endblock %}
+
+{% block project-integrations-active %}active{% endblock %}
+{% block project_edit_content_header %}{% trans "Integrations" %}{% endblock %}
+
+{% block project_edit_content %}
+  <p>
+    Manage external integrations for project webhooks.
+  </p>
+
+  <form method="post" action="{% url 'projects_integrations_sync' project_slug=project.slug %}">
+    {% csrf_token %}
+    <input type="submit" value="{% trans "Resync Webhook" %}">
+  </form>
+
+  <h3>Recent Activity</h3>
+
+  <div class="module-list-wrapper httpexchanges">
+    <ul>
+      {% for exchange in object_list %}
+        <li class="module-item">
+          <span class="status status-{% if exchange.failed %}fail{% else %}pass{% endif %}">{{ exchange.status_code }}</span>
+          <a href="{% url 'projects_integrations_exchange_detail' exchange_pk=exchange.pk project_slug=project.slug %}">
+            {% blocktrans with date=exchange.date|timesince %}
+              {{ date }} ago
+            {% endblocktrans %}
+          </a>
+        </li>
+      {% empty %}
+        <li class="module-item">
+          <span class="quiet">
+            {% trans 'There is no recent activity' %}
+          </span>
+        </li>
+      {% endfor %}
+    </ul>
+  </div>
+{% endblock %}

readthedocs/templates/projects/project_edit_base.html

@@ -22,6 +22,7 @@
         <li class="{% block project-redirects-active %}{% endblock %}"><a href="{% url "projects_redirects" project.slug %}">{% trans "Redirects" %}</a></li>
         <li class="{% block project-translations-active %}{% endblock %}"><a href="{% url "projects_translations" project.slug %}">{% trans "Translations" %}</a></li>
         <li class="{% block project-subprojects-active %}{% endblock %}"><a href="{% url "projects_subprojects" project.slug %}">{% trans "Subprojects" %}</a></li>
+        <li class="{% block project-integrations-active %}{% endblock %}"><a href="{% url "projects_integrations" project.slug %}">{% trans "Integrations" %}</a></li>
         <li class="{% block project-notifications-active %}{% endblock %}"><a href="{% url "projects_notifications" project.slug %}">{% trans "Notifications" %}</a></li>
         <li class="{% block project-ads-active %}{% endblock %}"><a href="{% url "projects_advertising" project.slug %}">{% trans "Advertising" %} </a></li>
         {% if project.allow_comments %}

requirements/pip.txt

@@ -17,6 +17,7 @@ django-guardian==1.4.6
 django-extensions==1.7.4
 djangorestframework==3.5.4
 django-vanilla-views==1.0.4
+jsonfield==1.0.3
 pytest-django==2.8.0
 
 requests==2.3.0