Add modeling for webhooks, wrap webhook creation some with model creation (#2726)

* Add modeling for http transactions and wrappers around webhook views

* Transaction -> Exchange

* Move request payload normalization to util function
tox-dependencies
Anthony 2017-03-21 10:48:23 -07:00 committed by GitHub
parent 7f941040c1
commit ba644ddd31
21 changed files with 761 additions and 70 deletions

View File

@ -1029,3 +1029,105 @@ select.dropdown { display: none; }
.domain-machine { color: #999; }
.domain-canonical { font-weight: bold; }
/* Integrations */
div.module-list-wrapper.httptransactions li span.status {
padding: .2em .4em;
margin-right: .3em;
border-radius: .3em;
color: #fff;
}
div.module-list-wrapper.httptransactions li span.status.status-pass {
background: #5a5;
}
div.module-list-wrapper.httptransactions li span.status.status-fail {
background: #a55;
}
div.httptransaction dl dt {
display: inline-block;
font-weight: bold;
font-family: 'inconsolata', 'bitstream vera sans mono', 'andale mono', 'lucida console', monospace;
font-size: .9em;
}
div.httptransaction dl dd {
display: inline;
font-family: 'inconsolata', 'bitstream vera sans mono', 'andale mono', 'lucida console', monospace;
font-size: .9em;
}
div.httptransaction dl dd:after {
display: block;
content: '';
}
div.httptransaction div.highlight pre {
padding: 1em;
background: #f4f4f4;
border: 1px solid #ccc;
font-size: .9em;
}
/* Pygments */
div.highlight pre .hll { background-color: #ffffcc }
div.highlight pre .c { color: #60a0b0; font-style: italic } /* Comment */
div.highlight pre .err { border: 1px solid #FF0000 } /* Error */
div.highlight pre .k { color: #007020; font-weight: bold } /* Keyword */
div.highlight pre .o { color: #666666 } /* Operator */
div.highlight pre .cm { color: #60a0b0; font-style: italic } /* Comment.Multiline */
div.highlight pre .cp { color: #007020 } /* Comment.Preproc */
div.highlight pre .c1 { color: #60a0b0; font-style: italic } /* Comment.Single */
div.highlight pre .cs { color: #60a0b0; background-color: #fff0f0 } /* Comment.Special */
div.highlight pre .gd { color: #A00000 } /* Generic.Deleted */
div.highlight pre .ge { font-style: italic } /* Generic.Emph */
div.highlight pre .gr { color: #FF0000 } /* Generic.Error */
div.highlight pre .gh { color: #000080; font-weight: bold } /* Generic.Heading */
div.highlight pre .gi { color: #00A000 } /* Generic.Inserted */
div.highlight pre .go { color: #888888 } /* Generic.Output */
div.highlight pre .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */
div.highlight pre .gs { font-weight: bold } /* Generic.Strong */
div.highlight pre .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
div.highlight pre .gt { color: #0044DD } /* Generic.Traceback */
div.highlight pre .kc { color: #007020; font-weight: bold } /* Keyword.Constant */
div.highlight pre .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */
div.highlight pre .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */
div.highlight pre .kp { color: #007020 } /* Keyword.Pseudo */
div.highlight pre .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */
div.highlight pre .kt { color: #902000 } /* Keyword.Type */
div.highlight pre .m { color: #40a070 } /* Literal.Number */
div.highlight pre .s { color: #4070a0 } /* Literal.String */
div.highlight pre .na { color: #4070a0 } /* Name.Attribute */
div.highlight pre .nb { color: #007020 } /* Name.Builtin */
div.highlight pre .nc { color: #0e84b5; font-weight: bold } /* Name.Class */
div.highlight pre .no { color: #60add5 } /* Name.Constant */
div.highlight pre .nd { color: #555555; font-weight: bold } /* Name.Decorator */
div.highlight pre .ni { color: #d55537; font-weight: bold } /* Name.Entity */
div.highlight pre .ne { color: #007020 } /* Name.Exception */
div.highlight pre .nf { color: #06287e } /* Name.Function */
div.highlight pre .nl { color: #002070; font-weight: bold } /* Name.Label */
div.highlight pre .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */
div.highlight pre .nt { color: #062873; font-weight: bold } /* Name.Tag */
div.highlight pre .nv { color: #bb60d5 } /* Name.Variable */
div.highlight pre .ow { color: #007020; font-weight: bold } /* Operator.Word */
div.highlight pre .w { color: #bbbbbb } /* Text.Whitespace */
div.highlight pre .mb { color: #40a070 } /* Literal.Number.Bin */
div.highlight pre .mf { color: #40a070 } /* Literal.Number.Float */
div.highlight pre .mh { color: #40a070 } /* Literal.Number.Hex */
div.highlight pre .mi { color: #40a070 } /* Literal.Number.Integer */
div.highlight pre .mo { color: #40a070 } /* Literal.Number.Oct */
div.highlight pre .sb { color: #4070a0 } /* Literal.String.Backtick */
div.highlight pre .sc { color: #4070a0 } /* Literal.String.Char */
div.highlight pre .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */
div.highlight pre .s2 { color: #4070a0 } /* Literal.String.Double */
div.highlight pre .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */
div.highlight pre .sh { color: #4070a0 } /* Literal.String.Heredoc */
div.highlight pre .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */
div.highlight pre .sx { color: #c65d09 } /* Literal.String.Other */
div.highlight pre .sr { color: #235388 } /* Literal.String.Regex */
div.highlight pre .s1 { color: #4070a0 } /* Literal.String.Single */
div.highlight pre .ss { color: #517918 } /* Literal.String.Symbol */
div.highlight pre .bp { color: #007020 } /* Name.Builtin.Pseudo */
div.highlight pre .vc { color: #bb60d5 } /* Name.Variable.Class */
div.highlight pre .vg { color: #bb60d5 } /* Name.Variable.Global */
div.highlight pre .vi { color: #bb60d5 } /* Name.Variable.Instance */
div.highlight pre .il { color: #40a070 } /* Literal.Number.Integer.Long */

View File

@ -139,6 +139,10 @@ def _build_url(url, projects, branches):
def github_build(request):
"""GitHub webhook consumer
.. warning:: **DEPRECATED**
Use :py:cls:`readthedocs.restapi.views.intergrations.GitHubWebhookView`
instead of this view function
This will search for projects matching either a stripped down HTTP or SSH
URL. The search is error prone, use the API v2 webhook for new webhooks.
@ -189,6 +193,10 @@ def github_build(request):
def gitlab_build(request):
"""GitLab webhook consumer
.. warning:: **DEPRECATED**
Use :py:cls:`readthedocs.restapi.views.intergrations.GitLabWebhookView`
instead of this view function
Search project repository URLs using the site URL from GitLab webhook payload.
This search is error-prone, use the API v2 webhook view for new webhooks.
"""
@ -220,6 +228,10 @@ def gitlab_build(request):
def bitbucket_build(request):
"""Consume webhooks from multiple versions of Bitbucket's API
.. warning:: **DEPRECATED**
Use :py:cls:`readthedocs.restapi.views.intergrations.BitbucketWebhookView`
instead of this view function
New webhooks are set up with v2, but v1 webhooks will still point to this
endpoint. There are also "services" that point here and submit
``application/x-www-form-urlencoded`` data.
@ -277,6 +289,12 @@ def bitbucket_build(request):
@csrf_exempt
def generic_build(request, project_id_or_slug=None):
"""Generic webhook build endpoint
.. warning:: **DEPRECATED**
Use :py:cls:`readthedocs.restapi.views.intergrations.GenericWebhookView`
instead of this view function
"""
try:
project = Project.objects.get(pk=project_id_or_slug)
# Allow slugs too

View File

View File

@ -0,0 +1,37 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.12 on 2017-03-16 18:30
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
import jsonfield.fields
import uuid
class Migration(migrations.Migration):
initial = True
dependencies = [
('contenttypes', '0002_remove_content_type_name'),
]
operations = [
migrations.CreateModel(
name='HttpExchange',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('object_id', models.PositiveIntegerField()),
('date', models.DateTimeField(auto_now_add=True, verbose_name='Date')),
('request_headers', jsonfield.fields.JSONField(verbose_name='Request headers')),
('request_body', models.TextField(verbose_name='Request body')),
('response_headers', jsonfield.fields.JSONField(verbose_name='Request headers')),
('response_body', models.TextField(verbose_name='Response body')),
('status_code', models.IntegerField(default=200, verbose_name='Status code')),
('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
],
options={
'ordering': ['-date'],
},
),
]

View File

@ -0,0 +1,141 @@
"""Integration models for external services"""
import json
import uuid
from django.db import models, transaction
from django.utils.translation import ugettext_lazy as _
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.utils.safestring import mark_safe
from rest_framework import status
from jsonfield import JSONField
from pygments import highlight
from pygments.lexers import JsonLexer
from pygments.formatters import HtmlFormatter
from .utils import normalize_request_payload
class HttpExchangeManager(models.Manager):
"""HTTP exchange manager methods"""
@transaction.atomic
def from_exchange(self, req, resp, related_object, payload=None):
"""Create object from Django request and response objects
If an explicit Request ``payload`` is not specified, the payload will be
determined directly from the Request object. This makes a good effort to
normalize the data, however we don't enforce that the payload is JSON
:param req: Request object to store
:type req: HttpRequest
:param resp: Response object to store
:type resp: HttpResponse
:param related_object: Object to use for generic relation
:param payload: Alternate payload object to store
:type payload: dict
"""
request_payload = payload
if request_payload is None:
request_payload = normalize_request_payload(req)
try:
request_body = json.dumps(request_payload, sort_keys=True)
except TypeError:
request_body = str(request_payload)
# This is the rawest form of request header we have, the WSGI
# headers. HTTP headers are prefixed with `HTTP_`, which we remove,
# and because the keys are all uppercase, we'll normalize them to
# title case-y hyphen separated values.
request_headers = dict(
(key[5:].title().replace('_', '-'), str(val))
for (key, val) in req.META.items()
if key.startswith('HTTP_')
)
request_headers['Content-Type'] = req.content_type
response_payload = resp.data if hasattr(resp, 'data') else resp.content
try:
response_body = json.dumps(response_payload, sort_keys=True)
except TypeError:
response_body = str(response_payload)
response_headers = dict(resp.items())
fields = {
'status_code': resp.status_code,
'request_headers': request_headers,
'request_body': request_body,
'response_body': response_body,
'response_headers': response_headers,
}
fields['related_object'] = related_object
obj = self.create(**fields)
self.delete_limit(related_object)
return obj
def delete_limit(self, related_object, limit=10):
queryset = self.filter(
content_type=ContentType.objects.get(
app_label=related_object._meta.app_label,
model=related_object._meta.model_name,
),
object_id=related_object.pk
)
for exchange in queryset[limit:]:
exchange.delete()
class HttpExchange(models.Model):
"""HTTP request/response exchange"""
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
object_id = models.PositiveIntegerField()
related_object = GenericForeignKey('content_type', 'object_id')
date = models.DateTimeField(_('Date'), auto_now_add=True)
request_headers = JSONField(_('Request headers'))
request_body = models.TextField(_('Request body'))
response_headers = JSONField(_('Request headers'))
response_body = models.TextField(_('Response body'))
status_code = models.IntegerField(
_('Status code'), default=status.HTTP_200_OK
)
objects = HttpExchangeManager()
class Meta:
ordering = ['-date']
def __unicode__(self):
return _('Exchange {0}').format(self.pk)
@property
def failed(self):
# Assume anything that isn't 2xx level status code is an error
return int(self.status_code / 100) != 2
def formatted_json(self, field):
"""Try to return pretty printed and Pygment highlighted code"""
value = getattr(self, field) or ''
try:
json_value = json.dumps(json.loads(value), sort_keys=True, indent=2)
formatter = HtmlFormatter()
html = highlight(json_value, JsonLexer(), formatter)
return mark_safe(html)
except (ValueError, TypeError):
return value
@property
def formatted_request_body(self):
return self.formatted_json('request_body')
@property
def formatted_response_body(self):
return self.formatted_json('response_body')

View File

@ -0,0 +1,22 @@
"""Integration utility functions"""
def normalize_request_payload(request):
"""Normalize the request body, hopefully to JSON
This will attempt to return a JSON body, backing down to a string body next.
:param request: HTTP request object
:type request: django.http.HttpRequest
:returns: The request body as a string
:rtype: str
"""
request_payload = getattr(request, 'data', {})
if request.content_type != 'application/json':
# Here, request_body can be a dict or a MergeDict. Probably best to
# normalize everything first
try:
request_payload = dict(request_payload.items())
except AttributeError:
pass
return request_payload

View File

@ -16,6 +16,11 @@ def attach_webhook(project, request=None):
service = service_cls
break
else:
messages.error(
request,
_('Webhook activation failed. '
'There are no connected services for this project.')
)
return None
user_accounts = service.for_user(request.user)
@ -25,18 +30,19 @@ def attach_webhook(project, request=None):
messages.success(request, _('Webhook activated'))
project.has_valid_webhook = True
project.save()
break
return True
# No valid account found
if user_accounts:
messages.error(
request,
_('Webhook activation failed. Make sure you have permissions to set it.')
)
else:
if user_accounts:
messages.error(
request,
_('Webhook activation failed. Make sure you have permissions to set it.')
)
else:
messages.error(
request,
_('No accounts available to set webhook on. '
'Please connect your {network} account.'.format(
network=service.adapter().get_provider().name
))
)
messages.error(
request,
_('No accounts available to set webhook on. '
'Please connect your {network} account.'.format(
network=service.adapter().get_provider().name
))
)
return False

View File

@ -7,6 +7,7 @@ from readthedocs.projects.views.private import (
ProjectDashboard, ImportView,
ProjectUpdate, ProjectAdvancedUpdate,
DomainList, DomainCreate, DomainDelete, DomainUpdate,
IntegrationList, IntegrationExchangeDetail, IntegrationWebhookSync,
ProjectAdvertisingUpdate)
from readthedocs.projects.backends.views import ImportWizardView, ImportDemoView
@ -105,10 +106,6 @@ urlpatterns = [
private.project_redirects_delete,
name='projects_redirects_delete'),
url(r'^(?P<project_slug>[-\w]+)/resync_webhook/$',
private.project_resync_webhook,
name='projects_resync_webhook'),
url(r'^(?P<project_slug>[-\w]+)/advertising/$',
ProjectAdvertisingUpdate.as_view(),
name='projects_advertising'),
@ -130,3 +127,17 @@ domain_urls = [
]
urlpatterns += domain_urls
integration_urls = [
url(r'^(?P<project_slug>[-\w]+)/integrations/$',
IntegrationList.as_view(),
name='projects_integrations'),
url(r'^(?P<project_slug>[-\w]+)/integrations/exchange/(?P<exchange_pk>[-\w]+)/$',
IntegrationExchangeDetail.as_view(),
name='projects_integrations_exchange_detail'),
url(r'^(?P<project_slug>[-\w]+)/integrations/sync/$',
IntegrationWebhookSync.as_view(),
name='projects_integrations_sync'),
]
urlpatterns += integration_urls

View File

@ -79,9 +79,6 @@ class ProjectAdminMixin(object):
kwargs['project'] = self.get_project()
return self.form_class(data, files, **kwargs)
def get_success_url(self, **kwargs):
return reverse('projects_domains', args=[self.get_project().slug])
class ProjectSpamMixin(object):

View File

@ -5,8 +5,8 @@ import logging
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.contrib import messages
from django.contrib.contenttypes.models import ContentType
from django.core.urlresolvers import reverse
from django.conf import settings
from django.http import (HttpResponseRedirect, HttpResponseNotAllowed,
Http404, HttpResponseBadRequest)
from django.shortcuts import get_object_or_404, render_to_response, render
@ -18,7 +18,7 @@ from django.middleware.csrf import get_token
from formtools.wizard.views import SessionWizardView
from allauth.socialaccount.models import SocialAccount
from vanilla import CreateView, DeleteView, UpdateView
from vanilla import CreateView, DeleteView, UpdateView, DetailView, GenericView
from readthedocs.bookmarks.models import Bookmark
from readthedocs.builds.models import Version
@ -27,6 +27,7 @@ from readthedocs.builds.filters import VersionFilter
from readthedocs.builds.models import VersionAlias
from readthedocs.core.utils import trigger_build, broadcast
from readthedocs.core.mixins import ListViewWithForm
from readthedocs.integrations.models import HttpExchange
from readthedocs.projects.forms import (
ProjectBasicsForm, ProjectExtraForm,
ProjectAdvancedForm, UpdateProjectForm, SubprojectForm,
@ -656,6 +657,9 @@ class DomainMixin(ProjectAdminMixin, PrivateViewMixin):
form_class = DomainForm
lookup_url_kwarg = 'domain_pk'
def get_success_url(self):
return reverse('projects_domains', args=[self.get_project().slug])
class DomainList(DomainMixin, ListViewWithForm):
pass
@ -673,25 +677,63 @@ class DomainDelete(DomainMixin, DeleteView):
pass
@login_required
def project_resync_webhook(request, project_slug):
class IntegrationMixin(object):
"""Project external service mixin for listing webhook objects
This mixin will be used more once we have modeling around webhooks and
external integrations.
"""
Resync a project webhook.
def get_success_url(self):
return reverse('projects_integrations', args=[self.get_project().slug])
def get_template_names(self):
if self.template_name:
return self.template_name
return 'projects/integration{0}.html'.format(self.template_name_suffix)
class IntegrationExchangeMixin(ProjectAdminMixin, PrivateViewMixin):
"""Project webhook exchange mixin for listing exchange objects"""
model = HttpExchange
lookup_url_kwarg = 'exchange_pk'
def get_queryset(self):
self.project = self.get_project()
return self.model.objects.filter(
content_type=ContentType.objects.filter(
app_label='projects',
model='project'
),
object_id=self.project.pk
)
class IntegrationList(IntegrationMixin, IntegrationExchangeMixin, ListView):
pass
class IntegrationExchangeDetail(IntegrationMixin, IntegrationExchangeMixin, DetailView):
template_name = 'projects/integration_exchange_detail.html'
class IntegrationWebhookSync(PrivateViewMixin, ProjectAdminMixin, GenericView):
"""Resync a project webhook
The signal will add a success/failure message on the request.
"""
project = get_object_or_404(Project.objects.for_admin_user(request.user),
slug=project_slug)
if request.method == 'POST':
attach_webhook(project=project, request=request)
return HttpResponseRedirect(reverse('projects_detail',
args=[project.slug]))
return render_to_response(
'projects/project_resync_webhook.html',
{'project': project},
context_instance=RequestContext(request)
)
def post(self, request, *args, **kwargs):
# pylint: disable=unused-argument
attach_webhook(project=self.get_project(), request=request)
return HttpResponseRedirect(self.get_success_url())
def get_success_url(self):
return reverse('projects_integrations', args=[self.get_project().slug])
class ProjectAdvertisingUpdate(PrivateViewMixin, UpdateView):

View File

@ -71,6 +71,9 @@ integration_urls = [
url(r'webhook/bitbucket/(?P<project_slug>{project_slug})/'.format(**pattern_opts),
integrations.BitbucketWebhookView.as_view(),
name='api_webhook_bitbucket'),
url(r'webhook/generic/(?P<project_slug>{project_slug})/'.format(**pattern_opts),
integrations.GenericWebhookView.as_view(),
name='api_webhook_generic'),
]
urlpatterns += function_urls

View File

@ -1,10 +1,11 @@
import json
import logging
from rest_framework import permissions
from rest_framework.views import APIView
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
from rest_framework.exceptions import ParseError
from rest_framework.exceptions import APIException, ParseError, NotFound
from django.shortcuts import get_object_or_404
from django.http import Http404
@ -12,6 +13,8 @@ from django.http import Http404
from readthedocs.core.views.hooks import build_branches
from readthedocs.core.signals import (webhook_github, webhook_bitbucket,
webhook_gitlab)
from readthedocs.integrations.models import HttpExchange
from readthedocs.integrations.utils import normalize_request_payload
from readthedocs.projects.models import Project
@ -28,20 +31,44 @@ class WebhookMixin(object):
renderer_classes = (JSONRenderer,)
def post(self, request, project_slug, format=None):
"""Set up webhook post view with request and project objects"""
self.request = request
self.project = None
try:
project = Project.objects.get(slug=project_slug)
resp = self.handle_webhook(request, project, request.data)
self.project = Project.objects.get(slug=project_slug)
resp = self.handle_webhook()
if resp is None:
log.info('Unhandled webhook event')
resp = {'detail': 'Unhandled webhook event'}
resp = Response(resp)
except Project.DoesNotExist:
raise Http404('Project does not exist')
return Response(resp)
raise NotFound('Project not found')
return resp
def handle_webhook(self, request, project, data=None):
def finalize_response(self, req, *args, **kwargs):
"""If the project was set on POST, store an HTTP exchange"""
resp = super(WebhookMixin, self).finalize_response(req, *args, **kwargs)
if hasattr(self, 'project') and self.project:
HttpExchange.objects.from_exchange(
req,
resp,
related_object=self.project,
payload=self.get_payload(),
)
return resp
def handle_webhook(self):
"""Handle webhook payload"""
raise NotImplementedError
def get_payload(self):
"""Don't specify any special handling of the payload data
The exchange will record ``request.data`` instead of assume any
special handling of the payload data
"""
return None
def get_response_push(self, project, branches):
"""Build branches on push events and return API response
@ -65,7 +92,7 @@ class WebhookMixin(object):
triggered = True if to_build else False
return {'build_triggered': triggered,
'project': project.slug,
'versions': to_build}
'versions': list(to_build)}
class GitHubWebhookView(WebhookMixin, APIView):
@ -84,15 +111,25 @@ class GitHubWebhookView(WebhookMixin, APIView):
}
"""
def handle_webhook(self, request, project, data=None):
def get_payload(self):
if self.request.content_type == 'application/x-www-form-urlencoded':
try:
return json.loads(self.request.data['payload'])
except ValueError:
pass
return normalize_request_payload(self.request)
def handle_webhook(self):
data = self.get_payload()
# Get event and trigger other webhook events
event = request.META.get('HTTP_X_GITHUB_EVENT', 'push')
webhook_github.send(Project, project=project, data=data, event=event)
event = self.request.META.get('HTTP_X_GITHUB_EVENT', 'push')
webhook_github.send(Project, project=self.project,
data=data, event=event)
# Handle push events and trigger builds
if event == GITHUB_PUSH:
try:
branches = [request.data['ref'].replace('refs/heads/', '')]
return self.get_response_push(project, branches)
branches = [data['ref'].replace('refs/heads/', '')]
return self.get_response_push(self.project, branches)
except KeyError:
raise ParseError('Parameter "ref" is required')
@ -112,15 +149,16 @@ class GitLabWebhookView(WebhookMixin, APIView):
}
"""
def handle_webhook(self, request, project, data=None):
def handle_webhook(self):
# Get event and trigger other webhook events
event = data.get('object_kind', GITLAB_PUSH)
webhook_gitlab.send(Project, project=project, data=data, event=event)
event = self.request.data.get('object_kind', GITLAB_PUSH)
webhook_gitlab.send(Project, project=self.project,
data=self.request.data, event=event)
# Handle push events and trigger builds
if event == GITLAB_PUSH:
try:
branches = [request.data['ref'].replace('refs/heads/', '')]
return self.get_response_push(project, branches)
branches = [self.request.data['ref'].replace('refs/heads/', '')]
return self.get_response_push(self.project, branches)
except KeyError:
raise ParseError('Parameter "ref" is required')
@ -148,16 +186,39 @@ class BitbucketWebhookView(WebhookMixin, APIView):
}
"""
def handle_webhook(self, request, project, data=None):
def handle_webhook(self):
# Get event and trigger other webhook events
event = request.META.get('HTTP_X_EVENT_KEY', BITBUCKET_PUSH)
webhook_bitbucket.send(Project, project=project, data=data, event=event)
event = self.request.META.get('HTTP_X_EVENT_KEY', BITBUCKET_PUSH)
webhook_bitbucket.send(Project, project=self.project,
data=self.request.data, event=event)
# Handle push events and trigger builds
if event == BITBUCKET_PUSH:
try:
changes = data['push']['changes']
changes = self.request.data['push']['changes']
branches = [change['new']['name']
for change in changes]
return self.get_response_push(project, branches)
return self.get_response_push(self.project, branches)
except KeyError:
raise ParseError('Invalid request')
class GenericWebhookView(WebhookMixin, APIView):
"""Generic webhook consumer
Expects the following JSON::
{
"branches": ["master"]
}
"""
def handle_webhook(self):
try:
branches = list(self.request.data.get(
'branches',
[self.project.get_default_branch()]
))
return self.get_response_push(self.project, branches)
except TypeError:
raise ParseError('Invalid request')

View File

@ -0,0 +1,141 @@
import django_dynamic_fixture as fixture
from django.test import TestCase, RequestFactory
from django.contrib.contenttypes.models import ContentType
from rest_framework.test import APIClient
from rest_framework.test import APIRequestFactory
from rest_framework.response import Response
from readthedocs.integrations.models import HttpExchange
from readthedocs.projects.models import Project
class HttpExchangeTests(TestCase):
"""Test HttpExchange model by using existing views
This doesn't mock out a req/resp cycle, as manually creating these outside
views misses a number of attributes on the request object.
"""
def test_exchange_json_request_body(self):
client = APIClient()
client.login(username='super', password='test')
project = fixture.get(Project, main_language_project=None)
resp = client.post(
'/api/v2/webhook/github/{0}/'.format(project.slug),
{'ref': 'exchange_json'},
format='json'
)
exchange = HttpExchange.objects.get(
content_type=ContentType.objects.filter(
app_label='projects',
model='project'
),
object_id=project.pk
)
self.assertEqual(
exchange.request_body,
'{"ref": "exchange_json"}'
)
self.assertEqual(
exchange.request_headers,
{u'Content-Type': u'application/json; charset=None',
u'Cookie': u''}
)
self.assertEqual(
exchange.response_body,
('{{"build_triggered": false, "project": "{0}", "versions": []}}'
.format(project.slug)),
)
self.assertEqual(
exchange.response_headers,
{u'Allow': u'POST, OPTIONS',
u'Content-Type': u'text/html; charset=utf-8'}
)
def test_exchange_form_request_body(self):
client = APIClient()
client.login(username='super', password='test')
project = fixture.get(Project, main_language_project=None)
resp = client.post(
'/api/v2/webhook/github/{0}/'.format(project.slug),
'payload=%7B%22ref%22%3A+%22exchange_form%22%7D',
content_type='application/x-www-form-urlencoded',
)
exchange = HttpExchange.objects.get(
content_type=ContentType.objects.filter(
app_label='projects',
model='project'
),
object_id=project.pk
)
self.assertEqual(
exchange.request_body,
'{"ref": "exchange_form"}'
)
self.assertEqual(
exchange.request_headers,
{u'Content-Type': u'application/x-www-form-urlencoded',
u'Cookie': u''}
)
self.assertEqual(
exchange.response_body,
('{{"build_triggered": false, "project": "{0}", "versions": []}}'
.format(project.slug)),
)
self.assertEqual(
exchange.response_headers,
{u'Allow': u'POST, OPTIONS',
u'Content-Type': u'text/html; charset=utf-8'}
)
def test_extraneous_exchanges_deleted_in_correct_order(self):
client = APIClient()
client.login(username='super', password='test')
project = fixture.get(Project, main_language_project=None)
self.assertEqual(
HttpExchange.objects.filter(
content_type=ContentType.objects.get(
app_label='projects',
model='project',
),
object_id=project.pk
).count(),
0
)
for _ in range(10):
resp = client.post(
'/api/v2/webhook/github/{0}/'.format(project.slug),
{'ref': 'deleted'},
format='json'
)
for _ in range(10):
resp = client.post(
'/api/v2/webhook/github/{0}/'.format(project.slug),
{'ref': 'preserved'},
format='json'
)
self.assertEqual(
HttpExchange.objects.filter(
content_type=ContentType.objects.get(
app_label='projects',
model='project',
),
object_id=project.pk
).count(),
10
)
self.assertEqual(
HttpExchange.objects.filter(
content_type=ContentType.objects.get(
app_label='projects',
model='project',
),
object_id=project.pk,
request_body='{"ref": "preserved"}',
).count(),
10
)

View File

@ -6,6 +6,7 @@ from django.core.urlresolvers import reverse
from readthedocs.builds.models import Build, VersionAlias, BuildCommandResult
from readthedocs.comments.models import DocumentComment, NodeSnapshot
from readthedocs.integrations.models import HttpExchange
from readthedocs.projects.models import Project, Domain
from readthedocs.oauth.models import RemoteRepository, RemoteOrganization
from readthedocs.rtd_tests.utils import create_user
@ -59,7 +60,15 @@ class URLAccessMixin(object):
if self.context_data and getattr(response, 'context'):
self._test_context(response)
for (key, val) in response_attrs.items():
self.assertEqual(getattr(response, key), val)
resp_val = getattr(response, key)
self.assertEqual(
resp_val,
val,
('Attribute mismatch for view {view} ({path}): '
'{key} != {expected} (got {value})'
.format(view=name, path=path, key=key, expected=val,
value=resp_val))
)
return response
def _test_context(self, response):
@ -95,17 +104,17 @@ class URLAccessMixin(object):
raise Exception('URL argument not in test kwargs. Please add `%s`' % key)
added_kwargs[key] = self.default_kwargs[key]
path = reverse(name, kwargs=added_kwargs)
print "Tested %s (%s)" % (name, path)
self.assertResponse(path=path, name=name)
print "Passed %s (%s)" % (name, path)
added_kwargs = {}
def setUp(self):
# Previous Fixtures
self.owner = create_user(username='owner', password='test')
self.tester = create_user(username='tester', password='test')
self.pip = get(Project, slug='pip', users=[self.owner], privacy_level='public')
self.private = get(Project, slug='private', privacy_level='private')
self.pip = get(Project, slug='pip', users=[self.owner],
privacy_level='public', main_language_project=None)
self.private = get(Project, slug='private', privacy_level='private',
main_language_project=None)
class ProjectMixin(URLAccessMixin):
@ -115,9 +124,17 @@ class ProjectMixin(URLAccessMixin):
self.build = get(Build, project=self.pip)
self.tag = get(Tag, slug='coolness')
self.alias = get(VersionAlias, slug='that_alias', project=self.pip)
self.subproject = get(Project, slug='sub', language='ja', users=[self.owner])
self.subproject = get(Project, slug='sub', language='ja',
users=[self.owner], main_language_project=None)
self.pip.add_subproject(self.subproject)
self.pip.translations.add(self.subproject)
# For whatever reason, fixtures hates JSONField
self.webhook_exchange = HttpExchange.objects.create(
related_object=self.pip,
request_headers='{"foo": "bar"}',
response_headers='{"foo": "bar"}',
status_code=200,
)
self.domain = get(Domain, url='http://docs.foobar.com', project=self.pip)
self.default_kwargs = {
'project_slug': self.pip.slug,
@ -129,6 +146,7 @@ class ProjectMixin(URLAccessMixin):
'child_slug': self.subproject.slug,
'build_pk': self.build.pk,
'domain_pk': self.domain.pk,
'exchange_pk': self.webhook_exchange.pk,
}
@ -204,6 +222,7 @@ class PrivateProjectAdminAccessTest(PrivateProjectMixin, TestCase):
'/dashboard/pip/users/delete/': {'status_code': 405},
'/dashboard/pip/notifications/delete/': {'status_code': 405},
'/dashboard/pip/redirects/delete/': {'status_code': 405},
'/dashboard/pip/integrations/sync/': {'status_code': 405},
}
def login(self):
@ -229,6 +248,7 @@ class PrivateProjectUserAccessTest(PrivateProjectMixin, TestCase):
'/dashboard/pip/users/delete/': {'status_code': 405},
'/dashboard/pip/notifications/delete/': {'status_code': 405},
'/dashboard/pip/redirects/delete/': {'status_code': 405},
'/dashboard/pip/integrations/sync/': {'status_code': 405},
}
# Filtered out by queryset on projects that we don't own.
@ -298,6 +318,7 @@ class APIMixin(URLAccessMixin):
'api_webhook_github': {'status_code': 405},
'api_webhook_gitlab': {'status_code': 405},
'api_webhook_bitbucket': {'status_code': 405},
'api_webhook_generic': {'status_code': 405},
'remoteorganization-detail': {'status_code': 404},
'remoterepository-detail': {'status_code': 404},
}

View File

@ -104,6 +104,7 @@ class CommunityBaseSettings(Settings):
'readthedocs.donate',
'readthedocs.payments',
'readthedocs.notifications',
'readthedocs.integrations',
# allauth
'allauth',

View File

@ -28,15 +28,14 @@
{% if not project.has_valid_webhook and request.user|is_admin:project %}
<p class="build-failure">
{% url "projects_resync_webhook" project.slug as resync_url %}
{% url "projects_integrations_sync" project.slug as sync_url %}
{% blocktrans %}
This repository doesn't have a valid webhook set up.
That means it won't be rebuilt on commits to the repository.
This repository doesn't have a valid webhook set up,
commits won't trigger new builds for this project.
<br>
You can <a href='{{ resync_url }}'>resync your webhook</a> to fix this.
You can <a href='{{ sync_url }}'>sync your webhook</a> to fix this.
{% endblocktrans %}
</p>
<br>
{% endif %}
{% if project.skip %}

View File

@ -0,0 +1,41 @@
{% extends "projects/project_edit_base.html" %}
{% load i18n %}
{% block title %}{% trans "Integrations" %}{% endblock %}
{% block nav-dashboard %} class="active"{% endblock %}
{% block editing-option-edit-integrations %}class="active"{% endblock %}
{% block project-integrations-active %}active{% endblock %}
{% block project_edit_content_header %}{% trans "Exchange" %}{% endblock %}
{% block project_edit_content %}
<h4>Response</h4>
<div class="httpexchange">
<dl class="term-list-flat http-header">
<dt>Status:</dt>
<dd>{{ httpexchange.status_code }}</dd>
{% for header, value in httpexchange.response_headers.items %}
<dt>{{ header }}:</dt>
<dd>{{ value }}</dd>
{% endfor %}
</dl>
{{ httpexchange.formatted_response_body }}
</div>
<h4>Request</h4>
<div class="httpexchange">
<dl class="term-list-flat http-header">
{% for header, value in httpexchange.request_headers.items %}
<dt>{{ header }}:</dt>
<dd>{{ value }}</dd>
{% endfor %}
</dl>
{{ httpexchange.formatted_request_body }}
</div>
{% endblock %}

View File

@ -0,0 +1,46 @@
{% extends "projects/project_edit_base.html" %}
{% load i18n %}
{% block title %}{% trans "Integrations" %}{% endblock %}
{% block nav-dashboard %} class="active"{% endblock %}
{% block editing-option-edit-integrations %}class="active"{% endblock %}
{% block project-integrations-active %}active{% endblock %}
{% block project_edit_content_header %}{% trans "Integrations" %}{% endblock %}
{% block project_edit_content %}
<p>
Manage external integrations for project webhooks.
</p>
<form method="post" action="{% url 'projects_integrations_sync' project_slug=project.slug %}">
{% csrf_token %}
<input type="submit" value="{% trans "Resync Webhook" %}">
</form>
<h3>Recent Activity</h3>
<div class="module-list-wrapper httpexchanges">
<ul>
{% for exchange in object_list %}
<li class="module-item">
<span class="status status-{% if exchange.failed %}fail{% else %}pass{% endif %}">{{ exchange.status_code }}</span>
<a href="{% url 'projects_integrations_exchange_detail' exchange_pk=exchange.pk project_slug=project.slug %}">
{% blocktrans with date=exchange.date|timesince %}
{{ date }} ago
{% endblocktrans %}
</a>
</li>
{% empty %}
<li class="module-item">
<span class="quiet">
{% trans 'There is no recent activity' %}
</span>
</li>
{% endfor %}
</ul>
</div>
{% endblock %}

View File

@ -22,6 +22,7 @@
<li class="{% block project-redirects-active %}{% endblock %}"><a href="{% url "projects_redirects" project.slug %}">{% trans "Redirects" %}</a></li>
<li class="{% block project-translations-active %}{% endblock %}"><a href="{% url "projects_translations" project.slug %}">{% trans "Translations" %}</a></li>
<li class="{% block project-subprojects-active %}{% endblock %}"><a href="{% url "projects_subprojects" project.slug %}">{% trans "Subprojects" %}</a></li>
<li class="{% block project-integrations-active %}{% endblock %}"><a href="{% url "projects_integrations" project.slug %}">{% trans "Integrations" %}</a></li>
<li class="{% block project-notifications-active %}{% endblock %}"><a href="{% url "projects_notifications" project.slug %}">{% trans "Notifications" %}</a></li>
<li class="{% block project-ads-active %}{% endblock %}"><a href="{% url "projects_advertising" project.slug %}">{% trans "Advertising" %} </a></li>
{% if project.allow_comments %}

View File

@ -17,6 +17,7 @@ django-guardian==1.4.6
django-extensions==1.7.4
djangorestframework==3.5.4
django-vanilla-views==1.0.4
jsonfield==1.0.3
pytest-django==2.8.0
requests==2.3.0