Split up OAuth services into own path, allow for extension of classes
parent
61a6ecc91b
commit
fb75fd09e5
|
@ -7,6 +7,8 @@
|
|||
.idea
|
||||
.vagrant
|
||||
.tox
|
||||
.rope_project/
|
||||
.ropeproject/
|
||||
_build
|
||||
cnames
|
||||
bower_components/
|
||||
|
@ -35,4 +37,3 @@ whoosh_index
|
|||
xml_output
|
||||
public_cnames
|
||||
public_symlinks
|
||||
.rope_project/
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
from django.core.management.base import BaseCommand
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from readthedocs.oauth.utils import GitHubService
|
||||
from readthedocs.oauth.services import GitHubService
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
|
|
|
@ -0,0 +1,13 @@
|
|||
"""Conditional classes for OAuth services"""
|
||||
|
||||
from django.utils.module_loading import import_by_path
|
||||
from django.conf import settings
|
||||
|
||||
GitHubService = import_by_path(
|
||||
getattr(settings, 'OAUTH_GITHUB_SERVICE',
|
||||
'readthedocs.oauth.services.github.GitHubService'))
|
||||
BitbucketService = import_by_path(
|
||||
getattr(settings, 'OAUTH_BITBUCKET_SERVICE',
|
||||
'readthedocs.oauth.services.bitbucket.BitbucketService'))
|
||||
|
||||
registry = [GitHubService, BitbucketService]
|
|
@ -0,0 +1,141 @@
|
|||
"""OAuth utility functions"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from django.conf import settings
|
||||
from requests_oauthlib import OAuth2Session
|
||||
from allauth.socialaccount.models import SocialAccount
|
||||
|
||||
|
||||
DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public')
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Service(object):
|
||||
|
||||
"""Service mapping for local accounts
|
||||
|
||||
:param user: User to use in token lookup and session creation
|
||||
:param account: :py:cls:`SocialAccount` instance for user
|
||||
"""
|
||||
|
||||
adapter = None
|
||||
url_pattern = None
|
||||
|
||||
def __init__(self, user, account):
|
||||
self.session = None
|
||||
self.user = user
|
||||
self.account = account
|
||||
|
||||
@classmethod
|
||||
def for_user(cls, user):
|
||||
"""Create instance if user has an account for the provider"""
|
||||
try:
|
||||
account = SocialAccount.objects.get(
|
||||
user=user,
|
||||
provider=cls.adapter.provider_id
|
||||
)
|
||||
return cls(user=user, account=account)
|
||||
except SocialAccount.DoesNotExist:
|
||||
return None
|
||||
|
||||
def get_adapter(self):
|
||||
return self.adapter
|
||||
|
||||
@property
|
||||
def provider_id(self):
|
||||
return self.get_adapter().provider_id
|
||||
|
||||
def get_session(self):
|
||||
if self.session is None:
|
||||
self.create_session()
|
||||
return self.session
|
||||
|
||||
def create_session(self):
|
||||
"""Create OAuth session for user
|
||||
|
||||
This configures the OAuth session based on the :py:cls:`SocialToken`
|
||||
attributes. If there is an ``expires_at``, treat the session as an auto
|
||||
renewing token. Some providers expire tokens after as little as 2
|
||||
hours.
|
||||
"""
|
||||
token = self.account.socialtoken_set.first()
|
||||
if token is None:
|
||||
return None
|
||||
|
||||
token_config = {
|
||||
'access_token': str(token.token),
|
||||
'token_type': 'bearer',
|
||||
}
|
||||
if token.expires_at is not None:
|
||||
token_expires = (token.expires_at - datetime.now()).total_seconds()
|
||||
token_config.update({
|
||||
'refresh_token': str(token.token_secret),
|
||||
'expires_in': token_expires,
|
||||
})
|
||||
|
||||
self.session = OAuth2Session(
|
||||
client_id=token.app.client_id,
|
||||
token=token_config,
|
||||
auto_refresh_kwargs={
|
||||
'client_id': token.app.client_id,
|
||||
'client_secret': token.app.secret,
|
||||
},
|
||||
auto_refresh_url=self.get_adapter().access_token_url,
|
||||
token_updater=self.token_updater(token)
|
||||
)
|
||||
|
||||
return self.session or None
|
||||
|
||||
def token_updater(self, token):
|
||||
"""Update token given data from OAuth response
|
||||
|
||||
Expect the following response into the closure::
|
||||
|
||||
{
|
||||
u'token_type': u'bearer',
|
||||
u'scopes': u'webhook repository team account',
|
||||
u'refresh_token': u'...',
|
||||
u'access_token': u'...',
|
||||
u'expires_in': 3600,
|
||||
u'expires_at': 1449218652.558185
|
||||
}
|
||||
"""
|
||||
|
||||
def _updater(data):
|
||||
token.token = data['access_token']
|
||||
token.expires_at = datetime.fromtimestamp(data['expires_at'])
|
||||
token.save()
|
||||
log.info('Updated token %s:', token)
|
||||
|
||||
return _updater
|
||||
|
||||
def sync(self, sync):
|
||||
raise NotImplementedError
|
||||
|
||||
def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
|
||||
organization=None):
|
||||
raise NotImplementedError
|
||||
|
||||
def create_organization(self, fields):
|
||||
raise NotImplementedError
|
||||
|
||||
def setup_webhook(self, project):
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def is_project_service(cls, project):
|
||||
"""Determine if this is the service the project is using
|
||||
|
||||
.. note::
|
||||
This should be deprecated in favor of attaching the
|
||||
:py:cls:`RemoteRepository` to the project instance. This is a slight
|
||||
improvement on the legacy check for webhooks
|
||||
"""
|
||||
# TODO Replace this check by keying project to remote repos
|
||||
return (
|
||||
cls.url_pattern is not None and
|
||||
cls.url_pattern.search(project.repo) is not None
|
||||
)
|
|
@ -0,0 +1,181 @@
|
|||
"""OAuth utility functions"""
|
||||
|
||||
import logging
|
||||
import json
|
||||
import re
|
||||
|
||||
from django.conf import settings
|
||||
from requests.exceptions import RequestException
|
||||
from allauth.socialaccount.providers.bitbucket_oauth2.views import (
|
||||
BitbucketOAuth2Adapter)
|
||||
|
||||
from readthedocs.builds import utils as build_utils
|
||||
|
||||
from ..models import RemoteOrganization, RemoteRepository
|
||||
from .base import Service
|
||||
|
||||
|
||||
DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public')
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BitbucketService(Service):
|
||||
|
||||
"""Provider service for Bitbucket"""
|
||||
|
||||
adapter = BitbucketOAuth2Adapter
|
||||
# TODO replace this with a less naive check
|
||||
url_pattern = re.compile(r'bitbucket.org\/')
|
||||
|
||||
def sync(self, sync):
|
||||
"""Import from Bitbucket"""
|
||||
if sync:
|
||||
self.sync_repositories()
|
||||
self.sync_teams()
|
||||
|
||||
def sync_repositories(self):
|
||||
# Get user repos
|
||||
try:
|
||||
repos = self.paginate(
|
||||
'https://bitbucket.org/api/2.0/repositories/?role=member')
|
||||
for repo in repos:
|
||||
self.create_repository(repo)
|
||||
except (TypeError, ValueError) as e:
|
||||
log.error('Error syncing Bitbucket repositories: %s',
|
||||
str(e), exc_info=True)
|
||||
raise Exception('Could not sync your Bitbucket repositories, '
|
||||
'try reconnecting your account')
|
||||
|
||||
# Because privileges aren't returned with repository data, run query
|
||||
# again for repositories that user has admin role for, and update
|
||||
# existing repositories.
|
||||
try:
|
||||
resp = self.paginate(
|
||||
'https://bitbucket.org/api/2.0/repositories/?role=admin')
|
||||
repos = (
|
||||
RemoteRepository.objects
|
||||
.filter(users=self.user,
|
||||
full_name__in=[r['full_name'] for r in resp],
|
||||
account=self.account)
|
||||
)
|
||||
for repo in repos:
|
||||
repo.admin = True
|
||||
repo.save()
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
def sync_teams(self):
|
||||
"""Sync Bitbucket teams and team repositories for user token"""
|
||||
try:
|
||||
teams = self.paginate(
|
||||
'https://api.bitbucket.org/2.0/teams/?role=member'
|
||||
)
|
||||
for team in teams:
|
||||
org = self.create_organization(team)
|
||||
repos = self.paginate(team['links']['repositories']['href'])
|
||||
for repo in repos:
|
||||
self.create_repository(repo, organization=org)
|
||||
except ValueError as e:
|
||||
log.error('Error syncing Bitbucket organizations: %s',
|
||||
str(e), exc_info=True)
|
||||
raise Exception('Could not sync your Bitbucket team repositories, '
|
||||
'try reconnecting your account')
|
||||
|
||||
def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
|
||||
organization=None):
|
||||
"""Update or create a repository from Bitbucket
|
||||
|
||||
This looks up existing repositories based on the full repository name,
|
||||
that is the username and repository name.
|
||||
|
||||
.. note::
|
||||
The :py:data:`admin` property is not set during creation, as
|
||||
permissions are not part of the returned repository data from
|
||||
Bitbucket.
|
||||
"""
|
||||
if (fields['is_private'] is True and privacy == 'private' or
|
||||
fields['is_private'] is False and privacy == 'public'):
|
||||
repo, _ = RemoteRepository.objects.get_or_create(
|
||||
full_name=fields['full_name'],
|
||||
account=self.account,
|
||||
)
|
||||
if repo.organization and repo.organization != organization:
|
||||
log.debug('Not importing %s because mismatched orgs' %
|
||||
fields['name'])
|
||||
return None
|
||||
else:
|
||||
repo.organization = organization
|
||||
repo.users.add(self.user)
|
||||
repo.name = fields['name']
|
||||
repo.description = fields['description']
|
||||
repo.private = fields['is_private']
|
||||
repo.clone_url = fields['links']['clone'][0]['href']
|
||||
repo.ssh_url = fields['links']['clone'][1]['href']
|
||||
if repo.private:
|
||||
repo.clone_url = repo.ssh_url
|
||||
repo.html_url = fields['links']['html']['href']
|
||||
repo.vcs = fields['scm']
|
||||
repo.account = self.account
|
||||
|
||||
avatar_url = fields['links']['avatar']['href'] or ''
|
||||
repo.avatar_url = re.sub(r'\/16\/$', r'/32/', avatar_url)
|
||||
|
||||
repo.json = json.dumps(fields)
|
||||
repo.save()
|
||||
return repo
|
||||
else:
|
||||
log.debug('Not importing %s because mismatched type' %
|
||||
fields['name'])
|
||||
|
||||
def create_organization(self, fields):
|
||||
organization, _ = RemoteOrganization.objects.get_or_create(
|
||||
slug=fields.get('username'),
|
||||
account=self.account,
|
||||
)
|
||||
organization.name = fields.get('display_name')
|
||||
organization.email = fields.get('email')
|
||||
organization.avatar_url = fields['links']['avatar']['href']
|
||||
organization.html_url = fields['links']['html']['href']
|
||||
organization.json = json.dumps(fields)
|
||||
organization.account = self.account
|
||||
organization.users.add(self.user)
|
||||
organization.save()
|
||||
return organization
|
||||
|
||||
def paginate(self, url):
|
||||
"""Combines results from Bitbucket pagination
|
||||
|
||||
:param url: start url to get the data from.
|
||||
"""
|
||||
resp = self.get_session().get(url)
|
||||
data = resp.json()
|
||||
results = data.get('values', [])
|
||||
next_url = data.get('next')
|
||||
if next_url:
|
||||
results.extend(self.paginate(next_url))
|
||||
return results
|
||||
|
||||
def setup_webhook(self, project):
|
||||
session = self.get_session()
|
||||
owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo)
|
||||
data = {
|
||||
'type': 'POST',
|
||||
'url': 'https://{domain}/bitbucket'.format(domain=settings.PRODUCTION_DOMAIN),
|
||||
}
|
||||
try:
|
||||
resp = session.post(
|
||||
'https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/services'.format(
|
||||
owner=owner, repo=repo
|
||||
),
|
||||
data=data,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
log.info('Created Bitbucket webhook: project={project}'
|
||||
.format(project=project))
|
||||
return True
|
||||
except RequestException:
|
||||
pass
|
||||
else:
|
||||
log.exception('Bitbucket webhook creation failed', exc_info=True)
|
||||
return False
|
|
@ -0,0 +1,183 @@
|
|||
"""OAuth utility functions"""
|
||||
|
||||
import logging
|
||||
import json
|
||||
import re
|
||||
|
||||
from django.conf import settings
|
||||
from requests.exceptions import RequestException
|
||||
from allauth.socialaccount.models import SocialToken
|
||||
from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter
|
||||
|
||||
from readthedocs.builds import utils as build_utils
|
||||
from readthedocs.restapi.client import api
|
||||
|
||||
from ..models import RemoteOrganization, RemoteRepository
|
||||
from .base import Service
|
||||
|
||||
|
||||
DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public')
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GitHubService(Service):
|
||||
|
||||
"""Provider service for GitHub"""
|
||||
|
||||
adapter = GitHubOAuth2Adapter
|
||||
# TODO replace this with a less naive check
|
||||
url_pattern = re.compile(r'^github\.com\/')
|
||||
|
||||
def sync(self, sync):
|
||||
"""Sync repositories and organizations"""
|
||||
if sync:
|
||||
self.sync_repositories()
|
||||
self.sync_organizations()
|
||||
|
||||
def sync_repositories(self):
|
||||
"""Get repositories for GitHub user via OAuth token"""
|
||||
repos = self.paginate('https://api.github.com/user/repos?per_page=100')
|
||||
try:
|
||||
for repo in repos:
|
||||
self.create_repository(repo)
|
||||
except (TypeError, ValueError) as e:
|
||||
log.error('Error syncing GitHub repositories: %s',
|
||||
str(e), exc_info=True)
|
||||
raise Exception('Could not sync your GitHub repositories, '
|
||||
'try reconnecting your account')
|
||||
|
||||
def sync_organizations(self):
|
||||
"""Sync GitHub organizations and organization repositories"""
|
||||
try:
|
||||
orgs = self.paginate('https://api.github.com/user/orgs')
|
||||
for org in orgs:
|
||||
org_resp = self.get_session().get(org['url'])
|
||||
org_obj = self.create_organization(org_resp.json())
|
||||
# Add repos
|
||||
# TODO ?per_page=100
|
||||
org_repos = self.paginate(
|
||||
'{org_url}/repos'.format(org_url=org['url'])
|
||||
)
|
||||
for repo in org_repos:
|
||||
self.create_repository(repo, organization=org_obj)
|
||||
except (TypeError, ValueError) as e:
|
||||
log.error('Error syncing GitHub organizations: %s',
|
||||
str(e), exc_info=True)
|
||||
raise Exception('Could not sync your GitHub organizations, '
|
||||
'try reconnecting your account')
|
||||
|
||||
def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
|
||||
organization=None):
|
||||
if (
|
||||
(privacy == 'private') or
|
||||
(fields['private'] is False and privacy == 'public')):
|
||||
repo, _ = RemoteRepository.objects.get_or_create(
|
||||
full_name=fields['full_name'],
|
||||
account=self.account,
|
||||
)
|
||||
if repo.organization and repo.organization != organization:
|
||||
log.debug('Not importing %s because mismatched orgs' %
|
||||
fields['name'])
|
||||
return None
|
||||
else:
|
||||
repo.organization = organization
|
||||
repo.users.add(self.user)
|
||||
repo.name = fields['name']
|
||||
repo.description = fields['description']
|
||||
repo.ssh_url = fields['ssh_url']
|
||||
repo.html_url = fields['html_url']
|
||||
repo.private = fields['private']
|
||||
if repo.private:
|
||||
repo.clone_url = fields['ssh_url']
|
||||
else:
|
||||
repo.clone_url = fields['clone_url']
|
||||
repo.admin = fields.get('permissions', {}).get('admin', False)
|
||||
repo.vcs = 'git'
|
||||
repo.account = self.account
|
||||
repo.avatar_url = fields.get('owner', {}).get('avatar_url')
|
||||
repo.json = json.dumps(fields)
|
||||
repo.save()
|
||||
return repo
|
||||
else:
|
||||
log.debug('Not importing %s because mismatched type' %
|
||||
fields['name'])
|
||||
|
||||
def create_organization(self, fields):
|
||||
organization, _ = RemoteOrganization.objects.get_or_create(
|
||||
slug=fields.get('login'),
|
||||
account=self.account,
|
||||
)
|
||||
organization.html_url = fields.get('html_url')
|
||||
organization.name = fields.get('name')
|
||||
organization.email = fields.get('email')
|
||||
organization.avatar_url = fields.get('avatar_url')
|
||||
organization.json = json.dumps(fields)
|
||||
organization.account = self.account
|
||||
organization.users.add(self.user)
|
||||
organization.save()
|
||||
return organization
|
||||
|
||||
def paginate(self, url):
|
||||
"""Combines return from GitHub pagination
|
||||
|
||||
:param url: start url to get the data from.
|
||||
|
||||
See https://developer.github.com/v3/#pagination
|
||||
"""
|
||||
resp = self.get_session().get(url)
|
||||
result = resp.json()
|
||||
next_url = resp.links.get('next', {}).get('url')
|
||||
if next_url:
|
||||
result.extend(self.paginate(next_url))
|
||||
return result
|
||||
|
||||
def setup_webhook(self, project):
|
||||
"""Set up GitHub webhook for project
|
||||
|
||||
:param project: Project instance to set up webhook for
|
||||
"""
|
||||
session = self.get_session()
|
||||
owner, repo = build_utils.get_github_username_repo(url=project.repo)
|
||||
data = json.dumps({
|
||||
'name': 'readthedocs',
|
||||
'active': True,
|
||||
'config': {'url': 'https://{domain}/github'.format(domain=settings.PRODUCTION_DOMAIN)}
|
||||
})
|
||||
resp = None
|
||||
try:
|
||||
resp = session.post(
|
||||
'https://api.github.com/repos/{owner}/{repo}/hooks'.format(owner=owner, repo=repo),
|
||||
data=data,
|
||||
headers={'content-type': 'application/json'}
|
||||
)
|
||||
if resp.status_code == 201:
|
||||
log.info('Created GitHub webhook: project={project}'
|
||||
.format(project=project))
|
||||
return True
|
||||
except RequestException:
|
||||
pass
|
||||
else:
|
||||
log.exception('GitHub Hook creation failed', exc_info=True)
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def get_token_for_project(cls, project, force_local=False):
|
||||
"""Get access token for project by iterating over project users"""
|
||||
# TODO why does this only target GitHub?
|
||||
if not getattr(settings, 'ALLOW_PRIVATE_REPOS', False):
|
||||
return None
|
||||
token = None
|
||||
try:
|
||||
if getattr(settings, 'DONT_HIT_DB', True) and not force_local:
|
||||
token = api.project(project.pk).token().get()['token']
|
||||
else:
|
||||
for user in project.users.all():
|
||||
tokens = SocialToken.objects.filter(
|
||||
account__user=user,
|
||||
app__provider=cls.adapter.provider_id)
|
||||
if tokens.exists():
|
||||
token = tokens[0].token
|
||||
except Exception:
|
||||
log.error('Failed to get token for user', exc_info=True)
|
||||
return token
|
|
@ -6,7 +6,7 @@ from djcelery import celery as celery_app
|
|||
from readthedocs.core.utils.tasks import PublicTask
|
||||
from readthedocs.core.utils.tasks import permission_check
|
||||
from readthedocs.core.utils.tasks import user_id_matches
|
||||
from .utils import services
|
||||
from .services import registry
|
||||
|
||||
|
||||
@permission_check(user_id_matches)
|
||||
|
@ -16,7 +16,7 @@ class SyncRemoteRepositories(PublicTask):
|
|||
|
||||
def run_public(self, user_id):
|
||||
user = User.objects.get(pk=user_id)
|
||||
for service_cls in services:
|
||||
for service_cls in registry:
|
||||
service = service_cls.for_user(user)
|
||||
if service is not None:
|
||||
service.sync(sync=True)
|
||||
|
|
|
@ -1,478 +0,0 @@
|
|||
"""OAuth utility functions"""
|
||||
|
||||
import logging
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
from django.conf import settings
|
||||
from requests_oauthlib import OAuth2Session
|
||||
from requests.exceptions import RequestException
|
||||
from allauth.socialaccount.models import SocialAccount
|
||||
from allauth.socialaccount.providers.bitbucket_oauth2.views import (
|
||||
BitbucketOAuth2Adapter)
|
||||
from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter
|
||||
|
||||
from readthedocs.builds import utils as build_utils
|
||||
from readthedocs.restapi.client import api
|
||||
|
||||
from .models import RemoteOrganization, RemoteRepository
|
||||
|
||||
|
||||
DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public')
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Service(object):
|
||||
|
||||
"""Service mapping for local accounts
|
||||
|
||||
:param user: User to use in token lookup and session creation
|
||||
:param account: :py:cls:`SocialAccount` instance for user
|
||||
"""
|
||||
|
||||
adapter = None
|
||||
url_pattern = None
|
||||
|
||||
def __init__(self, user, account):
|
||||
self.session = None
|
||||
self.user = user
|
||||
self.account = account
|
||||
|
||||
@classmethod
|
||||
def for_user(cls, user):
|
||||
"""Create instance if user has an account for the provider"""
|
||||
try:
|
||||
account = SocialAccount.objects.get(
|
||||
user=user,
|
||||
provider=cls.adapter.provider_id
|
||||
)
|
||||
return cls(user=user, account=account)
|
||||
except SocialAccount.DoesNotExist:
|
||||
return None
|
||||
|
||||
def get_adapter(self):
|
||||
return self.adapter
|
||||
|
||||
@property
|
||||
def provider_id(self):
|
||||
return self.get_adapter().provider_id
|
||||
|
||||
def get_session(self):
|
||||
if self.session is None:
|
||||
self.create_session()
|
||||
return self.session
|
||||
|
||||
def create_session(self):
|
||||
"""Create OAuth session for user
|
||||
|
||||
This configures the OAuth session based on the :py:cls:`SocialToken`
|
||||
attributes. If there is an ``expires_at``, treat the session as an auto
|
||||
renewing token. Some providers expire tokens after as little as 2
|
||||
hours.
|
||||
"""
|
||||
token = self.account.socialtoken_set.first()
|
||||
if token is None:
|
||||
return None
|
||||
|
||||
token_config = {
|
||||
'access_token': str(token.token),
|
||||
'token_type': 'bearer',
|
||||
}
|
||||
if token.expires_at is not None:
|
||||
token_expires = (token.expires_at - datetime.now()).total_seconds()
|
||||
token_config.update({
|
||||
'refresh_token': str(token.token_secret),
|
||||
'expires_in': token_expires,
|
||||
})
|
||||
|
||||
self.session = OAuth2Session(
|
||||
client_id=token.app.client_id,
|
||||
token=token_config,
|
||||
auto_refresh_kwargs={
|
||||
'client_id': token.app.client_id,
|
||||
'client_secret': token.app.secret,
|
||||
},
|
||||
auto_refresh_url=self.get_adapter().access_token_url,
|
||||
token_updater=self.token_updater(token)
|
||||
)
|
||||
|
||||
return self.session or None
|
||||
|
||||
def token_updater(self, token):
|
||||
"""Update token given data from OAuth response
|
||||
|
||||
Expect the following response into the closure::
|
||||
|
||||
{
|
||||
u'token_type': u'bearer',
|
||||
u'scopes': u'webhook repository team account',
|
||||
u'refresh_token': u'...',
|
||||
u'access_token': u'...',
|
||||
u'expires_in': 3600,
|
||||
u'expires_at': 1449218652.558185
|
||||
}
|
||||
"""
|
||||
|
||||
def _updater(data):
|
||||
token.token = data['access_token']
|
||||
token.expires_at = datetime.fromtimestamp(data['expires_at'])
|
||||
token.save()
|
||||
log.info('Updated token %s:', token)
|
||||
|
||||
return _updater
|
||||
|
||||
def sync(self, sync):
|
||||
raise NotImplementedError
|
||||
|
||||
def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
|
||||
organization=None):
|
||||
raise NotImplementedError
|
||||
|
||||
def create_organization(self, fields):
|
||||
raise NotImplementedError
|
||||
|
||||
def setup_webhook(self, project):
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def is_project_service(cls, project):
|
||||
"""Determine if this is the service the project is using
|
||||
|
||||
.. note::
|
||||
This should be deprecated in favor of attaching the
|
||||
:py:cls:`RemoteRepository` to the project instance. This is a slight
|
||||
improvement on the legacy check for webhooks
|
||||
"""
|
||||
# TODO Replace this check by keying project to remote repos
|
||||
return (
|
||||
cls.url_pattern is not None and
|
||||
cls.url_pattern.search(project.repo) is not None
|
||||
)
|
||||
|
||||
|
||||
class GitHubService(Service):
|
||||
|
||||
"""Provider service for GitHub"""
|
||||
|
||||
adapter = GitHubOAuth2Adapter
|
||||
# TODO replace this with a less naive check
|
||||
url_pattern = re.compile(r'^github\.com\/')
|
||||
|
||||
def sync(self, sync):
|
||||
"""Sync repositories and organizations"""
|
||||
if sync:
|
||||
self.sync_repositories()
|
||||
self.sync_organizations()
|
||||
|
||||
def sync_repositories(self):
|
||||
"""Get repositories for GitHub user via OAuth token"""
|
||||
repos = self.paginate('https://api.github.com/user/repos?per_page=100')
|
||||
try:
|
||||
for repo in repos:
|
||||
self.create_repository(repo)
|
||||
except (TypeError, ValueError) as e:
|
||||
log.error('Error syncing GitHub repositories: %s',
|
||||
str(e), exc_info=True)
|
||||
raise Exception('Could not sync your GitHub repositories, '
|
||||
'try reconnecting your account')
|
||||
|
||||
def sync_organizations(self):
|
||||
"""Sync GitHub organizations and organization repositories"""
|
||||
try:
|
||||
orgs = self.paginate('https://api.github.com/user/orgs')
|
||||
for org in orgs:
|
||||
org_resp = self.get_session().get(org['url'])
|
||||
org_obj = self.create_organization(org_resp.json())
|
||||
# Add repos
|
||||
# TODO ?per_page=100
|
||||
org_repos = self.paginate(
|
||||
'{org_url}/repos'.format(org_url=org['url'])
|
||||
)
|
||||
for repo in org_repos:
|
||||
self.create_repository(repo, organization=org_obj)
|
||||
except (TypeError, ValueError) as e:
|
||||
log.error('Error syncing GitHub organizations: %s',
|
||||
str(e), exc_info=True)
|
||||
raise Exception('Could not sync your GitHub organizations, '
|
||||
'try reconnecting your account')
|
||||
|
||||
def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
|
||||
organization=None):
|
||||
if (
|
||||
(privacy == 'private') or
|
||||
(fields['private'] is False and privacy == 'public')):
|
||||
repo, _ = RemoteRepository.objects.get_or_create(
|
||||
full_name=fields['full_name'],
|
||||
account=self.account,
|
||||
)
|
||||
if repo.organization and repo.organization != organization:
|
||||
log.debug('Not importing %s because mismatched orgs' %
|
||||
fields['name'])
|
||||
return None
|
||||
else:
|
||||
repo.organization = organization
|
||||
repo.users.add(self.user)
|
||||
repo.name = fields['name']
|
||||
repo.description = fields['description']
|
||||
repo.ssh_url = fields['ssh_url']
|
||||
repo.html_url = fields['html_url']
|
||||
repo.private = fields['private']
|
||||
if repo.private:
|
||||
repo.clone_url = fields['ssh_url']
|
||||
else:
|
||||
repo.clone_url = fields['clone_url']
|
||||
repo.admin = fields.get('permissions', {}).get('admin', False)
|
||||
repo.vcs = 'git'
|
||||
repo.account = self.account
|
||||
repo.avatar_url = fields.get('owner', {}).get('avatar_url')
|
||||
repo.json = json.dumps(fields)
|
||||
repo.save()
|
||||
return repo
|
||||
else:
|
||||
log.debug('Not importing %s because mismatched type' %
|
||||
fields['name'])
|
||||
|
||||
def create_organization(self, fields):
|
||||
organization, _ = RemoteOrganization.objects.get_or_create(
|
||||
slug=fields.get('login'),
|
||||
account=self.account,
|
||||
)
|
||||
organization.html_url = fields.get('html_url')
|
||||
organization.name = fields.get('name')
|
||||
organization.email = fields.get('email')
|
||||
organization.avatar_url = fields.get('avatar_url')
|
||||
organization.json = json.dumps(fields)
|
||||
organization.account = self.account
|
||||
organization.users.add(self.user)
|
||||
organization.save()
|
||||
return organization
|
||||
|
||||
def paginate(self, url):
|
||||
"""Combines return from GitHub pagination
|
||||
|
||||
:param url: start url to get the data from.
|
||||
|
||||
See https://developer.github.com/v3/#pagination
|
||||
"""
|
||||
resp = self.get_session().get(url)
|
||||
result = resp.json()
|
||||
next_url = resp.links.get('next', {}).get('url')
|
||||
if next_url:
|
||||
result.extend(self.paginate(next_url))
|
||||
return result
|
||||
|
||||
def setup_webhook(self, project):
|
||||
"""Set up GitHub webhook for project
|
||||
|
||||
:param project: Project instance to set up webhook for
|
||||
"""
|
||||
session = self.get_session()
|
||||
owner, repo = build_utils.get_github_username_repo(url=project.repo)
|
||||
data = json.dumps({
|
||||
'name': 'readthedocs',
|
||||
'active': True,
|
||||
'config': {'url': 'https://{domain}/github'.format(domain=settings.PRODUCTION_DOMAIN)}
|
||||
})
|
||||
resp = None
|
||||
try:
|
||||
resp = session.post(
|
||||
'https://api.github.com/repos/{owner}/{repo}/hooks'.format(owner=owner, repo=repo),
|
||||
data=data,
|
||||
headers={'content-type': 'application/json'}
|
||||
)
|
||||
if resp.status_code == 201:
|
||||
log.info('Created GitHub webhook: project={project}'
|
||||
.format(project=project))
|
||||
return True
|
||||
except RequestException:
|
||||
pass
|
||||
else:
|
||||
log.exception('GitHub Hook creation failed', exc_info=True)
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def get_token_for_project(cls, project, force_local=False):
|
||||
"""Get access token for project by iterating over project users"""
|
||||
# TODO why does this only target GitHub?
|
||||
if not getattr(settings, 'ALLOW_PRIVATE_REPOS', False):
|
||||
return None
|
||||
token = None
|
||||
try:
|
||||
if getattr(settings, 'DONT_HIT_DB', True) and not force_local:
|
||||
token = api.project(project.pk).token().get()['token']
|
||||
else:
|
||||
for user in project.users.all():
|
||||
tokens = SocialToken.objects.filter(
|
||||
account__user=user,
|
||||
app__provider=cls.adapter.provider_id)
|
||||
if tokens.exists():
|
||||
token = tokens[0].token
|
||||
except Exception:
|
||||
log.error('Failed to get token for user', exc_info=True)
|
||||
return token
|
||||
|
||||
|
||||
class BitbucketService(Service):
|
||||
|
||||
"""Provider service for Bitbucket"""
|
||||
|
||||
adapter = BitbucketOAuth2Adapter
|
||||
# TODO replace this with a less naive check
|
||||
url_pattern = re.compile(r'bitbucket.org\/')
|
||||
|
||||
def sync(self, sync):
|
||||
"""Import from Bitbucket"""
|
||||
if sync:
|
||||
self.sync_repositories()
|
||||
self.sync_teams()
|
||||
|
||||
def sync_repositories(self):
|
||||
# Get user repos
|
||||
try:
|
||||
repos = self.paginate(
|
||||
'https://bitbucket.org/api/2.0/repositories/?role=member')
|
||||
for repo in repos:
|
||||
self.create_repository(repo)
|
||||
except (TypeError, ValueError) as e:
|
||||
log.error('Error syncing Bitbucket repositories: %s',
|
||||
str(e), exc_info=True)
|
||||
raise Exception('Could not sync your Bitbucket repositories, '
|
||||
'try reconnecting your account')
|
||||
|
||||
# Because privileges aren't returned with repository data, run query
|
||||
# again for repositories that user has admin role for, and update
|
||||
# existing repositories.
|
||||
try:
|
||||
resp = self.paginate(
|
||||
'https://bitbucket.org/api/2.0/repositories/?role=admin')
|
||||
repos = (
|
||||
RemoteRepository.objects
|
||||
.filter(users=self.user,
|
||||
full_name__in=[r['full_name'] for r in resp],
|
||||
account=self.account)
|
||||
)
|
||||
for repo in repos:
|
||||
repo.admin = True
|
||||
repo.save()
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
def sync_teams(self):
|
||||
"""Sync Bitbucket teams and team repositories for user token"""
|
||||
try:
|
||||
teams = self.paginate(
|
||||
'https://api.bitbucket.org/2.0/teams/?role=member'
|
||||
)
|
||||
for team in teams:
|
||||
org = self.create_organization(team)
|
||||
repos = self.paginate(team['links']['repositories']['href'])
|
||||
for repo in repos:
|
||||
self.create_repository(repo, organization=org)
|
||||
except ValueError as e:
|
||||
log.error('Error syncing Bitbucket organizations: %s',
|
||||
str(e), exc_info=True)
|
||||
raise Exception('Could not sync your Bitbucket team repositories, '
|
||||
'try reconnecting your account')
|
||||
|
||||
def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
|
||||
organization=None):
|
||||
"""Update or create a repository from Bitbucket
|
||||
|
||||
This looks up existing repositories based on the full repository name,
|
||||
that is the username and repository name.
|
||||
|
||||
.. note::
|
||||
The :py:data:`admin` property is not set during creation, as
|
||||
permissions are not part of the returned repository data from
|
||||
Bitbucket.
|
||||
"""
|
||||
if (fields['is_private'] is True and privacy == 'private' or
|
||||
fields['is_private'] is False and privacy == 'public'):
|
||||
repo, _ = RemoteRepository.objects.get_or_create(
|
||||
full_name=fields['full_name'],
|
||||
account=self.account,
|
||||
)
|
||||
if repo.organization and repo.organization != organization:
|
||||
log.debug('Not importing %s because mismatched orgs' %
|
||||
fields['name'])
|
||||
return None
|
||||
else:
|
||||
repo.organization = organization
|
||||
repo.users.add(self.user)
|
||||
repo.name = fields['name']
|
||||
repo.description = fields['description']
|
||||
repo.private = fields['is_private']
|
||||
repo.clone_url = fields['links']['clone'][0]['href']
|
||||
repo.ssh_url = fields['links']['clone'][1]['href']
|
||||
if repo.private:
|
||||
repo.clone_url = repo.ssh_url
|
||||
repo.html_url = fields['links']['html']['href']
|
||||
repo.vcs = fields['scm']
|
||||
repo.account = self.account
|
||||
|
||||
avatar_url = fields['links']['avatar']['href'] or ''
|
||||
repo.avatar_url = re.sub(r'\/16\/$', r'/32/', avatar_url)
|
||||
|
||||
repo.json = json.dumps(fields)
|
||||
repo.save()
|
||||
return repo
|
||||
else:
|
||||
log.debug('Not importing %s because mismatched type' %
|
||||
fields['name'])
|
||||
|
||||
def create_organization(self, fields):
|
||||
organization, _ = RemoteOrganization.objects.get_or_create(
|
||||
slug=fields.get('username'),
|
||||
account=self.account,
|
||||
)
|
||||
organization.name = fields.get('display_name')
|
||||
organization.email = fields.get('email')
|
||||
organization.avatar_url = fields['links']['avatar']['href']
|
||||
organization.html_url = fields['links']['html']['href']
|
||||
organization.json = json.dumps(fields)
|
||||
organization.account = self.account
|
||||
organization.users.add(self.user)
|
||||
organization.save()
|
||||
return organization
|
||||
|
||||
def paginate(self, url):
|
||||
"""Combines results from Bitbucket pagination
|
||||
|
||||
:param url: start url to get the data from.
|
||||
"""
|
||||
resp = self.get_session().get(url)
|
||||
data = resp.json()
|
||||
results = data.get('values', [])
|
||||
next_url = data.get('next')
|
||||
if next_url:
|
||||
results.extend(self.paginate(next_url))
|
||||
return results
|
||||
|
||||
def setup_webhook(self, project):
|
||||
session = self.get_session()
|
||||
owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo)
|
||||
data = {
|
||||
'type': 'POST',
|
||||
'url': 'https://{domain}/bitbucket'.format(domain=settings.PRODUCTION_DOMAIN),
|
||||
}
|
||||
try:
|
||||
resp = session.post(
|
||||
'https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/services'.format(
|
||||
owner=owner, repo=repo
|
||||
),
|
||||
data=data,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
log.info('Created Bitbucket webhook: project={project}'
|
||||
.format(project=project))
|
||||
return True
|
||||
except RequestException:
|
||||
pass
|
||||
else:
|
||||
log.exception('Bitbucket webhook creation failed', exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
services = [GitHubService, BitbucketService]
|
|
@ -7,7 +7,7 @@ from django.contrib import messages
|
|||
from django.dispatch import receiver
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from readthedocs.oauth.utils import services
|
||||
from readthedocs.oauth.services import registry
|
||||
|
||||
|
||||
before_vcs = django.dispatch.Signal(providing_args=["version"])
|
||||
|
@ -28,7 +28,7 @@ def handle_project_import(sender, **kwargs):
|
|||
project = sender
|
||||
request = kwargs.get('request')
|
||||
|
||||
for service_cls in services:
|
||||
for service_cls in registry:
|
||||
if service_cls.is_project_service(project):
|
||||
service = service_cls.for_user(request.user)
|
||||
if service.setup_webhook(project):
|
||||
|
|
|
@ -11,7 +11,7 @@ from readthedocs.builds.constants import TAG
|
|||
from readthedocs.builds.filters import VersionFilter
|
||||
from readthedocs.builds.models import Build, BuildCommandResult, Version
|
||||
from readthedocs.core.utils import trigger_build
|
||||
from readthedocs.oauth.utils import GitHubService, services
|
||||
from readthedocs.oauth.services import GitHubService, registry
|
||||
from readthedocs.oauth.models import RemoteOrganization, RemoteRepository
|
||||
from readthedocs.builds.constants import STABLE
|
||||
from readthedocs.projects.filters import ProjectFilter, DomainFilter
|
||||
|
@ -230,7 +230,7 @@ class RemoteOrganizationViewSet(viewsets.ReadOnlyModelViewSet):
|
|||
def get_queryset(self):
|
||||
return (self.model.objects.api(self.request.user)
|
||||
.filter(account__provider__in=[service.adapter.provider_id
|
||||
for service in services]))
|
||||
for service in registry]))
|
||||
|
||||
|
||||
class RemoteRepositoryViewSet(viewsets.ReadOnlyModelViewSet):
|
||||
|
@ -245,7 +245,7 @@ class RemoteRepositoryViewSet(viewsets.ReadOnlyModelViewSet):
|
|||
if org is not None:
|
||||
query = query.filter(organization__pk=org)
|
||||
query = query.filter(account__provider__in=[service.adapter.provider_id
|
||||
for service in services])
|
||||
for service in registry])
|
||||
return query
|
||||
|
||||
def get_paginate_by(self):
|
||||
|
|
|
@ -5,7 +5,7 @@ from allauth.socialaccount.models import SocialToken
|
|||
|
||||
from readthedocs.projects.models import Project
|
||||
|
||||
from readthedocs.oauth.utils import GitHubService
|
||||
from readthedocs.oauth.services import GitHubService
|
||||
from readthedocs.oauth.models import RemoteRepository, RemoteOrganization
|
||||
|
||||
|
||||
|
|
Loading…
Reference in New Issue