Split up OAuth services into own path, allow for extension of classes

.gitignore

@@ -7,6 +7,8 @@
 .idea
 .vagrant
 .tox
+.rope_project/
+.ropeproject/
 _build
 cnames
 bower_components/
@@ -35,4 +37,3 @@ whoosh_index
 xml_output
 public_cnames
 public_symlinks
-.rope_project/

readthedocs/core/management/commands/import_github.py

@@ -1,7 +1,7 @@
 from django.core.management.base import BaseCommand
 from django.contrib.auth.models import User
 
-from readthedocs.oauth.utils import GitHubService
+from readthedocs.oauth.services import GitHubService
 
 
 class Command(BaseCommand):

readthedocs/oauth/services/__init__.py

@@ -0,0 +1,13 @@
+"""Conditional classes for OAuth services"""
+
+from django.utils.module_loading import import_by_path
+from django.conf import settings
+
+GitHubService = import_by_path(
+    getattr(settings, 'OAUTH_GITHUB_SERVICE',
+            'readthedocs.oauth.services.github.GitHubService'))
+BitbucketService = import_by_path(
+    getattr(settings, 'OAUTH_BITBUCKET_SERVICE',
+            'readthedocs.oauth.services.bitbucket.BitbucketService'))
+
+registry = [GitHubService, BitbucketService]

readthedocs/oauth/services/base.py

@@ -0,0 +1,141 @@
+"""OAuth utility functions"""
+
+import logging
+from datetime import datetime
+
+from django.conf import settings
+from requests_oauthlib import OAuth2Session
+from allauth.socialaccount.models import SocialAccount
+
+
+DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public')
+
+log = logging.getLogger(__name__)
+
+
+class Service(object):
+
+    """Service mapping for local accounts
+
+    :param user: User to use in token lookup and session creation
+    :param account: :py:cls:`SocialAccount` instance for user
+    """
+
+    adapter = None
+    url_pattern = None
+
+    def __init__(self, user, account):
+        self.session = None
+        self.user = user
+        self.account = account
+
+    @classmethod
+    def for_user(cls, user):
+        """Create instance if user has an account for the provider"""
+        try:
+            account = SocialAccount.objects.get(
+                user=user,
+                provider=cls.adapter.provider_id
+            )
+            return cls(user=user, account=account)
+        except SocialAccount.DoesNotExist:
+            return None
+
+    def get_adapter(self):
+        return self.adapter
+
+    @property
+    def provider_id(self):
+        return self.get_adapter().provider_id
+
+    def get_session(self):
+        if self.session is None:
+            self.create_session()
+        return self.session
+
+    def create_session(self):
+        """Create OAuth session for user
+
+        This configures the OAuth session based on the :py:cls:`SocialToken`
+        attributes. If there is an ``expires_at``, treat the session as an auto
+        renewing token. Some providers expire tokens after as little as 2
+        hours.
+        """
+        token = self.account.socialtoken_set.first()
+        if token is None:
+            return None
+
+        token_config = {
+            'access_token': str(token.token),
+            'token_type': 'bearer',
+        }
+        if token.expires_at is not None:
+            token_expires = (token.expires_at - datetime.now()).total_seconds()
+            token_config.update({
+                'refresh_token': str(token.token_secret),
+                'expires_in': token_expires,
+            })
+
+        self.session = OAuth2Session(
+            client_id=token.app.client_id,
+            token=token_config,
+            auto_refresh_kwargs={
+                'client_id': token.app.client_id,
+                'client_secret': token.app.secret,
+            },
+            auto_refresh_url=self.get_adapter().access_token_url,
+            token_updater=self.token_updater(token)
+        )
+
+        return self.session or None
+
+    def token_updater(self, token):
+        """Update token given data from OAuth response
+
+        Expect the following response into the closure::
+
+            {
+                u'token_type': u'bearer',
+                u'scopes': u'webhook repository team account',
+                u'refresh_token': u'...',
+                u'access_token': u'...',
+                u'expires_in': 3600,
+                u'expires_at': 1449218652.558185
+            }
+        """
+
+        def _updater(data):
+            token.token = data['access_token']
+            token.expires_at = datetime.fromtimestamp(data['expires_at'])
+            token.save()
+            log.info('Updated token %s:', token)
+
+        return _updater
+
+    def sync(self, sync):
+        raise NotImplementedError
+
+    def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
+                          organization=None):
+        raise NotImplementedError
+
+    def create_organization(self, fields):
+        raise NotImplementedError
+
+    def setup_webhook(self, project):
+        raise NotImplementedError
+
+    @classmethod
+    def is_project_service(cls, project):
+        """Determine if this is the service the project is using
+
+        .. note::
+            This should be deprecated in favor of attaching the
+            :py:cls:`RemoteRepository` to the project instance. This is a slight
+            improvement on the legacy check for webhooks
+        """
+        # TODO Replace this check by keying project to remote repos
+        return (
+            cls.url_pattern is not None and
+            cls.url_pattern.search(project.repo) is not None
+        )

readthedocs/oauth/services/bitbucket.py

@@ -0,0 +1,181 @@
+"""OAuth utility functions"""
+
+import logging
+import json
+import re
+
+from django.conf import settings
+from requests.exceptions import RequestException
+from allauth.socialaccount.providers.bitbucket_oauth2.views import (
+    BitbucketOAuth2Adapter)
+
+from readthedocs.builds import utils as build_utils
+
+from ..models import RemoteOrganization, RemoteRepository
+from .base import Service
+
+
+DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public')
+
+log = logging.getLogger(__name__)
+
+
+class BitbucketService(Service):
+
+    """Provider service for Bitbucket"""
+
+    adapter = BitbucketOAuth2Adapter
+    # TODO replace this with a less naive check
+    url_pattern = re.compile(r'bitbucket.org\/')
+
+    def sync(self, sync):
+        """Import from Bitbucket"""
+        if sync:
+            self.sync_repositories()
+            self.sync_teams()
+
+    def sync_repositories(self):
+        # Get user repos
+        try:
+            repos = self.paginate(
+                'https://bitbucket.org/api/2.0/repositories/?role=member')
+            for repo in repos:
+                self.create_repository(repo)
+        except (TypeError, ValueError) as e:
+            log.error('Error syncing Bitbucket repositories: %s',
+                      str(e), exc_info=True)
+            raise Exception('Could not sync your Bitbucket repositories, '
+                            'try reconnecting your account')
+
+        # Because privileges aren't returned with repository data, run query
+        # again for repositories that user has admin role for, and update
+        # existing repositories.
+        try:
+            resp = self.paginate(
+                'https://bitbucket.org/api/2.0/repositories/?role=admin')
+            repos = (
+                RemoteRepository.objects
+                .filter(users=self.user,
+                        full_name__in=[r['full_name'] for r in resp],
+                        account=self.account)
+            )
+            for repo in repos:
+                repo.admin = True
+                repo.save()
+        except (TypeError, ValueError):
+            pass
+
+    def sync_teams(self):
+        """Sync Bitbucket teams and team repositories for user token"""
+        try:
+            teams = self.paginate(
+                'https://api.bitbucket.org/2.0/teams/?role=member'
+            )
+            for team in teams:
+                org = self.create_organization(team)
+                repos = self.paginate(team['links']['repositories']['href'])
+                for repo in repos:
+                    self.create_repository(repo, organization=org)
+        except ValueError as e:
+            log.error('Error syncing Bitbucket organizations: %s',
+                      str(e), exc_info=True)
+            raise Exception('Could not sync your Bitbucket team repositories, '
+                            'try reconnecting your account')
+
+    def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
+                          organization=None):
+        """Update or create a repository from Bitbucket
+
+        This looks up existing repositories based on the full repository name,
+        that is the username and repository name.
+
+        .. note::
+            The :py:data:`admin` property is not set during creation, as
+            permissions are not part of the returned repository data from
+            Bitbucket.
+        """
+        if (fields['is_private'] is True and privacy == 'private' or
+                fields['is_private'] is False and privacy == 'public'):
+            repo, _ = RemoteRepository.objects.get_or_create(
+                full_name=fields['full_name'],
+                account=self.account,
+            )
+            if repo.organization and repo.organization != organization:
+                log.debug('Not importing %s because mismatched orgs' %
+                          fields['name'])
+                return None
+            else:
+                repo.organization = organization
+            repo.users.add(self.user)
+            repo.name = fields['name']
+            repo.description = fields['description']
+            repo.private = fields['is_private']
+            repo.clone_url = fields['links']['clone'][0]['href']
+            repo.ssh_url = fields['links']['clone'][1]['href']
+            if repo.private:
+                repo.clone_url = repo.ssh_url
+            repo.html_url = fields['links']['html']['href']
+            repo.vcs = fields['scm']
+            repo.account = self.account
+
+            avatar_url = fields['links']['avatar']['href'] or ''
+            repo.avatar_url = re.sub(r'\/16\/$', r'/32/', avatar_url)
+
+            repo.json = json.dumps(fields)
+            repo.save()
+            return repo
+        else:
+            log.debug('Not importing %s because mismatched type' %
+                      fields['name'])
+
+    def create_organization(self, fields):
+        organization, _ = RemoteOrganization.objects.get_or_create(
+            slug=fields.get('username'),
+            account=self.account,
+        )
+        organization.name = fields.get('display_name')
+        organization.email = fields.get('email')
+        organization.avatar_url = fields['links']['avatar']['href']
+        organization.html_url = fields['links']['html']['href']
+        organization.json = json.dumps(fields)
+        organization.account = self.account
+        organization.users.add(self.user)
+        organization.save()
+        return organization
+
+    def paginate(self, url):
+        """Combines results from Bitbucket pagination
+
+        :param url: start url to get the data from.
+        """
+        resp = self.get_session().get(url)
+        data = resp.json()
+        results = data.get('values', [])
+        next_url = data.get('next')
+        if next_url:
+            results.extend(self.paginate(next_url))
+        return results
+
+    def setup_webhook(self, project):
+        session = self.get_session()
+        owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo)
+        data = {
+            'type': 'POST',
+            'url': 'https://{domain}/bitbucket'.format(domain=settings.PRODUCTION_DOMAIN),
+        }
+        try:
+            resp = session.post(
+                'https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/services'.format(
+                    owner=owner, repo=repo
+                ),
+                data=data,
+            )
+            if resp.status_code == 200:
+                log.info('Created Bitbucket webhook: project={project}'
+                         .format(project=project))
+                return True
+        except RequestException:
+            pass
+        else:
+            log.exception('Bitbucket webhook creation failed', exc_info=True)
+            return False

readthedocs/oauth/services/github.py

@@ -0,0 +1,183 @@
+"""OAuth utility functions"""
+
+import logging
+import json
+import re
+
+from django.conf import settings
+from requests.exceptions import RequestException
+from allauth.socialaccount.models import SocialToken
+from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter
+
+from readthedocs.builds import utils as build_utils
+from readthedocs.restapi.client import api
+
+from ..models import RemoteOrganization, RemoteRepository
+from .base import Service
+
+
+DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public')
+
+log = logging.getLogger(__name__)
+
+
+class GitHubService(Service):
+
+    """Provider service for GitHub"""
+
+    adapter = GitHubOAuth2Adapter
+    # TODO replace this with a less naive check
+    url_pattern = re.compile(r'^github\.com\/')
+
+    def sync(self, sync):
+        """Sync repositories and organizations"""
+        if sync:
+            self.sync_repositories()
+            self.sync_organizations()
+
+    def sync_repositories(self):
+        """Get repositories for GitHub user via OAuth token"""
+        repos = self.paginate('https://api.github.com/user/repos?per_page=100')
+        try:
+            for repo in repos:
+                self.create_repository(repo)
+        except (TypeError, ValueError) as e:
+            log.error('Error syncing GitHub repositories: %s',
+                      str(e), exc_info=True)
+            raise Exception('Could not sync your GitHub repositories, '
+                            'try reconnecting your account')
+
+    def sync_organizations(self):
+        """Sync GitHub organizations and organization repositories"""
+        try:
+            orgs = self.paginate('https://api.github.com/user/orgs')
+            for org in orgs:
+                org_resp = self.get_session().get(org['url'])
+                org_obj = self.create_organization(org_resp.json())
+                # Add repos
+                # TODO ?per_page=100
+                org_repos = self.paginate(
+                    '{org_url}/repos'.format(org_url=org['url'])
+                )
+                for repo in org_repos:
+                    self.create_repository(repo, organization=org_obj)
+        except (TypeError, ValueError) as e:
+            log.error('Error syncing GitHub organizations: %s',
+                      str(e), exc_info=True)
+            raise Exception('Could not sync your GitHub organizations, '
+                            'try reconnecting your account')
+
+    def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
+                          organization=None):
+        if (
+                (privacy == 'private') or
+                (fields['private'] is False and privacy == 'public')):
+            repo, _ = RemoteRepository.objects.get_or_create(
+                full_name=fields['full_name'],
+                account=self.account,
+            )
+            if repo.organization and repo.organization != organization:
+                log.debug('Not importing %s because mismatched orgs' %
+                          fields['name'])
+                return None
+            else:
+                repo.organization = organization
+            repo.users.add(self.user)
+            repo.name = fields['name']
+            repo.description = fields['description']
+            repo.ssh_url = fields['ssh_url']
+            repo.html_url = fields['html_url']
+            repo.private = fields['private']
+            if repo.private:
+                repo.clone_url = fields['ssh_url']
+            else:
+                repo.clone_url = fields['clone_url']
+            repo.admin = fields.get('permissions', {}).get('admin', False)
+            repo.vcs = 'git'
+            repo.account = self.account
+            repo.avatar_url = fields.get('owner', {}).get('avatar_url')
+            repo.json = json.dumps(fields)
+            repo.save()
+            return repo
+        else:
+            log.debug('Not importing %s because mismatched type' %
+                      fields['name'])
+
+    def create_organization(self, fields):
+        organization, _ = RemoteOrganization.objects.get_or_create(
+            slug=fields.get('login'),
+            account=self.account,
+        )
+        organization.html_url = fields.get('html_url')
+        organization.name = fields.get('name')
+        organization.email = fields.get('email')
+        organization.avatar_url = fields.get('avatar_url')
+        organization.json = json.dumps(fields)
+        organization.account = self.account
+        organization.users.add(self.user)
+        organization.save()
+        return organization
+
+    def paginate(self, url):
+        """Combines return from GitHub pagination
+
+        :param url: start url to get the data from.
+
+        See https://developer.github.com/v3/#pagination
+        """
+        resp = self.get_session().get(url)
+        result = resp.json()
+        next_url = resp.links.get('next', {}).get('url')
+        if next_url:
+            result.extend(self.paginate(next_url))
+        return result
+
+    def setup_webhook(self, project):
+        """Set up GitHub webhook for project
+
+        :param project: Project instance to set up webhook for
+        """
+        session = self.get_session()
+        owner, repo = build_utils.get_github_username_repo(url=project.repo)
+        data = json.dumps({
+            'name': 'readthedocs',
+            'active': True,
+            'config': {'url': 'https://{domain}/github'.format(domain=settings.PRODUCTION_DOMAIN)}
+        })
+        resp = None
+        try:
+            resp = session.post(
+                'https://api.github.com/repos/{owner}/{repo}/hooks'.format(owner=owner, repo=repo),
+                data=data,
+                headers={'content-type': 'application/json'}
+            )
+            if resp.status_code == 201:
+                log.info('Created GitHub webhook: project={project}'
+                         .format(project=project))
+                return True
+        except RequestException:
+            pass
+        else:
+            log.exception('GitHub Hook creation failed', exc_info=True)
+            return False
+
+    @classmethod
+    def get_token_for_project(cls, project, force_local=False):
+        """Get access token for project by iterating over project users"""
+        # TODO why does this only target GitHub?
+        if not getattr(settings, 'ALLOW_PRIVATE_REPOS', False):
+            return None
+        token = None
+        try:
+            if getattr(settings, 'DONT_HIT_DB', True) and not force_local:
+                token = api.project(project.pk).token().get()['token']
+            else:
+                for user in project.users.all():
+                    tokens = SocialToken.objects.filter(
+                        account__user=user,
+                        app__provider=cls.adapter.provider_id)
+                    if tokens.exists():
+                        token = tokens[0].token
+        except Exception:
+            log.error('Failed to get token for user', exc_info=True)
+        return token

readthedocs/oauth/tasks.py

@@ -6,7 +6,7 @@ from djcelery import celery as celery_app
 from readthedocs.core.utils.tasks import PublicTask
 from readthedocs.core.utils.tasks import permission_check
 from readthedocs.core.utils.tasks import user_id_matches
-from .utils import services
+from .services import registry
 
 
 @permission_check(user_id_matches)
@@ -16,7 +16,7 @@ class SyncRemoteRepositories(PublicTask):
 
     def run_public(self, user_id):
         user = User.objects.get(pk=user_id)
-        for service_cls in services:
+        for service_cls in registry:
             service = service_cls.for_user(user)
             if service is not None:
                 service.sync(sync=True)

readthedocs/oauth/utils.py

@@ -1,478 +0,0 @@
-"""OAuth utility functions"""
-
-import logging
-import json
-import re
-from datetime import datetime
-
-from django.conf import settings
-from requests_oauthlib import OAuth2Session
-from requests.exceptions import RequestException
-from allauth.socialaccount.models import SocialAccount
-from allauth.socialaccount.providers.bitbucket_oauth2.views import (
-    BitbucketOAuth2Adapter)
-from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter
-
-from readthedocs.builds import utils as build_utils
-from readthedocs.restapi.client import api
-
-from .models import RemoteOrganization, RemoteRepository
-
-
-DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public')
-
-log = logging.getLogger(__name__)
-
-
-class Service(object):
-
-    """Service mapping for local accounts
-
-    :param user: User to use in token lookup and session creation
-    :param account: :py:cls:`SocialAccount` instance for user
-    """
-
-    adapter = None
-    url_pattern = None
-
-    def __init__(self, user, account):
-        self.session = None
-        self.user = user
-        self.account = account
-
-    @classmethod
-    def for_user(cls, user):
-        """Create instance if user has an account for the provider"""
-        try:
-            account = SocialAccount.objects.get(
-                user=user,
-                provider=cls.adapter.provider_id
-            )
-            return cls(user=user, account=account)
-        except SocialAccount.DoesNotExist:
-            return None
-
-    def get_adapter(self):
-        return self.adapter
-
-    @property
-    def provider_id(self):
-        return self.get_adapter().provider_id
-
-    def get_session(self):
-        if self.session is None:
-            self.create_session()
-        return self.session
-
-    def create_session(self):
-        """Create OAuth session for user
-
-        This configures the OAuth session based on the :py:cls:`SocialToken`
-        attributes. If there is an ``expires_at``, treat the session as an auto
-        renewing token. Some providers expire tokens after as little as 2
-        hours.
-        """
-        token = self.account.socialtoken_set.first()
-        if token is None:
-            return None
-
-        token_config = {
-            'access_token': str(token.token),
-            'token_type': 'bearer',
-        }
-        if token.expires_at is not None:
-            token_expires = (token.expires_at - datetime.now()).total_seconds()
-            token_config.update({
-                'refresh_token': str(token.token_secret),
-                'expires_in': token_expires,
-            })
-
-        self.session = OAuth2Session(
-            client_id=token.app.client_id,
-            token=token_config,
-            auto_refresh_kwargs={
-                'client_id': token.app.client_id,
-                'client_secret': token.app.secret,
-            },
-            auto_refresh_url=self.get_adapter().access_token_url,
-            token_updater=self.token_updater(token)
-        )
-
-        return self.session or None
-
-    def token_updater(self, token):
-        """Update token given data from OAuth response
-
-        Expect the following response into the closure::
-
-            {
-                u'token_type': u'bearer',
-                u'scopes': u'webhook repository team account',
-                u'refresh_token': u'...',
-                u'access_token': u'...',
-                u'expires_in': 3600,
-                u'expires_at': 1449218652.558185
-            }
-        """
-
-        def _updater(data):
-            token.token = data['access_token']
-            token.expires_at = datetime.fromtimestamp(data['expires_at'])
-            token.save()
-            log.info('Updated token %s:', token)
-
-        return _updater
-
-    def sync(self, sync):
-        raise NotImplementedError
-
-    def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
-                          organization=None):
-        raise NotImplementedError
-
-    def create_organization(self, fields):
-        raise NotImplementedError
-
-    def setup_webhook(self, project):
-        raise NotImplementedError
-
-    @classmethod
-    def is_project_service(cls, project):
-        """Determine if this is the service the project is using
-
-        .. note::
-            This should be deprecated in favor of attaching the
-            :py:cls:`RemoteRepository` to the project instance. This is a slight
-            improvement on the legacy check for webhooks
-        """
-        # TODO Replace this check by keying project to remote repos
-        return (
-            cls.url_pattern is not None and
-            cls.url_pattern.search(project.repo) is not None
-        )
-
-
-class GitHubService(Service):
-
-    """Provider service for GitHub"""
-
-    adapter = GitHubOAuth2Adapter
-    # TODO replace this with a less naive check
-    url_pattern = re.compile(r'^github\.com\/')
-
-    def sync(self, sync):
-        """Sync repositories and organizations"""
-        if sync:
-            self.sync_repositories()
-            self.sync_organizations()
-
-    def sync_repositories(self):
-        """Get repositories for GitHub user via OAuth token"""
-        repos = self.paginate('https://api.github.com/user/repos?per_page=100')
-        try:
-            for repo in repos:
-                self.create_repository(repo)
-        except (TypeError, ValueError) as e:
-            log.error('Error syncing GitHub repositories: %s',
-                      str(e), exc_info=True)
-            raise Exception('Could not sync your GitHub repositories, '
-                            'try reconnecting your account')
-
-    def sync_organizations(self):
-        """Sync GitHub organizations and organization repositories"""
-        try:
-            orgs = self.paginate('https://api.github.com/user/orgs')
-            for org in orgs:
-                org_resp = self.get_session().get(org['url'])
-                org_obj = self.create_organization(org_resp.json())
-                # Add repos
-                # TODO ?per_page=100
-                org_repos = self.paginate(
-                    '{org_url}/repos'.format(org_url=org['url'])
-                )
-                for repo in org_repos:
-                    self.create_repository(repo, organization=org_obj)
-        except (TypeError, ValueError) as e:
-            log.error('Error syncing GitHub organizations: %s',
-                      str(e), exc_info=True)
-            raise Exception('Could not sync your GitHub organizations, '
-                            'try reconnecting your account')
-
-    def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
-                          organization=None):
-        if (
-                (privacy == 'private') or
-                (fields['private'] is False and privacy == 'public')):
-            repo, _ = RemoteRepository.objects.get_or_create(
-                full_name=fields['full_name'],
-                account=self.account,
-            )
-            if repo.organization and repo.organization != organization:
-                log.debug('Not importing %s because mismatched orgs' %
-                          fields['name'])
-                return None
-            else:
-                repo.organization = organization
-            repo.users.add(self.user)
-            repo.name = fields['name']
-            repo.description = fields['description']
-            repo.ssh_url = fields['ssh_url']
-            repo.html_url = fields['html_url']
-            repo.private = fields['private']
-            if repo.private:
-                repo.clone_url = fields['ssh_url']
-            else:
-                repo.clone_url = fields['clone_url']
-            repo.admin = fields.get('permissions', {}).get('admin', False)
-            repo.vcs = 'git'
-            repo.account = self.account
-            repo.avatar_url = fields.get('owner', {}).get('avatar_url')
-            repo.json = json.dumps(fields)
-            repo.save()
-            return repo
-        else:
-            log.debug('Not importing %s because mismatched type' %
-                      fields['name'])
-
-    def create_organization(self, fields):
-        organization, _ = RemoteOrganization.objects.get_or_create(
-            slug=fields.get('login'),
-            account=self.account,
-        )
-        organization.html_url = fields.get('html_url')
-        organization.name = fields.get('name')
-        organization.email = fields.get('email')
-        organization.avatar_url = fields.get('avatar_url')
-        organization.json = json.dumps(fields)
-        organization.account = self.account
-        organization.users.add(self.user)
-        organization.save()
-        return organization
-
-    def paginate(self, url):
-        """Combines return from GitHub pagination
-
-        :param url: start url to get the data from.
-
-        See https://developer.github.com/v3/#pagination
-        """
-        resp = self.get_session().get(url)
-        result = resp.json()
-        next_url = resp.links.get('next', {}).get('url')
-        if next_url:
-            result.extend(self.paginate(next_url))
-        return result
-
-    def setup_webhook(self, project):
-        """Set up GitHub webhook for project
-
-        :param project: Project instance to set up webhook for
-        """
-        session = self.get_session()
-        owner, repo = build_utils.get_github_username_repo(url=project.repo)
-        data = json.dumps({
-            'name': 'readthedocs',
-            'active': True,
-            'config': {'url': 'https://{domain}/github'.format(domain=settings.PRODUCTION_DOMAIN)}
-        })
-        resp = None
-        try:
-            resp = session.post(
-                'https://api.github.com/repos/{owner}/{repo}/hooks'.format(owner=owner, repo=repo),
-                data=data,
-                headers={'content-type': 'application/json'}
-            )
-            if resp.status_code == 201:
-                log.info('Created GitHub webhook: project={project}'
-                         .format(project=project))
-                return True
-        except RequestException:
-            pass
-        else:
-            log.exception('GitHub Hook creation failed', exc_info=True)
-            return False
-
-    @classmethod
-    def get_token_for_project(cls, project, force_local=False):
-        """Get access token for project by iterating over project users"""
-        # TODO why does this only target GitHub?
-        if not getattr(settings, 'ALLOW_PRIVATE_REPOS', False):
-            return None
-        token = None
-        try:
-            if getattr(settings, 'DONT_HIT_DB', True) and not force_local:
-                token = api.project(project.pk).token().get()['token']
-            else:
-                for user in project.users.all():
-                    tokens = SocialToken.objects.filter(
-                        account__user=user,
-                        app__provider=cls.adapter.provider_id)
-                    if tokens.exists():
-                        token = tokens[0].token
-        except Exception:
-            log.error('Failed to get token for user', exc_info=True)
-        return token
-
-
-class BitbucketService(Service):
-
-    """Provider service for Bitbucket"""
-
-    adapter = BitbucketOAuth2Adapter
-    # TODO replace this with a less naive check
-    url_pattern = re.compile(r'bitbucket.org\/')
-
-    def sync(self, sync):
-        """Import from Bitbucket"""
-        if sync:
-            self.sync_repositories()
-            self.sync_teams()
-
-    def sync_repositories(self):
-        # Get user repos
-        try:
-            repos = self.paginate(
-                'https://bitbucket.org/api/2.0/repositories/?role=member')
-            for repo in repos:
-                self.create_repository(repo)
-        except (TypeError, ValueError) as e:
-            log.error('Error syncing Bitbucket repositories: %s',
-                      str(e), exc_info=True)
-            raise Exception('Could not sync your Bitbucket repositories, '
-                            'try reconnecting your account')
-
-        # Because privileges aren't returned with repository data, run query
-        # again for repositories that user has admin role for, and update
-        # existing repositories.
-        try:
-            resp = self.paginate(
-                'https://bitbucket.org/api/2.0/repositories/?role=admin')
-            repos = (
-                RemoteRepository.objects
-                .filter(users=self.user,
-                        full_name__in=[r['full_name'] for r in resp],
-                        account=self.account)
-            )
-            for repo in repos:
-                repo.admin = True
-                repo.save()
-        except (TypeError, ValueError):
-            pass
-
-    def sync_teams(self):
-        """Sync Bitbucket teams and team repositories for user token"""
-        try:
-            teams = self.paginate(
-                'https://api.bitbucket.org/2.0/teams/?role=member'
-            )
-            for team in teams:
-                org = self.create_organization(team)
-                repos = self.paginate(team['links']['repositories']['href'])
-                for repo in repos:
-                    self.create_repository(repo, organization=org)
-        except ValueError as e:
-            log.error('Error syncing Bitbucket organizations: %s',
-                      str(e), exc_info=True)
-            raise Exception('Could not sync your Bitbucket team repositories, '
-                            'try reconnecting your account')
-
-    def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
-                          organization=None):
-        """Update or create a repository from Bitbucket
-
-        This looks up existing repositories based on the full repository name,
-        that is the username and repository name.
-
-        .. note::
-            The :py:data:`admin` property is not set during creation, as
-            permissions are not part of the returned repository data from
-            Bitbucket.
-        """
-        if (fields['is_private'] is True and privacy == 'private' or
-                fields['is_private'] is False and privacy == 'public'):
-            repo, _ = RemoteRepository.objects.get_or_create(
-                full_name=fields['full_name'],
-                account=self.account,
-            )
-            if repo.organization and repo.organization != organization:
-                log.debug('Not importing %s because mismatched orgs' %
-                          fields['name'])
-                return None
-            else:
-                repo.organization = organization
-            repo.users.add(self.user)
-            repo.name = fields['name']
-            repo.description = fields['description']
-            repo.private = fields['is_private']
-            repo.clone_url = fields['links']['clone'][0]['href']
-            repo.ssh_url = fields['links']['clone'][1]['href']
-            if repo.private:
-                repo.clone_url = repo.ssh_url
-            repo.html_url = fields['links']['html']['href']
-            repo.vcs = fields['scm']
-            repo.account = self.account
-
-            avatar_url = fields['links']['avatar']['href'] or ''
-            repo.avatar_url = re.sub(r'\/16\/$', r'/32/', avatar_url)
-
-            repo.json = json.dumps(fields)
-            repo.save()
-            return repo
-        else:
-            log.debug('Not importing %s because mismatched type' %
-                      fields['name'])
-
-    def create_organization(self, fields):
-        organization, _ = RemoteOrganization.objects.get_or_create(
-            slug=fields.get('username'),
-            account=self.account,
-        )
-        organization.name = fields.get('display_name')
-        organization.email = fields.get('email')
-        organization.avatar_url = fields['links']['avatar']['href']
-        organization.html_url = fields['links']['html']['href']
-        organization.json = json.dumps(fields)
-        organization.account = self.account
-        organization.users.add(self.user)
-        organization.save()
-        return organization
-
-    def paginate(self, url):
-        """Combines results from Bitbucket pagination
-
-        :param url: start url to get the data from.
-        """
-        resp = self.get_session().get(url)
-        data = resp.json()
-        results = data.get('values', [])
-        next_url = data.get('next')
-        if next_url:
-            results.extend(self.paginate(next_url))
-        return results
-
-    def setup_webhook(self, project):
-        session = self.get_session()
-        owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo)
-        data = {
-            'type': 'POST',
-            'url': 'https://{domain}/bitbucket'.format(domain=settings.PRODUCTION_DOMAIN),
-        }
-        try:
-            resp = session.post(
-                'https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/services'.format(
-                    owner=owner, repo=repo
-                ),
-                data=data,
-            )
-            if resp.status_code == 200:
-                log.info('Created Bitbucket webhook: project={project}'
-                         .format(project=project))
-                return True
-        except RequestException:
-            pass
-        else:
-            log.exception('Bitbucket webhook creation failed', exc_info=True)
-            return False
-
-
-services = [GitHubService, BitbucketService]

readthedocs/projects/signals.py

@@ -7,7 +7,7 @@ from django.contrib import messages
 from django.dispatch import receiver
 from django.utils.translation import ugettext_lazy as _
 
-from readthedocs.oauth.utils import services
+from readthedocs.oauth.services import registry
 
 
 before_vcs = django.dispatch.Signal(providing_args=["version"])
@@ -28,7 +28,7 @@ def handle_project_import(sender, **kwargs):
     project = sender
     request = kwargs.get('request')
 
-    for service_cls in services:
+    for service_cls in registry:
         if service_cls.is_project_service(project):
             service = service_cls.for_user(request.user)
             if service.setup_webhook(project):

readthedocs/restapi/views/model_views.py

@@ -11,7 +11,7 @@ from readthedocs.builds.constants import TAG
 from readthedocs.builds.filters import VersionFilter
 from readthedocs.builds.models import Build, BuildCommandResult, Version
 from readthedocs.core.utils import trigger_build
-from readthedocs.oauth.utils import GitHubService, services
+from readthedocs.oauth.services import GitHubService, registry
 from readthedocs.oauth.models import RemoteOrganization, RemoteRepository
 from readthedocs.builds.constants import STABLE
 from readthedocs.projects.filters import ProjectFilter, DomainFilter
@@ -230,7 +230,7 @@ class RemoteOrganizationViewSet(viewsets.ReadOnlyModelViewSet):
     def get_queryset(self):
         return (self.model.objects.api(self.request.user)
                 .filter(account__provider__in=[service.adapter.provider_id
-                                               for service in services]))
+                                               for service in registry]))
 
 
 class RemoteRepositoryViewSet(viewsets.ReadOnlyModelViewSet):
@@ -245,7 +245,7 @@ class RemoteRepositoryViewSet(viewsets.ReadOnlyModelViewSet):
         if org is not None:
             query = query.filter(organization__pk=org)
         query = query.filter(account__provider__in=[service.adapter.provider_id
-                                                    for service in services])
+                                                    for service in registry])
         return query
 
     def get_paginate_by(self):

readthedocs/rtd_tests/tests/test_oauth.py

@@ -5,7 +5,7 @@ from allauth.socialaccount.models import SocialToken
 
 from readthedocs.projects.models import Project
 
-from readthedocs.oauth.utils import GitHubService
+from readthedocs.oauth.services import GitHubService
 from readthedocs.oauth.models import RemoteRepository, RemoteOrganization