diff --git a/.gitignore b/.gitignore index 63ba96916..a2e56a8f2 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ .idea .vagrant .tox +.rope_project/ +.ropeproject/ _build cnames bower_components/ @@ -35,4 +37,3 @@ whoosh_index xml_output public_cnames public_symlinks -.rope_project/ diff --git a/readthedocs/core/management/commands/import_github.py b/readthedocs/core/management/commands/import_github.py index f30820774..6d5dc3dbe 100644 --- a/readthedocs/core/management/commands/import_github.py +++ b/readthedocs/core/management/commands/import_github.py @@ -1,7 +1,7 @@ from django.core.management.base import BaseCommand from django.contrib.auth.models import User -from readthedocs.oauth.utils import GitHubService +from readthedocs.oauth.services import GitHubService class Command(BaseCommand): diff --git a/readthedocs/oauth/services/__init__.py b/readthedocs/oauth/services/__init__.py new file mode 100644 index 000000000..37ff9ca78 --- /dev/null +++ b/readthedocs/oauth/services/__init__.py @@ -0,0 +1,13 @@ +"""Conditional classes for OAuth services""" + +from django.utils.module_loading import import_by_path +from django.conf import settings + +GitHubService = import_by_path( + getattr(settings, 'OAUTH_GITHUB_SERVICE', + 'readthedocs.oauth.services.github.GitHubService')) +BitbucketService = import_by_path( + getattr(settings, 'OAUTH_BITBUCKET_SERVICE', + 'readthedocs.oauth.services.bitbucket.BitbucketService')) + +registry = [GitHubService, BitbucketService] diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py new file mode 100644 index 000000000..f3f357543 --- /dev/null +++ b/readthedocs/oauth/services/base.py @@ -0,0 +1,141 @@ +"""OAuth utility functions""" + +import logging +from datetime import datetime + +from django.conf import settings +from requests_oauthlib import OAuth2Session +from allauth.socialaccount.models import SocialAccount + + +DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public') + +log = logging.getLogger(__name__) + + +class Service(object): + + """Service mapping for local accounts + + :param user: User to use in token lookup and session creation + :param account: :py:cls:`SocialAccount` instance for user + """ + + adapter = None + url_pattern = None + + def __init__(self, user, account): + self.session = None + self.user = user + self.account = account + + @classmethod + def for_user(cls, user): + """Create instance if user has an account for the provider""" + try: + account = SocialAccount.objects.get( + user=user, + provider=cls.adapter.provider_id + ) + return cls(user=user, account=account) + except SocialAccount.DoesNotExist: + return None + + def get_adapter(self): + return self.adapter + + @property + def provider_id(self): + return self.get_adapter().provider_id + + def get_session(self): + if self.session is None: + self.create_session() + return self.session + + def create_session(self): + """Create OAuth session for user + + This configures the OAuth session based on the :py:cls:`SocialToken` + attributes. If there is an ``expires_at``, treat the session as an auto + renewing token. Some providers expire tokens after as little as 2 + hours. + """ + token = self.account.socialtoken_set.first() + if token is None: + return None + + token_config = { + 'access_token': str(token.token), + 'token_type': 'bearer', + } + if token.expires_at is not None: + token_expires = (token.expires_at - datetime.now()).total_seconds() + token_config.update({ + 'refresh_token': str(token.token_secret), + 'expires_in': token_expires, + }) + + self.session = OAuth2Session( + client_id=token.app.client_id, + token=token_config, + auto_refresh_kwargs={ + 'client_id': token.app.client_id, + 'client_secret': token.app.secret, + }, + auto_refresh_url=self.get_adapter().access_token_url, + token_updater=self.token_updater(token) + ) + + return self.session or None + + def token_updater(self, token): + """Update token given data from OAuth response + + Expect the following response into the closure:: + + { + u'token_type': u'bearer', + u'scopes': u'webhook repository team account', + u'refresh_token': u'...', + u'access_token': u'...', + u'expires_in': 3600, + u'expires_at': 1449218652.558185 + } + """ + + def _updater(data): + token.token = data['access_token'] + token.expires_at = datetime.fromtimestamp(data['expires_at']) + token.save() + log.info('Updated token %s:', token) + + return _updater + + def sync(self, sync): + raise NotImplementedError + + def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL, + organization=None): + raise NotImplementedError + + def create_organization(self, fields): + raise NotImplementedError + + def setup_webhook(self, project): + raise NotImplementedError + + @classmethod + def is_project_service(cls, project): + """Determine if this is the service the project is using + + .. note:: + This should be deprecated in favor of attaching the + :py:cls:`RemoteRepository` to the project instance. This is a slight + improvement on the legacy check for webhooks + """ + # TODO Replace this check by keying project to remote repos + return ( + cls.url_pattern is not None and + cls.url_pattern.search(project.repo) is not None + ) diff --git a/readthedocs/oauth/services/bitbucket.py b/readthedocs/oauth/services/bitbucket.py new file mode 100644 index 000000000..0fcaac51d --- /dev/null +++ b/readthedocs/oauth/services/bitbucket.py @@ -0,0 +1,181 @@ +"""OAuth utility functions""" + +import logging +import json +import re + +from django.conf import settings +from requests.exceptions import RequestException +from allauth.socialaccount.providers.bitbucket_oauth2.views import ( + BitbucketOAuth2Adapter) + +from readthedocs.builds import utils as build_utils + +from ..models import RemoteOrganization, RemoteRepository +from .base import Service + + +DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public') + +log = logging.getLogger(__name__) + + +class BitbucketService(Service): + + """Provider service for Bitbucket""" + + adapter = BitbucketOAuth2Adapter + # TODO replace this with a less naive check + url_pattern = re.compile(r'bitbucket.org\/') + + def sync(self, sync): + """Import from Bitbucket""" + if sync: + self.sync_repositories() + self.sync_teams() + + def sync_repositories(self): + # Get user repos + try: + repos = self.paginate( + 'https://bitbucket.org/api/2.0/repositories/?role=member') + for repo in repos: + self.create_repository(repo) + except (TypeError, ValueError) as e: + log.error('Error syncing Bitbucket repositories: %s', + str(e), exc_info=True) + raise Exception('Could not sync your Bitbucket repositories, ' + 'try reconnecting your account') + + # Because privileges aren't returned with repository data, run query + # again for repositories that user has admin role for, and update + # existing repositories. + try: + resp = self.paginate( + 'https://bitbucket.org/api/2.0/repositories/?role=admin') + repos = ( + RemoteRepository.objects + .filter(users=self.user, + full_name__in=[r['full_name'] for r in resp], + account=self.account) + ) + for repo in repos: + repo.admin = True + repo.save() + except (TypeError, ValueError): + pass + + def sync_teams(self): + """Sync Bitbucket teams and team repositories for user token""" + try: + teams = self.paginate( + 'https://api.bitbucket.org/2.0/teams/?role=member' + ) + for team in teams: + org = self.create_organization(team) + repos = self.paginate(team['links']['repositories']['href']) + for repo in repos: + self.create_repository(repo, organization=org) + except ValueError as e: + log.error('Error syncing Bitbucket organizations: %s', + str(e), exc_info=True) + raise Exception('Could not sync your Bitbucket team repositories, ' + 'try reconnecting your account') + + def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL, + organization=None): + """Update or create a repository from Bitbucket + + This looks up existing repositories based on the full repository name, + that is the username and repository name. + + .. note:: + The :py:data:`admin` property is not set during creation, as + permissions are not part of the returned repository data from + Bitbucket. + """ + if (fields['is_private'] is True and privacy == 'private' or + fields['is_private'] is False and privacy == 'public'): + repo, _ = RemoteRepository.objects.get_or_create( + full_name=fields['full_name'], + account=self.account, + ) + if repo.organization and repo.organization != organization: + log.debug('Not importing %s because mismatched orgs' % + fields['name']) + return None + else: + repo.organization = organization + repo.users.add(self.user) + repo.name = fields['name'] + repo.description = fields['description'] + repo.private = fields['is_private'] + repo.clone_url = fields['links']['clone'][0]['href'] + repo.ssh_url = fields['links']['clone'][1]['href'] + if repo.private: + repo.clone_url = repo.ssh_url + repo.html_url = fields['links']['html']['href'] + repo.vcs = fields['scm'] + repo.account = self.account + + avatar_url = fields['links']['avatar']['href'] or '' + repo.avatar_url = re.sub(r'\/16\/$', r'/32/', avatar_url) + + repo.json = json.dumps(fields) + repo.save() + return repo + else: + log.debug('Not importing %s because mismatched type' % + fields['name']) + + def create_organization(self, fields): + organization, _ = RemoteOrganization.objects.get_or_create( + slug=fields.get('username'), + account=self.account, + ) + organization.name = fields.get('display_name') + organization.email = fields.get('email') + organization.avatar_url = fields['links']['avatar']['href'] + organization.html_url = fields['links']['html']['href'] + organization.json = json.dumps(fields) + organization.account = self.account + organization.users.add(self.user) + organization.save() + return organization + + def paginate(self, url): + """Combines results from Bitbucket pagination + + :param url: start url to get the data from. + """ + resp = self.get_session().get(url) + data = resp.json() + results = data.get('values', []) + next_url = data.get('next') + if next_url: + results.extend(self.paginate(next_url)) + return results + + def setup_webhook(self, project): + session = self.get_session() + owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo) + data = { + 'type': 'POST', + 'url': 'https://{domain}/bitbucket'.format(domain=settings.PRODUCTION_DOMAIN), + } + try: + resp = session.post( + 'https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/services'.format( + owner=owner, repo=repo + ), + data=data, + ) + if resp.status_code == 200: + log.info('Created Bitbucket webhook: project={project}' + .format(project=project)) + return True + except RequestException: + pass + else: + log.exception('Bitbucket webhook creation failed', exc_info=True) + return False diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py new file mode 100644 index 000000000..aaab806a4 --- /dev/null +++ b/readthedocs/oauth/services/github.py @@ -0,0 +1,183 @@ +"""OAuth utility functions""" + +import logging +import json +import re + +from django.conf import settings +from requests.exceptions import RequestException +from allauth.socialaccount.models import SocialToken +from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter + +from readthedocs.builds import utils as build_utils +from readthedocs.restapi.client import api + +from ..models import RemoteOrganization, RemoteRepository +from .base import Service + + +DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public') + +log = logging.getLogger(__name__) + + +class GitHubService(Service): + + """Provider service for GitHub""" + + adapter = GitHubOAuth2Adapter + # TODO replace this with a less naive check + url_pattern = re.compile(r'^github\.com\/') + + def sync(self, sync): + """Sync repositories and organizations""" + if sync: + self.sync_repositories() + self.sync_organizations() + + def sync_repositories(self): + """Get repositories for GitHub user via OAuth token""" + repos = self.paginate('https://api.github.com/user/repos?per_page=100') + try: + for repo in repos: + self.create_repository(repo) + except (TypeError, ValueError) as e: + log.error('Error syncing GitHub repositories: %s', + str(e), exc_info=True) + raise Exception('Could not sync your GitHub repositories, ' + 'try reconnecting your account') + + def sync_organizations(self): + """Sync GitHub organizations and organization repositories""" + try: + orgs = self.paginate('https://api.github.com/user/orgs') + for org in orgs: + org_resp = self.get_session().get(org['url']) + org_obj = self.create_organization(org_resp.json()) + # Add repos + # TODO ?per_page=100 + org_repos = self.paginate( + '{org_url}/repos'.format(org_url=org['url']) + ) + for repo in org_repos: + self.create_repository(repo, organization=org_obj) + except (TypeError, ValueError) as e: + log.error('Error syncing GitHub organizations: %s', + str(e), exc_info=True) + raise Exception('Could not sync your GitHub organizations, ' + 'try reconnecting your account') + + def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL, + organization=None): + if ( + (privacy == 'private') or + (fields['private'] is False and privacy == 'public')): + repo, _ = RemoteRepository.objects.get_or_create( + full_name=fields['full_name'], + account=self.account, + ) + if repo.organization and repo.organization != organization: + log.debug('Not importing %s because mismatched orgs' % + fields['name']) + return None + else: + repo.organization = organization + repo.users.add(self.user) + repo.name = fields['name'] + repo.description = fields['description'] + repo.ssh_url = fields['ssh_url'] + repo.html_url = fields['html_url'] + repo.private = fields['private'] + if repo.private: + repo.clone_url = fields['ssh_url'] + else: + repo.clone_url = fields['clone_url'] + repo.admin = fields.get('permissions', {}).get('admin', False) + repo.vcs = 'git' + repo.account = self.account + repo.avatar_url = fields.get('owner', {}).get('avatar_url') + repo.json = json.dumps(fields) + repo.save() + return repo + else: + log.debug('Not importing %s because mismatched type' % + fields['name']) + + def create_organization(self, fields): + organization, _ = RemoteOrganization.objects.get_or_create( + slug=fields.get('login'), + account=self.account, + ) + organization.html_url = fields.get('html_url') + organization.name = fields.get('name') + organization.email = fields.get('email') + organization.avatar_url = fields.get('avatar_url') + organization.json = json.dumps(fields) + organization.account = self.account + organization.users.add(self.user) + organization.save() + return organization + + def paginate(self, url): + """Combines return from GitHub pagination + + :param url: start url to get the data from. + + See https://developer.github.com/v3/#pagination + """ + resp = self.get_session().get(url) + result = resp.json() + next_url = resp.links.get('next', {}).get('url') + if next_url: + result.extend(self.paginate(next_url)) + return result + + def setup_webhook(self, project): + """Set up GitHub webhook for project + + :param project: Project instance to set up webhook for + """ + session = self.get_session() + owner, repo = build_utils.get_github_username_repo(url=project.repo) + data = json.dumps({ + 'name': 'readthedocs', + 'active': True, + 'config': {'url': 'https://{domain}/github'.format(domain=settings.PRODUCTION_DOMAIN)} + }) + resp = None + try: + resp = session.post( + 'https://api.github.com/repos/{owner}/{repo}/hooks'.format(owner=owner, repo=repo), + data=data, + headers={'content-type': 'application/json'} + ) + if resp.status_code == 201: + log.info('Created GitHub webhook: project={project}' + .format(project=project)) + return True + except RequestException: + pass + else: + log.exception('GitHub Hook creation failed', exc_info=True) + return False + + @classmethod + def get_token_for_project(cls, project, force_local=False): + """Get access token for project by iterating over project users""" + # TODO why does this only target GitHub? + if not getattr(settings, 'ALLOW_PRIVATE_REPOS', False): + return None + token = None + try: + if getattr(settings, 'DONT_HIT_DB', True) and not force_local: + token = api.project(project.pk).token().get()['token'] + else: + for user in project.users.all(): + tokens = SocialToken.objects.filter( + account__user=user, + app__provider=cls.adapter.provider_id) + if tokens.exists(): + token = tokens[0].token + except Exception: + log.error('Failed to get token for user', exc_info=True) + return token diff --git a/readthedocs/oauth/tasks.py b/readthedocs/oauth/tasks.py index 754fa6d3b..6225867bb 100644 --- a/readthedocs/oauth/tasks.py +++ b/readthedocs/oauth/tasks.py @@ -6,7 +6,7 @@ from djcelery import celery as celery_app from readthedocs.core.utils.tasks import PublicTask from readthedocs.core.utils.tasks import permission_check from readthedocs.core.utils.tasks import user_id_matches -from .utils import services +from .services import registry @permission_check(user_id_matches) @@ -16,7 +16,7 @@ class SyncRemoteRepositories(PublicTask): def run_public(self, user_id): user = User.objects.get(pk=user_id) - for service_cls in services: + for service_cls in registry: service = service_cls.for_user(user) if service is not None: service.sync(sync=True) diff --git a/readthedocs/oauth/utils.py b/readthedocs/oauth/utils.py deleted file mode 100644 index 9a968ecc7..000000000 --- a/readthedocs/oauth/utils.py +++ /dev/null @@ -1,478 +0,0 @@ -"""OAuth utility functions""" - -import logging -import json -import re -from datetime import datetime - -from django.conf import settings -from requests_oauthlib import OAuth2Session -from requests.exceptions import RequestException -from allauth.socialaccount.models import SocialAccount -from allauth.socialaccount.providers.bitbucket_oauth2.views import ( - BitbucketOAuth2Adapter) -from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter - -from readthedocs.builds import utils as build_utils -from readthedocs.restapi.client import api - -from .models import RemoteOrganization, RemoteRepository - - -DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public') - -log = logging.getLogger(__name__) - - -class Service(object): - - """Service mapping for local accounts - - :param user: User to use in token lookup and session creation - :param account: :py:cls:`SocialAccount` instance for user - """ - - adapter = None - url_pattern = None - - def __init__(self, user, account): - self.session = None - self.user = user - self.account = account - - @classmethod - def for_user(cls, user): - """Create instance if user has an account for the provider""" - try: - account = SocialAccount.objects.get( - user=user, - provider=cls.adapter.provider_id - ) - return cls(user=user, account=account) - except SocialAccount.DoesNotExist: - return None - - def get_adapter(self): - return self.adapter - - @property - def provider_id(self): - return self.get_adapter().provider_id - - def get_session(self): - if self.session is None: - self.create_session() - return self.session - - def create_session(self): - """Create OAuth session for user - - This configures the OAuth session based on the :py:cls:`SocialToken` - attributes. If there is an ``expires_at``, treat the session as an auto - renewing token. Some providers expire tokens after as little as 2 - hours. - """ - token = self.account.socialtoken_set.first() - if token is None: - return None - - token_config = { - 'access_token': str(token.token), - 'token_type': 'bearer', - } - if token.expires_at is not None: - token_expires = (token.expires_at - datetime.now()).total_seconds() - token_config.update({ - 'refresh_token': str(token.token_secret), - 'expires_in': token_expires, - }) - - self.session = OAuth2Session( - client_id=token.app.client_id, - token=token_config, - auto_refresh_kwargs={ - 'client_id': token.app.client_id, - 'client_secret': token.app.secret, - }, - auto_refresh_url=self.get_adapter().access_token_url, - token_updater=self.token_updater(token) - ) - - return self.session or None - - def token_updater(self, token): - """Update token given data from OAuth response - - Expect the following response into the closure:: - - { - u'token_type': u'bearer', - u'scopes': u'webhook repository team account', - u'refresh_token': u'...', - u'access_token': u'...', - u'expires_in': 3600, - u'expires_at': 1449218652.558185 - } - """ - - def _updater(data): - token.token = data['access_token'] - token.expires_at = datetime.fromtimestamp(data['expires_at']) - token.save() - log.info('Updated token %s:', token) - - return _updater - - def sync(self, sync): - raise NotImplementedError - - def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL, - organization=None): - raise NotImplementedError - - def create_organization(self, fields): - raise NotImplementedError - - def setup_webhook(self, project): - raise NotImplementedError - - @classmethod - def is_project_service(cls, project): - """Determine if this is the service the project is using - - .. note:: - This should be deprecated in favor of attaching the - :py:cls:`RemoteRepository` to the project instance. This is a slight - improvement on the legacy check for webhooks - """ - # TODO Replace this check by keying project to remote repos - return ( - cls.url_pattern is not None and - cls.url_pattern.search(project.repo) is not None - ) - - -class GitHubService(Service): - - """Provider service for GitHub""" - - adapter = GitHubOAuth2Adapter - # TODO replace this with a less naive check - url_pattern = re.compile(r'^github\.com\/') - - def sync(self, sync): - """Sync repositories and organizations""" - if sync: - self.sync_repositories() - self.sync_organizations() - - def sync_repositories(self): - """Get repositories for GitHub user via OAuth token""" - repos = self.paginate('https://api.github.com/user/repos?per_page=100') - try: - for repo in repos: - self.create_repository(repo) - except (TypeError, ValueError) as e: - log.error('Error syncing GitHub repositories: %s', - str(e), exc_info=True) - raise Exception('Could not sync your GitHub repositories, ' - 'try reconnecting your account') - - def sync_organizations(self): - """Sync GitHub organizations and organization repositories""" - try: - orgs = self.paginate('https://api.github.com/user/orgs') - for org in orgs: - org_resp = self.get_session().get(org['url']) - org_obj = self.create_organization(org_resp.json()) - # Add repos - # TODO ?per_page=100 - org_repos = self.paginate( - '{org_url}/repos'.format(org_url=org['url']) - ) - for repo in org_repos: - self.create_repository(repo, organization=org_obj) - except (TypeError, ValueError) as e: - log.error('Error syncing GitHub organizations: %s', - str(e), exc_info=True) - raise Exception('Could not sync your GitHub organizations, ' - 'try reconnecting your account') - - def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL, - organization=None): - if ( - (privacy == 'private') or - (fields['private'] is False and privacy == 'public')): - repo, _ = RemoteRepository.objects.get_or_create( - full_name=fields['full_name'], - account=self.account, - ) - if repo.organization and repo.organization != organization: - log.debug('Not importing %s because mismatched orgs' % - fields['name']) - return None - else: - repo.organization = organization - repo.users.add(self.user) - repo.name = fields['name'] - repo.description = fields['description'] - repo.ssh_url = fields['ssh_url'] - repo.html_url = fields['html_url'] - repo.private = fields['private'] - if repo.private: - repo.clone_url = fields['ssh_url'] - else: - repo.clone_url = fields['clone_url'] - repo.admin = fields.get('permissions', {}).get('admin', False) - repo.vcs = 'git' - repo.account = self.account - repo.avatar_url = fields.get('owner', {}).get('avatar_url') - repo.json = json.dumps(fields) - repo.save() - return repo - else: - log.debug('Not importing %s because mismatched type' % - fields['name']) - - def create_organization(self, fields): - organization, _ = RemoteOrganization.objects.get_or_create( - slug=fields.get('login'), - account=self.account, - ) - organization.html_url = fields.get('html_url') - organization.name = fields.get('name') - organization.email = fields.get('email') - organization.avatar_url = fields.get('avatar_url') - organization.json = json.dumps(fields) - organization.account = self.account - organization.users.add(self.user) - organization.save() - return organization - - def paginate(self, url): - """Combines return from GitHub pagination - - :param url: start url to get the data from. - - See https://developer.github.com/v3/#pagination - """ - resp = self.get_session().get(url) - result = resp.json() - next_url = resp.links.get('next', {}).get('url') - if next_url: - result.extend(self.paginate(next_url)) - return result - - def setup_webhook(self, project): - """Set up GitHub webhook for project - - :param project: Project instance to set up webhook for - """ - session = self.get_session() - owner, repo = build_utils.get_github_username_repo(url=project.repo) - data = json.dumps({ - 'name': 'readthedocs', - 'active': True, - 'config': {'url': 'https://{domain}/github'.format(domain=settings.PRODUCTION_DOMAIN)} - }) - resp = None - try: - resp = session.post( - 'https://api.github.com/repos/{owner}/{repo}/hooks'.format(owner=owner, repo=repo), - data=data, - headers={'content-type': 'application/json'} - ) - if resp.status_code == 201: - log.info('Created GitHub webhook: project={project}' - .format(project=project)) - return True - except RequestException: - pass - else: - log.exception('GitHub Hook creation failed', exc_info=True) - return False - - @classmethod - def get_token_for_project(cls, project, force_local=False): - """Get access token for project by iterating over project users""" - # TODO why does this only target GitHub? - if not getattr(settings, 'ALLOW_PRIVATE_REPOS', False): - return None - token = None - try: - if getattr(settings, 'DONT_HIT_DB', True) and not force_local: - token = api.project(project.pk).token().get()['token'] - else: - for user in project.users.all(): - tokens = SocialToken.objects.filter( - account__user=user, - app__provider=cls.adapter.provider_id) - if tokens.exists(): - token = tokens[0].token - except Exception: - log.error('Failed to get token for user', exc_info=True) - return token - - -class BitbucketService(Service): - - """Provider service for Bitbucket""" - - adapter = BitbucketOAuth2Adapter - # TODO replace this with a less naive check - url_pattern = re.compile(r'bitbucket.org\/') - - def sync(self, sync): - """Import from Bitbucket""" - if sync: - self.sync_repositories() - self.sync_teams() - - def sync_repositories(self): - # Get user repos - try: - repos = self.paginate( - 'https://bitbucket.org/api/2.0/repositories/?role=member') - for repo in repos: - self.create_repository(repo) - except (TypeError, ValueError) as e: - log.error('Error syncing Bitbucket repositories: %s', - str(e), exc_info=True) - raise Exception('Could not sync your Bitbucket repositories, ' - 'try reconnecting your account') - - # Because privileges aren't returned with repository data, run query - # again for repositories that user has admin role for, and update - # existing repositories. - try: - resp = self.paginate( - 'https://bitbucket.org/api/2.0/repositories/?role=admin') - repos = ( - RemoteRepository.objects - .filter(users=self.user, - full_name__in=[r['full_name'] for r in resp], - account=self.account) - ) - for repo in repos: - repo.admin = True - repo.save() - except (TypeError, ValueError): - pass - - def sync_teams(self): - """Sync Bitbucket teams and team repositories for user token""" - try: - teams = self.paginate( - 'https://api.bitbucket.org/2.0/teams/?role=member' - ) - for team in teams: - org = self.create_organization(team) - repos = self.paginate(team['links']['repositories']['href']) - for repo in repos: - self.create_repository(repo, organization=org) - except ValueError as e: - log.error('Error syncing Bitbucket organizations: %s', - str(e), exc_info=True) - raise Exception('Could not sync your Bitbucket team repositories, ' - 'try reconnecting your account') - - def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL, - organization=None): - """Update or create a repository from Bitbucket - - This looks up existing repositories based on the full repository name, - that is the username and repository name. - - .. note:: - The :py:data:`admin` property is not set during creation, as - permissions are not part of the returned repository data from - Bitbucket. - """ - if (fields['is_private'] is True and privacy == 'private' or - fields['is_private'] is False and privacy == 'public'): - repo, _ = RemoteRepository.objects.get_or_create( - full_name=fields['full_name'], - account=self.account, - ) - if repo.organization and repo.organization != organization: - log.debug('Not importing %s because mismatched orgs' % - fields['name']) - return None - else: - repo.organization = organization - repo.users.add(self.user) - repo.name = fields['name'] - repo.description = fields['description'] - repo.private = fields['is_private'] - repo.clone_url = fields['links']['clone'][0]['href'] - repo.ssh_url = fields['links']['clone'][1]['href'] - if repo.private: - repo.clone_url = repo.ssh_url - repo.html_url = fields['links']['html']['href'] - repo.vcs = fields['scm'] - repo.account = self.account - - avatar_url = fields['links']['avatar']['href'] or '' - repo.avatar_url = re.sub(r'\/16\/$', r'/32/', avatar_url) - - repo.json = json.dumps(fields) - repo.save() - return repo - else: - log.debug('Not importing %s because mismatched type' % - fields['name']) - - def create_organization(self, fields): - organization, _ = RemoteOrganization.objects.get_or_create( - slug=fields.get('username'), - account=self.account, - ) - organization.name = fields.get('display_name') - organization.email = fields.get('email') - organization.avatar_url = fields['links']['avatar']['href'] - organization.html_url = fields['links']['html']['href'] - organization.json = json.dumps(fields) - organization.account = self.account - organization.users.add(self.user) - organization.save() - return organization - - def paginate(self, url): - """Combines results from Bitbucket pagination - - :param url: start url to get the data from. - """ - resp = self.get_session().get(url) - data = resp.json() - results = data.get('values', []) - next_url = data.get('next') - if next_url: - results.extend(self.paginate(next_url)) - return results - - def setup_webhook(self, project): - session = self.get_session() - owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo) - data = { - 'type': 'POST', - 'url': 'https://{domain}/bitbucket'.format(domain=settings.PRODUCTION_DOMAIN), - } - try: - resp = session.post( - 'https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/services'.format( - owner=owner, repo=repo - ), - data=data, - ) - if resp.status_code == 200: - log.info('Created Bitbucket webhook: project={project}' - .format(project=project)) - return True - except RequestException: - pass - else: - log.exception('Bitbucket webhook creation failed', exc_info=True) - return False - - -services = [GitHubService, BitbucketService] diff --git a/readthedocs/projects/signals.py b/readthedocs/projects/signals.py index 0300a484a..1e54ce945 100644 --- a/readthedocs/projects/signals.py +++ b/readthedocs/projects/signals.py @@ -7,7 +7,7 @@ from django.contrib import messages from django.dispatch import receiver from django.utils.translation import ugettext_lazy as _ -from readthedocs.oauth.utils import services +from readthedocs.oauth.services import registry before_vcs = django.dispatch.Signal(providing_args=["version"]) @@ -28,7 +28,7 @@ def handle_project_import(sender, **kwargs): project = sender request = kwargs.get('request') - for service_cls in services: + for service_cls in registry: if service_cls.is_project_service(project): service = service_cls.for_user(request.user) if service.setup_webhook(project): diff --git a/readthedocs/restapi/views/model_views.py b/readthedocs/restapi/views/model_views.py index 73c3a9a39..4c36173da 100644 --- a/readthedocs/restapi/views/model_views.py +++ b/readthedocs/restapi/views/model_views.py @@ -11,7 +11,7 @@ from readthedocs.builds.constants import TAG from readthedocs.builds.filters import VersionFilter from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build -from readthedocs.oauth.utils import GitHubService, services +from readthedocs.oauth.services import GitHubService, registry from readthedocs.oauth.models import RemoteOrganization, RemoteRepository from readthedocs.builds.constants import STABLE from readthedocs.projects.filters import ProjectFilter, DomainFilter @@ -230,7 +230,7 @@ class RemoteOrganizationViewSet(viewsets.ReadOnlyModelViewSet): def get_queryset(self): return (self.model.objects.api(self.request.user) .filter(account__provider__in=[service.adapter.provider_id - for service in services])) + for service in registry])) class RemoteRepositoryViewSet(viewsets.ReadOnlyModelViewSet): @@ -245,7 +245,7 @@ class RemoteRepositoryViewSet(viewsets.ReadOnlyModelViewSet): if org is not None: query = query.filter(organization__pk=org) query = query.filter(account__provider__in=[service.adapter.provider_id - for service in services]) + for service in registry]) return query def get_paginate_by(self): diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index cee2fc908..ec2ea258f 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -5,7 +5,7 @@ from allauth.socialaccount.models import SocialToken from readthedocs.projects.models import Project -from readthedocs.oauth.utils import GitHubService +from readthedocs.oauth.services import GitHubService from readthedocs.oauth.models import RemoteRepository, RemoteOrganization