diff --git a/pyproject.toml b/pyproject.toml index 12bbdd9e..b29a500d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "jsonschema[format-nongpl]", "json-e>=2.5.0", "PyYAML", - "taskcluster>=40", + "taskcluster>=106", "taskcluster-taskgraph", ] diff --git a/scriptworker.yaml.tmpl b/scriptworker.yaml.tmpl index 79357b4b..5514d284 100644 --- a/scriptworker.yaml.tmpl +++ b/scriptworker.yaml.tmpl @@ -45,9 +45,15 @@ verify_cot_signature: false # Chain of Trust job type, e.g. signing cot_job_type: scriptworker cot_product: firefox -# Calls to Github API are limited to 60 an hour. Using an API token allows to raise the limit to -# 5000 per hour. https://developer.github.com/v3/#rate-limiting -github_oauth_token: somegithubtoken + +# Scriptworker first tries to obtain a repository scoped token from Taskcluster's auth service, +# using the Github App registered under this name. +# github_app_name: read +# +# `github_oauth_token` is used as a fallback if that fails (e.g. missing scopes or app not +# configured). Without either, calls to the Github API are unauthenticated and limited to 60 an +# hour. See https://developer.github.com/v3/#rate-limiting +# github_oauth_token: somegithubtoken #----------------------------------------------------------------------------------------------- diff --git a/src/scriptworker/constants.py b/src/scriptworker/constants.py index 013620bd..d38f0ef1 100644 --- a/src/scriptworker/constants.py +++ b/src/scriptworker/constants.py @@ -79,7 +79,12 @@ "max_chain_length": 20, # Calls to Github API are limited to 60 an hour. Using an API token allows to raise the limit to # 5000 per hour. https://developer.github.com/v3/#rate-limiting + # Scriptworker first tries to obtain a repository scoped token from Taskcluster's auth + # service, falling back to this token if that fails. "github_oauth_token": "", + # The name of the Github App registered with Taskcluster's auth service, used to obtain + # a repository scoped token via `auth.githubRepoToken`. + "github_app_name": "read", # ed25519 settings "ed25519_private_key_path": "...", "ed25519_public_keys": immutabledict( diff --git a/src/scriptworker/cot/verify.py b/src/scriptworker/cot/verify.py index 1a33481f..373f9a72 100644 --- a/src/scriptworker/cot/verify.py +++ b/src/scriptworker/cot/verify.py @@ -20,7 +20,6 @@ from copy import deepcopy from urllib.parse import urlparse -import aiohttp import dictdiffer import jsone from immutabledict import immutabledict @@ -40,7 +39,7 @@ from scriptworker.context import Context from scriptworker.ed25519 import ed25519_public_key_from_string, verify_ed25519_signature from scriptworker.exceptions import BaseDownloadError, CoTError, ScriptWorkerEd25519Error -from scriptworker.github import GitHubRepository, extract_github_repo_full_name, extract_github_repo_owner_and_name, extract_github_repo_ssh_url +from scriptworker.github import GitHubRepository, extract_github_repo_full_name, extract_github_repo_owner_and_name, extract_github_repo_ssh_url, is_github_url from scriptworker.log import contextual_log_handler, get_chain_of_trust_log_filename from scriptworker.task import ( get_action_callback_name, @@ -1118,7 +1117,7 @@ async def _get_additional_github_releases_jsone_context(decision_link): repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url) tag_name = get_revision(task, source_env_prefix) - github_repo = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"]) + github_repo = GitHubRepository(context, repo_owner, repo_name) release_data = await github_repo.get_release(tag_name) # The release data expose by the API[1] is not the same as the original event[2]. That's why @@ -1200,17 +1199,16 @@ async def _get_additional_github_pull_request_jsone_context(decision_link): repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1) repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url) pull_request_number = get_pull_request_number(task, source_env_prefix) - token = context.config["github_oauth_token"] - github_repo = GitHubRepository(repo_owner, repo_name, token) - repo_definition = github_repo.definition + github_repo = GitHubRepository(context, repo_owner, repo_name) + repo_definition = await github_repo.get_definition() # We need to query the repository where the pull request was made to extract # pull request data. The pull request could be created on the same repo as # the commit, or an upstream repo. We can compare the base and head repo URLs # to infer where the pull request lives. if repo_definition["fork"] and base_repo_url != repo_url: - github_repo = GitHubRepository(owner=repo_definition["parent"]["owner"]["login"], repo_name=repo_definition["parent"]["name"], token=token) + github_repo = GitHubRepository(context, repo_definition["parent"]["owner"]["login"], repo_definition["parent"]["name"]) pull_request_data = await github_repo.get_pull_request(pull_request_number) # Even though pull_request_data['head']['repo']['pushed_at'] does exist, @@ -1245,7 +1243,7 @@ async def _get_additional_github_push_jsone_context(decision_link): repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url) commit_hash = get_revision(task, source_env_prefix) - github_repo = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"]) + github_repo = GitHubRepository(context, repo_owner, repo_name) commit_data = await github_repo.get_commit(commit_hash) committer = commit_data["committer"] or {} @@ -1399,16 +1397,11 @@ def build_taskcluster_yml_url(link): """ source_env_prefix = link.context.config["source_env_prefix"] repo_url = get_repo(link.task, source_env_prefix) - repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1) revision = get_revision(link.task, source_env_prefix) repo_parts = urlparse(repo_url) - if repo_parts.netloc == "github.com": - user, repo_name = extract_github_repo_owner_and_name(repo_url) - url = f"https://raw.githubusercontent.com/{user}/{repo_name}/{revision}/.taskcluster.yml" - elif repo_parts.netloc == "hg.mozilla.org": - url = f"{repo_parts.scheme}://{repo_parts.netloc}{repo_parts.path}/raw-file/{revision}/.taskcluster.yml" - else: + if repo_parts.netloc != "hg.mozilla.org": raise CoTError("Unsupported VCS server!") + url = f"{repo_parts.scheme}://{repo_parts.netloc}{repo_parts.path}/raw-file/{revision}/.taskcluster.yml" log.debug(f"{link.name} .taskcluster.yml is at {url}") return url @@ -1430,19 +1423,20 @@ async def get_in_tree_template(link): """ context = link.context - source_url = build_taskcluster_yml_url(link) - repo_url = get_repo(link.task, link.context.config["source_env_prefix"]) + source_env_prefix = context.config["source_env_prefix"] + repo_url = get_repo(link.task, source_env_prefix) + repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1) - auth = None - if ( - (repo_url.startswith(("ssh://", "git@github.com")) or any(vcs_rule.get("require_secret") for vcs_rule in context.config["trusted_vcs_rules"])) - and "github.com" in repo_url - and context.config.get("github_oauth_token") - ): - auth = aiohttp.BasicAuth(context.config["github_oauth_token"]) + if is_github_url(repo_url): + revision = get_revision(link.task, source_env_prefix) + repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url) + github_repo = GitHubRepository(context, repo_owner, repo_name) + content = await github_repo.get_file_contents(".taskcluster.yml", ref=revision) + return load_json_or_yaml(content, file_type="yaml") + + source_url = build_taskcluster_yml_url(link) url_hash = hashlib.sha1(source_url.encode("ascii")).hexdigest() - tmpl = await load_json_or_yaml_from_url(context, source_url, os.path.join(context.config["work_dir"], "{}_taskcluster.yml".format(url_hash)), auth=auth) - return tmpl + return await load_json_or_yaml_from_url(context, source_url, os.path.join(context.config["work_dir"], "{}_taskcluster.yml".format(url_hash))) def _get_action_from_actions_json(all_actions, callback_name): diff --git a/src/scriptworker/github.py b/src/scriptworker/github.py index 3acf7b04..2a8322e2 100644 --- a/src/scriptworker/github.py +++ b/src/scriptworker/github.py @@ -6,6 +6,8 @@ from github3 import GitHub from github3.exceptions import GitHubException +from taskcluster.aio import Auth +from taskcluster.exceptions import TaskclusterFailure from scriptworker.exceptions import ConfigError from scriptworker.utils import get_parts_of_url_path, get_single_item_from_sequence, retry_async_decorator, retry_request, retry_sync @@ -23,30 +25,96 @@ class GitHubRepository: """Wrapper around GitHub API. Used to access public data.""" - def __init__(self, owner, repo_name, token=""): - """Build the GitHub API URL which points to the definition of the repository. + GITHUB_PERMISSIONS = {"contents": "read", "metadata": "read", "pull_requests": "read"} + + def __init__(self, context, owner, repo_name): + """Store the repository coordinates. The github3 repository object is built lazily. Args: - owner (str): the owner's GitHub username + context (scriptworker.context.Context): the scriptworker context + owner (str): the owner of the repository repo_name (str): the name of the repository - token (str): the GitHub API token + + """ + self._context = context + self._owner = owner + self._repo_name = repo_name + self._repository_cache = None + self._repository_lock = asyncio.Lock() + + async def _get_repository(self): + """Build and cache the github3 repository object. Returns: - dict: a representation of the repo definition + github3.repos.repo.Repository: the github3 repository object """ - github = retry_sync(GitHub, kwargs={"token": token}, sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS) - self._github_repository = retry_sync(github.repository, args=(owner, repo_name), sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS) + async with self._repository_lock: + if self._repository_cache is None: + token = await self._get_token(self._context, self._owner, self._repo_name) + github = retry_sync(GitHub, kwargs={"token": token}, sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS) + self._repository_cache = retry_sync(github.repository, args=(self._owner, self._repo_name), sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS) + + return self._repository_cache + + async def _get_token(self, context, owner, repo_name): + """Get a repository scoped GitHub token from Taskcluster's auth service. + + Falls back to ``context.config["github_oauth_token"]`` if the auth service call + fails, e.g. because of missing scopes. + + Args: + context (scriptworker.context.Context): the scriptworker context + owner (str): the owner of the repository + repo_name (str): the name of the repository - @property - def definition(self): + Returns: + str: the scoped GitHub token, or the fallback token + + """ + if not context.credentials: + return context.config.get("github_oauth_token", "") + + try: + auth = Auth(options={"rootUrl": context.config["taskcluster_root_url"], "credentials": context.credentials}) + response = await auth.githubRepoToken( + context.config["github_app_name"], owner, payload={"repositories": [repo_name], "permissions": self.GITHUB_PERMISSIONS} + ) + return response["token"] + except TaskclusterFailure as e: + # TODO When opening a PR from a fork, we're guaranteed to hit this + # fallback as the task won't have auth service scopes for the repo + # fork. We'll need to improve this before we can stop depending on + # `github_oauth_token`. + log.warning(f"Could not obtain Github token from Taskcluster for {owner}/{repo_name}, falling back to `github_oauth_token`: {e}") + return context.config.get("github_oauth_token", "") + + async def get_definition(self): """Fetch the definition of the repository, exposed by the GitHub API. Returns: dict: a representation of the repo definition """ - return self._github_repository.as_dict() + repository = await self._get_repository() + return repository.as_dict() + + @retry_async_decorator(retry_exceptions=GitHubException) + async def get_file_contents(self, path, ref=None): + """Fetch the decoded contents of a file in the repository. + + Args: + path (str): the path to the file, relative to the repository root + ref (str, optional): the commit/branch/tag to read the file from. + Defaults to the repository's default branch. + + Returns: + str: the decoded contents of the file + + """ + repository = await self._get_repository() + contents = repository.file_contents(path, ref=ref) + return contents.decoded.decode("utf-8") @retry_async_decorator(retry_exceptions=GitHubException) async def get_commit(self, commit_hash): @@ -59,7 +127,8 @@ async def get_commit(self, commit_hash): dict: a representation of the commit """ - return self._github_repository.commit(commit_hash).as_dict() + repository = await self._get_repository() + return repository.commit(commit_hash).as_dict() @retry_async_decorator(retry_exceptions=GitHubException) async def get_pull_request(self, pull_request_number): @@ -72,7 +141,8 @@ async def get_pull_request(self, pull_request_number): dict: a representation of the pull request """ - return self._github_repository.pull_request(pull_request_number).as_dict() + repository = await self._get_repository() + return repository.pull_request(pull_request_number).as_dict() @retry_async_decorator(retry_exceptions=GitHubException) async def get_release(self, tag_name): @@ -85,7 +155,8 @@ async def get_release(self, tag_name): dict: a representation of the tag """ - return self._github_repository.release_from_tag(tag_name).as_dict() + repository = await self._get_repository() + return repository.release_from_tag(tag_name).as_dict() @retry_async_decorator(retry_exceptions=GitHubException) async def get_tag_hash(self, tag_name): @@ -98,8 +169,9 @@ async def get_tag_hash(self, tag_name): str: the commit hash linked by the tag """ + repository = await self._get_repository() tag_object = get_single_item_from_sequence( - sequence=self._github_repository.tags(), + sequence=repository.tags(), condition=lambda tag: tag.name == tag_name, no_item_error_message='No tag "{}" exist'.format(tag_name), too_many_item_error_message='Too many tags "{}" found'.format(tag_name), @@ -128,7 +200,8 @@ async def has_commit_landed_on_repository(self, context, revision): if not _is_git_full_hash(revision): revision = await self.get_tag_hash(tag_name=revision) - html_text = await _fetch_github_branch_commits_data(context, self._github_repository.html_url, revision) + repository = await self._get_repository() + html_text = await _fetch_github_branch_commits_data(context, repository.html_url, revision) # https://github.com/{repo_owner}/{repo_name}/branch_commits/{revision} just returns some \n # when the commit hasn't landed on the origin repo. Otherwise, some HTML data is returned - it diff --git a/src/scriptworker/task.py b/src/scriptworker/task.py index 19fc6660..7284c3ce 100644 --- a/src/scriptworker/task.py +++ b/src/scriptworker/task.py @@ -550,7 +550,7 @@ async def is_pull_request(context, task): if not revision and can_skip: continue - github_repository = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"]) + github_repository = GitHubRepository(context, repo_owner, repo_name) conditions.append(not await github_repository.has_commit_landed_on_repository(context, revision)) return any(conditions) diff --git a/tests/test_cot_verify.py b/tests/test_cot_verify.py index c5e1dd74..2083a007 100644 --- a/tests/test_cot_verify.py +++ b/tests/test_cot_verify.py @@ -9,9 +9,8 @@ import time from copy import deepcopy from functools import partial -from unittest.mock import MagicMock +from unittest.mock import ANY, MagicMock -import aiohttp import jsone import pytest from immutabledict import immutabledict @@ -325,7 +324,7 @@ def cot_load(string, is_path=False, parent_dir=None, **kwargs): elif string.endswith("actions.json"): return load_json_or_yaml(os.path.join(parent_dir, "actions.json"), is_path=True) else: - return load_json_or_yaml(string) + return load_json_or_yaml(string, **kwargs) cotv2_load = partial(cot_load, parent_dir=COTV2_DIR) @@ -1130,7 +1129,7 @@ async def get_release_mock(release_name, *args, **kwargs): context = await cotverify.populate_jsone_context(mobile_chain, mobile_github_release_link, mobile_github_release_link, tasks_for="github-release") - github_repo_class_mock.assert_called_once_with("mozilla-mobile", "reference-browser", "fakegithubtoken") + github_repo_class_mock.assert_called_once_with(ANY, "mozilla-mobile", "reference-browser") del context["as_slugid"] assert context == { "event": { @@ -1210,7 +1209,7 @@ async def get_commit_mock(commit_hash, *args, **kwargs): context = await cotverify.populate_jsone_context(mobile_chain, mobile_github_push_link, mobile_github_push_link, tasks_for="github-push") - github_repo_class_mock.assert_called_once_with("mozilla-mobile", "reference-browser", "fakegithubtoken") + github_repo_class_mock.assert_called_once_with(ANY, "mozilla-mobile", "reference-browser") del context["as_slugid"] assert context == { "event": { @@ -1287,7 +1286,11 @@ async def test_populate_jsone_context_github_pull_request( github_repo_mock = MagicMock() repo_definition = {"fork": True, "parent": {"name": "reference-browser", "owner": {"login": "mozilla-mobile"}}} repo_definition.update(extra_repo_definition) - github_repo_mock.definition = repo_definition + + async def get_definition_mock(): + return repo_definition + + github_repo_mock.get_definition = get_definition_mock mobile_github_pull_request_link.task["extra"]["tasks_for"] = tasks_for mobile_github_pull_request_link.task["payload"]["env"].update(extra_env) @@ -1320,10 +1323,10 @@ async def get_pull_request_mock(pull_request_number, *args, **kwargs): mobile_chain_pull_request, mobile_github_pull_request_link, mobile_github_pull_request_link, tasks_for=tasks_for ) - github_repo_class_mock.assert_any_call("JohanLorenzo", "reference-browser", "fakegithubtoken") + github_repo_class_mock.assert_any_call(ANY, "JohanLorenzo", "reference-browser") if expected_use_parent: - github_repo_class_mock.assert_any_call(owner="mozilla-mobile", repo_name="reference-browser", token="fakegithubtoken") + github_repo_class_mock.assert_any_call(ANY, "mozilla-mobile", "reference-browser") assert len(github_repo_class_mock.call_args_list) == 2 else: assert len(github_repo_class_mock.call_args_list) == 1 @@ -1569,12 +1572,13 @@ async def get_commit(self, commit): "committer": {"login": "Callek"}, } + async def get_file_contents(self, path, ref=None): + assert path == ".taskcluster.yml" + assert ref == "330ea928b42ff2403fc99cd3e596d13294fe8775" + with open(os.path.join(COTV4_DIR, "private_github_.taskcluster.yml")) as fh: + return fh.read() + async def mocked_load_url(context, url, path, parent_path=COTV4_DIR, **kwargs): - if path.endswith("taskcluster.yml"): - assert kwargs.get("auth") - assert isinstance(kwargs["auth"], aiohttp.BasicAuth) - assert kwargs["auth"].login == "fakegithubtoken" - return load_json_or_yaml(os.path.join(parent_path, "private_github_.taskcluster.yml"), is_path=True, file_type="yaml") raise NotImplementedError() mocker.patch.object(cotverify, "load_json_or_yaml_from_url", new=mocked_load_url) @@ -1594,50 +1598,53 @@ def test_build_taskcluster_yml_url_unknown_server(decision_link): @pytest.mark.asyncio -@pytest.mark.parametrize("use_auth", (None, True)) @pytest.mark.parametrize( - "source_repo,revision,expected_url", + "source_repo", ( - ( - "ssh://github.com/mozilla-mobile/mozilla-vpn-client", - "330ea928b42ff2403fc99cd3e596d13294fe8775", - "https://raw.githubusercontent.com/mozilla-mobile/mozilla-vpn-client/330ea928b42ff2403fc99cd3e596d13294fe8775/.taskcluster.yml", - ), - ( - "git@github.com:mozilla-mobile/mozilla-vpn-client", - "330ea928b42ff2403fc99cd3e596d13294fe8775", - "https://raw.githubusercontent.com/mozilla-mobile/mozilla-vpn-client/330ea928b42ff2403fc99cd3e596d13294fe8775/.taskcluster.yml", - ), - ( - "https://hg.mozilla.org/ci/taskgraph-try", - "a9afa8aa11cf1431d4e6ef06c2a08d19e271c6ea", - "https://hg.mozilla.org/ci/taskgraph-try/raw-file/a9afa8aa11cf1431d4e6ef06c2a08d19e271c6ea/.taskcluster.yml", - ), + "ssh://github.com/mozilla-mobile/mozilla-vpn-client", + "git@github.com:mozilla-mobile/mozilla-vpn-client", ), ) -async def test_get_in_tree_template_auth_morphing(vpn_chain, mocker, use_auth, source_repo, revision, expected_url): +async def test_get_in_tree_template_github(vpn_chain, mocker, source_repo): name = "decision" task_id = "VUTfOIPFQWaGHf7sIbgTEg" - if not use_auth: - del vpn_chain.context.config["github_oauth_token"] + revision = "330ea928b42ff2403fc99cd3e596d13294fe8775" link = cotverify.LinkOfTrust(vpn_chain.context, name, task_id) - async def mocked_load_url(context, url, path, parent_path=COTV2_DIR, **kwargs): + github_repo_mock = MagicMock() + + async def get_file_contents_mock(path, ref=None): + assert path == ".taskcluster.yml" + assert ref == revision + return "tasks: []\n" + + github_repo_mock.get_file_contents = get_file_contents_mock + github_repo_class_mock = mocker.patch.object(cotverify, "GitHubRepository", return_value=github_repo_mock) + mocker.patch.object(cotverify, "get_repo", new=lambda x, y: source_repo) + mocker.patch.object(cotverify, "get_revision", new=lambda x, y: revision) + + tmpl = await cotverify.get_in_tree_template(link) + + github_repo_class_mock.assert_called_once_with(vpn_chain.context, "mozilla-mobile", "mozilla-vpn-client") + assert tmpl == {"tasks": []} + + +@pytest.mark.asyncio +async def test_get_in_tree_template_hg(decision_link, mocker): + source_repo = "https://hg.mozilla.org/ci/taskgraph-try" + revision = "a9afa8aa11cf1431d4e6ef06c2a08d19e271c6ea" + expected_url = "https://hg.mozilla.org/ci/taskgraph-try/raw-file/a9afa8aa11cf1431d4e6ef06c2a08d19e271c6ea/.taskcluster.yml" + + async def mocked_load_url(context, url, path, **kwargs): assert url == expected_url - if use_auth and "github.com" in source_repo: - assert kwargs.get("auth") - assert isinstance(kwargs["auth"], aiohttp.BasicAuth) - assert kwargs["auth"].login == "fakegithubtoken" - return "some_template" - else: - assert not kwargs.get("auth") - return "some_template_no_auth" + return "some_template" mocker.patch.object(cotverify, "load_json_or_yaml_from_url", new=mocked_load_url) mocker.patch.object(cotverify, "get_repo", new=lambda x, y: source_repo) mocker.patch.object(cotverify, "get_revision", new=lambda x, y: revision) - await cotverify.get_in_tree_template(link) + tmpl = await cotverify.get_in_tree_template(decision_link) + assert tmpl == "some_template" @pytest.mark.asyncio @@ -1748,9 +1755,19 @@ async def test_get_pr_action_context_and_template(mocker, mobile_chain, github_p """Test that pr-action tasks generate correct context and template with pr-action specific values.""" mobile_chain.context.config["min_cot_version"] = 3 + class MockedGitHubRepository(object): + def __init__(self, *args, **kwargs): + pass + + async def get_file_contents(self, path, ref=None): + assert path == ".taskcluster.yml" + with open(os.path.join(COTV4_DIR, ".taskcluster.yml")) as fh: + return fh.read() + mocker.patch.object(cotverify, "load_json_or_yaml_from_url", new=cotv4_load_url) mocker.patch.object(swcontext, "load_json_or_yaml_from_url", new=cotv4_load_url) mocker.patch.object(cotverify, "load_json_or_yaml", new=cotv4_load) + mocker.patch.object(cotverify, "GitHubRepository", new=MockedGitHubRepository) mobile_chain.links = list(set([mobile_github_push_link, github_pr_action_link])) diff --git a/tests/test_github.py b/tests/test_github.py index 4d315f76..2464b01a 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -1,9 +1,10 @@ import asyncio from copy import copy from types import SimpleNamespace -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest +import taskcluster.exceptions from scriptworker import github from scriptworker.exceptions import ConfigError, ScriptWorkerRetryException @@ -31,7 +32,15 @@ def vpn_context(vpn_private_rw_context): @pytest.fixture(scope="function") -def github_repository(mocker): +def token_context(): + return SimpleNamespace( + config={"github_oauth_token": "fallback-token", "github_app_name": "read", "taskcluster_root_url": "https://tc.example.com"}, + credentials={"a": "b"}, + ) + + +@pytest.fixture(scope="function") +def github_repository(mocker, token_context): github_repository_mock = MagicMock() github_repository_mock.__name__ = "GithubRepositoryMock" github_repository_mock.html_url = "https://github.com/some-user/some-repo/" @@ -43,28 +52,61 @@ def github_repository(mocker): github_instance_mock.repository.return_value = github_repository_mock github_class_mock = mocker.patch.object(github, "GitHub", return_value=github_instance_mock) github_class_mock.__name__ = github_class_mock.name - yield github.GitHubRepository("some-user", "some-repo") + mocker.patch.object(github, "Auth", side_effect=taskcluster.exceptions.TaskclusterFailure("disabled in tests")) + yield github.GitHubRepository(token_context, "some-user", "some-repo") + + +@pytest.mark.asyncio +async def test_constructor(mocker, token_context): + github_instance_mock = MagicMock() + github_instance_mock.repository.__name__ = "github_instance_repository_mock" + github_class_mock = mocker.patch.object(github, "GitHub", return_value=github_instance_mock) + github_class_mock.__name__ = github_class_mock.name + mocker.patch.object(github, "Auth", side_effect=taskcluster.exceptions.TaskclusterFailure("disabled in tests")) + + repo = github.GitHubRepository(token_context, "some-user", "some-repo") + await repo._get_repository() + + github_class_mock.assert_called_once_with(token="fallback-token") + github_instance_mock.repository.assert_called_once_with("some-user", "some-repo") +@pytest.mark.asyncio @pytest.mark.parametrize( - "args, expected_class_kwargs", ((("some-user", "some-repo", "some-token"), {"token": "some-token"}), (("some-user", "some-repo"), {"token": ""})) + "raises, expected_token", + ( + (False, "scoped-token"), + (True, "fallback-token"), + ), ) -def test_constructor(mocker, args, expected_class_kwargs): +async def test_constructor_with_context(mocker, token_context, raises, expected_token): github_instance_mock = MagicMock() github_instance_mock.repository.__name__ = "github_instance_repository_mock" github_class_mock = mocker.patch.object(github, "GitHub", return_value=github_instance_mock) github_class_mock.__name__ = github_class_mock.name - github.GitHubRepository(*args) + auth_instance_mock = MagicMock() + if raises: + auth_instance_mock.githubRepoToken = AsyncMock(side_effect=taskcluster.exceptions.TaskclusterRestFailure("missing scopes", None, status_code=403)) + else: + auth_instance_mock.githubRepoToken = AsyncMock(return_value={"token": "scoped-token", "expires": "2020-01-01T00:00:00Z"}) + auth_class_mock = mocker.patch.object(github, "Auth", return_value=auth_instance_mock) - github_class_mock.assert_called_once_with(**expected_class_kwargs) - github_instance_mock.repository.assert_called_once_with("some-user", "some-repo") + repo = github.GitHubRepository(token_context, "some-user", "some-repo") + await repo._get_repository() + + github_class_mock.assert_called_once_with(token=expected_token) + auth_class_mock.assert_called_once_with(options={"rootUrl": "https://tc.example.com", "credentials": {"a": "b"}}) + auth_instance_mock.githubRepoToken.assert_called_once_with( + "read", "some-user", payload={"repositories": ["some-repo"], "permissions": github.GitHubRepository.GITHUB_PERMISSIONS} + ) retry_count = {} -def test_constructor_uses_retry_sync(mocker): +@pytest.mark.asyncio +async def test_constructor_uses_retry_sync(mocker, token_context): global retry_count retry_count["fail_first"] = 0 @@ -80,34 +122,41 @@ def fail_first(*args, **kwargs): github_class_mock = mocker.patch.object(github, "GitHub", side_effect=fail_first) github_class_mock.__name__ = github_class_mock.name + mocker.patch.object(github, "Auth", side_effect=taskcluster.exceptions.TaskclusterFailure("disabled in tests")) mocker.patch.object(github, "_GITHUB_LIBRARY_SLEEP_TIME_KWARGS", {"delay_factor": 0.1}) - github.GitHubRepository("some-user", "some-repo", "some-token") + repo = github.GitHubRepository(token_context, "some-user", "some-repo") + await repo._get_repository() assert retry_count["fail_first"] == 2 -def test_get_definition(github_repository): - github_repository._github_repository.as_dict.return_value = {"foo": "bar"} - assert github_repository.definition == {"foo": "bar"} - github_repository._github_repository.as_dict.assert_called_once_with() +@pytest.mark.asyncio +async def test_get_definition(github_repository): + repository = await github_repository._get_repository() + repository.as_dict.return_value = {"foo": "bar"} + assert await github_repository.get_definition() == {"foo": "bar"} + repository.as_dict.assert_called_once_with() @pytest.mark.asyncio async def test_get_commit(github_repository): await github_repository.get_commit("somehash") - github_repository._github_repository.commit.assert_called_once_with("somehash") + repository = await github_repository._get_repository() + repository.commit.assert_called_once_with("somehash") @pytest.mark.asyncio async def test_get_pull_request(github_repository): await github_repository.get_pull_request(1) - github_repository._github_repository.pull_request.assert_called_once_with(1) + repository = await github_repository._get_repository() + repository.pull_request.assert_called_once_with(1) @pytest.mark.asyncio async def test_get_release(github_repository): await github_repository.get_release("some-tag") - github_repository._github_repository.release_from_tag.assert_called_once_with("some-tag") + repository = await github_repository._get_repository() + repository.release_from_tag.assert_called_once_with("some-tag") @pytest.mark.parametrize( @@ -128,7 +177,8 @@ async def test_get_release(github_repository): ) @pytest.mark.asyncio async def test_get_tag_hash(github_repository, tags, raises, expected): - github_repository._github_repository.tags.return_value = tags + repository = await github_repository._get_repository() + repository.tags.return_value = tags if raises: with pytest.raises(ValueError):