From d90e63682ce1f9647a289839ea9fd175ef7ea1ab Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Wed, 2 Sep 2026 11:36:21 -0400 Subject: [PATCH 1/2] Bug 2068710 - Bump taskcluster dependency to at least 106.0.0 This is the first version that implements the auth service's ability to hand out Github tokens. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 12bbdd9e..b29a500d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "jsonschema[format-nongpl]", "json-e>=2.5.0", "PyYAML", - "taskcluster>=40", + "taskcluster>=106", "taskcluster-taskgraph", ] From 1afe69b7d60c9acdaf86feeccfb9c6ae3d5fef41 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Wed, 2 Sep 2026 11:48:35 -0400 Subject: [PATCH 2/2] Bug 2068710 - Use Taskcluster Auth service to obtain Github token if possible If there's an error (e.g missing scopes or app not installed on the target repo), we fallback to the existing github_oauth_token. --- scriptworker.yaml.tmpl | 12 +++- src/scriptworker/constants.py | 5 ++ src/scriptworker/cot/verify.py | 46 +++++++-------- src/scriptworker/github.py | 103 ++++++++++++++++++++++++++++----- src/scriptworker/task.py | 2 +- tests/test_cot_verify.py | 103 +++++++++++++++++++-------------- tests/test_github.py | 86 +++++++++++++++++++++------ 7 files changed, 251 insertions(+), 106 deletions(-) diff --git a/scriptworker.yaml.tmpl b/scriptworker.yaml.tmpl index 79357b4b..5514d284 100644 --- a/scriptworker.yaml.tmpl +++ b/scriptworker.yaml.tmpl @@ -45,9 +45,15 @@ verify_cot_signature: false # Chain of Trust job type, e.g. signing cot_job_type: scriptworker cot_product: firefox -# Calls to Github API are limited to 60 an hour. Using an API token allows to raise the limit to -# 5000 per hour. https://developer.github.com/v3/#rate-limiting -github_oauth_token: somegithubtoken + +# Scriptworker first tries to obtain a repository scoped token from Taskcluster's auth service, +# using the Github App registered under this name. +# github_app_name: read +# +# `github_oauth_token` is used as a fallback if that fails (e.g. missing scopes or app not +# configured). Without either, calls to the Github API are unauthenticated and limited to 60 an +# hour. See https://developer.github.com/v3/#rate-limiting +# github_oauth_token: somegithubtoken #----------------------------------------------------------------------------------------------- diff --git a/src/scriptworker/constants.py b/src/scriptworker/constants.py index 013620bd..d38f0ef1 100644 --- a/src/scriptworker/constants.py +++ b/src/scriptworker/constants.py @@ -79,7 +79,12 @@ "max_chain_length": 20, # Calls to Github API are limited to 60 an hour. Using an API token allows to raise the limit to # 5000 per hour. https://developer.github.com/v3/#rate-limiting + # Scriptworker first tries to obtain a repository scoped token from Taskcluster's auth + # service, falling back to this token if that fails. "github_oauth_token": "", + # The name of the Github App registered with Taskcluster's auth service, used to obtain + # a repository scoped token via `auth.githubRepoToken`. + "github_app_name": "read", # ed25519 settings "ed25519_private_key_path": "...", "ed25519_public_keys": immutabledict( diff --git a/src/scriptworker/cot/verify.py b/src/scriptworker/cot/verify.py index 1a33481f..373f9a72 100644 --- a/src/scriptworker/cot/verify.py +++ b/src/scriptworker/cot/verify.py @@ -20,7 +20,6 @@ from copy import deepcopy from urllib.parse import urlparse -import aiohttp import dictdiffer import jsone from immutabledict import immutabledict @@ -40,7 +39,7 @@ from scriptworker.context import Context from scriptworker.ed25519 import ed25519_public_key_from_string, verify_ed25519_signature from scriptworker.exceptions import BaseDownloadError, CoTError, ScriptWorkerEd25519Error -from scriptworker.github import GitHubRepository, extract_github_repo_full_name, extract_github_repo_owner_and_name, extract_github_repo_ssh_url +from scriptworker.github import GitHubRepository, extract_github_repo_full_name, extract_github_repo_owner_and_name, extract_github_repo_ssh_url, is_github_url from scriptworker.log import contextual_log_handler, get_chain_of_trust_log_filename from scriptworker.task import ( get_action_callback_name, @@ -1118,7 +1117,7 @@ async def _get_additional_github_releases_jsone_context(decision_link): repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url) tag_name = get_revision(task, source_env_prefix) - github_repo = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"]) + github_repo = GitHubRepository(context, repo_owner, repo_name) release_data = await github_repo.get_release(tag_name) # The release data expose by the API[1] is not the same as the original event[2]. That's why @@ -1200,17 +1199,16 @@ async def _get_additional_github_pull_request_jsone_context(decision_link): repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1) repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url) pull_request_number = get_pull_request_number(task, source_env_prefix) - token = context.config["github_oauth_token"] - github_repo = GitHubRepository(repo_owner, repo_name, token) - repo_definition = github_repo.definition + github_repo = GitHubRepository(context, repo_owner, repo_name) + repo_definition = await github_repo.get_definition() # We need to query the repository where the pull request was made to extract # pull request data. The pull request could be created on the same repo as # the commit, or an upstream repo. We can compare the base and head repo URLs # to infer where the pull request lives. if repo_definition["fork"] and base_repo_url != repo_url: - github_repo = GitHubRepository(owner=repo_definition["parent"]["owner"]["login"], repo_name=repo_definition["parent"]["name"], token=token) + github_repo = GitHubRepository(context, repo_definition["parent"]["owner"]["login"], repo_definition["parent"]["name"]) pull_request_data = await github_repo.get_pull_request(pull_request_number) # Even though pull_request_data['head']['repo']['pushed_at'] does exist, @@ -1245,7 +1243,7 @@ async def _get_additional_github_push_jsone_context(decision_link): repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url) commit_hash = get_revision(task, source_env_prefix) - github_repo = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"]) + github_repo = GitHubRepository(context, repo_owner, repo_name) commit_data = await github_repo.get_commit(commit_hash) committer = commit_data["committer"] or {} @@ -1399,16 +1397,11 @@ def build_taskcluster_yml_url(link): """ source_env_prefix = link.context.config["source_env_prefix"] repo_url = get_repo(link.task, source_env_prefix) - repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1) revision = get_revision(link.task, source_env_prefix) repo_parts = urlparse(repo_url) - if repo_parts.netloc == "github.com": - user, repo_name = extract_github_repo_owner_and_name(repo_url) - url = f"https://raw.githubusercontent.com/{user}/{repo_name}/{revision}/.taskcluster.yml" - elif repo_parts.netloc == "hg.mozilla.org": - url = f"{repo_parts.scheme}://{repo_parts.netloc}{repo_parts.path}/raw-file/{revision}/.taskcluster.yml" - else: + if repo_parts.netloc != "hg.mozilla.org": raise CoTError("Unsupported VCS server!") + url = f"{repo_parts.scheme}://{repo_parts.netloc}{repo_parts.path}/raw-file/{revision}/.taskcluster.yml" log.debug(f"{link.name} .taskcluster.yml is at {url}") return url @@ -1430,19 +1423,20 @@ async def get_in_tree_template(link): """ context = link.context - source_url = build_taskcluster_yml_url(link) - repo_url = get_repo(link.task, link.context.config["source_env_prefix"]) + source_env_prefix = context.config["source_env_prefix"] + repo_url = get_repo(link.task, source_env_prefix) + repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1) - auth = None - if ( - (repo_url.startswith(("ssh://", "git@github.com")) or any(vcs_rule.get("require_secret") for vcs_rule in context.config["trusted_vcs_rules"])) - and "github.com" in repo_url - and context.config.get("github_oauth_token") - ): - auth = aiohttp.BasicAuth(context.config["github_oauth_token"]) + if is_github_url(repo_url): + revision = get_revision(link.task, source_env_prefix) + repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url) + github_repo = GitHubRepository(context, repo_owner, repo_name) + content = await github_repo.get_file_contents(".taskcluster.yml", ref=revision) + return load_json_or_yaml(content, file_type="yaml") + + source_url = build_taskcluster_yml_url(link) url_hash = hashlib.sha1(source_url.encode("ascii")).hexdigest() - tmpl = await load_json_or_yaml_from_url(context, source_url, os.path.join(context.config["work_dir"], "{}_taskcluster.yml".format(url_hash)), auth=auth) - return tmpl + return await load_json_or_yaml_from_url(context, source_url, os.path.join(context.config["work_dir"], "{}_taskcluster.yml".format(url_hash))) def _get_action_from_actions_json(all_actions, callback_name): diff --git a/src/scriptworker/github.py b/src/scriptworker/github.py index 3acf7b04..2a8322e2 100644 --- a/src/scriptworker/github.py +++ b/src/scriptworker/github.py @@ -6,6 +6,8 @@ from github3 import GitHub from github3.exceptions import GitHubException +from taskcluster.aio import Auth +from taskcluster.exceptions import TaskclusterFailure from scriptworker.exceptions import ConfigError from scriptworker.utils import get_parts_of_url_path, get_single_item_from_sequence, retry_async_decorator, retry_request, retry_sync @@ -23,30 +25,96 @@ class GitHubRepository: """Wrapper around GitHub API. Used to access public data.""" - def __init__(self, owner, repo_name, token=""): - """Build the GitHub API URL which points to the definition of the repository. + GITHUB_PERMISSIONS = {"contents": "read", "metadata": "read", "pull_requests": "read"} + + def __init__(self, context, owner, repo_name): + """Store the repository coordinates. The github3 repository object is built lazily. Args: - owner (str): the owner's GitHub username + context (scriptworker.context.Context): the scriptworker context + owner (str): the owner of the repository repo_name (str): the name of the repository - token (str): the GitHub API token + + """ + self._context = context + self._owner = owner + self._repo_name = repo_name + self._repository_cache = None + self._repository_lock = asyncio.Lock() + + async def _get_repository(self): + """Build and cache the github3 repository object. Returns: - dict: a representation of the repo definition + github3.repos.repo.Repository: the github3 repository object """ - github = retry_sync(GitHub, kwargs={"token": token}, sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS) - self._github_repository = retry_sync(github.repository, args=(owner, repo_name), sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS) + async with self._repository_lock: + if self._repository_cache is None: + token = await self._get_token(self._context, self._owner, self._repo_name) + github = retry_sync(GitHub, kwargs={"token": token}, sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS) + self._repository_cache = retry_sync(github.repository, args=(self._owner, self._repo_name), sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS) + + return self._repository_cache + + async def _get_token(self, context, owner, repo_name): + """Get a repository scoped GitHub token from Taskcluster's auth service. + + Falls back to ``context.config["github_oauth_token"]`` if the auth service call + fails, e.g. because of missing scopes. + + Args: + context (scriptworker.context.Context): the scriptworker context + owner (str): the owner of the repository + repo_name (str): the name of the repository - @property - def definition(self): + Returns: + str: the scoped GitHub token, or the fallback token + + """ + if not context.credentials: + return context.config.get("github_oauth_token", "") + + try: + auth = Auth(options={"rootUrl": context.config["taskcluster_root_url"], "credentials": context.credentials}) + response = await auth.githubRepoToken( + context.config["github_app_name"], owner, payload={"repositories": [repo_name], "permissions": self.GITHUB_PERMISSIONS} + ) + return response["token"] + except TaskclusterFailure as e: + # TODO When opening a PR from a fork, we're guaranteed to hit this + # fallback as the task won't have auth service scopes for the repo + # fork. We'll need to improve this before we can stop depending on + # `github_oauth_token`. + log.warning(f"Could not obtain Github token from Taskcluster for {owner}/{repo_name}, falling back to `github_oauth_token`: {e}") + return context.config.get("github_oauth_token", "") + + async def get_definition(self): """Fetch the definition of the repository, exposed by the GitHub API. Returns: dict: a representation of the repo definition """ - return self._github_repository.as_dict() + repository = await self._get_repository() + return repository.as_dict() + + @retry_async_decorator(retry_exceptions=GitHubException) + async def get_file_contents(self, path, ref=None): + """Fetch the decoded contents of a file in the repository. + + Args: + path (str): the path to the file, relative to the repository root + ref (str, optional): the commit/branch/tag to read the file from. + Defaults to the repository's default branch. + + Returns: + str: the decoded contents of the file + + """ + repository = await self._get_repository() + contents = repository.file_contents(path, ref=ref) + return contents.decoded.decode("utf-8") @retry_async_decorator(retry_exceptions=GitHubException) async def get_commit(self, commit_hash): @@ -59,7 +127,8 @@ async def get_commit(self, commit_hash): dict: a representation of the commit """ - return self._github_repository.commit(commit_hash).as_dict() + repository = await self._get_repository() + return repository.commit(commit_hash).as_dict() @retry_async_decorator(retry_exceptions=GitHubException) async def get_pull_request(self, pull_request_number): @@ -72,7 +141,8 @@ async def get_pull_request(self, pull_request_number): dict: a representation of the pull request """ - return self._github_repository.pull_request(pull_request_number).as_dict() + repository = await self._get_repository() + return repository.pull_request(pull_request_number).as_dict() @retry_async_decorator(retry_exceptions=GitHubException) async def get_release(self, tag_name): @@ -85,7 +155,8 @@ async def get_release(self, tag_name): dict: a representation of the tag """ - return self._github_repository.release_from_tag(tag_name).as_dict() + repository = await self._get_repository() + return repository.release_from_tag(tag_name).as_dict() @retry_async_decorator(retry_exceptions=GitHubException) async def get_tag_hash(self, tag_name): @@ -98,8 +169,9 @@ async def get_tag_hash(self, tag_name): str: the commit hash linked by the tag """ + repository = await self._get_repository() tag_object = get_single_item_from_sequence( - sequence=self._github_repository.tags(), + sequence=repository.tags(), condition=lambda tag: tag.name == tag_name, no_item_error_message='No tag "{}" exist'.format(tag_name), too_many_item_error_message='Too many tags "{}" found'.format(tag_name), @@ -128,7 +200,8 @@ async def has_commit_landed_on_repository(self, context, revision): if not _is_git_full_hash(revision): revision = await self.get_tag_hash(tag_name=revision) - html_text = await _fetch_github_branch_commits_data(context, self._github_repository.html_url, revision) + repository = await self._get_repository() + html_text = await _fetch_github_branch_commits_data(context, repository.html_url, revision) # https://github.com/{repo_owner}/{repo_name}/branch_commits/{revision} just returns some \n # when the commit hasn't landed on the origin repo. Otherwise, some HTML data is returned - it diff --git a/src/scriptworker/task.py b/src/scriptworker/task.py index 19fc6660..7284c3ce 100644 --- a/src/scriptworker/task.py +++ b/src/scriptworker/task.py @@ -550,7 +550,7 @@ async def is_pull_request(context, task): if not revision and can_skip: continue - github_repository = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"]) + github_repository = GitHubRepository(context, repo_owner, repo_name) conditions.append(not await github_repository.has_commit_landed_on_repository(context, revision)) return any(conditions) diff --git a/tests/test_cot_verify.py b/tests/test_cot_verify.py index c5e1dd74..2083a007 100644 --- a/tests/test_cot_verify.py +++ b/tests/test_cot_verify.py @@ -9,9 +9,8 @@ import time from copy import deepcopy from functools import partial -from unittest.mock import MagicMock +from unittest.mock import ANY, MagicMock -import aiohttp import jsone import pytest from immutabledict import immutabledict @@ -325,7 +324,7 @@ def cot_load(string, is_path=False, parent_dir=None, **kwargs): elif string.endswith("actions.json"): return load_json_or_yaml(os.path.join(parent_dir, "actions.json"), is_path=True) else: - return load_json_or_yaml(string) + return load_json_or_yaml(string, **kwargs) cotv2_load = partial(cot_load, parent_dir=COTV2_DIR) @@ -1130,7 +1129,7 @@ async def get_release_mock(release_name, *args, **kwargs): context = await cotverify.populate_jsone_context(mobile_chain, mobile_github_release_link, mobile_github_release_link, tasks_for="github-release") - github_repo_class_mock.assert_called_once_with("mozilla-mobile", "reference-browser", "fakegithubtoken") + github_repo_class_mock.assert_called_once_with(ANY, "mozilla-mobile", "reference-browser") del context["as_slugid"] assert context == { "event": { @@ -1210,7 +1209,7 @@ async def get_commit_mock(commit_hash, *args, **kwargs): context = await cotverify.populate_jsone_context(mobile_chain, mobile_github_push_link, mobile_github_push_link, tasks_for="github-push") - github_repo_class_mock.assert_called_once_with("mozilla-mobile", "reference-browser", "fakegithubtoken") + github_repo_class_mock.assert_called_once_with(ANY, "mozilla-mobile", "reference-browser") del context["as_slugid"] assert context == { "event": { @@ -1287,7 +1286,11 @@ async def test_populate_jsone_context_github_pull_request( github_repo_mock = MagicMock() repo_definition = {"fork": True, "parent": {"name": "reference-browser", "owner": {"login": "mozilla-mobile"}}} repo_definition.update(extra_repo_definition) - github_repo_mock.definition = repo_definition + + async def get_definition_mock(): + return repo_definition + + github_repo_mock.get_definition = get_definition_mock mobile_github_pull_request_link.task["extra"]["tasks_for"] = tasks_for mobile_github_pull_request_link.task["payload"]["env"].update(extra_env) @@ -1320,10 +1323,10 @@ async def get_pull_request_mock(pull_request_number, *args, **kwargs): mobile_chain_pull_request, mobile_github_pull_request_link, mobile_github_pull_request_link, tasks_for=tasks_for ) - github_repo_class_mock.assert_any_call("JohanLorenzo", "reference-browser", "fakegithubtoken") + github_repo_class_mock.assert_any_call(ANY, "JohanLorenzo", "reference-browser") if expected_use_parent: - github_repo_class_mock.assert_any_call(owner="mozilla-mobile", repo_name="reference-browser", token="fakegithubtoken") + github_repo_class_mock.assert_any_call(ANY, "mozilla-mobile", "reference-browser") assert len(github_repo_class_mock.call_args_list) == 2 else: assert len(github_repo_class_mock.call_args_list) == 1 @@ -1569,12 +1572,13 @@ async def get_commit(self, commit): "committer": {"login": "Callek"}, } + async def get_file_contents(self, path, ref=None): + assert path == ".taskcluster.yml" + assert ref == "330ea928b42ff2403fc99cd3e596d13294fe8775" + with open(os.path.join(COTV4_DIR, "private_github_.taskcluster.yml")) as fh: + return fh.read() + async def mocked_load_url(context, url, path, parent_path=COTV4_DIR, **kwargs): - if path.endswith("taskcluster.yml"): - assert kwargs.get("auth") - assert isinstance(kwargs["auth"], aiohttp.BasicAuth) - assert kwargs["auth"].login == "fakegithubtoken" - return load_json_or_yaml(os.path.join(parent_path, "private_github_.taskcluster.yml"), is_path=True, file_type="yaml") raise NotImplementedError() mocker.patch.object(cotverify, "load_json_or_yaml_from_url", new=mocked_load_url) @@ -1594,50 +1598,53 @@ def test_build_taskcluster_yml_url_unknown_server(decision_link): @pytest.mark.asyncio -@pytest.mark.parametrize("use_auth", (None, True)) @pytest.mark.parametrize( - "source_repo,revision,expected_url", + "source_repo", ( - ( - "ssh://github.com/mozilla-mobile/mozilla-vpn-client", - "330ea928b42ff2403fc99cd3e596d13294fe8775", - "https://raw.githubusercontent.com/mozilla-mobile/mozilla-vpn-client/330ea928b42ff2403fc99cd3e596d13294fe8775/.taskcluster.yml", - ), - ( - "git@github.com:mozilla-mobile/mozilla-vpn-client", - "330ea928b42ff2403fc99cd3e596d13294fe8775", - "https://raw.githubusercontent.com/mozilla-mobile/mozilla-vpn-client/330ea928b42ff2403fc99cd3e596d13294fe8775/.taskcluster.yml", - ), - ( - "https://hg.mozilla.org/ci/taskgraph-try", - "a9afa8aa11cf1431d4e6ef06c2a08d19e271c6ea", - "https://hg.mozilla.org/ci/taskgraph-try/raw-file/a9afa8aa11cf1431d4e6ef06c2a08d19e271c6ea/.taskcluster.yml", - ), + "ssh://github.com/mozilla-mobile/mozilla-vpn-client", + "git@github.com:mozilla-mobile/mozilla-vpn-client", ), ) -async def test_get_in_tree_template_auth_morphing(vpn_chain, mocker, use_auth, source_repo, revision, expected_url): +async def test_get_in_tree_template_github(vpn_chain, mocker, source_repo): name = "decision" task_id = "VUTfOIPFQWaGHf7sIbgTEg" - if not use_auth: - del vpn_chain.context.config["github_oauth_token"] + revision = "330ea928b42ff2403fc99cd3e596d13294fe8775" link = cotverify.LinkOfTrust(vpn_chain.context, name, task_id) - async def mocked_load_url(context, url, path, parent_path=COTV2_DIR, **kwargs): + github_repo_mock = MagicMock() + + async def get_file_contents_mock(path, ref=None): + assert path == ".taskcluster.yml" + assert ref == revision + return "tasks: []\n" + + github_repo_mock.get_file_contents = get_file_contents_mock + github_repo_class_mock = mocker.patch.object(cotverify, "GitHubRepository", return_value=github_repo_mock) + mocker.patch.object(cotverify, "get_repo", new=lambda x, y: source_repo) + mocker.patch.object(cotverify, "get_revision", new=lambda x, y: revision) + + tmpl = await cotverify.get_in_tree_template(link) + + github_repo_class_mock.assert_called_once_with(vpn_chain.context, "mozilla-mobile", "mozilla-vpn-client") + assert tmpl == {"tasks": []} + + +@pytest.mark.asyncio +async def test_get_in_tree_template_hg(decision_link, mocker): + source_repo = "https://hg.mozilla.org/ci/taskgraph-try" + revision = "a9afa8aa11cf1431d4e6ef06c2a08d19e271c6ea" + expected_url = "https://hg.mozilla.org/ci/taskgraph-try/raw-file/a9afa8aa11cf1431d4e6ef06c2a08d19e271c6ea/.taskcluster.yml" + + async def mocked_load_url(context, url, path, **kwargs): assert url == expected_url - if use_auth and "github.com" in source_repo: - assert kwargs.get("auth") - assert isinstance(kwargs["auth"], aiohttp.BasicAuth) - assert kwargs["auth"].login == "fakegithubtoken" - return "some_template" - else: - assert not kwargs.get("auth") - return "some_template_no_auth" + return "some_template" mocker.patch.object(cotverify, "load_json_or_yaml_from_url", new=mocked_load_url) mocker.patch.object(cotverify, "get_repo", new=lambda x, y: source_repo) mocker.patch.object(cotverify, "get_revision", new=lambda x, y: revision) - await cotverify.get_in_tree_template(link) + tmpl = await cotverify.get_in_tree_template(decision_link) + assert tmpl == "some_template" @pytest.mark.asyncio @@ -1748,9 +1755,19 @@ async def test_get_pr_action_context_and_template(mocker, mobile_chain, github_p """Test that pr-action tasks generate correct context and template with pr-action specific values.""" mobile_chain.context.config["min_cot_version"] = 3 + class MockedGitHubRepository(object): + def __init__(self, *args, **kwargs): + pass + + async def get_file_contents(self, path, ref=None): + assert path == ".taskcluster.yml" + with open(os.path.join(COTV4_DIR, ".taskcluster.yml")) as fh: + return fh.read() + mocker.patch.object(cotverify, "load_json_or_yaml_from_url", new=cotv4_load_url) mocker.patch.object(swcontext, "load_json_or_yaml_from_url", new=cotv4_load_url) mocker.patch.object(cotverify, "load_json_or_yaml", new=cotv4_load) + mocker.patch.object(cotverify, "GitHubRepository", new=MockedGitHubRepository) mobile_chain.links = list(set([mobile_github_push_link, github_pr_action_link])) diff --git a/tests/test_github.py b/tests/test_github.py index 4d315f76..2464b01a 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -1,9 +1,10 @@ import asyncio from copy import copy from types import SimpleNamespace -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest +import taskcluster.exceptions from scriptworker import github from scriptworker.exceptions import ConfigError, ScriptWorkerRetryException @@ -31,7 +32,15 @@ def vpn_context(vpn_private_rw_context): @pytest.fixture(scope="function") -def github_repository(mocker): +def token_context(): + return SimpleNamespace( + config={"github_oauth_token": "fallback-token", "github_app_name": "read", "taskcluster_root_url": "https://tc.example.com"}, + credentials={"a": "b"}, + ) + + +@pytest.fixture(scope="function") +def github_repository(mocker, token_context): github_repository_mock = MagicMock() github_repository_mock.__name__ = "GithubRepositoryMock" github_repository_mock.html_url = "https://github.com/some-user/some-repo/" @@ -43,28 +52,61 @@ def github_repository(mocker): github_instance_mock.repository.return_value = github_repository_mock github_class_mock = mocker.patch.object(github, "GitHub", return_value=github_instance_mock) github_class_mock.__name__ = github_class_mock.name - yield github.GitHubRepository("some-user", "some-repo") + mocker.patch.object(github, "Auth", side_effect=taskcluster.exceptions.TaskclusterFailure("disabled in tests")) + yield github.GitHubRepository(token_context, "some-user", "some-repo") + + +@pytest.mark.asyncio +async def test_constructor(mocker, token_context): + github_instance_mock = MagicMock() + github_instance_mock.repository.__name__ = "github_instance_repository_mock" + github_class_mock = mocker.patch.object(github, "GitHub", return_value=github_instance_mock) + github_class_mock.__name__ = github_class_mock.name + mocker.patch.object(github, "Auth", side_effect=taskcluster.exceptions.TaskclusterFailure("disabled in tests")) + + repo = github.GitHubRepository(token_context, "some-user", "some-repo") + await repo._get_repository() + + github_class_mock.assert_called_once_with(token="fallback-token") + github_instance_mock.repository.assert_called_once_with("some-user", "some-repo") +@pytest.mark.asyncio @pytest.mark.parametrize( - "args, expected_class_kwargs", ((("some-user", "some-repo", "some-token"), {"token": "some-token"}), (("some-user", "some-repo"), {"token": ""})) + "raises, expected_token", + ( + (False, "scoped-token"), + (True, "fallback-token"), + ), ) -def test_constructor(mocker, args, expected_class_kwargs): +async def test_constructor_with_context(mocker, token_context, raises, expected_token): github_instance_mock = MagicMock() github_instance_mock.repository.__name__ = "github_instance_repository_mock" github_class_mock = mocker.patch.object(github, "GitHub", return_value=github_instance_mock) github_class_mock.__name__ = github_class_mock.name - github.GitHubRepository(*args) + auth_instance_mock = MagicMock() + if raises: + auth_instance_mock.githubRepoToken = AsyncMock(side_effect=taskcluster.exceptions.TaskclusterRestFailure("missing scopes", None, status_code=403)) + else: + auth_instance_mock.githubRepoToken = AsyncMock(return_value={"token": "scoped-token", "expires": "2020-01-01T00:00:00Z"}) + auth_class_mock = mocker.patch.object(github, "Auth", return_value=auth_instance_mock) - github_class_mock.assert_called_once_with(**expected_class_kwargs) - github_instance_mock.repository.assert_called_once_with("some-user", "some-repo") + repo = github.GitHubRepository(token_context, "some-user", "some-repo") + await repo._get_repository() + + github_class_mock.assert_called_once_with(token=expected_token) + auth_class_mock.assert_called_once_with(options={"rootUrl": "https://tc.example.com", "credentials": {"a": "b"}}) + auth_instance_mock.githubRepoToken.assert_called_once_with( + "read", "some-user", payload={"repositories": ["some-repo"], "permissions": github.GitHubRepository.GITHUB_PERMISSIONS} + ) retry_count = {} -def test_constructor_uses_retry_sync(mocker): +@pytest.mark.asyncio +async def test_constructor_uses_retry_sync(mocker, token_context): global retry_count retry_count["fail_first"] = 0 @@ -80,34 +122,41 @@ def fail_first(*args, **kwargs): github_class_mock = mocker.patch.object(github, "GitHub", side_effect=fail_first) github_class_mock.__name__ = github_class_mock.name + mocker.patch.object(github, "Auth", side_effect=taskcluster.exceptions.TaskclusterFailure("disabled in tests")) mocker.patch.object(github, "_GITHUB_LIBRARY_SLEEP_TIME_KWARGS", {"delay_factor": 0.1}) - github.GitHubRepository("some-user", "some-repo", "some-token") + repo = github.GitHubRepository(token_context, "some-user", "some-repo") + await repo._get_repository() assert retry_count["fail_first"] == 2 -def test_get_definition(github_repository): - github_repository._github_repository.as_dict.return_value = {"foo": "bar"} - assert github_repository.definition == {"foo": "bar"} - github_repository._github_repository.as_dict.assert_called_once_with() +@pytest.mark.asyncio +async def test_get_definition(github_repository): + repository = await github_repository._get_repository() + repository.as_dict.return_value = {"foo": "bar"} + assert await github_repository.get_definition() == {"foo": "bar"} + repository.as_dict.assert_called_once_with() @pytest.mark.asyncio async def test_get_commit(github_repository): await github_repository.get_commit("somehash") - github_repository._github_repository.commit.assert_called_once_with("somehash") + repository = await github_repository._get_repository() + repository.commit.assert_called_once_with("somehash") @pytest.mark.asyncio async def test_get_pull_request(github_repository): await github_repository.get_pull_request(1) - github_repository._github_repository.pull_request.assert_called_once_with(1) + repository = await github_repository._get_repository() + repository.pull_request.assert_called_once_with(1) @pytest.mark.asyncio async def test_get_release(github_repository): await github_repository.get_release("some-tag") - github_repository._github_repository.release_from_tag.assert_called_once_with("some-tag") + repository = await github_repository._get_repository() + repository.release_from_tag.assert_called_once_with("some-tag") @pytest.mark.parametrize( @@ -128,7 +177,8 @@ async def test_get_release(github_repository): ) @pytest.mark.asyncio async def test_get_tag_hash(github_repository, tags, raises, expected): - github_repository._github_repository.tags.return_value = tags + repository = await github_repository._get_repository() + repository.tags.return_value = tags if raises: with pytest.raises(ValueError):