diff --git a/dciclient/v1/api/file.py b/dciclient/v1/api/file.py index 79ce877b..7ceabeea 100644 --- a/dciclient/v1/api/file.py +++ b/dciclient/v1/api/file.py @@ -1,6 +1,6 @@ # -*- encoding: utf-8 -*- # -# Copyright 2015-2016 Red Hat, Inc. +# Copyright 2015-2026 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -55,7 +55,7 @@ def create( "DCI-JOBSTATE-ID": jobstate_id, "DCI-MD5": md5, "DCI-JOB-ID": job_id, - "Content-Type": mime + "Content-Type": mime, } headers = utils.sanitize_kwargs(**headers) uri = "%s/%s" % (context.dci_cs_api, RESOURCE) diff --git a/dciclient/v1/api/redact.py b/dciclient/v1/api/redact.py new file mode 100644 index 00000000..492335a3 --- /dev/null +++ b/dciclient/v1/api/redact.py @@ -0,0 +1,108 @@ +# -*- encoding: utf-8 -*- +# +# Copyright 2026 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import re +from io import StringIO + +logger = logging.getLogger(__name__) + +REDACT_REPLACEMENT = "***REDACTED***" + +DEFAULT_REDACT_PATTERNS = [ + # GitHub tokens + # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats + (r"gh[hopru]_[a-zA-Z0-9]{36}", REDACT_REPLACEMENT), + (r"github_pat_[a-zA-Z0-9_]{82}", REDACT_REPLACEMENT), + (r"ghs_[A-Za-z0-9\.\-_]{36,}", REDACT_REPLACEMENT), + # DCI credentials + (r"remoteci/[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}", REDACT_REPLACEMENT), + (r"DCI\.[a-zA-Z0-9]{60}", REDACT_REPLACEMENT), + # Pull secrets (JSON and YAML) + (r'("auth"\s*:\s*")[^"]*(")', r"\1" + REDACT_REPLACEMENT + r"\2"), + (r"(auth:\s+)\S+", r"\1" + REDACT_REPLACEMENT), +] + + +def should_redact(redact=False): + """Determine whether redaction should be applied. + + The DCI_REDACT environment variable takes precedence if set. + Otherwise the redact parameter value is used. + """ + env_val = os.environ.get("DCI_REDACT") + if env_val is not None: + return env_val.lower() in ("1", "true", "yes") + return redact + + +def get_patterns(): + """Return a list of compiled patterns to redact. + + When DCI_REDACT_PATTERNS is set, its colon-separated values are used + as patterns (each replaced with REDACT_REPLACEMENT). Custom patterns + replace defaults entirely. + """ + env_val = os.environ.get("DCI_REDACT_PATTERNS") + if env_val is not None: + raw_patterns = [(p, REDACT_REPLACEMENT) for p in env_val.split(":") if p] + else: + raw_patterns = DEFAULT_REDACT_PATTERNS + + compiled = [] + for pattern_str, replacement in raw_patterns: + try: + compiled.append((re.compile(pattern_str, re.MULTILINE), replacement)) + except re.error as e: + logger.warning("Invalid redact pattern %r: %s", pattern_str, e) + return compiled + + +def redact_stream(src, dst): + """Redact sensitive patterns from src writing the result to dst. + + Reads line by line from src and applies redaction patterns to each + line before writing to dst. Both src and dst must be text-mode + file-like objects. + """ + patterns = get_patterns() + for line in src: + for pattern, replacement in patterns: + line = pattern.sub(replacement, line) + dst.write(line) + + +def redact_file(input_path, output_path): + """Redact sensitive patterns from a file writing the result to another file. + + Opens input_path in text mode tolerating non-UTF-8 content, like binary + fragments in log files. + """ + with open(input_path, "r", encoding="utf-8", errors="ignore") as src, \ + open(output_path, "w", encoding="utf-8") as dst: + redact_stream(src, dst) + + +def redact_content(content): + """Redact sensitive patterns from a string. + + Returns a new string with all matching patterns replaced. + """ + src = StringIO(content) + dst = StringIO() + redact_stream(src, dst) + return dst.getvalue() diff --git a/python-dciclient.spec b/python-dciclient.spec index f6a87d0c..31fc4ef9 100644 --- a/python-dciclient.spec +++ b/python-dciclient.spec @@ -2,7 +2,7 @@ %global summary Python client for DCI control server for the remote CIs Name: python-%{srcname} -Version: 4.1.0 +Version: 4.2.0 Release: 1.VERS%{?dist} Summary: %{summary} @@ -59,6 +59,9 @@ install -d %{buildroot}%{_bindir} %{buildroot}%{_datadir}/python-%{srcname} %changelog +* Fri Aug 21 2026 Tony Garcia 4.2.0-1 +- Add redact to create file + * Tue Sep 09 2025 Yassine Lamgarchal 4.1.0-1 - add job-search to the cli diff --git a/tests/test_redact.py b/tests/test_redact.py new file mode 100644 index 00000000..da085941 --- /dev/null +++ b/tests/test_redact.py @@ -0,0 +1,256 @@ +# -*- encoding: utf-8 -*- +# +# Copyright 2026 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from io import StringIO + +from dciclient.v1.api import redact + +REDACTED = redact.REDACT_REPLACEMENT + + +class TestShouldRedact: + def test_default(self): + assert redact.should_redact() is False + + def test_explicit_true(self): + assert redact.should_redact(redact=True) is True + + def test_explicit_false(self): + assert redact.should_redact(redact=False) is False + + def test_env_var_true_overrides_param(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT", "1") + assert redact.should_redact(redact=False) is True + + def test_env_var_false_overrides_param(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT", "false") + assert redact.should_redact(redact=True) is False + + def test_env_var_zero(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT", "0") + assert redact.should_redact() is False + + def test_env_var_no(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT", "no") + assert redact.should_redact() is False + + def test_env_var_yes(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT", "yes") + assert redact.should_redact() is True + + +class TestGetPatterns: + def test_returns_default_patterns(self): + patterns = redact.get_patterns() + assert len(patterns) == len(redact.DEFAULT_REDACT_PATTERNS) + + def test_env_patterns_override_defaults(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT_PATTERNS", r"secret_\d+:token_\w+") + patterns = redact.get_patterns() + assert len(patterns) == 2 + + def test_env_empty_segments_ignored(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT_PATTERNS", r"secret_\d+::token_\w+:") + patterns = redact.get_patterns() + assert len(patterns) == 2 + + def test_invalid_pattern_warns(self, monkeypatch, caplog): + monkeypatch.setenv("DCI_REDACT_PATTERNS", "[invalid") + import logging + + with caplog.at_level(logging.WARNING): + patterns = redact.get_patterns() + assert len(patterns) == 0 + assert "Invalid redact pattern" in caplog.text + + +class TestRedactStream: + def test_redacts_token(self): + token = "ghp_" + "a" * 36 + src = StringIO("token=%s\n" % token) + dst = StringIO() + redact.redact_stream(src, dst) + assert token not in dst.getvalue() + assert REDACTED in dst.getvalue() + + def test_no_match_unchanged(self): + src = StringIO("nothing sensitive here\n") + dst = StringIO() + redact.redact_stream(src, dst) + assert dst.getvalue() == "nothing sensitive here\n" + + def test_empty_input(self): + src = StringIO("") + dst = StringIO() + redact.redact_stream(src, dst) + assert dst.getvalue() == "" + + def test_multiple_lines(self): + token = "ghp_" + "a" * 36 + src = StringIO("line1\ntoken=%s\nline3\n" % token) + dst = StringIO() + redact.redact_stream(src, dst) + output = dst.getvalue() + assert token not in output + assert "line1\n" in output + assert "\nline3\n" in output + + def test_multiple_tokens_across_lines(self): + token1 = "ghp_" + "a" * 36 + token2 = "ghp_" + "b" * 36 + src = StringIO("%s\n%s\n" % (token1, token2)) + dst = StringIO() + redact.redact_stream(src, dst) + output = dst.getvalue() + assert token1 not in output + assert token2 not in output + + def test_custom_patterns(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT_PATTERNS", "PASSWORD=\\S+:SECRET=\\S+") + src = StringIO("PASSWORD=hunter2 SECRET=monkey123\n") + dst = StringIO() + redact.redact_stream(src, dst) + output = dst.getvalue() + assert "hunter2" not in output + assert "monkey123" not in output + assert REDACTED in output + + +class TestRedactFile: + def test_redacts_token(self, tmp_path): + token = "ghp_" + "a" * 36 + input_file = tmp_path / "input.log" + output_file = tmp_path / "output.log" + input_file.write_text("token=%s\n" % token) + redact.redact_file(str(input_file), str(output_file)) + result = output_file.read_text() + assert token not in result + assert REDACTED in result + + def test_empty_file(self, tmp_path): + input_file = tmp_path / "input.log" + output_file = tmp_path / "output.log" + input_file.write_text("") + redact.redact_file(str(input_file), str(output_file)) + assert output_file.read_text() == "" + + def test_multiple_tokens_across_lines(self, tmp_path): + token1 = "ghp_" + "a" * 36 + token2 = "ghp_" + "b" * 36 + input_file = tmp_path / "input.log" + output_file = tmp_path / "output.log" + input_file.write_text("%s\n%s\n" % (token1, token2)) + redact.redact_file(str(input_file), str(output_file)) + result = output_file.read_text() + assert token1 not in result + assert token2 not in result + + def test_binary_tolerant(self, tmp_path): + input_file = tmp_path / "input.log" + output_file = tmp_path / "output.log" + input_file.write_bytes("valid line\n\x80\x81\xff\xfe\nmore text\n".encode("latin-1")) + redact.redact_file(str(input_file), str(output_file)) + result = output_file.read_text() + assert "valid line" in result + assert "more text" in result + + def test_no_match_unchanged(self, tmp_path): + input_file = tmp_path / "input.log" + output_file = tmp_path / "output.log" + input_file.write_text("nothing sensitive here\n") + redact.redact_file(str(input_file), str(output_file)) + assert output_file.read_text() == "nothing sensitive here\n" + + +class TestRedactContent: + def test_redact_ghp_token(self): + token = "ghp_" + "a" * 36 + result = redact.redact_content("token: %s end" % token) + assert token not in result + assert REDACTED in result + assert result == "token: %s end" % REDACTED + + def test_redact_github_pat_token(self): + token = "github_pat_" + "A" * 82 + result = redact.redact_content("auth=%s" % token) + assert token not in result + assert REDACTED in result + + def test_redact_gho_token(self): + token = "gho_" + "x" * 36 + result = redact.redact_content("GITHUB_TOKEN=%s" % token) + assert token not in result + assert REDACTED in result + + def test_redact_remoteci_uuid(self): + remoteci = "remoteci/12345678-1234-1234-1234-123456789abc" + result = redact.redact_content("client_id=%s" % remoteci) + assert remoteci not in result + assert REDACTED in result + + def test_redact_dci_secret(self): + secret = "DCI." + "a" * 60 + result = redact.redact_content("api_secret=%s" % secret) + assert secret not in result + assert REDACTED in result + + def test_redact_pull_secret_json(self): + content = '{"auths": {"registry.example.com": {"auth": "dXNlcjpwYXNz"}}}' + result = redact.redact_content(content) + assert "dXNlcjpwYXNz" not in result + assert '"auth": "%s"' % REDACTED in result + + def test_redact_pull_secret_json_with_spaces(self): + result = redact.redact_content('"auth" : "secretvalue"') + assert "secretvalue" not in result + assert REDACTED in result + + def test_redact_pull_secret_yaml(self): + result = redact.redact_content("auth: mysecrettoken") + assert "mysecrettoken" not in result + assert "auth: %s" % REDACTED in result + + def test_returns_str(self): + token = "ghp_" + "a" * 36 + result = redact.redact_content("token: %s" % token) + assert isinstance(result, str) + + def test_no_match_unchanged(self): + assert redact.redact_content("nothing sensitive here") == "nothing sensitive here" + + def test_empty_string(self): + assert redact.redact_content("") == "" + + def test_multiple_matches(self): + token1 = "ghp_" + "a" * 36 + token2 = "ghp_" + "b" * 36 + result = redact.redact_content("%s and %s" % (token1, token2)) + assert token1 not in result + assert token2 not in result + + def test_multiline_content(self): + token = "ghp_" + "c" * 36 + result = redact.redact_content("line1\ntoken=%s\nline3" % token) + assert token not in result + assert "line1\n" in result + assert "\nline3" in result + + def test_custom_patterns(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT_PATTERNS", "PASSWORD=\\S+:SECRET=\\S+") + result = redact.redact_content("PASSWORD=hunter2 something SECRET=monkey123 else") + assert "hunter2" not in result + assert "monkey123" not in result + assert REDACTED in result