From 63433b0e95402e7f2b93bd9caeb47afbbc4ff3e8 Mon Sep 17 00:00:00 2001 From: Tony Garcia Date: Wed, 18 Feb 2026 10:36:40 -0600 Subject: [PATCH] Add redact feature for file uploads Provide functions to strip sensitive data (tokens, credentials, pull secrets) from content before uploading to the DCI control server. A helper determines whether redaction is active, checking an environment variable first and falling back to a caller-provided default. This lets callers enable or disable redaction globally The API offers a file path and a string variant, both built on top of a line-by-line streaming. This keeps memory usage constant regardless of file size, which matters for large log and configuration files. Custom regex patterns can be supplied via environment variable, replacing the built-in defaults, to support project-specific secrets. Assisted-by: Claude Gitleaks-Sign: OC4zMC4wfDIwMjYtMDgtMTBUMjI6MTM6MTF8YzBlYjhlYjQ5YjVmMTk3N2VlOTJiZjk0NmM2MmZmZDUwMTY5ZDU4Yw== Gitleaks-Hash: 6f1eb83afc605505c0daf1dae6905745408162f062db4334044236205d3a36b8 --- dciclient/v1/api/file.py | 4 +- dciclient/v1/api/redact.py | 108 ++++++++++++++++ python-dciclient.spec | 5 +- tests/test_redact.py | 256 +++++++++++++++++++++++++++++++++++++ 4 files changed, 370 insertions(+), 3 deletions(-) create mode 100644 dciclient/v1/api/redact.py create mode 100644 tests/test_redact.py diff --git a/dciclient/v1/api/file.py b/dciclient/v1/api/file.py index 79ce877b..7ceabeea 100644 --- a/dciclient/v1/api/file.py +++ b/dciclient/v1/api/file.py @@ -1,6 +1,6 @@ # -*- encoding: utf-8 -*- # -# Copyright 2015-2016 Red Hat, Inc. +# Copyright 2015-2026 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -55,7 +55,7 @@ def create( "DCI-JOBSTATE-ID": jobstate_id, "DCI-MD5": md5, "DCI-JOB-ID": job_id, - "Content-Type": mime + "Content-Type": mime, } headers = utils.sanitize_kwargs(**headers) uri = "%s/%s" % (context.dci_cs_api, RESOURCE) diff --git a/dciclient/v1/api/redact.py b/dciclient/v1/api/redact.py new file mode 100644 index 00000000..492335a3 --- /dev/null +++ b/dciclient/v1/api/redact.py @@ -0,0 +1,108 @@ +# -*- encoding: utf-8 -*- +# +# Copyright 2026 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import re +from io import StringIO + +logger = logging.getLogger(__name__) + +REDACT_REPLACEMENT = "***REDACTED***" + +DEFAULT_REDACT_PATTERNS = [ + # GitHub tokens + # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats + (r"gh[hopru]_[a-zA-Z0-9]{36}", REDACT_REPLACEMENT), + (r"github_pat_[a-zA-Z0-9_]{82}", REDACT_REPLACEMENT), + (r"ghs_[A-Za-z0-9\.\-_]{36,}", REDACT_REPLACEMENT), + # DCI credentials + (r"remoteci/[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}", REDACT_REPLACEMENT), + (r"DCI\.[a-zA-Z0-9]{60}", REDACT_REPLACEMENT), + # Pull secrets (JSON and YAML) + (r'("auth"\s*:\s*")[^"]*(")', r"\1" + REDACT_REPLACEMENT + r"\2"), + (r"(auth:\s+)\S+", r"\1" + REDACT_REPLACEMENT), +] + + +def should_redact(redact=False): + """Determine whether redaction should be applied. + + The DCI_REDACT environment variable takes precedence if set. + Otherwise the redact parameter value is used. + """ + env_val = os.environ.get("DCI_REDACT") + if env_val is not None: + return env_val.lower() in ("1", "true", "yes") + return redact + + +def get_patterns(): + """Return a list of compiled patterns to redact. + + When DCI_REDACT_PATTERNS is set, its colon-separated values are used + as patterns (each replaced with REDACT_REPLACEMENT). Custom patterns + replace defaults entirely. + """ + env_val = os.environ.get("DCI_REDACT_PATTERNS") + if env_val is not None: + raw_patterns = [(p, REDACT_REPLACEMENT) for p in env_val.split(":") if p] + else: + raw_patterns = DEFAULT_REDACT_PATTERNS + + compiled = [] + for pattern_str, replacement in raw_patterns: + try: + compiled.append((re.compile(pattern_str, re.MULTILINE), replacement)) + except re.error as e: + logger.warning("Invalid redact pattern %r: %s", pattern_str, e) + return compiled + + +def redact_stream(src, dst): + """Redact sensitive patterns from src writing the result to dst. + + Reads line by line from src and applies redaction patterns to each + line before writing to dst. Both src and dst must be text-mode + file-like objects. + """ + patterns = get_patterns() + for line in src: + for pattern, replacement in patterns: + line = pattern.sub(replacement, line) + dst.write(line) + + +def redact_file(input_path, output_path): + """Redact sensitive patterns from a file writing the result to another file. + + Opens input_path in text mode tolerating non-UTF-8 content, like binary + fragments in log files. + """ + with open(input_path, "r", encoding="utf-8", errors="ignore") as src, \ + open(output_path, "w", encoding="utf-8") as dst: + redact_stream(src, dst) + + +def redact_content(content): + """Redact sensitive patterns from a string. + + Returns a new string with all matching patterns replaced. + """ + src = StringIO(content) + dst = StringIO() + redact_stream(src, dst) + return dst.getvalue() diff --git a/python-dciclient.spec b/python-dciclient.spec index f6a87d0c..0fc15643 100644 --- a/python-dciclient.spec +++ b/python-dciclient.spec @@ -2,7 +2,7 @@ %global summary Python client for DCI control server for the remote CIs Name: python-%{srcname} -Version: 4.1.0 +Version: 4.2.0 Release: 1.VERS%{?dist} Summary: %{summary} @@ -59,6 +59,9 @@ install -d %{buildroot}%{_bindir} %{buildroot}%{_datadir}/python-%{srcname} %changelog +* Fri Jun 21 2026 Tony Garcia 4.2.0-1 +- Add redact to create file + * Tue Sep 09 2025 Yassine Lamgarchal 4.1.0-1 - add job-search to the cli diff --git a/tests/test_redact.py b/tests/test_redact.py new file mode 100644 index 00000000..da085941 --- /dev/null +++ b/tests/test_redact.py @@ -0,0 +1,256 @@ +# -*- encoding: utf-8 -*- +# +# Copyright 2026 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from io import StringIO + +from dciclient.v1.api import redact + +REDACTED = redact.REDACT_REPLACEMENT + + +class TestShouldRedact: + def test_default(self): + assert redact.should_redact() is False + + def test_explicit_true(self): + assert redact.should_redact(redact=True) is True + + def test_explicit_false(self): + assert redact.should_redact(redact=False) is False + + def test_env_var_true_overrides_param(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT", "1") + assert redact.should_redact(redact=False) is True + + def test_env_var_false_overrides_param(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT", "false") + assert redact.should_redact(redact=True) is False + + def test_env_var_zero(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT", "0") + assert redact.should_redact() is False + + def test_env_var_no(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT", "no") + assert redact.should_redact() is False + + def test_env_var_yes(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT", "yes") + assert redact.should_redact() is True + + +class TestGetPatterns: + def test_returns_default_patterns(self): + patterns = redact.get_patterns() + assert len(patterns) == len(redact.DEFAULT_REDACT_PATTERNS) + + def test_env_patterns_override_defaults(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT_PATTERNS", r"secret_\d+:token_\w+") + patterns = redact.get_patterns() + assert len(patterns) == 2 + + def test_env_empty_segments_ignored(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT_PATTERNS", r"secret_\d+::token_\w+:") + patterns = redact.get_patterns() + assert len(patterns) == 2 + + def test_invalid_pattern_warns(self, monkeypatch, caplog): + monkeypatch.setenv("DCI_REDACT_PATTERNS", "[invalid") + import logging + + with caplog.at_level(logging.WARNING): + patterns = redact.get_patterns() + assert len(patterns) == 0 + assert "Invalid redact pattern" in caplog.text + + +class TestRedactStream: + def test_redacts_token(self): + token = "ghp_" + "a" * 36 + src = StringIO("token=%s\n" % token) + dst = StringIO() + redact.redact_stream(src, dst) + assert token not in dst.getvalue() + assert REDACTED in dst.getvalue() + + def test_no_match_unchanged(self): + src = StringIO("nothing sensitive here\n") + dst = StringIO() + redact.redact_stream(src, dst) + assert dst.getvalue() == "nothing sensitive here\n" + + def test_empty_input(self): + src = StringIO("") + dst = StringIO() + redact.redact_stream(src, dst) + assert dst.getvalue() == "" + + def test_multiple_lines(self): + token = "ghp_" + "a" * 36 + src = StringIO("line1\ntoken=%s\nline3\n" % token) + dst = StringIO() + redact.redact_stream(src, dst) + output = dst.getvalue() + assert token not in output + assert "line1\n" in output + assert "\nline3\n" in output + + def test_multiple_tokens_across_lines(self): + token1 = "ghp_" + "a" * 36 + token2 = "ghp_" + "b" * 36 + src = StringIO("%s\n%s\n" % (token1, token2)) + dst = StringIO() + redact.redact_stream(src, dst) + output = dst.getvalue() + assert token1 not in output + assert token2 not in output + + def test_custom_patterns(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT_PATTERNS", "PASSWORD=\\S+:SECRET=\\S+") + src = StringIO("PASSWORD=hunter2 SECRET=monkey123\n") + dst = StringIO() + redact.redact_stream(src, dst) + output = dst.getvalue() + assert "hunter2" not in output + assert "monkey123" not in output + assert REDACTED in output + + +class TestRedactFile: + def test_redacts_token(self, tmp_path): + token = "ghp_" + "a" * 36 + input_file = tmp_path / "input.log" + output_file = tmp_path / "output.log" + input_file.write_text("token=%s\n" % token) + redact.redact_file(str(input_file), str(output_file)) + result = output_file.read_text() + assert token not in result + assert REDACTED in result + + def test_empty_file(self, tmp_path): + input_file = tmp_path / "input.log" + output_file = tmp_path / "output.log" + input_file.write_text("") + redact.redact_file(str(input_file), str(output_file)) + assert output_file.read_text() == "" + + def test_multiple_tokens_across_lines(self, tmp_path): + token1 = "ghp_" + "a" * 36 + token2 = "ghp_" + "b" * 36 + input_file = tmp_path / "input.log" + output_file = tmp_path / "output.log" + input_file.write_text("%s\n%s\n" % (token1, token2)) + redact.redact_file(str(input_file), str(output_file)) + result = output_file.read_text() + assert token1 not in result + assert token2 not in result + + def test_binary_tolerant(self, tmp_path): + input_file = tmp_path / "input.log" + output_file = tmp_path / "output.log" + input_file.write_bytes("valid line\n\x80\x81\xff\xfe\nmore text\n".encode("latin-1")) + redact.redact_file(str(input_file), str(output_file)) + result = output_file.read_text() + assert "valid line" in result + assert "more text" in result + + def test_no_match_unchanged(self, tmp_path): + input_file = tmp_path / "input.log" + output_file = tmp_path / "output.log" + input_file.write_text("nothing sensitive here\n") + redact.redact_file(str(input_file), str(output_file)) + assert output_file.read_text() == "nothing sensitive here\n" + + +class TestRedactContent: + def test_redact_ghp_token(self): + token = "ghp_" + "a" * 36 + result = redact.redact_content("token: %s end" % token) + assert token not in result + assert REDACTED in result + assert result == "token: %s end" % REDACTED + + def test_redact_github_pat_token(self): + token = "github_pat_" + "A" * 82 + result = redact.redact_content("auth=%s" % token) + assert token not in result + assert REDACTED in result + + def test_redact_gho_token(self): + token = "gho_" + "x" * 36 + result = redact.redact_content("GITHUB_TOKEN=%s" % token) + assert token not in result + assert REDACTED in result + + def test_redact_remoteci_uuid(self): + remoteci = "remoteci/12345678-1234-1234-1234-123456789abc" + result = redact.redact_content("client_id=%s" % remoteci) + assert remoteci not in result + assert REDACTED in result + + def test_redact_dci_secret(self): + secret = "DCI." + "a" * 60 + result = redact.redact_content("api_secret=%s" % secret) + assert secret not in result + assert REDACTED in result + + def test_redact_pull_secret_json(self): + content = '{"auths": {"registry.example.com": {"auth": "dXNlcjpwYXNz"}}}' + result = redact.redact_content(content) + assert "dXNlcjpwYXNz" not in result + assert '"auth": "%s"' % REDACTED in result + + def test_redact_pull_secret_json_with_spaces(self): + result = redact.redact_content('"auth" : "secretvalue"') + assert "secretvalue" not in result + assert REDACTED in result + + def test_redact_pull_secret_yaml(self): + result = redact.redact_content("auth: mysecrettoken") + assert "mysecrettoken" not in result + assert "auth: %s" % REDACTED in result + + def test_returns_str(self): + token = "ghp_" + "a" * 36 + result = redact.redact_content("token: %s" % token) + assert isinstance(result, str) + + def test_no_match_unchanged(self): + assert redact.redact_content("nothing sensitive here") == "nothing sensitive here" + + def test_empty_string(self): + assert redact.redact_content("") == "" + + def test_multiple_matches(self): + token1 = "ghp_" + "a" * 36 + token2 = "ghp_" + "b" * 36 + result = redact.redact_content("%s and %s" % (token1, token2)) + assert token1 not in result + assert token2 not in result + + def test_multiline_content(self): + token = "ghp_" + "c" * 36 + result = redact.redact_content("line1\ntoken=%s\nline3" % token) + assert token not in result + assert "line1\n" in result + assert "\nline3" in result + + def test_custom_patterns(self, monkeypatch): + monkeypatch.setenv("DCI_REDACT_PATTERNS", "PASSWORD=\\S+:SECRET=\\S+") + result = redact.redact_content("PASSWORD=hunter2 something SECRET=monkey123 else") + assert "hunter2" not in result + assert "monkey123" not in result + assert REDACTED in result