From 5cbbd2434f0cce94d1be563d8e317c34f13a5718 Mon Sep 17 00:00:00 2001 From: Nikhil Arora Date: Sat, 29 Aug 2026 12:22:19 +0530 Subject: [PATCH] Add plugin pipeline and plugins Introduce a Plugin base (before_log/after_log/on_error) and three concrete plugins: ContextPlugin, RedactPlugin, and SamplingPlugin. Logger now accepts plugins, exposes .use(), runs before_log hooks (which may return None to drop a record), dispatches to transports, then runs after_log; plugin exceptions are caught and routed to the plugin's on_error so a broken plugin can't crash logging. Added unit tests, updated README/CHANGELOG/CONTRIBUTING, bumped version to 0.1.3, and adjusted exports in __init__.py and a minor HTTPTransport doc tweak. --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- CHANGELOG.md | 18 ++++-- CONTRIBUTING.md | 5 +- README.md | 32 +++++++-- logquill/__init__.py | 10 ++- logquill/context_plugin.py | 20 ++++++ logquill/http_transport.py | 2 +- logquill/logger.py | 32 +++++++++ logquill/plugin.py | 22 +++++++ logquill/redact_plugin.py | 28 ++++++++ logquill/sampling_plugin.py | 20 ++++++ pyproject.toml | 2 +- tests/test_context_plugin.py | 20 ++++++ tests/test_plugin.py | 107 +++++++++++++++++++++++++++++++ tests/test_redact_plugin.py | 31 +++++++++ tests/test_sampling_plugin.py | 32 +++++++++ 16 files changed, 367 insertions(+), 16 deletions(-) create mode 100644 logquill/context_plugin.py create mode 100644 logquill/plugin.py create mode 100644 logquill/redact_plugin.py create mode 100644 logquill/sampling_plugin.py create mode 100644 tests/test_context_plugin.py create mode 100644 tests/test_plugin.py create mode 100644 tests/test_redact_plugin.py create mode 100644 tests/test_sampling_plugin.py diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 54b0d37..15b3288 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -15,5 +15,5 @@ ## Scope - diff --git a/CHANGELOG.md b/CHANGELOG.md index 6046d4e..5178c68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project are documented in this file. ## Unreleased +- Plugin pipeline: `Plugin` base (`before_log`/`after_log`/`on_error`, + all optional to override), `ContextPlugin` (merges fixed context into + `meta`), `RedactPlugin` (replaces sensitive `meta` values by key, case- + insensitive), and `SamplingPlugin` (probabilistically drops records). + `Logger` now accepts `plugins=[...]` and gained `.use(plugin)` to register + one and chain. A plugin hook that raises is caught, routed to that same + plugin's `on_error`, and the pipeline continues — a broken plugin can't + crash logging, verified by test. - Added `.github/dependabot.yml`: weekly version updates for `pip` dependencies and GitHub Actions. - Added GitHub issue templates: `.github/ISSUE_TEMPLATE/bug_report.yml`, @@ -12,23 +20,23 @@ All notable changes to this project are documented in this file. - Added `.github/SECURITY.md`: supported-versions policy and instructions to report vulnerabilities via GitHub's private vulnerability reporting instead of public issues. Linked from the README. -- Phase 2 transports: `Transport` base (`format`/`write`/`close`), `ConsoleTransport` +- Transports: `Transport` base (`format`/`write`/`close`), `ConsoleTransport` (colorized, ERROR/FATAL to stderr), `FileTransport` (size-based rotation), and `HTTPTransport` (batched, newline-delimited JSON over stdlib `urllib`, with an injectable `sender` for tests or alternate backends). `Logger` now accepts `transports=[...]` and dispatches each record to them synchronously, and gained `.close()` to close all attached transports. Dispatch is still synchronous — - the non-blocking queue/async path is Phase 4. Also added `CollectingTransport`, - an in-memory transport for tests. + a non-blocking queue/async path isn't implemented yet. Also added + `CollectingTransport`, an in-memory transport for tests. - Added `CODE_OF_CONDUCT.md` (Contributor Covenant v2.1), `.github/CODEOWNERS`, `.github/PULL_REQUEST_TEMPLATE.md`, and `CONTRIBUTING.md` documenting the PR workflow (branch naming, scoping, review/CI requirements, squash-merge). -- Phase 1 core API: `Level` (TRACE/DEBUG/INFO/WARN/ERROR/FATAL, matching +- Core API: `Level` (TRACE/DEBUG/INFO/WARN/ERROR/FATAL, matching logquill-js's numeric weights), `parse_level()`, the `LogRecord` shape, `Logger` with `.trace()/.debug()/.info()/.warn()/.error()/.fatal()` and `.set_level()`, and a `Formatter` protocol with a `JSONFormatter` implementation. Log calls return the record dict (or `None` when filtered - by level) — no transports or dispatch yet, that's Phase 2. + by level) — no transports or dispatch yet. - Repo scaffold: `pyproject.toml`, package skeleton, dev tooling (ruff, mypy --strict, pytest), pre-commit hooks, and CI workflow. - Packaging metadata: expanded classifiers (OS, Topic) and keywords, added an `Issues` project URL, and fixed the `Homepage`/`Repository`/`Changelog` URLs to point at the actual `nikhilvdev/logquill-python` GitHub repo instead of a stale placeholder org. - Added a pepy.tech download-count badge to the README for tracking installs. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f20c9bf..6a23d5f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,9 +21,8 @@ pytest - **Branch from `main`**, name branches by intent: `feat/…`, `fix/…`, `docs/…`, `chore/…` (e.g. `feat/rotating-file-transport`). -- **Keep PRs scoped to one phase or concern** where possible. A PR that - mixes an unrelated refactor with a feature is harder to review and harder - to revert. +- **Keep PRs scoped to one concern** where possible. A PR that mixes an + unrelated refactor with a feature is harder to review and harder to revert. - **Every PR must satisfy this definition of done** before it's ready for review: 1. Type hints throughout, `mypy --strict` clean on the public API diff --git a/README.md b/README.md index 51f4cc1..276921e 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,9 @@ Sibling to [`logquill` on npm](https://www.npmjs.com/package/logquill) across a Python + Node stack. Status: pre-release, under active development. The core `Logger`, level -filtering, and transports are implemented; plugins and non-blocking async -dispatch are not yet — see `CHANGELOG.md` for what's landed so far. +filtering, transports, and the plugin pipeline are implemented; +non-blocking async dispatch is not yet — see `CHANGELOG.md` for what's +landed so far. ## Features @@ -23,9 +24,10 @@ dispatch are not yet — see `CHANGELOG.md` for what's landed so far. - **Cross-language record shape** — identical JSON shape and level names/weights as [`logquill` on npm](https://www.npmjs.com/package/logquill) - **Pluggable transports** — `ConsoleTransport` (colorized, stderr for errors), `FileTransport` (rotation), `HTTPTransport` (batched); write your own by subclassing `Transport` - **Pluggable formatters** — `JSONFormatter` out of the box; implement `format(record) -> str` for your own +- **Plugin pipeline** — `ContextPlugin`, `RedactPlugin`, `SamplingPlugin` out of the box; a broken plugin can't crash logging - **Zero required runtime dependencies** — stdlib only; `aiohttp` is opt-in, for async HTTP - **Typed throughout** — `mypy --strict` clean on the public API -- *(planned)* plugin pipeline (redaction, sampling, rate limiting), non-blocking async dispatch, `contextvars`-based context propagation — see `CHANGELOG.md` +- *(planned)* non-blocking async dispatch, `contextvars`-based context propagation — see `CHANGELOG.md` ## Install @@ -66,7 +68,7 @@ print(JSONFormatter().format(record)) Attach transports to a `Logger` to actually write records somewhere. Each record is dispatched to every attached transport synchronously (non-blocking -dispatch lands in a later phase): +dispatch isn't implemented yet): ```python from logquill import ConsoleTransport, FileTransport, HTTPTransport, Logger @@ -99,6 +101,28 @@ logger.info("hello") assert sink.records[0]["message"] == "hello" ``` +## Plugins + +Plugins hook into the pipeline around each log call: `before_log(record)` can +transform a record or return `None` to drop it, `after_log(record)` runs once +it's been dispatched to every transport, and `on_error(exc, record)` catches +anything a plugin's own hooks raise — a broken plugin can't take down logging. + +```python +from logquill import ContextPlugin, Logger, RedactPlugin, SamplingPlugin + +logger = Logger("app") +logger.use(ContextPlugin(service="api", env="prod")) # merged into every record's meta +logger.use(RedactPlugin(keys=["password", "token"])) # replaces matching meta values +logger.use(SamplingPlugin(0.1)) # keep ~10% of records that reach this point + +logger.info("login attempt", user_id=42, password="hunter2") +# meta: {'service': 'api', 'env': 'prod', 'user_id': 42, 'password': '***'} +# (unless this call was one of the ~90% sampling dropped, in which case it's None) +``` + +Write your own by subclassing `Plugin`; override only the hooks you need. + ## Development ```bash diff --git a/logquill/__init__.py b/logquill/__init__.py index 4e3d2aa..3b40671 100644 --- a/logquill/__init__.py +++ b/logquill/__init__.py @@ -1,17 +1,22 @@ from logquill.console_transport import ConsoleTransport +from logquill.context_plugin import ContextPlugin from logquill.file_transport import FileTransport from logquill.formatter import Formatter, JSONFormatter from logquill.http_transport import HTTPTransport from logquill.levels import Level, parse_level from logquill.logger import Logger +from logquill.plugin import Plugin from logquill.records import LogRecord +from logquill.redact_plugin import RedactPlugin +from logquill.sampling_plugin import SamplingPlugin from logquill.transport import CollectingTransport, Transport -__version__ = "0.1.2" +__version__ = "0.1.3" __all__ = [ "CollectingTransport", "ConsoleTransport", + "ContextPlugin", "FileTransport", "Formatter", "HTTPTransport", @@ -19,6 +24,9 @@ "Level", "LogRecord", "Logger", + "Plugin", + "RedactPlugin", + "SamplingPlugin", "Transport", "parse_level", "__version__", diff --git a/logquill/context_plugin.py b/logquill/context_plugin.py new file mode 100644 index 0000000..ddfff18 --- /dev/null +++ b/logquill/context_plugin.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from typing import Any + +from logquill.plugin import Plugin +from logquill.records import LogRecord + + +class ContextPlugin(Plugin): + """Injects fixed key/value pairs into every record's `meta`. + + A value already present in a record's own `meta` wins over the fixed context. + """ + + def __init__(self, **context: Any) -> None: + self.context = context + + def before_log(self, record: LogRecord) -> LogRecord | None: + record["meta"] = {**self.context, **record["meta"]} + return record diff --git a/logquill/http_transport.py b/logquill/http_transport.py index 014d81c..e351ca7 100644 --- a/logquill/http_transport.py +++ b/logquill/http_transport.py @@ -27,7 +27,7 @@ class HTTPTransport(Transport): Uses `urllib` (stdlib) by default so the core package stays dependency-free. Pass `sender` to swap in a fake for tests, or a different backend (e.g. an - aiohttp-based one, once the async dispatch path from Phase 4 exists). + aiohttp-based one, once a non-blocking async dispatch path exists). """ def __init__( diff --git a/logquill/logger.py b/logquill/logger.py index 9590c0d..e40f290 100644 --- a/logquill/logger.py +++ b/logquill/logger.py @@ -1,8 +1,10 @@ from __future__ import annotations +import contextlib from typing import Any from logquill.levels import Level, parse_level +from logquill.plugin import Plugin from logquill.records import LogRecord, create_record from logquill.transport import Transport @@ -13,10 +15,12 @@ def __init__( name: str, level: int | str | Level = Level.INFO, transports: list[Transport] | None = None, + plugins: list[Plugin] | None = None, ) -> None: self.name = name self._level = parse_level(level) self.transports: list[Transport] = list(transports) if transports else [] + self.plugins: list[Plugin] = list(plugins) if plugins else [] @property def level(self) -> Level: @@ -25,17 +29,45 @@ def level(self) -> Level: def set_level(self, level: int | str | Level) -> None: self._level = parse_level(level) + def use(self, plugin: Plugin) -> Logger: + """Register a plugin. Returns `self` so calls can be chained.""" + self.plugins.append(plugin) + return self + def close(self) -> None: """Close every attached transport. Call on shutdown to flush buffered writes.""" for transport in self.transports: transport.close() + def _notify_error(self, plugin: Plugin, exc: Exception, record: LogRecord) -> None: + # a broken error handler must not crash logging either + with contextlib.suppress(Exception): + plugin.on_error(exc, record) + def _log(self, level: Level, message: str, meta: dict[str, Any]) -> LogRecord | None: if level < self._level: return None record = create_record(level=level, logger=self.name, message=message, meta=meta) + + for plugin in self.plugins: + try: + result = plugin.before_log(record) + except Exception as exc: + self._notify_error(plugin, exc, record) + continue + if result is None: + return None + record = result + for transport in self.transports: transport.write(transport.format(record), record) + + for plugin in self.plugins: + try: + plugin.after_log(record) + except Exception as exc: + self._notify_error(plugin, exc, record) + return record def trace(self, message: str, **meta: Any) -> LogRecord | None: diff --git a/logquill/plugin.py b/logquill/plugin.py new file mode 100644 index 0000000..c6d7494 --- /dev/null +++ b/logquill/plugin.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from logquill.records import LogRecord + + +class Plugin: + """Base class for the plugin pipeline: `before_log`, `after_log`, `on_error`. + + Override only the hooks you need — the rest default to no-ops. A plugin + hook that raises cannot crash logging: the pipeline catches it, routes it + to `on_error`, and moves on. + """ + + def before_log(self, record: LogRecord) -> LogRecord | None: + """Return a (possibly modified) record, or `None` to drop it.""" + return record + + def after_log(self, record: LogRecord) -> None: + """Called after the record has been dispatched to every transport.""" + + def on_error(self, exc: Exception, record: LogRecord) -> None: + """Called when one of this plugin's own hooks raises.""" diff --git a/logquill/redact_plugin.py b/logquill/redact_plugin.py new file mode 100644 index 0000000..3fe7616 --- /dev/null +++ b/logquill/redact_plugin.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from collections.abc import Iterable + +from logquill.plugin import Plugin +from logquill.records import LogRecord + +DEFAULT_REDACTED_KEYS = frozenset({"password", "token", "secret", "api_key", "authorization"}) + + +class RedactPlugin(Plugin): + """Replaces sensitive `meta` values, matched by key (case-insensitive), with a placeholder.""" + + def __init__( + self, + keys: Iterable[str] = DEFAULT_REDACTED_KEYS, + replacement: str = "***", + ) -> None: + self.keys = {key.lower() for key in keys} + self.replacement = replacement + + def before_log(self, record: LogRecord) -> LogRecord | None: + meta = record["meta"] + record["meta"] = { + key: self.replacement if key.lower() in self.keys else value + for key, value in meta.items() + } + return record diff --git a/logquill/sampling_plugin.py b/logquill/sampling_plugin.py new file mode 100644 index 0000000..c46ff19 --- /dev/null +++ b/logquill/sampling_plugin.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import random +from typing import Callable + +from logquill.plugin import Plugin +from logquill.records import LogRecord + + +class SamplingPlugin(Plugin): + """Keeps roughly `rate` of records (0.0-1.0), dropping the rest.""" + + def __init__(self, rate: float, rng: Callable[[], float] | None = None) -> None: + if not 0.0 <= rate <= 1.0: + raise ValueError(f"rate must be between 0 and 1, got {rate!r}") + self.rate = rate + self._rng = rng or random.random + + def before_log(self, record: LogRecord) -> LogRecord | None: + return record if self._rng() < self.rate else None diff --git a/pyproject.toml b/pyproject.toml index 0de71b3..b4f0b01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "logquill" -version = "0.1.2" +version = "0.1.3" description = "A structured, leveled logging framework with pluggable transports and a plugin pipeline." readme = "README.md" license = "MIT" diff --git a/tests/test_context_plugin.py b/tests/test_context_plugin.py new file mode 100644 index 0000000..79e1c33 --- /dev/null +++ b/tests/test_context_plugin.py @@ -0,0 +1,20 @@ +from logquill.context_plugin import ContextPlugin +from logquill.logger import Logger + + +def test_injects_fixed_context_into_meta() -> None: + logger = Logger("app.test", plugins=[ContextPlugin(service="api", env="prod")]) + + record = logger.info("hello", user_id=42) + + assert record is not None + assert record["meta"] == {"service": "api", "env": "prod", "user_id": 42} + + +def test_call_site_meta_overrides_fixed_context() -> None: + logger = Logger("app.test", plugins=[ContextPlugin(env="prod")]) + + record = logger.info("hello", env="staging") + + assert record is not None + assert record["meta"]["env"] == "staging" diff --git a/tests/test_plugin.py b/tests/test_plugin.py new file mode 100644 index 0000000..3fb6f0d --- /dev/null +++ b/tests/test_plugin.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +from logquill.logger import Logger +from logquill.plugin import Plugin +from logquill.records import LogRecord +from logquill.transport import CollectingTransport + + +class UppercasePlugin(Plugin): + def before_log(self, record: LogRecord) -> LogRecord | None: + record["message"] = record["message"].upper() + return record + + +class DroppingPlugin(Plugin): + def before_log(self, record: LogRecord) -> LogRecord | None: + return None + + +class SpyPlugin(Plugin): + def __init__(self) -> None: + self.errors: list[tuple[Exception, LogRecord]] = [] + self.after_log_calls: list[LogRecord] = [] + + def after_log(self, record: LogRecord) -> None: + self.after_log_calls.append(record) + + def on_error(self, exc: Exception, record: LogRecord) -> None: + self.errors.append((exc, record)) + + +class BrokenBeforeLogPlugin(SpyPlugin): + def before_log(self, record: LogRecord) -> LogRecord | None: + raise RuntimeError("boom") + + +class BrokenAfterLogPlugin(SpyPlugin): + def after_log(self, record: LogRecord) -> None: + super().after_log(record) + raise RuntimeError("boom") + + +def test_before_log_can_transform_the_record() -> None: + sink = CollectingTransport() + logger = Logger("app.test", transports=[sink], plugins=[UppercasePlugin()]) + + record = logger.info("hello") + + assert record is not None + assert record["message"] == "HELLO" + assert sink.records == [record] + + +def test_before_log_returning_none_drops_the_record() -> None: + sink = CollectingTransport() + logger = Logger("app.test", transports=[sink], plugins=[DroppingPlugin()]) + + assert logger.info("hello") is None + assert sink.records == [] + + +def test_broken_before_log_does_not_crash_logging() -> None: + sink = CollectingTransport() + broken = BrokenBeforeLogPlugin() + logger = Logger("app.test", transports=[sink], plugins=[broken]) + + record = logger.info("hello") + + assert record is not None + assert sink.records == [record] + assert len(broken.errors) == 1 + assert isinstance(broken.errors[0][0], RuntimeError) + + +def test_broken_before_log_does_not_stop_remaining_plugins_from_running() -> None: + sink = CollectingTransport() + plugins = [BrokenBeforeLogPlugin(), UppercasePlugin()] + logger = Logger("app.test", transports=[sink], plugins=plugins) + + record = logger.info("hello") + + assert record is not None + assert record["message"] == "HELLO" + + +def test_broken_after_log_does_not_crash_logging_or_skip_remaining_plugins() -> None: + broken = BrokenAfterLogPlugin() + spy = SpyPlugin() + logger = Logger("app.test", plugins=[broken, spy]) + + record = logger.info("hello") + + assert record is not None + assert broken.after_log_calls == [record] + assert len(broken.errors) == 1 + assert spy.after_log_calls == [record] + + +def test_use_registers_a_plugin_and_returns_self_for_chaining() -> None: + sink = CollectingTransport() + logger = Logger("app.test", transports=[sink]) + + result = logger.use(UppercasePlugin()) + + assert result is logger + logger.info("hi") + assert sink.records[0]["message"] == "HI" diff --git a/tests/test_redact_plugin.py b/tests/test_redact_plugin.py new file mode 100644 index 0000000..006c29e --- /dev/null +++ b/tests/test_redact_plugin.py @@ -0,0 +1,31 @@ +from logquill.logger import Logger +from logquill.redact_plugin import RedactPlugin + + +def test_redacts_default_sensitive_keys() -> None: + logger = Logger("app.test", plugins=[RedactPlugin()]) + + record = logger.info("login", password="hunter2", user_id=42) + + assert record is not None + assert record["meta"]["password"] == "***" + assert record["meta"]["user_id"] == 42 + + +def test_matches_keys_case_insensitively() -> None: + logger = Logger("app.test", plugins=[RedactPlugin()]) + + record = logger.info("login", Password="hunter2") + + assert record is not None + assert record["meta"]["Password"] == "***" + + +def test_custom_keys_and_replacement() -> None: + logger = Logger("app.test", plugins=[RedactPlugin(keys=["ssn"], replacement="[REDACTED]")]) + + record = logger.info("submit", ssn="123-45-6789", password="not redacted here") + + assert record is not None + assert record["meta"]["ssn"] == "[REDACTED]" + assert record["meta"]["password"] == "not redacted here" diff --git a/tests/test_sampling_plugin.py b/tests/test_sampling_plugin.py new file mode 100644 index 0000000..2fcaee2 --- /dev/null +++ b/tests/test_sampling_plugin.py @@ -0,0 +1,32 @@ +import pytest + +from logquill.logger import Logger +from logquill.sampling_plugin import SamplingPlugin + + +def test_rate_zero_drops_everything() -> None: + logger = Logger("app.test", plugins=[SamplingPlugin(0.0)]) + + assert logger.info("hello") is None + + +def test_rate_one_keeps_everything() -> None: + logger = Logger("app.test", plugins=[SamplingPlugin(1.0)]) + + assert logger.info("hello") is not None + + +def test_invalid_rate_raises() -> None: + with pytest.raises(ValueError): + SamplingPlugin(1.5) + + +def test_custom_rng_controls_keep_or_drop() -> None: + keep = SamplingPlugin(0.5, rng=lambda: 0.1) + drop = SamplingPlugin(0.5, rng=lambda: 0.9) + + logger_keep = Logger("app.test", plugins=[keep]) + logger_drop = Logger("app.test", plugins=[drop]) + + assert logger_keep.info("hello") is not None + assert logger_drop.info("hello") is None