From 0b019c7c0ad0e1e568e6edf9934b140b8151630e Mon Sep 17 00:00:00 2001 From: Carlos Ortiz Date: Thu, 27 Aug 2026 12:47:05 -0600 Subject: [PATCH] Add monotone feature constraints with build-time verification (closes #8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Training: train_whitebox(monotone_constraints=...) switches the whitebox backend to HistGradientBoostingRegressor (monotonic_cst enforced during growth); the unconstrained GradientBoostingRegressor path is untouched. New sklearn_hist extractor with random-row parity validation; categorical splits are refused at the model level. Verification is trainer-independent: build_artifact re-checks the declared directions against the quantized integer trees (per-tree, any depth) and refuses to build on violation, then records the signs as model.monotone_constraints — optional, hash-covered, spec section 3.1. Validation gains check 9, re-verifying from the artifact alone. At depth <= 2, scorecard_monotone_report certifies the aggregate direction on the printed scorecard tables via worst-case bin-to-bin increments. sweep_whitebox(monotone_constraints=...) measures the monotonicity premium. Docs: tuning guide section, FAQ entry, spec 3.1, nine-check tables. Tests use a teacher with a deliberate local inversion so the unconstrained whitebox provably violates and the verifier provably catches it. --- CHANGELOG.md | 14 ++ CONTRIBUTING.md | 3 +- README.md | 3 +- docs/ARTIFACT_SPEC.md | 22 ++- docs/faq.md | 14 ++ docs/howto/tuning.md | 51 ++++++ docs/howto/validate.md | 6 +- docs/reference/api.md | 6 + docs/reference/cli.md | 2 +- src/compileml/artifact/build.py | 25 +++ src/compileml/cli.py | 3 +- src/compileml/compile/__init__.py | 8 + src/compileml/compile/distill.py | 46 +++++- src/compileml/compile/extract.py | 95 ++++++++++- src/compileml/compile/monotone.py | 216 ++++++++++++++++++++++++ src/compileml/tune/sweeps.py | 7 + src/compileml/validate/framework.py | 25 ++- tests/test_monotone.py | 248 ++++++++++++++++++++++++++++ tests/test_validate.py | 7 +- 19 files changed, 779 insertions(+), 22 deletions(-) create mode 100644 src/compileml/compile/monotone.py create mode 100644 tests/test_monotone.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bdd14f0..806d4f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ inside each artifact (`schema_version`). ## [Unreleased] +### Added +- Monotone feature constraints ([#8](https://github.com/orgoca/CompileML/issues/8)): + `train_whitebox(monotone_constraints=...)` switches the whitebox backend to + `HistGradientBoostingRegressor` (extraction parity tested; the + unconstrained `GradientBoostingRegressor` path is byte-identical to + before). `build_artifact(monotone_constraints=...)` verifies the declared + directions against the compiled integer trees — refusing to build on any + violation, independent of the trainer — and records them as + `model.monotone_constraints` (hash-covered, spec §3.1). Validation gains + check 9, re-verifying the declaration from the artifact alone; at depth + ≤ 2, `scorecard_monotone_report` certifies the aggregate direction on the + printed scorecard tables. `sweep_whitebox(monotone_constraints=...)` + measures the monotonicity premium. + ## [0.1.1] - 2026-08-16 First complete release. Supersedes 0.1.0, whose wheel carried a stale diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0e97b4f..62447ee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,12 +9,11 @@ The [open issues](https://github.com/orgoca/CompileML/issues) are written to be picked up: each one states the problem, why it matters in a regulated lending context, a proposed approach, acceptance criteria, and the files involved. -Three of them are, in my view, what stands between this being interesting +Two of them are, in my view, what stands between this being interesting infrastructure and something a risk function could actually adopt: | | | |---|---| -| [#8](https://github.com/orgoca/CompileML/issues/8) | **Monotone feature constraints.** Today the distilled whitebox cannot enforce them, so a compiled scorecard may show a bin where more delinquency scores *better*. That is a scorecard a committee rejects on sight. | | [#9](https://github.com/orgoca/CompileML/issues/9) | **Stability monitoring (PSI/CSI/drift).** The validation framework checks an artifact at a point in time; model risk management is about what happens next. | | [#10](https://github.com/orgoca/CompileML/issues/10) | **Fair lending.** Disparate impact testing, plus disparity decomposition over the exact attributions — something the reconciliation identity makes possible here in a way it is not elsewhere. | diff --git a/README.md b/README.md index 786659c..e0431aa 100644 --- a/README.md +++ b/README.md @@ -305,7 +305,8 @@ It checks: 5. bad-rate monotonicity; 6. band-ladder churn; 7. explanation stability; -8. reason-code coverage. +8. reason-code coverage; +9. declared monotone directions, re-verified against the shipped trees. These checks run against the compiled artifact through the same runtime used for production decisions. There is no separate notebook implementation allowed to become "almost the same" over time. diff --git a/docs/ARTIFACT_SPEC.md b/docs/ARTIFACT_SPEC.md index 99d855f..929dee6 100644 --- a/docs/ARTIFACT_SPEC.md +++ b/docs/ARTIFACT_SPEC.md @@ -96,7 +96,8 @@ model, not of the float model it came from. "right": [4, 3, -1, -1, 6, -1, -1], "value_micro": [0, 0, -18342, 21077, 0, -4410, 33590] } - ] + ], + "monotone_constraints": [1, 0, -1, ...] // OPTIONAL (§3.1) }, "calibration": { // OPTIONAL (null allowed) "mode": "linear_int", // or "step" @@ -138,6 +139,22 @@ model, not of the float model it came from. payload. Interior nodes have `value_micro == 0`. Thresholds are float64 and MUST round-trip exactly through JSON (shortest-repr serialization). +### 3.1 Monotone constraints (optional) + +`model.monotone_constraints`, when present, is a list of length +`n_features` over `{-1, 0, +1}`: the declared direction of the compiled +score in each feature (+1 non-decreasing, −1 non-increasing, 0 +unconstrained). The field is covered by the hash (§9) like everything +else. + +The declaration is a *verified property of the shipped trees*, not a +training-time promise: builders MUST NOT emit the field unless the +quantized ensemble satisfies it (CompileML re-verifies tree-by-tree at +build and refuses otherwise), and validators re-verify it from the +artifact alone — validation check 9. Runtimes ignore the field; it +changes no decision, only what can be claimed about them. Absent field +means no directions are declared. + ## 4. Scoring (normative) Input: `x`, an array of float64 of length `n_features`, ordered by @@ -376,6 +393,9 @@ identified by its hash, and the hash is the unit of governance. --- *Changelog* +- **v2, additive** — optional `model.monotone_constraints` (§3.1): declared, + build-verified monotone directions. Absent field means unconstrained; + `schema_version` unchanged. - **v2** — integer-quantized leaves, integer calibration, half-micro exact attribution with largest-remainder display rounding, missing-value policy, hash-verified loads. Supersedes a pre-release float-scoring layout that diff --git a/docs/faq.md b/docs/faq.md index 6f8997c..645c4fd 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -46,6 +46,20 @@ or `compileml scorecard decision.json --format csv`. The printed tables re-sum to every production decision bit-for-bit; a validator can reproduce scores in a spreadsheet. +### Can I force a direction — "more delinquency must never score better"? + +Yes. Pass `monotone_constraints` (per-feature −1/0/+1) to `train_whitebox` +and the whitebox is trained with scikit-learn's histogram GBM, which enforces +directions during growth. Pass the same declaration to `build_artifact` and +it is *re-verified against the compiled integer trees* — the build refuses on +any violation, whatever trainer produced the model — then recorded in the +artifact under the hash. Validation check 9 repeats the verification from the +artifact alone, and at depth ≤ 2 the printed scorecard certifies the +aggregate direction (`scorecard_monotone_report`). Constraints cost fidelity +wherever the teacher genuinely wiggles; [measure the +premium](howto/tuning.md#the-monotonicity-premium-measured) instead of +guessing it. + ### Why not just use SHAP? TreeSHAP is exact for trees and a fine analysis tool — the differences are diff --git a/docs/howto/tuning.md b/docs/howto/tuning.md index 59c3545..e617e2f 100644 --- a/docs/howto/tuning.md +++ b/docs/howto/tuning.md @@ -169,6 +169,57 @@ spreadsheet. Above depth 2, `build_scorecard` raises instead of approximating — the same boundary as exact attribution, for the same reason. +## Enforcing monotone directions + +A compiled scorecard with a bin where more delinquency scores *better* is a +scorecard a committee rejects on sight — even when the wiggle is statistically +justified. Declare the directions and the whitebox is trained with +scikit-learn's `HistGradientBoostingRegressor`, which enforces them during +tree growth: + +```python +model, metrics = train_whitebox( + X, teacher_latent, + monotone_constraints={"UTIL": +1, "TENURE": -1}, # or a [-1, 0, +1, ...] list +) +artifact = build_artifact( + model, feature_names, baseline, edges, + monotone_constraints={"UTIL": +1, "TENURE": -1}, + ..., +) +``` + +Name-keyed dicts work at `train_whitebox` when `X` is a DataFrame; with bare +arrays, key by index. Without constraints, nothing changes — the classic +`GradientBoostingRegressor` path is untouched. + +The declaration at `build_artifact` is not a training-time promise passed +along: the builder re-verifies the *quantized integer trees* against it, +tree by tree, and refuses to emit the artifact on any violation — whatever +trainer produced the model. The verified signs are recorded in the artifact +(`model.monotone_constraints`, hash-covered), validation check 9 re-verifies +them from the artifact alone, and at depth ≤ 2 the scorecard's own tables +certify the aggregate direction (`scorecard_monotone_report`) — a check a +validator can repeat in a spreadsheet. + +### The monotonicity premium, measured + +Constraints cost fidelity wherever the teacher genuinely wiggles, and the +two backends also regularize differently (histogram binning, leaf-size +defaults), so do not guess the cost — measure it: + +```python +rows_free = sweep_whitebox(X, latent, y, X_val=Xv, y_val=yv, teacher_latent_val=lv) +rows_mono = sweep_whitebox(X, latent, y, X_val=Xv, y_val=yv, teacher_latent_val=lv, + monotone_constraints={"UTIL": +1, "TENURE": -1}) +``` + +Diff the `gini_retention_pct` column at your chosen configuration. If the +premium is small, the teacher's wiggle was noise and the constraint bought +committee-credibility for free; if it is large, the teacher has learned a +genuinely non-monotone pattern, and *that* is worth investigating before any +constraint is imposed. + ## Defaults, for the impatient `train_whitebox(n_estimators=30, max_depth=2)` and `n_bands=10` are sane diff --git a/docs/howto/validate.md b/docs/howto/validate.md index f1ebe86..af94492 100644 --- a/docs/howto/validate.md +++ b/docs/howto/validate.md @@ -24,7 +24,7 @@ Or gate a pipeline on the CLI's exit code: compileml validate decision.json --csv holdout.csv --y-col DEFAULT --require-reasons ``` -## The eight checks +## The nine checks | # | Check | What it proves | Needs | |---|---|---|---| @@ -36,9 +36,11 @@ compileml validate decision.json --csv holdout.csv --y-col DEFAULT --require-rea | 6 | churn baseline | bootstrap ladder stability, measured with fixed-point edges | X + latent_train | | 7 | explainability stability | top-k reason sets stable under small input perturbation, using the runtime's explainer | X | | 8 | reason coverage | dictionary coverage of feature names; optional hard gate | nothing | +| 9 | monotone constraints | declared directions re-verified against the shipped integer trees ([spec §3.1](../ARTIFACT_SPEC.md)) | nothing | Checks lacking inputs **skip** (reported as skipped, not passed silently); -check 1 and check 8 always run. +checks 1 and 8 always run, and check 9 runs whenever the artifact declares +constraints — it needs no data because the trees themselves are the evidence. ## Evidence, not verdicts diff --git a/docs/reference/api.md b/docs/reference/api.md index c7eeae9..702591e 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -8,6 +8,12 @@ ::: compileml.compile.quantize_model +::: compileml.compile.normalize_constraints + +::: compileml.compile.verify_monotone_constraints + +::: compileml.compile.scorecard_monotone_report + ## Bands ::: compileml.bands.quantile_bands diff --git a/docs/reference/cli.md b/docs/reference/cli.md index d4d2a10..8a00ad1 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -50,7 +50,7 @@ handled per the artifact's missing policy. compileml validate decision.json --csv holdout.csv --y-col DEFAULT --require-reasons ``` -Runs the [eight-check framework](../howto/validate.md); prints the full +Runs the [nine-check framework](../howto/validate.md); prints the full evidence report as JSON; exits non-zero if any check fails — suitable as a CI deployment gate. diff --git a/src/compileml/artifact/build.py b/src/compileml/artifact/build.py index 1e67d88..7cfa4c3 100644 --- a/src/compileml/artifact/build.py +++ b/src/compileml/artifact/build.py @@ -11,6 +11,7 @@ from compileml import __version__ from compileml.artifact.calibration import fit_isotonic_table from compileml.compile.extract import extract_trees, score_float +from compileml.compile.monotone import normalize_constraints, verify_monotone_constraints from compileml.compile.quantize import ( max_depth, quantization_error_bound, @@ -52,6 +53,7 @@ def build_artifact( display_names: dict | None = None, feature_meta: list | None = None, missing_policy: str = "baseline", + monotone_constraints=None, metadata: dict | None = None, scale: int = 1000, micro_scale: int = 1_000_000, @@ -82,6 +84,13 @@ def build_artifact( for consumer-facing notices. Coverage below 100% warns and is recorded in metadata (spec §7.6). missing_policy: "baseline" (impute at decision time) or "reject". + monotone_constraints: Declared directions per feature: a sequence of + -1/0/+1 in feature order, or a dict keyed by feature name or + index. The compiled integer trees are *verified* against the + declaration (spec §3.1) — any violation raises, whatever + trainer produced the model — and the signs are recorded in the + artifact under ``model.monotone_constraints``, covered by the + hash. Validation check 9 re-verifies on the artifact alone. X_sample: Optional sample rows; enables the measured quantization report and the latent-range check. @@ -118,6 +127,22 @@ def build_artifact( for tree in extracted.trees: tree["threshold"] = [round(t, int(threshold_decimals)) for t in tree["threshold"]] model_int = quantize_model(extracted, micro_scale=micro_scale) + + # --- monotone constraints: verify against the shipped trees, then record -- + cst = normalize_constraints(monotone_constraints, len(names), feature_names=names) + if cst is not None: + report = verify_monotone_constraints(model_int, cst) + if not report["ok"]: + constrained = [names[i] for i, sign in enumerate(cst) if sign] + raise ValueError( + f"monotone constraint violated by the compiled trees: " + f"{report['n_violations']} violation(s) across {constrained}. " + "First examples: " + f"{report['violations'][:3]}. Retrain with " + "train_whitebox(..., monotone_constraints=...) to enforce them." + ) + model_int["monotone_constraints"] = cst + depth = max_depth(model_int) if depth > 2: warnings.warn( diff --git a/src/compileml/cli.py b/src/compileml/cli.py index 1226e3e..78acfa0 100644 --- a/src/compileml/cli.py +++ b/src/compileml/cli.py @@ -57,6 +57,7 @@ def cmd_inspect(args) -> int: }, "calibration_mode": (artifact.get("calibration") or {}).get("mode"), "missing_policy": artifact["features"].get("missing_policy"), + "monotone_constraints": artifact["model"].get("monotone_constraints"), "reason_coverage": meta.get("reason_coverage"), "model_family": meta.get("model_family"), "compileml_version": meta.get("compileml_version"), @@ -252,7 +253,7 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--out", help="output file (default: stdout)") p.set_defaults(func=cmd_scorecard) - p = sub.add_parser("validate", help="run the 8-check validation framework") + p = sub.add_parser("validate", help="run the 9-check validation framework") p.add_argument("artifact") p.add_argument("--csv", help="validation CSV (features + outcome)") p.add_argument("--y-col", help="outcome column name in --csv") diff --git a/src/compileml/compile/__init__.py b/src/compileml/compile/__init__.py index 5325dba..0167d2d 100644 --- a/src/compileml/compile/__init__.py +++ b/src/compileml/compile/__init__.py @@ -11,16 +11,24 @@ score_float, validate_extraction, ) +from compileml.compile.monotone import ( + normalize_constraints, + scorecard_monotone_report, + verify_monotone_constraints, +) from compileml.compile.quantize import max_depth, quantization_error_bound, quantize_model, rha __all__ = [ "ExtractedModel", "extract_trees", "max_depth", + "normalize_constraints", "quantization_error_bound", "quantize_model", "rha", "score_float", + "scorecard_monotone_report", "train_whitebox", "validate_extraction", + "verify_monotone_constraints", ] diff --git a/src/compileml/compile/distill.py b/src/compileml/compile/distill.py index c7a49d7..e2a9639 100644 --- a/src/compileml/compile/distill.py +++ b/src/compileml/compile/distill.py @@ -5,6 +5,11 @@ to reproduce the teacher's latent. Depth ≤ 2 keeps the artifact's attribution *exact* (spec §7.3) — going deeper trades exactness for fidelity and is warned about loudly. + +With ``monotone_constraints`` the whitebox is trained with +``HistGradientBoostingRegressor`` (the only sklearn GBM that enforces +``monotonic_cst`` during growth); without them the classic +``GradientBoostingRegressor`` path is untouched, byte for byte. """ from __future__ import annotations @@ -13,7 +18,9 @@ import numpy as np from scipy.stats import pearsonr, spearmanr -from sklearn.ensemble import GradientBoostingRegressor +from sklearn.ensemble import GradientBoostingRegressor, HistGradientBoostingRegressor + +from compileml.compile.monotone import normalize_constraints def train_whitebox( @@ -25,11 +32,18 @@ def train_whitebox( learning_rate: float = 0.2, random_state: int = 42, loss: str = "squared_error", -) -> tuple[GradientBoostingRegressor, dict]: + monotone_constraints=None, +): """Fit a whitebox GBM to a teacher's latent scores. Returns (model, metrics) where metrics quantifies distillation fidelity on the training data (pearson, spearman, mae, rmse, prediction range). + + ``monotone_constraints`` takes a per-feature sequence of -1/0/+1 or a + dict keyed by feature index (or by name, when ``X`` is a DataFrame + carrying column names). Any nonzero sign switches + the backend to ``HistGradientBoostingRegressor``; ``None`` (or all + zeros) keeps the classic ``GradientBoostingRegressor``. """ if max_depth > 2: warnings.warn( @@ -40,13 +54,27 @@ def train_whitebox( X_arr = np.asarray(X, dtype=float) y = np.asarray(teacher_latent, dtype=float).reshape(-1) - model = GradientBoostingRegressor( - n_estimators=n_estimators, - max_depth=max_depth, - learning_rate=learning_rate, - random_state=random_state, - loss=loss, - ) + feature_names = list(X.columns) if hasattr(X, "columns") else None + cst = normalize_constraints(monotone_constraints, X_arr.shape[1], feature_names=feature_names) + if cst is not None: + model = HistGradientBoostingRegressor( + max_iter=n_estimators, + max_depth=max_depth, + learning_rate=learning_rate, + monotonic_cst=cst, + early_stopping=False, + max_leaf_nodes=None, + random_state=random_state, + loss=loss, + ) + else: + model = GradientBoostingRegressor( + n_estimators=n_estimators, + max_depth=max_depth, + learning_rate=learning_rate, + random_state=random_state, + loss=loss, + ) model.fit(X_arr, y) y_hat = np.clip(model.predict(X_arr), 0.0, 1.0) diff --git a/src/compileml/compile/extract.py b/src/compileml/compile/extract.py index 48603db..cc84fee 100644 --- a/src/compileml/compile/extract.py +++ b/src/compileml/compile/extract.py @@ -1,6 +1,7 @@ """Tree extraction from fitted models into portable float arrays. -Supported families: sklearn gradient boosting, XGBoost, LightGBM. +Supported families: sklearn gradient boosting (classic and hist), +XGBoost, LightGBM. Extraction produces float trees plus (base, learning_rate); quantization to the integer artifact model happens afterwards in ``quantize.py``. @@ -29,7 +30,7 @@ class ExtractedModel: trees: list[dict] # feature/threshold/left/right/value lists per tree base: float learning_rate: float - family: str # "sklearn" | "xgboost" | "lightgbm" + family: str # "sklearn" | "sklearn_hist" | "xgboost" | "lightgbm" input_precision: str = "float64" n_features: int = 0 notes: list[str] = field(default_factory=list) @@ -60,6 +61,14 @@ def _is_sklearn_gbm(model) -> bool: return hasattr(model, "estimators_") +def _is_sklearn_hist(model) -> bool: + try: + from sklearn.ensemble import HistGradientBoostingRegressor + except ImportError: # pragma: no cover + return False + return isinstance(model, HistGradientBoostingRegressor) + + def _is_xgboost(model) -> bool: try: import xgboost as xgb @@ -106,6 +115,81 @@ def _extract_sklearn(model) -> ExtractedModel: return ExtractedModel(trees, base, lr, "sklearn", "float64", n_features) +# --------------------------------------------------------------------------- +# sklearn HistGradientBoosting +# --------------------------------------------------------------------------- + + +def _extract_sklearn_hist(model) -> ExtractedModel: + """HistGradientBoostingRegressor — the constrained-whitebox backend. + + Walks the private ``_predictors`` structure (version-fragile by + nature; the random-row parity gate in ``validate_extraction`` turns + any sklearn-internals change into a loud failure instead of silent + drift). Leaf values arrive pre-shrunk, so ``learning_rate`` is 1.0; + the base is ``_baseline_prediction``. Thresholds are float64 and the + split convention is ``x <= threshold -> left``, matching ours. + """ + n_features = int(model.n_features_in_) + is_cat = getattr(model, "is_categorical_", None) + if is_cat is not None and any(is_cat): + # Categorical splits are bitset-based AND remap feature_idx inside + # the predictor nodes — both break the artifact's numeric-threshold + # tree shape, so refuse at the model level before touching nodes. + raise ValueError( + "categorical splits in HistGradientBoosting are not supported; " + "encode categoricals numerically before distilling" + ) + trees = [] + missing_right_nodes = 0 + for predictors in model._predictors: + if len(predictors) != 1: + raise ValueError("multi-output HistGradientBoosting models are not supported") + nodes = predictors[0].nodes + n = len(nodes) + feature = [LEAF] * n + threshold = [0.0] * n + left = [-1] * n + right = [-1] * n + value = [0.0] * n + for i in range(n): + node = nodes[i] + if bool(node["is_leaf"]): + value[i] = float(node["value"]) + continue + if bool(node["is_categorical"]): + raise ValueError( + "categorical splits in HistGradientBoosting are not supported; " + "encode categoricals numerically before distilling" + ) + feature[i] = int(node["feature_idx"]) + threshold[i] = float(node["num_threshold"]) + left[i] = int(node["left"]) + right[i] = int(node["right"]) + if not bool(node["missing_go_to_left"]): + missing_right_nodes += 1 + trees.append( + { + "feature": feature, + "threshold": threshold, + "left": left, + "right": right, + "value": value, + } + ) + notes: list[str] = [] + if missing_right_nodes: + # Unlike XGBoost, HGB records a routing direction on every split even + # when training saw no NaN, so this is a note, not a warning: the + # artifact has no missing branch either way — missing_policy governs. + notes.append( + f"{missing_right_nodes} split(s) route missing values right in the source " + "model; the artifact has no missing branch (missing_policy governs)." + ) + base = float(np.ravel(model._baseline_prediction)[0]) + return ExtractedModel(trees, base, 1.0, "sklearn_hist", "float64", n_features, notes) + + # --------------------------------------------------------------------------- # XGBoost # --------------------------------------------------------------------------- @@ -255,6 +339,10 @@ def _extract_lightgbm(model) -> ExtractedModel: def extract_trees(model) -> ExtractedModel: """Extract a fitted model into float tree arrays; dispatches on family.""" + if _is_sklearn_hist(model): + extracted = _extract_sklearn_hist(model) + validate_extraction(extracted, model) + return extracted if _is_sklearn_gbm(model): return _extract_sklearn(model) if _is_xgboost(model): @@ -292,6 +380,9 @@ def validate_extraction( booster = model.booster_ if hasattr(model, "booster_") else model X = rng.standard_normal((n_val, extracted.n_features)).astype(np.float64) ref = np.asarray(booster.predict(X, raw_score=True), dtype=float) + elif extracted.family == "sklearn_hist": + X = rng.standard_normal((n_val, extracted.n_features)).astype(np.float64) + ref = np.asarray(model.predict(X), dtype=float) else: return # sklearn covered by unit tests against model.predict diff --git a/src/compileml/compile/monotone.py b/src/compileml/compile/monotone.py new file mode 100644 index 0000000..f70d358 --- /dev/null +++ b/src/compileml/compile/monotone.py @@ -0,0 +1,216 @@ +"""Independent verification of monotone feature constraints. + +Trainer-agnostic by design: these checks read the *quantized integer +trees* — the thing that actually ships — never the fitted estimator. A +constraint holds because the compiled arithmetic says so, not because a +training library promised to enforce it. + +Two layers: + +- :func:`verify_monotone_constraints` — per-tree, any depth. A sum of + monotone functions is monotone, so per-tree monotonicity is a + *sufficient* condition for the whole model; and because + ``HistGradientBoostingRegressor`` enforces constraints per tree during + growth, models it trains never false-alarm here. +- :func:`scorecard_monotone_report` — aggregate, depth ≤ 2 only. + Necessary *and* sufficient: it certifies the function a committee will + actually read (the printed scorecard), via the worst-case increment + ``Δmain + Σ min-over-partner-bins Δgrid ≥ 0`` per adjacent bin pair. + +Everything here is exact integer arithmetic on ``value_micro`` leaves. +""" + +from __future__ import annotations + +from bisect import bisect_left +from itertools import product + +LEAF = -2 + + +def normalize_constraints(constraints, n_features: int, feature_names=None) -> list[int] | None: + """Normalize to a list of -1/0/+1 per feature, or None if unconstrained. + + Accepts a positional sequence, a dict keyed by feature index, or — + when ``feature_names`` is given — a dict keyed by feature name. + """ + if constraints is None: + return None + if isinstance(constraints, dict): + cst = [0] * n_features + for key, sign in constraints.items(): + if isinstance(key, str): + if feature_names is None: + raise ValueError( + f"constraint key {key!r} is a name, but no feature names are " + "available here — use feature indices, or pass the dict to " + "build_artifact where names are known" + ) + try: + idx = list(feature_names).index(key) + except ValueError: + raise ValueError(f"unknown feature name in constraints: {key!r}") from None + else: + idx = int(key) + if not 0 <= idx < n_features: + raise ValueError( + f"constraint index {idx} out of range for {n_features} features" + ) + cst[idx] = int(sign) + else: + cst = [int(v) for v in constraints] + if len(cst) != n_features: + raise ValueError( + f"constraints length {len(cst)} does not match n_features {n_features}" + ) + bad = sorted({v for v in cst if v not in (-1, 0, 1)}) + if bad: + raise ValueError(f"constraint signs must be -1, 0 or +1; got {bad}") + return cst if any(cst) else None + + +def _tree_thresholds_by_feature(tree: dict) -> dict[int, list[float]]: + out: dict[int, list[float]] = {} + for f, t in zip(tree["feature"], tree["threshold"]): + if f != LEAF: + out.setdefault(int(f), []).append(float(t)) + return {f: sorted(set(ts)) for f, ts in out.items()} + + +def _representatives(thresholds: list[float]) -> list[float]: + """One probe value per interval of ``(-inf, t1], …, (tk, inf)``.""" + if not thresholds: + return [0.0] + return [thresholds[0], *thresholds[1:], thresholds[-1] + 1.0] + + +def _eval_tree(tree: dict, values: dict[int, float]) -> int: + node = 0 + while tree["feature"][node] != LEAF: + f = tree["feature"][node] + node = tree["left"][node] if values[f] <= tree["threshold"][node] else tree["right"][node] + return int(tree["value_micro"][node]) + + +def verify_monotone_constraints(model_int: dict, constraints) -> dict: + """Check every quantized tree against the declared constraint signs. + + Exact and cheap at any depth: a tree references at most ``depth`` + features, so its cell grid is tiny. Per-tree monotonicity is + sufficient for the whole model (monotone functions sum to a monotone + function). Returns ``{"ok", "n_violations", "violations", "method"}`` + with at most 20 example violations, each pinpointing the tree, the + feature, the fixed context, and the offending leaf-value sequence. + """ + cst = normalize_constraints(constraints, int(model_int["n_features"])) + if cst is None: + return {"ok": True, "n_violations": 0, "violations": [], "method": "per_tree"} + + violations: list[dict] = [] + n_total = 0 + for t_idx, tree in enumerate(model_int["trees"]): + thresholds = _tree_thresholds_by_feature(tree) + constrained = [f for f in thresholds if cst[f] != 0] + if not constrained: + continue + others = sorted(f for f in thresholds if cst[f] == 0) + for f in constrained: + sign = cst[f] + f_reps = _representatives(thresholds[f]) + # Fix every OTHER feature of this tree (constrained ones included: + # they get their own pass) and sweep f across its intervals. + fixed = [g for g in constrained if g != f] + others + fixed_reps = [_representatives(thresholds[g]) for g in fixed] + for combo in product(*fixed_reps) if fixed else [()]: + context = dict(zip(fixed, combo)) + seq = [] + for rep in f_reps: + context[f] = rep + seq.append(_eval_tree(tree, context)) + if any(sign * (b - a) < 0 for a, b in zip(seq, seq[1:])): + n_total += 1 + if len(violations) < 20: + violations.append( + { + "tree": t_idx, + "feature": int(f), + "sign": int(sign), + "context": {int(k): float(v) for k, v in context.items() if k != f}, + "leaf_values_micro": seq, + } + ) + return { + "ok": n_total == 0, + "n_violations": n_total, + "violations": violations, + "method": "per_tree", + } + + +def scorecard_monotone_report(scorecard: dict, constraints) -> dict: + """Aggregate (necessary-and-sufficient) check on a depth ≤ 2 scorecard. + + For each constrained feature *f* and each adjacent pair of its bins, + the worst-case increment of the full function is the main-effect delta + plus, for every interaction grid involving *f*, the minimum delta over + the partner's bins. The constraint holds iff every worst-case + increment carries the declared sign. This certifies the printed + tables themselves — a validator can repeat it in a spreadsheet. + """ + names = list(scorecard["feature_names"]) + cst = normalize_constraints(constraints, len(names), feature_names=names) + if cst is None: + return {"ok": True, "per_feature": {}, "method": "scorecard_aggregate"} + + per_feature: dict[str, dict] = {} + for f_idx, sign in enumerate(cst): + if sign == 0: + continue + name = names[f_idx] + + # This feature's global bin edges: union of its main-effect + # thresholds and its thresholds inside every interaction it joins. + edges: set[float] = set() + main = scorecard["main_effects"].get(name) + if main: + edges.update(main["thresholds"]) + joined = [] + for inter in scorecard["interactions"].values(): + if f_idx in inter["feature_indices"]: + axis = "row" if inter["feature_indices"][0] == f_idx else "col" + joined.append((axis, inter)) + edges.update(inter[f"{axis}_thresholds"]) + if not edges: + per_feature[name] = {"ok": True, "note": "feature unused by the model"} + continue + + reps = _representatives(sorted(edges)) + + def _main_at(rep: float, main=main) -> int: + if not main: + return 0 + return int(main["bins"][bisect_left(main["thresholds"], rep)]["points_micro"]) + + worst = None + ok = True + for a, b in zip(reps, reps[1:]): + increment = sign * (_main_at(b) - _main_at(a)) + for axis, inter in joined: + grid = inter["grid_micro"] + ia = bisect_left(inter[f"{axis}_thresholds"], a) + ib = bisect_left(inter[f"{axis}_thresholds"], b) + if axis == "row": + deltas = [sign * (grid[ib][c] - grid[ia][c]) for c in range(len(grid[0]))] + else: + deltas = [sign * (row[ib] - row[ia]) for row in grid] + increment += min(deltas) + worst = increment if worst is None else min(worst, increment) + if increment < 0: + ok = False + per_feature[name] = {"ok": ok, "worst_increment_micro": int(worst)} + + return { + "ok": all(v["ok"] for v in per_feature.values()), + "per_feature": per_feature, + "method": "scorecard_aggregate", + } diff --git a/src/compileml/tune/sweeps.py b/src/compileml/tune/sweeps.py index 703f3f4..5a1933c 100644 --- a/src/compileml/tune/sweeps.py +++ b/src/compileml/tune/sweeps.py @@ -45,6 +45,7 @@ def sweep_whitebox( X_val=None, y_val=None, teacher_latent_val=None, + monotone_constraints=None, explain_timing_rows: int = 3, ) -> list[dict]: """Grid-sweep whitebox capacity; measure what each configuration buys. @@ -56,6 +57,10 @@ def sweep_whitebox( Supply a holdout (``X_val`` / ``y_val`` / ``teacher_latent_val``) for honest numbers; in-sample retention flatters every configuration. + + Pass ``monotone_constraints`` to sweep the constrained backend instead — + run both and diff the retention column to measure the monotonicity + premium before committing to it. """ X_arr = np.asarray(X, dtype=float) t_lat = np.asarray(teacher_latent, dtype=float).reshape(-1) @@ -85,6 +90,7 @@ def sweep_whitebox( max_depth=depth, learning_rate=learning_rate, random_state=random_state, + monotone_constraints=monotone_constraints, ) latent_eval = np.clip(model.predict(X_eval), 0.0, 1.0) gini = 2 * float(roc_auc_score(y_eval, latent_eval)) - 1 @@ -113,6 +119,7 @@ def sweep_whitebox( "exact_attribution": depth <= 2, "model_kb": round(model_kb, 1), "explain_ms_per_row": round(float(np.median(times)), 2), + "constrained": monotone_constraints is not None, "in_sample": in_sample, } ) diff --git a/src/compileml/validate/framework.py b/src/compileml/validate/framework.py index a337180..9d18a70 100644 --- a/src/compileml/validate/framework.py +++ b/src/compileml/validate/framework.py @@ -1,4 +1,4 @@ -"""Pre-deployment validation — eight checks run against the artifact itself. +"""Pre-deployment validation — nine checks run against the artifact itself. The defining property of this framework: **every check exercises the same JSON document and the same runtime that production runs.** There is no @@ -22,6 +22,9 @@ 7. explainability_stability top-k reason sets stable under small input perturbation, using the runtime's explainer 8. reason_coverage reason dictionary coverage of feature names + 9. monotone_constraints declared directions re-verified against the + shipped integer trees (skipped when the + artifact declares none) """ from __future__ import annotations @@ -69,7 +72,7 @@ def validate_artifact( require_full_reason_coverage: bool = False, seed: int = 42, ) -> dict: - """Run the eight-check framework. Checks lacking inputs skip, not fail. + """Run the nine-check framework. Checks lacking inputs skip, not fail. Args: artifact_or_path: Artifact dict or path to its JSON (paths get the @@ -291,4 +294,22 @@ def bootstrap_cutoffs(): "required_full": require_full_reason_coverage, } + # -------------------------------------------- 9 monotone constraints + cst = artifact["model"].get("monotone_constraints") + if cst: + from compileml.compile.monotone import verify_monotone_constraints + + report = verify_monotone_constraints(artifact["model"], cst) + checks["9_monotone_constraints"] = { + "pass": bool(report["ok"]), + "skipped": False, + "constrained_features": [ + names[i] for i, sign in enumerate(cst) if sign and i < len(names) + ], + "n_violations": int(report["n_violations"]), + "violations": report["violations"][:5], + } + else: + checks["9_monotone_constraints"] = {"pass": True, "skipped": True} + return {"all_pass": all(c["pass"] for c in checks.values()), "checks": checks} diff --git a/tests/test_monotone.py b/tests/test_monotone.py new file mode 100644 index 0000000..2c959a6 --- /dev/null +++ b/tests/test_monotone.py @@ -0,0 +1,248 @@ +"""Monotone constraints: HGB backend, trainer-independent verification, check 9. + +The teacher used here has a deliberate *local inversion* on f0 +(``x0 - 1.2·sin(2.5·x0)`` dips while trending up), so an unconstrained +whitebox provably learns a non-monotone shape — the verifier must catch +it — while the constrained backend must not. +""" + +import json +import warnings + +import numpy as np +import pytest + +from compileml.artifact import build_artifact +from compileml.compile import ( + extract_trees, + normalize_constraints, + quantize_model, + score_float, + scorecard_monotone_report, + train_whitebox, + verify_monotone_constraints, +) +from compileml.runtime import decide +from compileml.runtime.io import canonical_hash +from compileml.scorecard import build_scorecard +from compileml.validate import validate_artifact + +RNG = np.random.default_rng(7) +N, P = 3000, 4 +FEATURES = ["util", "dpd", "tenure", "inq"] +CONSTRAINTS = {"util": 1} # by name, resolved at build time +EDGES = [0.0, 0.25, 0.5, 0.75, 1.0] + + +@pytest.fixture(scope="module") +def data(): + X = RNG.standard_normal((N, P)) + # Locally inverted in x0: monotone overall trend, non-monotone shape. + latent = 1.0 / (1.0 + np.exp(-(X[:, 0] - 1.2 * np.sin(2.5 * X[:, 0]) + 0.5 * X[:, 1]))) + y = (RNG.random(N) < latent).astype(int) + return X, latent, y + + +@pytest.fixture(scope="module") +def constrained_model(data): + X, latent, _ = data + model, metrics = train_whitebox( + X, latent, n_estimators=25, max_depth=2, monotone_constraints={0: 1} + ) + assert type(model).__name__ == "HistGradientBoostingRegressor" + # The constraint forbids tracking the teacher's deliberate dips, so + # fidelity sits below the unconstrained fit — a floor, not a claim. + assert metrics["spearman"] > 0.85 + return model + + +@pytest.fixture(scope="module") +def unconstrained_model(data): + X, latent, _ = data + model, _ = train_whitebox(X, latent, n_estimators=25, max_depth=2) + assert type(model).__name__ == "GradientBoostingRegressor" + return model + + +def _build(model, data, **kwargs): + X, latent, y = data + with warnings.catch_warnings(): + warnings.simplefilter("ignore") # reason coverage warned, tested elsewhere + return build_artifact( + model, + FEATURES, + np.median(X, axis=0), + EDGES, + calibration_latent=latent, + calibration_y=y, + X_sample=X[:200], + **kwargs, + ) + + +@pytest.fixture(scope="module") +def constrained_artifact(constrained_model, data): + return _build(constrained_model, data, monotone_constraints=CONSTRAINTS) + + +# ---------------------------------------------------------- normalization +def test_normalize_constraint_forms(): + assert normalize_constraints([0, 1, 0, -1], 4) == [0, 1, 0, -1] + assert normalize_constraints({1: 1, 3: -1}, 4) == [0, 1, 0, -1] + assert normalize_constraints({"b": 1}, 3, feature_names=["a", "b", "c"]) == [0, 1, 0] + assert normalize_constraints(None, 4) is None + assert normalize_constraints([0, 0, 0, 0], 4) is None # all-zeros = unconstrained + + +def test_normalize_constraint_rejects(): + with pytest.raises(ValueError, match="length"): + normalize_constraints([1, 0], 4) + with pytest.raises(ValueError, match="signs"): + normalize_constraints([1, 2, 0, 0], 4) + with pytest.raises(ValueError, match="out of range"): + normalize_constraints({7: 1}, 4) + with pytest.raises(ValueError, match="unknown feature name"): + normalize_constraints({"nope": 1}, 3, feature_names=["a", "b", "c"]) + with pytest.raises(ValueError, match="no feature names"): + normalize_constraints({"util": 1}, 4) + + +# ------------------------------------------------------------- extraction +def test_hist_extraction_parity(constrained_model, data): + X, _, _ = data + extracted = extract_trees(constrained_model) + assert extracted.family == "sklearn_hist" + assert extracted.learning_rate == 1.0 # HGB leaves arrive pre-shrunk + for row in X[:50]: + assert score_float(extracted, [float(v) for v in row]) == pytest.approx( + float(constrained_model.predict(row.reshape(1, -1))[0]), abs=1e-9 + ) + + +def test_hist_categorical_splits_rejected(data): + from sklearn.ensemble import HistGradientBoostingRegressor + + X, latent, _ = data + Xc = X.copy() + Xc[:, 3] = RNG.integers(0, 4, N) + model = HistGradientBoostingRegressor(max_iter=3, categorical_features=[3]) + model.fit(Xc, latent) + with pytest.raises(ValueError, match="categorical"): + extract_trees(model) + + +# ---------------------------------------------- verification has teeth +def test_verifier_catches_unconstrained_inversion(unconstrained_model): + """The inversion teacher makes an unconstrained whitebox non-monotone.""" + model_int = quantize_model(extract_trees(unconstrained_model)) + report = verify_monotone_constraints(model_int, [1, 0, 0, 0]) + assert not report["ok"] + assert report["n_violations"] > 0 + v = report["violations"][0] + assert v["feature"] == 0 and v["sign"] == 1 + # The reported leaf sequence really does decrease somewhere. + seq = v["leaf_values_micro"] + assert any(b < a for a, b in zip(seq, seq[1:])) + + +def test_verifier_passes_constrained_model(constrained_model): + model_int = quantize_model(extract_trees(constrained_model)) + report = verify_monotone_constraints(model_int, [1, 0, 0, 0]) + assert report["ok"] and report["n_violations"] == 0 + + +def test_build_refuses_violating_declaration(unconstrained_model, data): + with pytest.raises(ValueError, match="monotone constraint violated"): + _build(unconstrained_model, data, monotone_constraints=CONSTRAINTS) + + +# ------------------------------------------------------------ the artifact +def test_constraints_recorded_and_hashed(constrained_artifact): + assert constrained_artifact["model"]["monotone_constraints"] == [1, 0, 0, 0] + # Hash-covered: flipping the recorded sign breaks verification. + tampered = json.loads(json.dumps(constrained_artifact)) + tampered["model"]["monotone_constraints"] = [-1, 0, 0, 0] + assert canonical_hash(tampered) != constrained_artifact["artifact_hash"] + + +def test_constrained_rebuild_hash_identical(constrained_model, data): + """Same data, fresh fit + build → byte-identical artifact (determinism).""" + X, latent, _ = data + model2, _ = train_whitebox(X, latent, n_estimators=25, max_depth=2, monotone_constraints={0: 1}) + art2 = _build(model2, data, monotone_constraints=CONSTRAINTS) + art1 = _build(constrained_model, data, monotone_constraints=CONSTRAINTS) + assert art1["artifact_hash"] == art2["artifact_hash"] + + +def test_decide_runs_on_constrained_artifact(constrained_artifact, data): + X, _, _ = data + out = decide(constrained_artifact, [float(v) for v in X[0]]) + assert out["exact_attribution"] is True + assert out["attribution_residual_half_micro"] == 0 + + +# -------------------------------------------------------------- check 9 +def test_check9_passes_and_names_features(constrained_artifact): + result = validate_artifact(constrained_artifact) + c9 = result["checks"]["9_monotone_constraints"] + assert c9["pass"] and not c9["skipped"] + assert c9["constrained_features"] == ["util"] + + +def test_check9_skips_without_declaration(unconstrained_model, data): + art = _build(unconstrained_model, data) + c9 = validate_artifact(art)["checks"]["9_monotone_constraints"] + assert c9["pass"] and c9["skipped"] + + +def test_check9_fails_on_tampered_declaration(artifact): + """Hand-built fixture (conftest) is increasing in f0; declare the opposite.""" + doc = json.loads(json.dumps(artifact)) + del doc["artifact_hash"] + doc["model"]["monotone_constraints"] = [-1, 0, 0] + doc["artifact_hash"] = canonical_hash(doc) + result = validate_artifact(doc) + c9 = result["checks"]["9_monotone_constraints"] + assert not c9["pass"] and not result["all_pass"] + assert c9["n_violations"] > 0 + + doc["model"]["monotone_constraints"] = [1, 0, 0] + del doc["artifact_hash"] + doc["artifact_hash"] = canonical_hash(doc) + assert validate_artifact(doc)["checks"]["9_monotone_constraints"]["pass"] + + +# ------------------------------------------------- scorecard aggregate +def test_scorecard_aggregate_report(constrained_artifact): + scorecard = build_scorecard(constrained_artifact) + report = scorecard_monotone_report(scorecard, CONSTRAINTS) + assert report["ok"] + assert report["per_feature"]["util"]["ok"] + assert report["per_feature"]["util"]["worst_increment_micro"] >= 0 + + +def test_scorecard_aggregate_catches_inversion(unconstrained_model, data): + art = _build(unconstrained_model, data) + scorecard = build_scorecard(art) + report = scorecard_monotone_report(scorecard, CONSTRAINTS) + assert not report["ok"] + assert report["per_feature"]["util"]["worst_increment_micro"] < 0 + + +# --------------------------------------------------------- name plumbing +def test_train_whitebox_name_dict_needs_names(data): + X, latent, _ = data + with pytest.raises(ValueError, match="no feature names"): + train_whitebox(X, latent, n_estimators=2, monotone_constraints={"util": 1}) + + +def test_train_whitebox_accepts_dataframe_names(data): + pd = pytest.importorskip("pandas") + X, latent, _ = data + model, _ = train_whitebox( + pd.DataFrame(X, columns=FEATURES), + latent, + n_estimators=2, + monotone_constraints={"util": 1}, + ) + assert type(model).__name__ == "HistGradientBoostingRegressor" diff --git a/tests/test_validate.py b/tests/test_validate.py index a15da5d..0be8349 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -29,7 +29,10 @@ def fitted(): X = RNG.standard_normal((N, P)) teacher = 1.0 / (1.0 + np.exp(-(1.3 * X[:, 0] + 0.9 * X[:, 1] - 0.7 * X[:, 2]))) y = (RNG.random(N) < teacher).astype(int) - model, _ = train_whitebox(X, teacher, n_estimators=60, random_state=1) + # The teacher is exactly monotone in x0/x1/x2, so the fixture declares it: + # every one of the nine checks then runs (check 9 skips when undeclared). + cst = [1, 1, -1, 0, 0, 0] + model, _ = train_whitebox(X, teacher, n_estimators=60, random_state=1, monotone_constraints=cst) latent = np.clip(model.predict(X), 0, 1) spec = monotone_quantile_bands(latent, y, n_bands=8) artifact = build_artifact( @@ -40,6 +43,7 @@ def fitted(): calibration_latent=latent, calibration_y=y, reasons=REASONS, + monotone_constraints=cst, X_sample=X[:200], ) return X, y, model, latent, artifact @@ -58,6 +62,7 @@ def test_all_checks_pass(fitted): assert report["all_pass"], {k: v for k, v in report["checks"].items() if not v["pass"]} assert not any(c["skipped"] for c in report["checks"].values()) assert report["checks"]["8_reason_coverage"]["coverage"] == 1.0 + assert report["checks"]["9_monotone_constraints"]["constrained_features"] == ["x0", "x1", "x2"] def test_checks_skip_without_inputs(fitted):