From 988be1e03a157a5bfdfe7fa7423f25f370991bc0 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Wed, 2 Sep 2026 10:03:21 +0100 Subject: [PATCH 1/9] Updated statsmodels breaking changes 0.15.0 --- .../estimation/linear_regression_estimator.py | 2 +- pyproject.toml | 2 +- tests/discovery_tests/test_abstract_discovery.py | 2 +- tests/main_tests/test_ctf.py | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/causal_testing/estimation/linear_regression_estimator.py b/causal_testing/estimation/linear_regression_estimator.py index deba16ce..2c91ded7 100644 --- a/causal_testing/estimation/linear_regression_estimator.py +++ b/causal_testing/estimation/linear_regression_estimator.py @@ -155,7 +155,7 @@ def estimate_ate_calculated(self, df: pd.DataFrame) -> EffectEstimate: return EffectEstimate("ate", pd.Series(treatment_outcome["mean"] - control_outcome["mean"]), ci_low, ci_high) def _get_confidence_intervals(self, model, treatment): - confidence_intervals = model.conf_int(alpha=self.alpha, cols=None) + confidence_intervals = model.conf_int(alpha=self.alpha) ci_low, ci_high = ( pd.Series(confidence_intervals[0].loc[treatment]), pd.Series(confidence_intervals[1].loc[treatment]), diff --git a/pyproject.toml b/pyproject.toml index c33ced14..9fc006ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "pandas>=2.1", "scikit_learn~=1.4", "scipy>=1.12.0,<=1.17.1", - "statsmodels~=0.14", + "statsmodels~=0.15", "tabulate~=0.9", "pydot>=2.0", "pygad~=3.3", diff --git a/tests/discovery_tests/test_abstract_discovery.py b/tests/discovery_tests/test_abstract_discovery.py index 1721f400..eaf1c64b 100644 --- a/tests/discovery_tests/test_abstract_discovery.py +++ b/tests/discovery_tests/test_abstract_discovery.py @@ -274,7 +274,7 @@ def test_evaluate_tests_inestimable(self): "outcome": "completed", }, { - "result": TestOutcome.INESTIMABLE, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "color", "outcome": "completed", diff --git a/tests/main_tests/test_ctf.py b/tests/main_tests/test_ctf.py index 312d3acf..d58fda36 100644 --- a/tests/main_tests/test_ctf.py +++ b/tests/main_tests/test_ctf.py @@ -197,14 +197,14 @@ def test_ctf_evaluate_dag_inestimable(self): expected = pd.Series( { "FAIL": 1, - "FAIL_ci_high": 2, + "FAIL_ci_high": 1, "FAIL_ci_low": 0, - "INESTIMABLE": 1, - "INESTIMABLE_ci_high": 1, + "INESTIMABLE": 0, + "INESTIMABLE_ci_high": 0, "INESTIMABLE_ci_low": 0, - "PASS": 4, - "PASS_ci_high": 4, - "PASS_ci_low": 0, + "PASS": 5, + "PASS_ci_high": 5, + "PASS_ci_low": 2, } ).sort_index() pd.testing.assert_series_equal(results, expected) From f79d7f38453e7470cd04a2dc728a7c00b6a33ff2 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Wed, 2 Sep 2026 10:06:55 +0100 Subject: [PATCH 2/9] Updated PR template --- .github/PULL_REQUEST_TEMPLATE.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d4f94d52..fbeddcfc 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,9 @@ +## Summary +What does this PR do? + +## Main file changes +Summarise changes to main files to be reviewed. + ## Checklist Before you mark your PR as ready for review, please ensure you have completed the following. From 20fa2bc4d1af78aa505daa5dc8bc01bc7367c26b Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Mon, 7 Sep 2026 08:34:48 +0100 Subject: [PATCH 3/9] Fixed estimator and expected effect lookup --- causal_testing/causal_testing_framework.py | 44 ++++++++-------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/causal_testing/causal_testing_framework.py b/causal_testing/causal_testing_framework.py index 016b1d61..5d7dfe02 100644 --- a/causal_testing/causal_testing_framework.py +++ b/causal_testing/causal_testing_framework.py @@ -145,45 +145,33 @@ def create_causal_test(self, test: dict) -> CausalTestCase: :return: CausalTestCase object :raises: ValueError if invalid estimator or configuration is provided """ + # Create the estimator with correct parameters estimator_map = {ff.name: ff for ff in entry_points(group="estimators")} - effect_map = {ff.name: ff for ff in entry_points(group="causal_effects")} - if "estimator" not in test: - raise ValueError("Test configuration must specify an estimator") - - if test["estimator"] not in estimator_map: + raise ValueError("Test configuration must specify an estimator.") + estimator_kwargs = test["estimator"] + estimator_name = estimator_kwargs.pop("name") + if estimator_name not in estimator_map: raise ValueError( - f"Unsupported estimator {test['estimator']}. Supported: {sorted(estimator_map)}. " + f"Unsupported estimator {estimator_name}. Supported: {sorted(estimator_map)}. " "If you have implemented a custom estimator, you will need to add this to your entrypoints via your " "pyproject.toml file." ) + estimator = estimator_map.get(estimator_name).load()(**estimator_kwargs) - # Create the estimator with correct parameters - treatment_variable = test.get("treatment_variable") - outcome_variable = test.get("outcome_variable") - estimator_class = estimator_map.get(test["estimator"]).load() - estimator_kwargs = test.get("estimator_kwargs", {}) - effect_type = test.get("expected_effect", {}).get("effect_type", "direct") - - estimator = estimator_class( - treatment_variable=treatment_variable, - outcome_variable=outcome_variable, - treatment_value=test.get("treatment_value"), - control_value=test.get("control_value"), - alpha=test.get("alpha", 0.05), - **estimator_kwargs, - ) - - # Get effect type and create expected effect - expected_effect = test["expected_effect"] - effect_type = expected_effect.pop("name") - if effect_type not in effect_map: + # Create an effect with the corect parameters + effect_map = {ff.name: ff for ff in entry_points(group="causal_effects")} + if "expected_effect" not in test: + raise ValueError("Test configuration must specify an expected effect.") + expected_effect_kwargs = test["expected_effect"] + expected_effect_name = expected_effect_kwargs.pop("name") + if expected_effect_name not in effect_map: raise ValueError( - f"Unsupported causal effect {effect_type}. Supported: {sorted(effect_map)}. " + f"Unsupported causal effect {expected_effect_name}. Supported: {sorted(effect_map)}. " "If you have implemented a custom causal effect, you will need to add this to your entrypoints via " "your pyproject.toml file." ) - expected_effect = effect_map[effect_type].load()(**expected_effect) + expected_effect = effect_map[expected_effect_name].load()(**expected_effect_kwargs) return CausalTestCase( name=test.get("name"), From f78df20d5eae8afe2f146d254f1c78abb234f709 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Mon, 7 Sep 2026 09:04:47 +0100 Subject: [PATCH 4/9] Fixed tests --- .../instrumental_variable_estimator.py | 16 ++++----- tests/main_tests/test_ctf.py | 34 +++++++++++-------- tests/resources/data/tests.json | 27 +++++++++------ 3 files changed, 44 insertions(+), 33 deletions(-) diff --git a/causal_testing/estimation/instrumental_variable_estimator.py b/causal_testing/estimation/instrumental_variable_estimator.py index f4ffab37..e52515dc 100644 --- a/causal_testing/estimation/instrumental_variable_estimator.py +++ b/causal_testing/estimation/instrumental_variable_estimator.py @@ -25,8 +25,6 @@ def __init__( self, outcome_variable: str, treatment_variable: str, - treatment_value: float, - control_value: float, instrument: str, alpha: float = 0.05, bootstrap_size=100, @@ -34,8 +32,6 @@ def __init__( super().__init__( treatment_variable=treatment_variable, outcome_variable=outcome_variable, - treatment_value=treatment_value, - control_value=control_value, alpha=alpha, ) @@ -47,13 +43,17 @@ def add_modelling_assumptions(self): Add modelling assumptions to the estimator. This is a list of strings which list the modelling assumptions that must hold if the resulting causal inference is to be considered valid. """ - self.modelling_assumptions.append("""The instrument and the treatment, and the treatment and the outcome must be - related linearly in the form Y = aX + b.""") - self.modelling_assumptions.append("""The three IV conditions must hold + self.modelling_assumptions.append( + """The instrument and the treatment, and the treatment and the outcome must be + related linearly in the form Y = aX + b.""" + ) + self.modelling_assumptions.append( + """The three IV conditions must hold (i) Instrument is associated with treatment (ii) Instrument does not affect outcome except through its potential effect on treatment (iii) Instrument and outcome do not share causes - """) + """ + ) def iv_coefficient(self, df) -> float: """ diff --git a/tests/main_tests/test_ctf.py b/tests/main_tests/test_ctf.py index d58fda36..e4b75908 100644 --- a/tests/main_tests/test_ctf.py +++ b/tests/main_tests/test_ctf.py @@ -57,7 +57,7 @@ def test_create_test_case_invalid_estimator(self): "treatment_variable": "test_input", "outcome_variable": "test_output", "expected_effect": {"name": "NoEffect"}, - "estimator": "InvalidEstimator", + "estimator": {"name": "InvalidEstimator"}, } ) self.assertEqual( @@ -79,7 +79,7 @@ def test_create_test_case_no_estimator(self): } ) self.assertEqual( - "Test configuration must specify an estimator", + "Test configuration must specify an estimator.", str(e.exception), ) @@ -89,12 +89,14 @@ def test_create_test_case_invalid_effect(self): framework.load_data(self.data_paths) test = { "name": "test1", - "treatment_variable": "test_input", - "estimator": "LinearRegressionEstimator", + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "test_input", + "outcome_variable": "test_output", + "adjustment_set": [], + }, "effect_measure": "coefficient", - "outcome_variable": "test_output", "expected_effect": {"name": "InvalidEffect"}, - "estimator_kwargs": {"adjustment_set": []}, } with self.assertRaises(ValueError) as e: framework.create_causal_test(test) @@ -111,12 +113,14 @@ def test_create_test_case_effect_kwargs(self): framework.load_data(self.data_paths) test = { "name": "test1", - "treatment_variable": "test_input", - "estimator": "LinearRegressionEstimator", + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "test_input", + "outcome_variable": "test_output", + "adjustment_set": [], + }, "effect_measure": "coefficient", - "outcome_variable": "test_output", "expected_effect": {"name": "ExactValue", "value": 4}, - "estimator_kwargs": {"adjustment_set": []}, } test_case = framework.create_causal_test(test) self.assertEqual(test_case.expected_causal_effect.value, 4) @@ -127,12 +131,14 @@ def test_create_test_case_estimator_kwargs(self): framework.load_data(self.data_paths) test = { "name": "test1", - "treatment_variable": "test_input", - "estimator": "InstrumentalVariableEstimator", + "estimator": { + "name": "InstrumentalVariableEstimator", + "treatment_variable": "test_input", + "outcome_variable": "test_output", + "instrument": "instrumental_variable", + }, "effect_measure": "coefficient", - "outcome_variable": "test_output", "expected_effect": {"name": "SomeEffect"}, - "estimator_kwargs": {"instrument": "instrumental_variable"}, } test_case = framework.create_causal_test(test) self.assertEqual(test_case.estimator.instrument, "instrumental_variable") diff --git a/tests/resources/data/tests.json b/tests/resources/data/tests.json index 9ee0171f..d6f64e5b 100644 --- a/tests/resources/data/tests.json +++ b/tests/resources/data/tests.json @@ -1,23 +1,28 @@ { "tests": [{ "name": "test1", - "treatment_variable": "test_input", - "estimator": "LinearRegressionEstimator", + "estimator": { + "name": "LinearRegressionEstimator", + "outcome_variable": "test_output", + "treatment_variable": "test_input", + "adjustment_set": [] + }, "effect_measure": "coefficient", - "outcome_variable": "test_output", "expected_effect": {"name": "NoEffect"}, + "expected_effect": {"name": "NoEffect"}, "skip": false, - "query": "test_input > 0", - "estimator_kwargs": {"adjustment_set": []} - + "query": "test_input > 0" }, { "name": "test2", - "treatment_variable": "test_input", - "estimator": "LinearRegressionEstimator", + "estimator": { + "name":"LinearRegressionEstimator", + "treatment_variable": "test_input", + "outcome_variable": "test_output", + "adjustment_set": [] + }, + "expected_effect": {"name": "NoEffect"}, "effect_measure": "coefficient", - "outcome_variable": "test_output", "expected_effect": {"name": "NoEffect"}, "skip": true, - "query": "test_input <= 5", - "estimator_kwargs": {"adjustment_set": []} + "query": "test_input <= 5" }] } From bf629acda1ca3cf12a9b49a1c73ad2aba67dd2f0 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Mon, 7 Sep 2026 09:08:47 +0100 Subject: [PATCH 5/9] Added top level option to include causal test adequacy results in json output. --- causal_testing/__main__.py | 12 ++++++++++-- causal_testing/causal_testing_framework.py | 16 +++++++++++++--- causal_testing/testing/causal_test_case.py | 16 ++++++++-------- causal_testing/testing/causal_test_result.py | 6 ++++-- causal_testing/testing/data_adequacy.py | 6 +++--- 5 files changed, 38 insertions(+), 18 deletions(-) diff --git a/causal_testing/__main__.py b/causal_testing/__main__.py index ddc9641b..5053acb8 100644 --- a/causal_testing/__main__.py +++ b/causal_testing/__main__.py @@ -79,7 +79,15 @@ def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace: "-s", "--silent", action="store_true", - help="Do not crash on error. If set to true, errors are recorded as test results.", + help="Do not crash on error. If set to true, errors are recorded as test results. (Defaults to False)", + default=False, + ) + parser_test.add_argument( + "-R", + "--include-adequacy-results", + action="store_true", + help="Include_adequacy_results: Whether to include the effect estimate and test outcome for adequacy " + "bootstraps. (Defaults to False)", default=False, ) @@ -257,7 +265,7 @@ def main() -> None: logging.info("Running tests") framework.run_tests(silent=args.silent, adequacy=args.adequacy, bootstrap_size=args.bootstrap_size) - framework.save_results(args.output) + framework.save_results(args.output, include_adequacy_results=args.include_adequacy_results) logging.info("Causal testing completed successfully.") case Command.EVALUATE: diff --git a/causal_testing/causal_testing_framework.py b/causal_testing/causal_testing_framework.py index 5d7dfe02..ddb59563 100644 --- a/causal_testing/causal_testing_framework.py +++ b/causal_testing/causal_testing_framework.py @@ -255,8 +255,14 @@ def evaluate_dag(self, bootstrap_size: bool = 100, alpha: float = 0.05) -> pd.Se return pd.Series(results).sort_index() - def save_results(self, output_path) -> list: - """Save test results to JSON file in the expected format.""" + def save_results(self, output_path: str, include_adequacy_results: bool = False): + """ + Save test results to JSON file in the expected format. + + :param output_path: Path for output file (.json). + :param include_adequacy_results: Whether to include the effect estimate and test outcome for adequacy + bootstraps. + """ logger.info(f"Saving results to {output_path}") # Create parent directory if it doesn't exist @@ -264,6 +270,10 @@ def save_results(self, output_path) -> list: # Save to file with open(output_path, "w", encoding="utf-8") as f: - json.dump([test.to_dict() for test in self.test_cases], f, indent=2) + json.dump( + [test.to_dict(include_adequacy_results=include_adequacy_results) for test in self.test_cases], + f, + indent=2, + ) logger.info("Results saved successfully") diff --git a/causal_testing/testing/causal_test_case.py b/causal_testing/testing/causal_test_case.py index ec1de7b8..4531b67d 100644 --- a/causal_testing/testing/causal_test_case.py +++ b/causal_testing/testing/causal_test_case.py @@ -172,10 +172,11 @@ def estimate_effect(self, df: pd.DataFrame) -> CausalTestResult: estimate_effect = getattr(self.estimator, f"estimate_{self.effect_measure}") return estimate_effect(df) - def to_dict(self) -> dict: + def to_dict(self, include_adequacy_results: bool = False) -> dict: """ Convert the test case to a python dictionary for easy serialisation as JSON. + :bool include_adequacy_results: Whether to include the effect estimate and test outcome for adequacy bootstraps. :returns: A JSON serialisable dict representing the test case. """ test_case = { @@ -185,12 +186,11 @@ def to_dict(self) -> dict: "query": self.query, } - for label, attribute in [ - ("expected_effect", self.expected_causal_effect), - ("estimator", self.estimator), - ("result", self.result), - ]: - if attribute is not None: - test_case[label] = attribute.to_dict() + if self.expected_causal_effect is not None: + test_case["expected_effect"] = self.expected_causal_effect.to_dict() + if self.estimator is not None: + test_case["estimator"] = self.estimator.to_dict() + if self.result is not None: + test_case["result"] = self.result.to_dict(include_adequacy_results=include_adequacy_results) return test_case diff --git a/causal_testing/testing/causal_test_result.py b/causal_testing/testing/causal_test_result.py index 5bfb15c1..c5240943 100644 --- a/causal_testing/testing/causal_test_result.py +++ b/causal_testing/testing/causal_test_result.py @@ -34,10 +34,12 @@ def passed(self) -> bool: """ return self.outcome == TestOutcome.PASS - def to_dict(self): + def to_dict(self, include_adequacy_results: bool = False): """ Convert the result to a python dictionary for easy serialisation as JSON. + :param include_adequacy_results: Whether to include the effect estimate and test outcome for adequacy + bootstraps. :returns: A JSON serialisable dict representing the test result. """ @@ -47,6 +49,6 @@ def to_dict(self): effect_estimate = self.effect_estimate.to_dict() if self.effect_estimate else {} - adequacy = self.adequacy.to_dict() if self.adequacy else {} + adequacy = self.adequacy.to_dict(include_adequacy_results=include_adequacy_results) if self.adequacy else {} return outcome | effect_estimate | {"adequacy": adequacy} diff --git a/causal_testing/testing/data_adequacy.py b/causal_testing/testing/data_adequacy.py index 192ef1a1..b094e554 100644 --- a/causal_testing/testing/data_adequacy.py +++ b/causal_testing/testing/data_adequacy.py @@ -34,10 +34,10 @@ def __init__( self.successful = successful self.bootstrap_size = bootstrap_size - def to_dict(self, include_results: bool = False): + def to_dict(self, include_adequacy_results: bool = False): """ :returns: the adequacy object as a dictionary. - :param include_results: Whether to serialise the results. + :param include_adequacy_results: Whether to serialise the results. """ result = { "kurtosis": self.kurtosis.to_dict(), @@ -45,6 +45,6 @@ def to_dict(self, include_results: bool = False): "successful": self.successful, "bootstrap_size": self.bootstrap_size, } - if include_results: + if include_adequacy_results: return result | {"results": self.results.reset_index(drop=True).to_dict()} return result From 02c1bdac99bc0b71d58f91af5b14a6b14a3e07d9 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Mon, 7 Sep 2026 09:18:28 +0100 Subject: [PATCH 6/9] Fixed tutorial tests --- .../vaccinating_elderly/causal_tests.json | 354 ++++++++++++++---- 1 file changed, 277 insertions(+), 77 deletions(-) diff --git a/docs/source/tutorials/vaccinating_elderly/causal_tests.json b/docs/source/tutorials/vaccinating_elderly/causal_tests.json index 3c4656c0..4c8409a4 100644 --- a/docs/source/tutorials/vaccinating_elderly/causal_tests.json +++ b/docs/source/tutorials/vaccinating_elderly/causal_tests.json @@ -1,109 +1,309 @@ { - "tests": [ - { - "name": "max_doses _||_ cum_vaccinations", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "tests": [{ + "name": "max_doses _||_ vaccine", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "max_doses", + "outcome_variable": "vaccine", + "alpha": 0.05, + "formula": "vaccine ~ max_doses" + } + }, { + "name": "vaccine _||_ max_doses", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "vaccine", + "outcome_variable": "max_doses", + "alpha": 0.05, + "formula": "max_doses ~ vaccine" + } + }, { + "name": "max_doses _||_ cum_vaccinations", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "max_doses", "outcome_variable": "cum_vaccinations", - "expected_effect": {"name": "NoEffect"}, - "estimator_kwargs": {"formula": "cum_vaccinations ~ max_doses"}, "alpha": 0.05, - "skip": false + "formula": "cum_vaccinations ~ max_doses" + } + }, { + "name": "cum_vaccinations _||_ max_doses", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "cum_vaccinations", + "outcome_variable": "max_doses", + "alpha": 0.05, + "formula": "max_doses ~ cum_vaccinations" + } + }, { + "name": "max_doses _||_ cum_vaccinated", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 }, - { - "name": "max_doses _||_ cum_vaccinated", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "max_doses", "outcome_variable": "cum_vaccinated", - "expected_effect": {"name": "NoEffect"}, - "estimator_kwargs": {"formula": "cum_vaccinated ~ max_doses"}, "alpha": 0.05, - "skip": false + "formula": "cum_vaccinated ~ max_doses" + } + }, { + "name": "cum_vaccinated _||_ max_doses", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "cum_vaccinated", + "outcome_variable": "max_doses", + "alpha": 0.05, + "formula": "max_doses ~ cum_vaccinated" + } + }, { + "name": "max_doses _||_ cum_infections", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 }, - { - "name": "max_doses _||_ cum_infections", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "max_doses", "outcome_variable": "cum_infections", - "expected_effect": {"name": "NoEffect"}, - "estimator_kwargs": {"formula": "cum_infections ~ max_doses"}, "alpha": 0.05, - "skip": false + "formula": "cum_infections ~ max_doses" + } + }, { + "name": "cum_infections _||_ max_doses", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "cum_infections", + "outcome_variable": "max_doses", + "alpha": 0.05, + "formula": "max_doses ~ cum_infections" + } + }, { + "name": "vaccine -> cum_vaccinations", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "SomeEffect", + "effect_type": "direct" }, - { - "name": "vaccine --> cum_vaccinations", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "vaccine", "outcome_variable": "cum_vaccinations", - "expected_effect": {"name": "SomeEffect"}, - "estimator_kwargs": {"formula": "cum_vaccinations ~ vaccine"}, - "skip": false - }, - { - "name": "vaccine --> cum_vaccinated", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "alpha": 0.05, + "formula": "cum_vaccinations ~ vaccine" + } + }, { + "name": "vaccine -> cum_vaccinated", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "SomeEffect", + "effect_type": "direct" + }, + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "vaccine", "outcome_variable": "cum_vaccinated", - "expected_effect": {"name": "SomeEffect"}, - "estimator_kwargs": {"formula": "cum_vaccinated ~ vaccine"}, - "skip": false - }, - { - "name": "vaccine --> cum_infections", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "alpha": 0.05, + "formula": "cum_vaccinated ~ vaccine" + } + }, { + "name": "vaccine -> cum_infections", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "SomeEffect", + "effect_type": "direct" + }, + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "vaccine", "outcome_variable": "cum_infections", - "expected_effect": {"name": "SomeEffect"}, - "estimator_kwargs": {"formula": "cum_infections ~ vaccine"}, - "skip": false - }, - { - "name": "cum_vaccinations _||_ cum_vaccinated | ['vaccine']", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "alpha": 0.05, + "formula": "cum_infections ~ vaccine" + } + }, { + "name": "cum_vaccinations _||_ cum_vaccinated", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "cum_vaccinations", "outcome_variable": "cum_vaccinated", - "expected_effect": {"name": "NoEffect"}, - "estimator_kwargs": {"formula": "cum_vaccinated ~ cum_vaccinations + vaccine"}, "alpha": 0.05, - "skip": false + "adjustment_set": ["vaccine"], + "formula": "cum_vaccinated ~ cum_vaccinations + vaccine" + } + }, { + "name": "cum_vaccinated _||_ cum_vaccinations", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 }, - { - "name": "cum_vaccinations _||_ cum_infections | ['vaccine']", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "cum_vaccinated", + "outcome_variable": "cum_vaccinations", + "alpha": 0.05, + "adjustment_set": ["vaccine"], + "formula": "cum_vaccinations ~ cum_vaccinated + vaccine" + } + }, { + "name": "cum_vaccinations _||_ cum_infections", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "cum_vaccinations", "outcome_variable": "cum_infections", - "expected_effect": {"name": "NoEffect"}, - "estimator_kwargs": {"formula": "cum_infections ~ cum_vaccinations + vaccine"}, "alpha": 0.05, - "skip": false + "adjustment_set": ["vaccine"], + "formula": "cum_infections ~ cum_vaccinations + vaccine" + } + }, { + "name": "cum_infections _||_ cum_vaccinations", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 }, - { - "name": "cum_vaccinated _||_ cum_infections | ['vaccine']", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "cum_infections", + "outcome_variable": "cum_vaccinations", + "alpha": 0.05, + "adjustment_set": ["vaccine"], + "formula": "cum_vaccinations ~ cum_infections + vaccine" + } + }, { + "name": "cum_vaccinated _||_ cum_infections", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "cum_vaccinated", "outcome_variable": "cum_infections", - "expected_effect": {"name": "NoEffect"}, - "estimator_kwargs": {"formula": "cum_infections ~ cum_vaccinated + vaccine"}, "alpha": 0.05, - "skip": false + "adjustment_set": ["vaccine"], + "formula": "cum_infections ~ cum_vaccinated + vaccine" + } + }, { + "name": "cum_infections _||_ cum_vaccinated", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "cum_infections", + "outcome_variable": "cum_vaccinated", + "alpha": 0.05, + "adjustment_set": ["vaccine"], + "formula": "cum_vaccinated ~ cum_infections + vaccine" } - ] + }] } From 3d609ab87216366cc40771008e7c694dacc9c814 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Mon, 7 Sep 2026 09:31:28 +0100 Subject: [PATCH 7/9] Fixed tests again --- .../test_instrumental_variable_estimator.py | 6 ------ tests/testing_tests/test_causal_test_adequacy.py | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/estimation_tests/test_instrumental_variable_estimator.py b/tests/estimation_tests/test_instrumental_variable_estimator.py index a4e1edf3..10f72fca 100644 --- a/tests/estimation_tests/test_instrumental_variable_estimator.py +++ b/tests/estimation_tests/test_instrumental_variable_estimator.py @@ -25,8 +25,6 @@ def test_estimate_coefficient(self): iv_estimator = InstrumentalVariableEstimator( treatment_variable="X", outcome_variable="Y", - treatment_value=None, - control_value=None, instrument="Z", ) effect_estimate = iv_estimator.estimate_coefficient(self.df) @@ -38,8 +36,6 @@ def test_to_dict(self): iv_estimator = InstrumentalVariableEstimator( treatment_variable="X", outcome_variable="Y", - control_value=0, - treatment_value=1, instrument="Z", ) self.assertEqual( @@ -49,8 +45,6 @@ def test_to_dict(self): "treatment_variable": "X", "outcome_variable": "Y", "alpha": 0.05, - "control_value": 0, - "treatment_value": 1, "instrument": "Z", "bootstrap_size": 100, }, diff --git a/tests/testing_tests/test_causal_test_adequacy.py b/tests/testing_tests/test_causal_test_adequacy.py index 2aec4a40..28980abd 100644 --- a/tests/testing_tests/test_causal_test_adequacy.py +++ b/tests/testing_tests/test_causal_test_adequacy.py @@ -210,7 +210,7 @@ def test_to_dict(self): # Use json_normalize to avoid rounding errors pd.testing.assert_frame_equal( pd.json_normalize(expected_dict).round(2), - pd.json_normalize(adequacy_metric.to_dict(include_results=True)).round(2), + pd.json_normalize(adequacy_metric.to_dict(include_adequacy_results=True)).round(2), ) def test_dag_adequacy_dependent(self): From 9516dcbf91571f1be2b5323fe4e389637e1117e6 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Mon, 7 Sep 2026 09:31:36 +0100 Subject: [PATCH 8/9] Fixed tutorials --- .../vaccinating_elderly_tutorial.ipynb | 26 ++++++++++++------- .../visualise_causal_test_results.ipynb | 5 ++-- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/docs/source/tutorials/vaccinating_elderly/vaccinating_elderly_tutorial.ipynb b/docs/source/tutorials/vaccinating_elderly/vaccinating_elderly_tutorial.ipynb index b67fe911..93e43d38 100644 --- a/docs/source/tutorials/vaccinating_elderly/vaccinating_elderly_tutorial.ipynb +++ b/docs/source/tutorials/vaccinating_elderly/vaccinating_elderly_tutorial.ipynb @@ -129,15 +129,23 @@ "name": "stdout", "output_type": "stream", "text": [ - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Loading DAG from dag.dot\n", - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - DAG loaded with 5 nodes and 3 edges\n", - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Loading data from 1 source(s)\n", - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Initial data shape: (60, 16)\n", - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Loading test configurations from causal_tests.json\n", - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Running causal tests...\n", - "100%|████████████████████████████████████████████| 9/9 [00:00<00:00, 271.76it/s]\n", - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Saving results to causal_test_results.json\n", - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Results saved successfully\n" + "2026-09-07 09:30:58 - causal_testing.causal_testing_framework - INFO - Loading DAG from dag.dot\n", + "2026-09-07 09:30:58 - causal_testing.causal_testing_framework - INFO - DAG loaded with 5 nodes and 3 edges\n", + "2026-09-07 09:30:58 - causal_testing.causal_testing_framework - INFO - Loading data from 1 source(s)\n", + "2026-09-07 09:30:58 - causal_testing.causal_testing_framework - INFO - Initial data shape: (60, 16)\n", + "2026-09-07 09:30:58 - causal_testing.causal_testing_framework - INFO - Loading test configurations from causal_tests.json\n", + "2026-09-07 09:30:58 - causal_testing.causal_testing_framework - INFO - Running causal tests...\n", + " 0%| | 0/17 [00:00" + "
" ] }, "metadata": {}, @@ -190,6 +190,7 @@ "fig, ax = plt.subplots(nrows=len(causal_tests_results), ncols=1, figsize=[10,100])\n", "\n", "dag_copy = dag.copy()\n", + "dag_copy.ignore_cycles=True\n", "\n", "for i, test in enumerate(causal_tests_results):\n", " treatment_node = test['estimator']['treatment_variable']\n", From 358645cb4568c61e3d29ff01f792050d58445d86 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Mon, 7 Sep 2026 09:49:37 +0100 Subject: [PATCH 9/9] Added test for no expected effect and for generate and test --- tests/main_tests/test_ctf.py | 21 ++++++++++++++++++ tests/main_tests/test_main.py | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/tests/main_tests/test_ctf.py b/tests/main_tests/test_ctf.py index e4b75908..cf38563a 100644 --- a/tests/main_tests/test_ctf.py +++ b/tests/main_tests/test_ctf.py @@ -83,6 +83,27 @@ def test_create_test_case_no_estimator(self): str(e.exception), ) + def test_create_test_case_no_expected_effect(self): + framework = CausalTestingFramework() + framework.load_dag(self.dag_path) + framework.load_data(self.data_paths) + test = { + "name": "test1", + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "test_input", + "outcome_variable": "test_output", + "adjustment_set": [], + }, + "effect_measure": "coefficient", + } + with self.assertRaises(ValueError) as e: + framework.create_causal_test(test) + self.assertEqual( + "Test configuration must specify an expected effect.", + str(e.exception), + ) + def test_create_test_case_invalid_effect(self): framework = CausalTestingFramework() framework.load_dag(self.dag_path) diff --git a/tests/main_tests/test_main.py b/tests/main_tests/test_main.py index eddb4938..239abe44 100644 --- a/tests/main_tests/test_main.py +++ b/tests/main_tests/test_main.py @@ -110,6 +110,47 @@ def test_parse_args_bootstrap_size_explicit_adequacy(self): executed_tests = [test for test in log if not test.get("skip", False)] assert all(test["result"].get("bootstrap_size", 50) == 50 for test in executed_tests) + def test_parse_args_generate_and_test(self): + with tempfile.TemporaryDirectory() as tmp: + with patch( + "sys.argv", + [ + "causal_testing", + "generate", + "--dag-path", + str(self.dag_path), + "--data-paths", + str(self.data_paths[0]), + "--output", + os.path.join(tmp, "tests.json"), + ], + ): + main() + self.assertTrue(os.path.exists(os.path.join(tmp, "tests.json"))) + with patch( + "sys.argv", + [ + "causal_testing", + "test", + "--dag-path", + str(self.dag_path), + "--data-paths", + str(self.data_paths[0]), + "--test-config", + os.path.join(tmp, "tests.json"), + "--output", + str(self.output_path.parent / "main.json"), + "-A", + "-b", + "50", + ], + ): + main() + with open(self.output_path.parent / "main.json", encoding="utf-8") as f: + log = json.load(f) + executed_tests = [test for test in log if not test.get("skip", False)] + assert all(test["result"].get("bootstrap_size", 50) == 50 for test in executed_tests) + def test_parse_args_generation(self): with tempfile.TemporaryDirectory() as tmp: with patch(