From eb809d9874a3516e504a055e06c2181976c669c9 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 8 Jun 2026 22:03:54 -0700 Subject: [PATCH 01/90] initial calibration commit --- activitysim/cli/run.py | 26 +- activitysim/core/calibration.py | 1067 +++++++++++++++++++++++++++++++ 2 files changed, 1091 insertions(+), 2 deletions(-) create mode 100644 activitysim/core/calibration.py diff --git a/activitysim/cli/run.py b/activitysim/cli/run.py index ad91c4f167..289a27914f 100644 --- a/activitysim/cli/run.py +++ b/activitysim/cli/run.py @@ -14,7 +14,7 @@ import numpy as np -from activitysim.core import chunk, config, mem, timing, tracing, workflow +from activitysim.core import calibration, chunk, config, mem, timing, tracing, workflow from activitysim.core.configuration import FileSystem, Settings from activitysim.core.run_id import RunId @@ -418,7 +418,29 @@ def run(args): check_model_settings(state, extension_settings=extension_checker_settings) try: - if state.settings.multiprocess: + if calibration.calibration_enabled(state): + logger.info("run calibration workflow") + + calibration_result = calibration.run_calibration_loop( + state=state, + models=state.settings.models, + memory_sidecar_process=memory_sidecar_process, + ) + + logger.info( + "calibration workflow complete converged=%s completed_global_iterations=%s", + calibration_result.converged, + calibration_result.completed_global_iterations, + ) + + if state.settings.cleanup_pipeline_after_run: + state.checkpoint.cleanup() + else: + state.checkpoint.close_store() + + mem.log_global_hwm() + + elif state.settings.multiprocess: logger.info("run multiprocess simulation") from activitysim.core import mp_tasks diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py new file mode 100644 index 0000000000..902bdbba61 --- /dev/null +++ b/activitysim/core/calibration.py @@ -0,0 +1,1067 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import importlib +import importlib.util +import inspect +import json +import logging +import math +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +from pydantic import model_validator + +from activitysim.core import workflow +from activitysim.core.configuration import PydanticReadable +from activitysim.core.configuration.base import PydanticBase + +logger = logging.getLogger("calibration") + +CALIBRATION_SETTINGS_FILE_NAME = "calibration.yaml" +CALIBRATION_OUTPUT_DIR = "calibration" +CALIBRATION_PROGRESS_FILE = "calibration/calibration_progress.json" +CALIBRATION_ITERATION_FILE = "calibration/calibration_iteration_records.csv" +CALIBRATION_SUMMARY_FILE = "calibration/calibration_iteration_summary.csv" +CALIBRATION_FINAL_COEFFICIENTS_FILE = "calibration/final_calibrated_coefficients.csv" + +CALIBRATION_REQUIRED_COLUMNS = [ + "description", + "coefficient", + "model_value", + "target_value", + "hold_fast", + "min", + "max", + "damping", + "method", + "tolerance", +] + +MP_INJECTABLES = [ + "data_dir", + "configs_dir", + "data_model_dir", + "output_dir", + "cache_dir", + "settings_file_name", + "imported_extensions", + "run_timestamp", + "run_id", +] + + +class CalibrationRunSettings(PydanticBase): + """Run-control settings for calibration.""" + + resume_after: str + calibrate_models: list[str] + restart_after: list[str] = [] + + +class CalibrationReportsSettings(PydanticBase): + """Reporting settings for a calibrated component.""" + + generic: bool = True + bespoke: str | None = None + + +class CalibrationComponentSettings(PydanticBase): + """Settings for one calibratable model component.""" + + calibration_spec: str + helper_module: str | None = None + submodel_max_iterations: int = 1 + reports: CalibrationReportsSettings = CalibrationReportsSettings() + + +class CalibrationConfig(PydanticReadable): + """Top-level calibration configuration.""" + + enable: bool = False + max_iterations: int = 1 + run: CalibrationRunSettings + model_settings: dict[str, CalibrationComponentSettings] = {} + + @model_validator(mode="after") + def validate_model_settings(self): + """Validate that configured components are aligned with run settings.""" + for component in self.run.calibrate_models: + if component not in self.model_settings: + raise ValueError( + f"calibration model '{component}' is not in model_settings" + ) + + for component in self.run.restart_after: + if component not in self.run.calibrate_models: + raise ValueError( + f"restart_after component '{component}' is not in calibrate_models" + ) + + if self.max_iterations < 1: + raise ValueError("max_iterations must be >= 1") + + return self + + +@dataclass +class CalibrationComponentResult: + """Result details from calibrating one component.""" + + component: str + converged: bool + component_iterations: int + + +@dataclass +class CalibrationRunResult: + """Result details from a complete global calibration loop.""" + + converged: bool + completed_global_iterations: int + + +def read_calibration_settings(state: workflow.State) -> CalibrationConfig | None: + """Read and validate calibration settings if calibration.yaml exists.""" + return CalibrationConfig.read_settings_file( + state.filesystem, + CALIBRATION_SETTINGS_FILE_NAME, + mandatory=False, + ) + + +def calibration_enabled(state: workflow.State) -> bool: + """Return True when calibration.yaml exists and is enabled.""" + settings = read_calibration_settings(state) + return bool(settings and settings.enable) + + +def run_calibration_loop( + state: workflow.State, + models: list[str], + memory_sidecar_process=None, +) -> CalibrationRunResult: + """ + Run the global calibration workflow. + + This function intentionally minimizes changes to the existing run mechanics: + it always reuses ActivitySim's normal model execution paths and only adds + calibration orchestration around them. + """ + calibration_settings = read_calibration_settings(state) + if not calibration_settings or not calibration_settings.enable: + raise RuntimeError("calibration loop called while calibration is disabled") + + _ensure_calibration_output_dir(state) + + # If there is recoverable calibration progress from a prior interrupted run, + # continue from that iteration. Coefficient updates are persisted in config + # coefficient files, so restarting from a later global iteration is compatible + # with current checkpoint semantics. + progress = _read_progress(state) + start_global_iter = int(progress.get("next_global_iteration", 1)) if progress else 1 + + original_pipeline_name = state.filesystem.pipeline_file_name + + try: + for global_iter in range( + start_global_iter, calibration_settings.max_iterations + 1 + ): + logger.info( + "calibration global iteration %s/%s", + global_iter, + calibration_settings.max_iterations, + ) + + # Run ActivitySim normally from resume_after through production model steps. + _run_model_system_for_iteration( + state, + models=models, + resume_after=calibration_settings.run.resume_after, + global_iter=global_iter, + memory_sidecar_process=memory_sidecar_process, + ) + + all_converged = True + restart_triggered = False + + for component in calibration_settings.run.calibrate_models: + component_settings = calibration_settings.model_settings[component] + prior_step = _prior_step_name(models, component) + if prior_step is None: + prior_step = calibration_settings.run.resume_after + + component_result = _calibrate_component( + state=state, + component_name=component, + component_settings=component_settings, + prior_step=prior_step, + global_iter=global_iter, + ) + + all_converged = all_converged and component_result.converged + + if component in calibration_settings.run.restart_after: + # Restart global loop from resume_after after this component. + restart_triggered = True + break + + _write_progress( + state, + { + "next_global_iteration": global_iter + 1, + "last_completed_global_iteration": global_iter, + }, + ) + + if all_converged and not restart_triggered: + _write_final_coefficients_snapshot(state, calibration_settings) + _clear_progress(state) + return CalibrationRunResult( + converged=True, + completed_global_iterations=global_iter, + ) + + _write_final_coefficients_snapshot(state, calibration_settings) + _clear_progress(state) + return CalibrationRunResult( + converged=False, + completed_global_iterations=calibration_settings.max_iterations, + ) + finally: + state.filesystem.pipeline_file_name = original_pipeline_name + + +def _run_model_system_for_iteration( + state: workflow.State, + models: list[str], + resume_after: str, + global_iter: int, + memory_sidecar_process=None, +) -> None: + """Run the normal ActivitySim model flow for one global calibration iteration.""" + if global_iter > 1: + # Seed a fresh pipeline from the configured resume checkpoint to avoid + # duplicate checkpoint-name collisions across global calibration loops. + prior_pipeline = state.checkpoint.store.filename + state.checkpoint.close_store() + state.filesystem.pipeline_file_name = f"pipeline_calibration_iter_{global_iter}" + state.checkpoint.restore_from(prior_pipeline, checkpoint_name=resume_after) + + _run_in_configured_mode( + state, + models=models, + resume_after=resume_after, + memory_sidecar_process=memory_sidecar_process, + ) + + +def _calibrate_component( + state: workflow.State, + component_name: str, + component_settings: CalibrationComponentSettings, + prior_step: str, + global_iter: int, +) -> CalibrationComponentResult: + """Run iterative coefficient calibration for one component.""" + model_settings_file = _infer_model_settings_file(component_name) + model_settings = state.filesystem.read_model_settings( + model_settings_file, mandatory=True + ) + + coefficients_df = state.filesystem.read_model_coefficients( + model_settings=model_settings + ) + helper_symbols, bespoke_callable = _load_helper_symbols( + state, + component_settings, + ) + + calibration_spec_df = _read_calibration_spec( + state, component_settings.calibration_spec + ) + + utility_coeff_names = _extract_utility_coefficient_names(state, model_settings) + _validate_calibration_coefficients_against_utility_spec( + component_name, + calibration_spec_df, + utility_coeff_names, + ) + + coefficients_df = _ensure_coefficients_exist( + component_name, + calibration_spec_df, + coefficients_df, + ) + + _warn_if_initial_values_outside_bounds( + component_name, calibration_spec_df, coefficients_df + ) + + component_converged = False + component_iterations = 0 + + for component_iter in range(1, component_settings.submodel_max_iterations + 1): + component_iterations = component_iter + + # Re-run only this component from its prior checkpoint so model values + # reflect the current candidate coefficients for this component. + if state.settings.multiprocess: + # In multiprocess mode, preserve the standard multiprocess orchestration + # so table coalescing semantics match the initial global run path. + _run_in_configured_mode( + state, + models=state.settings.models, + resume_after=prior_step, + ) + else: + run_model_name = ( + f"{component_name}.calibration_component_iter={component_iter};" + f"calibration_global_iter={global_iter}" + ) + state.run(models=[run_model_name], resume_after=prior_step) + + eval_context = _build_expression_context(state, helper_symbols) + + ( + row_records, + summary_record, + new_coefficients_df, + component_converged, + ) = _evaluate_and_update( + component_name=component_name, + calibration_spec_df=calibration_spec_df, + coefficients_df=coefficients_df, + eval_context=eval_context, + global_iter=global_iter, + component_iter=component_iter, + ) + + coefficients_df = new_coefficients_df + + _persist_coefficients_to_config(state, model_settings, coefficients_df) + _append_iteration_records(state, row_records) + _append_summary_records(state, [summary_record]) + + if component_settings.reports.generic: + _write_generic_report(state, component_name, row_records) + + if bespoke_callable is not None: + # Preserve compatibility with helper modules that expect a global + # `state` symbol and/or no explicit arguments. + _run_bespoke_report(bespoke_callable, state) + + if component_converged: + break + + return CalibrationComponentResult( + component=component_name, + converged=component_converged, + component_iterations=component_iterations, + ) + + +def _read_calibration_spec(state: workflow.State, file_name: str) -> pd.DataFrame: + """Read calibration spec CSV and validate required columns.""" + path = state.filesystem.get_config_file_path(file_name) + df = pd.read_csv(path, comment="#") + + missing = [c for c in CALIBRATION_REQUIRED_COLUMNS if c not in df.columns] + if missing: + raise ValueError( + f"calibration_spec {file_name} is missing required columns: {missing}" + ) + + df = df[CALIBRATION_REQUIRED_COLUMNS].copy() + df["description"] = df["description"].astype(str) + df["coefficient"] = df["coefficient"].astype(str) + + # Normalize booleans and defaults to keep row math deterministic. + df["hold_fast"] = ( + df["hold_fast"] + .fillna(False) + .astype(str) + .str.strip() + .str.lower() + .isin(["1", "true", "t", "yes", "y"]) + ) + + for c in ["min", "max", "damping", "tolerance"]: + df[c] = pd.to_numeric(df[c], errors="coerce") + + df["method"] = df["method"].astype(str).str.strip().str.lower() + + bad_methods = df.loc[~df["method"].isin(["log_ratio", "odds_ratio"]), "method"] + if not bad_methods.empty: + raise ValueError( + f"unsupported calibration method(s): {sorted(set(bad_methods))}" + ) + + if df["damping"].isna().any(): + raise ValueError("calibration_spec damping must be numeric") + if df["tolerance"].isna().any(): + raise ValueError("calibration_spec tolerance must be numeric") + + return df + + +def _extract_utility_coefficient_names( + state: workflow.State, + model_settings: dict[str, Any] | Any, +) -> set[str]: + """ + Extract coefficient tokens from configured utility spec files. + + The extraction scans all settings keys ending with "SPEC" and parses + tokens from utility columns (all non-description/expression columns). + """ + names: set[str] = set() + + model_settings_dict = _settings_to_dict(model_settings) + spec_keys = [ + k for k in model_settings_dict.keys() if str(k).upper().endswith("SPEC") + ] + for key in spec_keys: + spec_file = model_settings_dict.get(key) + if not spec_file: + continue + + spec_path = state.filesystem.get_config_file_path(spec_file) + try: + raw = pd.read_csv(spec_path, comment="#") + except Exception: + # Do not fail hard on optional or model-specific supplemental specs. + continue + + utility_columns = [ + c + for c in raw.columns + if c + not in [ + "Description", + "Expression", + "Label", + "description", + "expression", + "label", + ] + ] + + for col in utility_columns: + for value in raw[col].dropna().tolist(): + if isinstance(value, (int, float, np.number)): + continue + text = str(value) + for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", text): + names.add(token) + + return names + + +def _validate_calibration_coefficients_against_utility_spec( + component_name: str, + calibration_spec_df: pd.DataFrame, + utility_coeff_names: set[str], +) -> None: + """Ensure calibration coefficients are referenced in utility specifications.""" + missing = sorted( + set(calibration_spec_df["coefficient"].tolist()) - set(utility_coeff_names) + ) + if missing: + raise ValueError( + f"calibration coefficients not present in model utility specification for {component_name}: {missing}" + ) + + +def _ensure_coefficients_exist( + component_name: str, + calibration_spec_df: pd.DataFrame, + coefficients_df: pd.DataFrame, +) -> pd.DataFrame: + """Add calibration coefficients missing from coefficient file with value 0.0.""" + coeffs = coefficients_df.copy() + + for coefficient_name in calibration_spec_df["coefficient"].tolist(): + if coefficient_name not in coeffs.index: + logger.warning( + "component %s coefficient %s missing from coefficient file, adding with value 0.0", + component_name, + coefficient_name, + ) + coeffs.loc[coefficient_name, "value"] = 0.0 + + coeffs["value"] = pd.to_numeric(coeffs["value"], errors="coerce") + if coeffs["value"].isna().any(): + bad = coeffs[coeffs["value"].isna()].index.tolist() + raise ValueError(f"non-numeric coefficient values found: {bad}") + + return coeffs + + +def _warn_if_initial_values_outside_bounds( + component_name: str, + calibration_spec_df: pd.DataFrame, + coefficients_df: pd.DataFrame, +) -> None: + """Warn when initial coefficient values violate provided bounds.""" + for _, row in calibration_spec_df.iterrows(): + coefficient_name = row["coefficient"] + current_value = float(coefficients_df.loc[coefficient_name, "value"]) + lower = row["min"] + upper = row["max"] + + if not pd.isna(lower) and current_value < float(lower): + logger.warning( + "component %s coefficient %s starts below min bound (%s < %s)", + component_name, + coefficient_name, + current_value, + lower, + ) + if not pd.isna(upper) and current_value > float(upper): + logger.warning( + "component %s coefficient %s starts above max bound (%s > %s)", + component_name, + coefficient_name, + current_value, + upper, + ) + + +def _build_expression_context( + state: workflow.State, + helper_symbols: dict[str, Any], +) -> dict[str, Any]: + """Create the evaluation context for model_value and target_value expressions.""" + context: dict[str, Any] = { + "state": state, + "np": np, + "pd": pd, + } + + # Load active tables into context for direct expression access. + for table_name in list(state.existing_table_names): + try: + context[table_name] = state.get_dataframe(table_name, as_copy=False) + except Exception: + # Some entries may not be available as dataframes in all contexts. + continue + + context.update(helper_symbols) + return context + + +def _evaluate_and_update( + component_name: str, + calibration_spec_df: pd.DataFrame, + coefficients_df: pd.DataFrame, + eval_context: dict[str, Any], + global_iter: int, + component_iter: int, +) -> tuple[list[dict[str, Any]], dict[str, Any], pd.DataFrame, bool]: + """Evaluate spec rows, update coefficients, and return detailed records.""" + updated = coefficients_df.copy() + records: list[dict[str, Any]] = [] + + max_difference = -math.inf + max_difference_component = "" + max_difference_coefficient = "" + max_change = -math.inf + max_change_component = "" + max_change_coefficient = "" + + num_converged = 0 + + for _, row in calibration_spec_df.iterrows(): + coefficient_name = row["coefficient"] + description = row["description"] + method = row["method"] + hold_fast = bool(row["hold_fast"]) + + prev_value = float(updated.loc[coefficient_name, "value"]) + + model_value = _eval_numeric_value( + row["model_value"], + eval_context, + component_name, + description, + "model_value", + ) + target_value = _eval_numeric_value( + row["target_value"], + eval_context, + component_name, + description, + "target_value", + ) + + difference = target_value - model_value + prc_difference = _safe_percent_difference(difference, target_value) + + tolerance = float(row["tolerance"]) + converged = abs(difference) <= tolerance + + damping = float(row["damping"]) + raw_delta = _compute_delta( + method=method, + model_value=model_value, + target_value=target_value, + damping=damping, + component_name=component_name, + description=description, + ) + + candidate_value = prev_value if hold_fast else prev_value + raw_delta + + under_min = False + over_max = False + + lower = row["min"] + upper = row["max"] + + if not pd.isna(lower) and candidate_value < float(lower): + candidate_value = float(lower) + under_min = True + if not pd.isna(upper) and candidate_value > float(upper): + candidate_value = float(upper) + over_max = True + + if not np.isfinite(candidate_value): + raise RuntimeError( + f"non-finite next coefficient for {component_name} / {description} / {coefficient_name}" + ) + + updated.loc[coefficient_name, "value"] = candidate_value + + abs_diff = abs(difference) + abs_change = abs(candidate_value - prev_value) + + if abs_diff > max_difference: + max_difference = abs_diff + max_difference_component = component_name + max_difference_coefficient = coefficient_name + + if abs_change > max_change: + max_change = abs_change + max_change_component = component_name + max_change_coefficient = coefficient_name + + if converged: + num_converged += 1 + + records.append( + { + "global_iter": global_iter, + "component_iter": component_iter, + "description": description, + "component": component_name, + "coefficient": coefficient_name, + "target_value": target_value, + "model_value": model_value, + "difference": difference, + "prc_difference": prc_difference, + "hold_fast": hold_fast, + "prev_coefficient": prev_value, + "next_coefficient": candidate_value, + "converged": converged, + "under_min": under_min, + "over_max": over_max, + } + ) + + total_rows = len(calibration_spec_df) + num_unconverged = total_rows - num_converged + component_converged = num_unconverged == 0 + + summary_record = { + "global_iter": global_iter, + "component_iter": component_iter, + "component": component_name, + "max_difference": max_difference if max_difference != -math.inf else 0.0, + "max_difference_component": max_difference_component, + "max_difference_coefficient": max_difference_coefficient, + "max_change": max_change if max_change != -math.inf else 0.0, + "max_change_component": max_change_component, + "max_change_coefficient": max_change_coefficient, + "num_converged_iter": num_converged, + "tot_converged": num_converged, + "num_unconverged": num_unconverged, + } + + return records, summary_record, updated, component_converged + + +def _eval_numeric_value( + raw_value: Any, + context: dict[str, Any], + component_name: str, + description: str, + field_name: str, +) -> float: + """Evaluate numeric or expression value and enforce finite numeric result.""" + if isinstance(raw_value, (int, float, np.number)) and not pd.isna(raw_value): + value = float(raw_value) + else: + try: + value = eval(str(raw_value), {}, context) + except Exception as err: + raise RuntimeError( + f"error evaluating {field_name} for {component_name} / {description}: {raw_value}" + ) from err + + try: + value = float(value) + except Exception as err: + raise RuntimeError( + f"{field_name} did not evaluate to a numeric value for {component_name} / {description}: {value}" + ) from err + + if not np.isfinite(value): + raise RuntimeError( + f"{field_name} evaluated to non-finite value for {component_name} / {description}: {value}" + ) + + return value + + +def _compute_delta( + method: str, + model_value: float, + target_value: float, + damping: float, + component_name: str, + description: str, +) -> float: + """Compute damped coefficient delta using selected method.""" + if damping < 0: + raise RuntimeError( + f"negative damping not allowed for {component_name} / {description}: {damping}" + ) + + if method == "log_ratio": + if model_value <= 0 or target_value <= 0: + raise RuntimeError( + f"log_ratio requires positive model and target values for {component_name} / {description}" + ) + delta = math.log(target_value / model_value) * damping + + elif method == "odds_ratio": + # Formula requested by the calibration outline. + numerator = (target_value * model_value) - target_value + denominator = (target_value * model_value) - model_value + + if numerator <= 0 or denominator <= 0: + raise RuntimeError( + f"odds_ratio produced invalid numerator/denominator for {component_name} / {description}" + ) + + ratio = numerator / denominator + if ratio <= 0 or not np.isfinite(ratio): + raise RuntimeError( + f"odds_ratio produced invalid ratio for {component_name} / {description}" + ) + + delta = math.log(ratio) * damping + + else: + raise RuntimeError(f"unsupported calibration method: {method}") + + if not np.isfinite(delta): + raise RuntimeError( + f"coefficient delta is non-finite for {component_name} / {description}" + ) + + return delta + + +def _safe_percent_difference(difference: float, target_value: float) -> float: + """Return a stable percentage difference with zero-target handling.""" + if target_value == 0: + return math.inf if difference != 0 else 0.0 + return (difference / target_value) * 100.0 + + +def _persist_coefficients_to_config( + state: workflow.State, + model_settings: dict[str, Any] | Any, + coefficients_df: pd.DataFrame, +) -> None: + """Write updated coefficients back to the component coefficient file in configs.""" + coeff_file = _setting_value(model_settings, "COEFFICIENTS") + if not coeff_file: + raise RuntimeError("component model settings missing COEFFICIENTS") + + output = coefficients_df.copy() + output.index.name = "coefficient_name" + + coeff_path = state.filesystem.get_config_file_path(coeff_file) + output.to_csv(coeff_path) + + +def _append_iteration_records( + state: workflow.State, records: list[dict[str, Any]] +) -> None: + """Append per-coefficient calibration iteration records.""" + if not records: + return + path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) + df = pd.DataFrame(records) + _append_csv(df, path) + + +def _append_summary_records( + state: workflow.State, records: list[dict[str, Any]] +) -> None: + """Append per-iteration summary records.""" + if not records: + return + path = state.get_output_file_path(CALIBRATION_SUMMARY_FILE) + df = pd.DataFrame(records) + _append_csv(df, path) + + +def _append_csv(df: pd.DataFrame, path: Path) -> None: + """Append a dataframe to a CSV file with header-once behavior.""" + os.makedirs(path.parent, exist_ok=True) + write_header = not path.exists() + df.to_csv(path, mode="a", index=False, header=write_header) + + +def _write_generic_report( + state: workflow.State, + component_name: str, + row_records: list[dict[str, Any]], +) -> None: + """Write a simple dashboard-friendly generic report for the current component iteration.""" + if not row_records: + return + + df = pd.DataFrame(row_records) + report = ( + df[ + [ + "global_iter", + "component_iter", + "component", + "description", + "difference", + "prc_difference", + "converged", + ] + ] + .copy() + .sort_values(["global_iter", "component_iter", "description"]) + ) + + path = state.get_output_file_path( + f"calibration/{component_name}_generic_report.csv" + ) + _append_csv(report, path) + + +def _run_bespoke_report(bespoke_callable, state: workflow.State) -> None: + """Run optional bespoke report callback from helper module.""" + try: + # Support either no-argument callback or callback(state). + sig = inspect.signature(bespoke_callable) + if len(sig.parameters) == 0: + bespoke_callable() + else: + bespoke_callable(state) + except TypeError: + bespoke_callable() + + +def _load_helper_symbols( + state: workflow.State, + component_settings: CalibrationComponentSettings, +) -> tuple[dict[str, Any], Any | None]: + """Load helper module and return evaluation symbols and bespoke function.""" + if not component_settings.helper_module: + return {}, None + + module = _load_helper_module(state, component_settings.helper_module) + symbols = { + name: obj for name, obj in vars(module).items() if not name.startswith("__") + } + + bespoke = None + if component_settings.reports and component_settings.reports.bespoke: + fn_name = component_settings.reports.bespoke + if not hasattr(module, fn_name): + raise RuntimeError( + f"helper module does not define bespoke function {fn_name}" + ) + bespoke = getattr(module, fn_name) + + return symbols, bespoke + + +def _load_helper_module(state: workflow.State, helper_module: str): + """Load helper module by file path or import path.""" + if helper_module.endswith(".py"): + helper_path = state.filesystem.get_config_file_path(helper_module) + module_name = Path(helper_module).stem + + spec = importlib.util.spec_from_file_location(module_name, helper_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"unable to load helper module from {helper_path}") + + module = importlib.util.module_from_spec(spec) + # Expose state as a global for compatibility with helper examples. + module.state = state + spec.loader.exec_module(module) + return module + + module = importlib.import_module(helper_module) + setattr(module, "state", state) + return module + + +def _infer_model_settings_file(component_name: str) -> str: + """Infer model settings yaml filename from component step name.""" + # This follows the dominant naming convention in the existing codebase. + if component_name.endswith("_simulate"): + base = component_name[: -len("_simulate")] + else: + base = component_name + return f"{base}.yaml" + + +def _settings_to_dict(model_settings: dict[str, Any] | Any) -> dict[str, Any]: + """Convert pydantic or dict model settings to a plain dictionary.""" + if isinstance(model_settings, dict): + return model_settings + if hasattr(model_settings, "model_dump"): + return model_settings.model_dump() + return dict(model_settings) + + +def _setting_value(model_settings: dict[str, Any] | Any, key: str, default=None): + """Read a setting value from dict-like or attribute-based settings.""" + if isinstance(model_settings, dict): + return model_settings.get(key, default) + return getattr(model_settings, key, default) + + +def _prior_step_name(models: list[str], component_name: str) -> str | None: + """Return the step name immediately preceding component_name in models.""" + if component_name not in models: + return None + idx = models.index(component_name) + if idx == 0: + return None + return models[idx - 1] + + +def _write_final_coefficients_snapshot( + state: workflow.State, + calibration_settings: CalibrationConfig, +) -> None: + """Write a combined final coefficients file snapshot for calibrated components.""" + frames = [] + for component_name in calibration_settings.run.calibrate_models: + model_settings_file = _infer_model_settings_file(component_name) + model_settings = state.filesystem.read_model_settings( + model_settings_file, mandatory=True + ) + coeff_df = state.filesystem.read_model_coefficients( + model_settings=model_settings + ).copy() + coeff_df = coeff_df.reset_index().rename(columns={"index": "coefficient_name"}) + coeff_df.insert(0, "component", component_name) + frames.append(coeff_df) + + if not frames: + return + + final_df = pd.concat(frames, ignore_index=True) + path = state.get_output_file_path(CALIBRATION_FINAL_COEFFICIENTS_FILE) + os.makedirs(path.parent, exist_ok=True) + final_df.to_csv(path, index=False) + + +def _ensure_calibration_output_dir(state: workflow.State) -> None: + """Ensure output/calibration exists.""" + path = state.get_output_file_path(CALIBRATION_OUTPUT_DIR) + os.makedirs(path, exist_ok=True) + + +def _read_progress(state: workflow.State) -> dict[str, Any] | None: + """Read persisted calibration progress metadata if it exists.""" + path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) + if not path.exists(): + return None + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _write_progress(state: workflow.State, payload: dict[str, Any]) -> None: + """Write calibration progress metadata.""" + path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) + os.makedirs(path.parent, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + + +def _clear_progress(state: workflow.State) -> None: + """Delete calibration progress metadata after successful completion.""" + path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) + if path.exists(): + path.unlink() + + +def _run_in_configured_mode( + state: workflow.State, + models: list[str], + resume_after: str | None, + memory_sidecar_process=None, +) -> None: + """Run models using the same single/multiprocess mode as the parent run.""" + if state.settings.multiprocess: + _run_multiprocess_with_overrides( + state, + models=models, + resume_after=resume_after, + ) + return + + state.run( + models=models, + resume_after=resume_after, + memory_sidecar_process=memory_sidecar_process, + ) + + +def _run_multiprocess_with_overrides( + state: workflow.State, + models: list[str], + resume_after: str | None, +) -> None: + """Run multiprocess with temporary settings overrides for calibration passes.""" + from activitysim.core import mp_tasks + + original_models = state.settings.models + original_resume_after = state.settings.resume_after + + state.settings.models = models + state.settings.resume_after = resume_after + + try: + injectables = {} + for key in MP_INJECTABLES: + try: + injectables[key] = state.get_injectable(key) + except KeyError: + pass + injectables["settings"] = state.settings + mp_tasks.run_multiprocess(state, injectables) + finally: + state.settings.models = original_models + state.settings.resume_after = original_resume_after From 087659786cebe36b0cdc24453052305b8a128ed3 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Tue, 9 Jun 2026 10:26:32 -0700 Subject: [PATCH 02/90] do not validate settings if file not found and not mandatory --- activitysim/core/configuration/filesystem.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/activitysim/core/configuration/filesystem.py b/activitysim/core/configuration/filesystem.py index 71074ac62d..0de99ef1f1 100644 --- a/activitysim/core/configuration/filesystem.py +++ b/activitysim/core/configuration/filesystem.py @@ -724,6 +724,7 @@ def backfill_settings(settings, backfill): file_name = "%s.yaml" % (file_name,) inheriting = False + settings_file_found = False settings = {} if isinstance(include_stack, list): source_file_paths = include_stack.copy() @@ -732,6 +733,7 @@ def backfill_settings(settings, backfill): for dir in configs_dir_list: file_path = os.path.join(dir, file_name) if os.path.exists(file_path): + settings_file_found = True if inheriting: # we must be inheriting logger.debug( @@ -835,7 +837,7 @@ def backfill_settings(settings, backfill): settings.pop("inherit_settings", None) settings.pop("include_settings", None) - if validator_class is not None: + if validator_class is not None and (mandatory or settings_file_found): settings = validator_class.model_validate(settings) if include_stack: From 6571db170406f1b9e6fbd1ab52d1880a9b9aed14 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 2 Jul 2026 14:07:47 -0400 Subject: [PATCH 03/90] calibration settings, bespoke reports changes --- activitysim/core/calibration.py | 11 +++++++---- activitysim/core/estimation.py | 5 +++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 902bdbba61..a570392fbe 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -79,6 +79,7 @@ class CalibrationComponentSettings(PydanticBase): helper_module: str | None = None submodel_max_iterations: int = 1 reports: CalibrationReportsSettings = CalibrationReportsSettings() + survey_file: str class CalibrationConfig(PydanticReadable): @@ -355,7 +356,7 @@ def _calibrate_component( if bespoke_callable is not None: # Preserve compatibility with helper modules that expect a global # `state` symbol and/or no explicit arguments. - _run_bespoke_report(bespoke_callable, state) + _run_bespoke_report(bespoke_callable, state, component_settings) if component_converged: break @@ -865,15 +866,17 @@ def _write_generic_report( _append_csv(report, path) -def _run_bespoke_report(bespoke_callable, state: workflow.State) -> None: +def _run_bespoke_report(bespoke_callable, state: workflow.State, component_settings: CalibrationComponentSettings) -> None: """Run optional bespoke report callback from helper module.""" try: - # Support either no-argument callback or callback(state). + # Support no-argument callback, callback(state), or callback(state, component_settings). sig = inspect.signature(bespoke_callable) if len(sig.parameters) == 0: bespoke_callable() - else: + elif len(sig.parameters) == 1: bespoke_callable(state) + else: + bespoke_callable(state, component_settings) except TypeError: bespoke_callable() diff --git a/activitysim/core/estimation.py b/activitysim/core/estimation.py index bbbe376ee4..9b6fcbab4c 100644 --- a/activitysim/core/estimation.py +++ b/activitysim/core/estimation.py @@ -798,8 +798,9 @@ def initialize_settings(self, state): if not self.settings: # if the model self.settings file is not found, we are not in estimation mode. self.enabled = False - else: - self.enabled = self.settings.enable + return + + self.enabled = self.settings.enable self.bundles = self.settings.bundles self.estimation_table_types = self.settings.estimation_table_types From 6ccf84d3750317aa6b90f8220a5c04783c0e6a58 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Fri, 3 Jul 2026 15:56:00 -0400 Subject: [PATCH 04/90] Fix global iterations --- activitysim/core/calibration.py | 129 +++++++++++++++++++++++--------- 1 file changed, 93 insertions(+), 36 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index a570392fbe..99eabc0187 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -12,7 +12,7 @@ import re from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Optional import numpy as np import pandas as pd @@ -60,9 +60,11 @@ class CalibrationRunSettings(PydanticBase): """Run-control settings for calibration.""" - resume_after: str + resume_after: Optional[str] = None calibrate_models: list[str] restart_after: list[str] = [] + global_iterations: int = 1 + complete_steps: bool = True class CalibrationReportsSettings(PydanticBase): @@ -86,7 +88,6 @@ class CalibrationConfig(PydanticReadable): """Top-level calibration configuration.""" enable: bool = False - max_iterations: int = 1 run: CalibrationRunSettings model_settings: dict[str, CalibrationComponentSettings] = {} @@ -105,7 +106,7 @@ def validate_model_settings(self): f"restart_after component '{component}' is not in calibrate_models" ) - if self.max_iterations < 1: + if self.run.global_iterations < 1: raise ValueError("max_iterations must be >= 1") return self @@ -159,6 +160,16 @@ def run_calibration_loop( if not calibration_settings or not calibration_settings.enable: raise RuntimeError("calibration loop called while calibration is disabled") + assert all( + [c in models for c in calibration_settings.run.calibrate_models] + ), f"settings.yaml steps list does not include calibration model{'s' if len([c for c in calibration_settings.run.calibrate_models if c not in models]) != 1 else ''} {[c for c in calibration_settings.run.calibrate_models if c not in models]}" + + # sort calibration models into main model order + calibration_settings.run.calibrate_models = sorted( + calibration_settings.run.calibrate_models, key=lambda x: models.index(x) + ) + first_calib_model_idx = models.index(calibration_settings.run.calibrate_models[0]) + _ensure_calibration_output_dir(state) # If there is recoverable calibration progress from a prior interrupted run, @@ -172,31 +183,48 @@ def run_calibration_loop( try: for global_iter in range( - start_global_iter, calibration_settings.max_iterations + 1 + start_global_iter, + start_global_iter + calibration_settings.run.global_iterations, ): logger.info( "calibration global iteration %s/%s", - global_iter, - calibration_settings.max_iterations, + global_iter - start_global_iter, + calibration_settings.run.global_iterations, ) # Run ActivitySim normally from resume_after through production model steps. - _run_model_system_for_iteration( + _run_precursor_components( state, - models=models, - resume_after=calibration_settings.run.resume_after, + models=models[:first_calib_model_idx], + resume_after=calibration_settings.run.resume_after + if global_iter == start_global_iter + else _prior_step_name( + models, calibration_settings.run.calibrate_models[0] + ), global_iter=global_iter, memory_sidecar_process=memory_sidecar_process, ) all_converged = True - restart_triggered = False + last_calibrated_component = None for component in calibration_settings.run.calibrate_models: component_settings = calibration_settings.model_settings[component] + prior_step = _prior_step_name(models, component) - if prior_step is None: - prior_step = calibration_settings.run.resume_after + + if last_calibrated_component is not None: + + # run all models b/w the last calibrated model and the current one + _run_intermediate_components( + state, + models=models[ + models.index(last_calibrated_component) + + 1 : models.index(component) + ], + resume_after=last_calibrated_component, + memory_sidecar_process=memory_sidecar_process, + ) component_result = _calibrate_component( state=state, @@ -208,10 +236,15 @@ def run_calibration_loop( all_converged = all_converged and component_result.converged - if component in calibration_settings.run.restart_after: - # Restart global loop from resume_after after this component. - restart_triggered = True - break + last_calibrated_component = component + + if calibration_settings.run.complete_steps: + _run_subsequent_components( + state, + models=models[models.index(last_calibrated_component) + 1 :], + resume_after=last_calibrated_component, + memory_sidecar_process=memory_sidecar_process, + ) _write_progress( state, @@ -221,25 +254,16 @@ def run_calibration_loop( }, ) - if all_converged and not restart_triggered: - _write_final_coefficients_snapshot(state, calibration_settings) - _clear_progress(state) - return CalibrationRunResult( - converged=True, - completed_global_iterations=global_iter, - ) - _write_final_coefficients_snapshot(state, calibration_settings) - _clear_progress(state) return CalibrationRunResult( converged=False, - completed_global_iterations=calibration_settings.max_iterations, + completed_global_iterations=calibration_settings.run.global_iterations, ) finally: state.filesystem.pipeline_file_name = original_pipeline_name -def _run_model_system_for_iteration( +def _run_precursor_components( state: workflow.State, models: list[str], resume_after: str, @@ -247,6 +271,10 @@ def _run_model_system_for_iteration( memory_sidecar_process=None, ) -> None: """Run the normal ActivitySim model flow for one global calibration iteration.""" + + assert (resume_after is None) or ( + resume_after in models + ), f"resume_after step {resume_after} not in models preceding calibration models" if global_iter > 1: # Seed a fresh pipeline from the configured resume checkpoint to avoid # duplicate checkpoint-name collisions across global calibration loops. @@ -254,7 +282,38 @@ def _run_model_system_for_iteration( state.checkpoint.close_store() state.filesystem.pipeline_file_name = f"pipeline_calibration_iter_{global_iter}" state.checkpoint.restore_from(prior_pipeline, checkpoint_name=resume_after) + else: + + _run_in_configured_mode( + state, + models=models, + resume_after=resume_after, + memory_sidecar_process=memory_sidecar_process, + ) + + +def _run_intermediate_components( + state: workflow.State, + models: list[str], + resume_after: str, + memory_sidecar_process=None, +) -> None: + # don't modify the pipeline, just run the models needed + _run_in_configured_mode( + state, + models=models, + resume_after=resume_after, + memory_sidecar_process=memory_sidecar_process, + ) + +def _run_subsequent_components( + state: workflow.State, + models: list[str], + resume_after: str, + memory_sidecar_process=None, +) -> None: + # don't modify the pipeline, just run the models needed _run_in_configured_mode( state, models=models, @@ -360,6 +419,7 @@ def _calibrate_component( if component_converged: break + state.checkpoint.add(component_name) return CalibrationComponentResult( component=component_name, @@ -866,7 +926,11 @@ def _write_generic_report( _append_csv(report, path) -def _run_bespoke_report(bespoke_callable, state: workflow.State, component_settings: CalibrationComponentSettings) -> None: +def _run_bespoke_report( + bespoke_callable, + state: workflow.State, + component_settings: CalibrationComponentSettings, +) -> None: """Run optional bespoke report callback from helper module.""" try: # Support no-argument callback, callback(state), or callback(state, component_settings). @@ -1013,13 +1077,6 @@ def _write_progress(state: workflow.State, payload: dict[str, Any]) -> None: json.dump(payload, f, indent=2) -def _clear_progress(state: workflow.State) -> None: - """Delete calibration progress metadata after successful completion.""" - path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) - if path.exists(): - path.unlink() - - def _run_in_configured_mode( state: workflow.State, models: list[str], From f2ea9478712265c3ad4c7ad230ab02824c027bbf Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Fri, 3 Jul 2026 18:09:51 -0400 Subject: [PATCH 05/90] Loading from 2+ global iters --- activitysim/core/calibration.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 99eabc0187..a575f50ce0 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -197,7 +197,7 @@ def run_calibration_loop( state, models=models[:first_calib_model_idx], resume_after=calibration_settings.run.resume_after - if global_iter == start_global_iter + if global_iter == 1 else _prior_step_name( models, calibration_settings.run.calibrate_models[0] ), @@ -255,6 +255,18 @@ def run_calibration_loop( ) _write_final_coefficients_snapshot(state, calibration_settings) + + iteration_records = pd.read_csv(state.get_output_file_path(CALIBRATION_ITERATION_FILE)) + + for component in iteration_records.component.unique(): + + ax = iteration_records.loc[iteration_records.component == component].set_index(['global_iter','component_iter','coefficient']).next_coefficient.unstack('coefficient').plot() + ax.xaxis.set_label("Component iteration") + ax.yaxis.set_label("Coefficient value") + + ax.legend(title="Coefficient label") + ax.figure.savefig(os.path.join(state.filesystem.output_dir,f"{component}_coefficient_progress.png")) + return CalibrationRunResult( converged=False, completed_global_iterations=calibration_settings.run.global_iterations, From 72758e6a170649501a72c22f3ab2f31bcb2c89e8 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Mon, 6 Jul 2026 09:37:16 -0400 Subject: [PATCH 06/90] Bugfix in plotting coefficient momentum --- activitysim/core/calibration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index a575f50ce0..29e0c8f657 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -261,8 +261,8 @@ def run_calibration_loop( for component in iteration_records.component.unique(): ax = iteration_records.loc[iteration_records.component == component].set_index(['global_iter','component_iter','coefficient']).next_coefficient.unstack('coefficient').plot() - ax.xaxis.set_label("Component iteration") - ax.yaxis.set_label("Coefficient value") + ax.xaxis.set_label_text("Component iteration") + ax.yaxis.set_label_text("Coefficient value") ax.legend(title="Coefficient label") ax.figure.savefig(os.path.join(state.filesystem.output_dir,f"{component}_coefficient_progress.png")) From b065459890398d72323a6bab026e54d9711e08c3 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Mon, 6 Jul 2026 09:44:44 -0400 Subject: [PATCH 07/90] Add coef_delta to calibration records --- activitysim/core/calibration.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 29e0c8f657..bfa90f798f 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -675,7 +675,7 @@ def _evaluate_and_update( ) difference = target_value - model_value - prc_difference = _safe_percent_difference(difference, target_value) + pct_difference = _safe_percent_difference(difference, target_value) tolerance = float(row["tolerance"]) converged = abs(difference) <= tolerance @@ -738,9 +738,10 @@ def _evaluate_and_update( "target_value": target_value, "model_value": model_value, "difference": difference, - "prc_difference": prc_difference, + "pct_difference": pct_difference, "hold_fast": hold_fast, "prev_coefficient": prev_value, + "coef_delta": abs_change, "next_coefficient": candidate_value, "converged": converged, "under_min": under_min, @@ -924,7 +925,7 @@ def _write_generic_report( "component", "description", "difference", - "prc_difference", + "pct_difference", "converged", ] ] From 3c135c7a389bc23dca2421653423ca4f7a4fe096 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Mon, 6 Jul 2026 16:45:51 -0400 Subject: [PATCH 08/90] Update graphs, handle zero modeled vals --- activitysim/core/calibration.py | 118 +++++++++++++++++++++++--------- 1 file changed, 87 insertions(+), 31 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index bfa90f798f..99b4116662 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -4,7 +4,6 @@ import importlib import importlib.util -import inspect import json import logging import math @@ -16,6 +15,7 @@ import numpy as np import pandas as pd +import matplotlib.pyplot as plt from pydantic import model_validator from activitysim.core import workflow @@ -31,6 +31,8 @@ CALIBRATION_SUMMARY_FILE = "calibration/calibration_iteration_summary.csv" CALIBRATION_FINAL_COEFFICIENTS_FILE = "calibration/final_calibrated_coefficients.csv" +DEFAULT_INCREMENT = 2.0 + CALIBRATION_REQUIRED_COLUMNS = [ "description", "coefficient", @@ -64,7 +66,7 @@ class CalibrationRunSettings(PydanticBase): calibrate_models: list[str] restart_after: list[str] = [] global_iterations: int = 1 - complete_steps: bool = True + complete_steps: bool = False class CalibrationReportsSettings(PydanticBase): @@ -238,7 +240,11 @@ def run_calibration_loop( last_calibrated_component = component - if calibration_settings.run.complete_steps: + if calibration_settings.run.complete_steps or ( + start_global_iter + calibration_settings.run.global_iterations + == global_iter + 1 + ): + # finish the full model chain _run_subsequent_components( state, models=models[models.index(last_calibrated_component) + 1 :], @@ -256,16 +262,64 @@ def run_calibration_loop( _write_final_coefficients_snapshot(state, calibration_settings) - iteration_records = pd.read_csv(state.get_output_file_path(CALIBRATION_ITERATION_FILE)) + iteration_records = ( + pd.read_csv(state.get_output_file_path(CALIBRATION_ITERATION_FILE)) + .set_index(["global_iter", "component_iter", "coefficient"]) + .sort_index() + ) for component in iteration_records.component.unique(): - ax = iteration_records.loc[iteration_records.component == component].set_index(['global_iter','component_iter','coefficient']).next_coefficient.unstack('coefficient').plot() + ax = ( + iteration_records.loc[iteration_records.component == component] + .next_coefficient.unstack("coefficient") + .plot() + ) ax.xaxis.set_label_text("Component iteration") ax.yaxis.set_label_text("Coefficient value") - + ax.legend(title="Coefficient label") - ax.figure.savefig(os.path.join(state.filesystem.output_dir,f"{component}_coefficient_progress.png")) + ax.figure.savefig( + os.path.join( + state.filesystem.output_dir, + "calibration", + f"{component}_coefficient_progress.png", + ) + ) + + last_global = iteration_records.index.get_level_values("global_iter")[-1] + last_comp = iteration_records.loc[last_global].index.get_level_values( + "component_iter" + )[-1] + + last_records = iteration_records.xs( + (last_global, last_comp), level=("global_iter", "component_iter") + )[["target_value", "model_value"]] + ax = last_records.plot.bar() + ax.xaxis.set_tick_params(rotation=45) + ax.xaxis.set_label_text("Component value") + plt.tight_layout() + ax.figure.savefig( + os.path.join( + state.filesystem.output_dir, + "calibration", + f"{component}_final_components.png", + ) + ) + + _ = plt.subplots() + + pct_diff = last_records.diff(axis=1).model_value / last_records.target_value + ax = pct_diff.plot.bar() + ax.xaxis.set_tick_params(rotation=45) + plt.tight_layout() + ax.figure.savefig( + os.path.join( + state.filesystem.output_dir, + "calibration", + f"{component}_final_pct_change.png", + ) + ) return CalibrationRunResult( converged=False, @@ -427,7 +481,8 @@ def _calibrate_component( if bespoke_callable is not None: # Preserve compatibility with helper modules that expect a global # `state` symbol and/or no explicit arguments. - _run_bespoke_report(bespoke_callable, state, component_settings) + kwargs = {"state": state, "component_settings": component_settings} + bespoke_callable(**kwargs) if component_converged: break @@ -657,6 +712,12 @@ def _evaluate_and_update( method = row["method"] hold_fast = bool(row["hold_fast"]) + default_increment = ( + row["default_increment"] + if "default_increment" in row.index + else DEFAULT_INCREMENT + ) + prev_value = float(updated.loc[coefficient_name, "value"]) model_value = _eval_numeric_value( @@ -688,6 +749,7 @@ def _evaluate_and_update( damping=damping, component_name=component_name, description=description, + default_increment=default_increment, ) candidate_value = prev_value if hold_fast else prev_value + raw_delta @@ -811,6 +873,7 @@ def _compute_delta( damping: float, component_name: str, description: str, + default_increment: float, ) -> float: """Compute damped coefficient delta using selected method.""" if damping < 0: @@ -820,9 +883,15 @@ def _compute_delta( if method == "log_ratio": if model_value <= 0 or target_value <= 0: - raise RuntimeError( - f"log_ratio requires positive model and target values for {component_name} / {description}" + logger.warning( + f"log_ratio requires positive model and target values for {component_name} / {description}. Falling back to default increment {default_increment}" ) + if model_value <= 0 and target_value > 0: + return default_increment + elif model_value > 0 and target_value <= 0: + return -default_increment + else: + return 0 delta = math.log(target_value / model_value) * damping elif method == "odds_ratio": @@ -831,9 +900,15 @@ def _compute_delta( denominator = (target_value * model_value) - model_value if numerator <= 0 or denominator <= 0: - raise RuntimeError( - f"odds_ratio produced invalid numerator/denominator for {component_name} / {description}" + logger.warning( + f"odds_ratio produced invalid numerator/denominator for {component_name} / {description}. Falling back to default increment {default_increment}" ) + if model_value <= 0 and target_value > 0: + return default_increment + elif model_value > 0 and target_value <= 0: + return -default_increment + else: + return 0 ratio = numerator / denominator if ratio <= 0 or not np.isfinite(ratio): @@ -939,25 +1014,6 @@ def _write_generic_report( _append_csv(report, path) -def _run_bespoke_report( - bespoke_callable, - state: workflow.State, - component_settings: CalibrationComponentSettings, -) -> None: - """Run optional bespoke report callback from helper module.""" - try: - # Support no-argument callback, callback(state), or callback(state, component_settings). - sig = inspect.signature(bespoke_callable) - if len(sig.parameters) == 0: - bespoke_callable() - elif len(sig.parameters) == 1: - bespoke_callable(state) - else: - bespoke_callable(state, component_settings) - except TypeError: - bespoke_callable() - - def _load_helper_symbols( state: workflow.State, component_settings: CalibrationComponentSettings, From b1db935cd9679d6c5501d3dc8caccc0650952b01 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Mon, 6 Jul 2026 17:29:35 -0400 Subject: [PATCH 09/90] Batch output graphs --- activitysim/core/calibration.py | 109 +++++++++++++++++++------------- 1 file changed, 64 insertions(+), 45 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 99b4116662..8dca21f413 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -32,6 +32,7 @@ CALIBRATION_FINAL_COEFFICIENTS_FILE = "calibration/final_calibrated_coefficients.csv" DEFAULT_INCREMENT = 2.0 +MAX_COEFFS_IN_GRAPH = 10 CALIBRATION_REQUIRED_COLUMNS = [ "description", @@ -270,56 +271,74 @@ def run_calibration_loop( for component in iteration_records.component.unique(): - ax = ( - iteration_records.loc[iteration_records.component == component] - .next_coefficient.unstack("coefficient") - .plot() - ) - ax.xaxis.set_label_text("Component iteration") - ax.yaxis.set_label_text("Coefficient value") - - ax.legend(title="Coefficient label") - ax.figure.savefig( - os.path.join( - state.filesystem.output_dir, - "calibration", - f"{component}_coefficient_progress.png", + recs = iteration_records.loc[iteration_records.component == component] + coefs = sorted(recs.index.get_level_values("coefficient").unique()) + n_sets = math.ceil(len(coefs) / MAX_COEFFS_IN_GRAPH) + for coef_set in range(n_sets): + set_coefs = coefs[ + coef_set + * MAX_COEFFS_IN_GRAPH : min( + len(coefs), (coef_set + 1) * MAX_COEFFS_IN_GRAPH + ) + ] + ax = ( + recs[recs.index.get_level_values("coefficient").isin(set_coefs)] + .next_coefficient.unstack("coefficient") + .plot() + ) + ax.xaxis.set_label_text("Component iteration") + ax.yaxis.set_label_text("Coefficient value") + + ax.legend(title="Coefficient label") + ax.figure.savefig( + os.path.join( + state.filesystem.output_dir, + "calibration", + f"{component}_coefficient_progress_set_{coef_set}.png", + ) ) - ) - last_global = iteration_records.index.get_level_values("global_iter")[-1] - last_comp = iteration_records.loc[last_global].index.get_level_values( - "component_iter" - )[-1] - - last_records = iteration_records.xs( - (last_global, last_comp), level=("global_iter", "component_iter") - )[["target_value", "model_value"]] - ax = last_records.plot.bar() - ax.xaxis.set_tick_params(rotation=45) - ax.xaxis.set_label_text("Component value") - plt.tight_layout() - ax.figure.savefig( - os.path.join( - state.filesystem.output_dir, - "calibration", - f"{component}_final_components.png", + last_global = recs[ + recs.index.get_level_values("coefficient").isin(set_coefs) + ].index.get_level_values("global_iter")[-1] + last_comp = ( + recs[recs.index.get_level_values("coefficient").isin(set_coefs)] + .loc[last_global] + .index.get_level_values("component_iter")[-1] ) - ) - _ = plt.subplots() - - pct_diff = last_records.diff(axis=1).model_value / last_records.target_value - ax = pct_diff.plot.bar() - ax.xaxis.set_tick_params(rotation=45) - plt.tight_layout() - ax.figure.savefig( - os.path.join( - state.filesystem.output_dir, - "calibration", - f"{component}_final_pct_change.png", + last_records = recs[ + recs.index.get_level_values("coefficient").isin(set_coefs) + ].xs((last_global, last_comp), level=("global_iter", "component_iter"))[ + ["target_value", "model_value"] + ] + ax = last_records.plot.bar() + ax.xaxis.set_tick_params(rotation=45) + ax.xaxis.set_label_text("Component value") + plt.tight_layout() + ax.figure.savefig( + os.path.join( + state.filesystem.output_dir, + "calibration", + f"{component}_final_components_set_{coef_set}.png", + ) + ) + + _ = plt.subplots() + + pct_diff = ( + last_records.diff(axis=1).model_value / last_records.target_value + ) + ax = pct_diff.plot.bar() + ax.xaxis.set_tick_params(rotation=45) + plt.tight_layout() + ax.figure.savefig( + os.path.join( + state.filesystem.output_dir, + "calibration", + f"{component}_final_pct_change_set_{coef_set}.png", + ) ) - ) return CalibrationRunResult( converged=False, From 0b1f72346d8b0d00247d46b75a8c09213f4d3213 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Mon, 6 Jul 2026 17:43:01 -0400 Subject: [PATCH 10/90] Minor plotting updates --- activitysim/core/calibration.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 8dca21f413..b501c342f5 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -24,6 +24,8 @@ logger = logging.getLogger("calibration") +plt.style.use("seaborn-v0_8-darkgrid") + CALIBRATION_SETTINGS_FILE_NAME = "calibration.yaml" CALIBRATION_OUTPUT_DIR = "calibration" CALIBRATION_PROGRESS_FILE = "calibration/calibration_progress.json" @@ -312,7 +314,7 @@ def run_calibration_loop( ].xs((last_global, last_comp), level=("global_iter", "component_iter"))[ ["target_value", "model_value"] ] - ax = last_records.plot.bar() + ax = last_records.plot.barh() ax.xaxis.set_tick_params(rotation=45) ax.xaxis.set_label_text("Component value") plt.tight_layout() @@ -329,8 +331,10 @@ def run_calibration_loop( pct_diff = ( last_records.diff(axis=1).model_value / last_records.target_value ) - ax = pct_diff.plot.bar() + ax = pct_diff.plot.barh() ax.xaxis.set_tick_params(rotation=45) + ax.xaxis.set_label_text("Coefficient") + ax.yaxis.set_label_text("% Change") plt.tight_layout() ax.figure.savefig( os.path.join( From 96527ff8d22b72b12633ab77e4d92635ac12b782 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:44:26 -0700 Subject: [PATCH 11/90] expose context to bespoke functions --- activitysim/core/calibration.py | 45 ++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index b501c342f5..068ff50ac1 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -427,7 +427,7 @@ def _calibrate_component( coefficients_df = state.filesystem.read_model_coefficients( model_settings=model_settings ) - helper_symbols, bespoke_callable = _load_helper_symbols( + helper_symbols, bespoke_callable, helper_module = _load_helper_symbols( state, component_settings, ) @@ -477,6 +477,7 @@ def _calibrate_component( state.run(models=[run_model_name], resume_after=prior_step) eval_context = _build_expression_context(state, helper_symbols) + _bind_context_to_helper_module_globals(helper_module, eval_context) ( row_records, @@ -703,11 +704,47 @@ def _build_expression_context( except Exception: # Some entries may not be available as dataframes in all contexts. continue + try: + network_los = state.get_injectable("network_los") + context["network_los"] = network_los + context["skim_dict"] = network_los.get_default_skim_dict() + except Exception: + # Network LOS may not be available in all contexts. + pass context.update(helper_symbols) return context +def _bind_context_to_helper_module_globals( + helper_module: Any | None, + eval_context: dict[str, Any], +) -> None: + """Bind eval context names into helper module globals. + + Helper functions referenced by calibration expressions resolve free names + from their module globals, not from eval locals. This syncs the current + evaluation context into the helper module globals each component iteration. + """ + if helper_module is None: + return + + excluded = { + "__builtins__", + "__name__", + "__package__", + "__loader__", + "__spec__", + "__file__", + "__cached__", + } + + filtered_context = { + key: value for key, value in eval_context.items() if key not in excluded + } + helper_module.__dict__.update(filtered_context) + + def _evaluate_and_update( component_name: str, calibration_spec_df: pd.DataFrame, @@ -1040,10 +1077,10 @@ def _write_generic_report( def _load_helper_symbols( state: workflow.State, component_settings: CalibrationComponentSettings, -) -> tuple[dict[str, Any], Any | None]: +) -> tuple[dict[str, Any], Any | None, Any | None]: """Load helper module and return evaluation symbols and bespoke function.""" if not component_settings.helper_module: - return {}, None + return {}, None, None module = _load_helper_module(state, component_settings.helper_module) symbols = { @@ -1059,7 +1096,7 @@ def _load_helper_symbols( ) bespoke = getattr(module, fn_name) - return symbols, bespoke + return symbols, bespoke, module def _load_helper_module(state: workflow.State, helper_module: str): From d347f9c7f25796c10d6fb5db30737842e287dd80 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 6 Jul 2026 21:44:41 -0700 Subject: [PATCH 12/90] output updates --- activitysim/core/calibration.py | 252 +++++++++++++++++++------------- 1 file changed, 147 insertions(+), 105 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 068ff50ac1..2fd2888f99 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -15,6 +15,7 @@ import numpy as np import pandas as pd +import matplotlib import matplotlib.pyplot as plt from pydantic import model_validator @@ -25,6 +26,7 @@ logger = logging.getLogger("calibration") plt.style.use("seaborn-v0_8-darkgrid") +matplotlib.use('Agg') # Forces non-interactive background rendering CALIBRATION_SETTINGS_FILE_NAME = "calibration.yaml" CALIBRATION_OUTPUT_DIR = "calibration" @@ -238,6 +240,7 @@ def run_calibration_loop( prior_step=prior_step, global_iter=global_iter, ) + _write_component_plots(state, component) all_converged = all_converged and component_result.converged @@ -265,85 +268,6 @@ def run_calibration_loop( _write_final_coefficients_snapshot(state, calibration_settings) - iteration_records = ( - pd.read_csv(state.get_output_file_path(CALIBRATION_ITERATION_FILE)) - .set_index(["global_iter", "component_iter", "coefficient"]) - .sort_index() - ) - - for component in iteration_records.component.unique(): - - recs = iteration_records.loc[iteration_records.component == component] - coefs = sorted(recs.index.get_level_values("coefficient").unique()) - n_sets = math.ceil(len(coefs) / MAX_COEFFS_IN_GRAPH) - for coef_set in range(n_sets): - set_coefs = coefs[ - coef_set - * MAX_COEFFS_IN_GRAPH : min( - len(coefs), (coef_set + 1) * MAX_COEFFS_IN_GRAPH - ) - ] - ax = ( - recs[recs.index.get_level_values("coefficient").isin(set_coefs)] - .next_coefficient.unstack("coefficient") - .plot() - ) - ax.xaxis.set_label_text("Component iteration") - ax.yaxis.set_label_text("Coefficient value") - - ax.legend(title="Coefficient label") - ax.figure.savefig( - os.path.join( - state.filesystem.output_dir, - "calibration", - f"{component}_coefficient_progress_set_{coef_set}.png", - ) - ) - - last_global = recs[ - recs.index.get_level_values("coefficient").isin(set_coefs) - ].index.get_level_values("global_iter")[-1] - last_comp = ( - recs[recs.index.get_level_values("coefficient").isin(set_coefs)] - .loc[last_global] - .index.get_level_values("component_iter")[-1] - ) - - last_records = recs[ - recs.index.get_level_values("coefficient").isin(set_coefs) - ].xs((last_global, last_comp), level=("global_iter", "component_iter"))[ - ["target_value", "model_value"] - ] - ax = last_records.plot.barh() - ax.xaxis.set_tick_params(rotation=45) - ax.xaxis.set_label_text("Component value") - plt.tight_layout() - ax.figure.savefig( - os.path.join( - state.filesystem.output_dir, - "calibration", - f"{component}_final_components_set_{coef_set}.png", - ) - ) - - _ = plt.subplots() - - pct_diff = ( - last_records.diff(axis=1).model_value / last_records.target_value - ) - ax = pct_diff.plot.barh() - ax.xaxis.set_tick_params(rotation=45) - ax.xaxis.set_label_text("Coefficient") - ax.yaxis.set_label_text("% Change") - plt.tight_layout() - ax.figure.savefig( - os.path.join( - state.filesystem.output_dir, - "calibration", - f"{component}_final_pct_change_set_{coef_set}.png", - ) - ) - return CalibrationRunResult( converged=False, completed_global_iterations=calibration_settings.run.global_iterations, @@ -476,7 +400,7 @@ def _calibrate_component( ) state.run(models=[run_model_name], resume_after=prior_step) - eval_context = _build_expression_context(state, helper_symbols) + eval_context = _build_expression_context(state, helper_symbols, component_name) _bind_context_to_helper_module_globals(helper_module, eval_context) ( @@ -496,20 +420,20 @@ def _calibrate_component( coefficients_df = new_coefficients_df _persist_coefficients_to_config(state, model_settings, coefficients_df) - _append_iteration_records(state, row_records) + _append_iteration_records(state, component_name, row_records) _append_summary_records(state, [summary_record]) if component_settings.reports.generic: _write_generic_report(state, component_name, row_records) if bespoke_callable is not None: - # Preserve compatibility with helper modules that expect a global - # `state` symbol and/or no explicit arguments. - kwargs = {"state": state, "component_settings": component_settings} - bespoke_callable(**kwargs) + # Helper module globals were already updated from eval_context, + # so bespoke functions can access tables/injectables directly. + bespoke_callable() if component_converged: break + state.checkpoint.add(component_name) return CalibrationComponentResult( @@ -689,12 +613,14 @@ def _warn_if_initial_values_outside_bounds( def _build_expression_context( state: workflow.State, helper_symbols: dict[str, Any], + component_name: str, ) -> dict[str, Any]: """Create the evaluation context for model_value and target_value expressions.""" context: dict[str, Any] = { "state": state, "np": np, "pd": pd, + "component_output_dir": _component_output_dir(state, component_name), } # Load active tables into context for direct expression access. @@ -758,10 +684,8 @@ def _evaluate_and_update( records: list[dict[str, Any]] = [] max_difference = -math.inf - max_difference_component = "" max_difference_coefficient = "" max_change = -math.inf - max_change_component = "" max_change_coefficient = "" num_converged = 0 @@ -814,18 +738,18 @@ def _evaluate_and_update( candidate_value = prev_value if hold_fast else prev_value + raw_delta - under_min = False - over_max = False + at_min = False + at_max = False lower = row["min"] upper = row["max"] - if not pd.isna(lower) and candidate_value < float(lower): + if not pd.isna(lower) and candidate_value <= float(lower): candidate_value = float(lower) - under_min = True - if not pd.isna(upper) and candidate_value > float(upper): + at_min = True + if not pd.isna(upper) and candidate_value >= float(upper): candidate_value = float(upper) - over_max = True + at_max = True if not np.isfinite(candidate_value): raise RuntimeError( @@ -839,12 +763,10 @@ def _evaluate_and_update( if abs_diff > max_difference: max_difference = abs_diff - max_difference_component = component_name max_difference_coefficient = coefficient_name if abs_change > max_change: max_change = abs_change - max_change_component = component_name max_change_coefficient = coefficient_name if converged: @@ -866,8 +788,8 @@ def _evaluate_and_update( "coef_delta": abs_change, "next_coefficient": candidate_value, "converged": converged, - "under_min": under_min, - "over_max": over_max, + "at_min": at_min, + "at_max": at_max, } ) @@ -880,10 +802,8 @@ def _evaluate_and_update( "component_iter": component_iter, "component": component_name, "max_difference": max_difference if max_difference != -math.inf else 0.0, - "max_difference_component": max_difference_component, "max_difference_coefficient": max_difference_coefficient, "max_change": max_change if max_change != -math.inf else 0.0, - "max_change_component": max_change_component, "max_change_coefficient": max_change_coefficient, "num_converged_iter": num_converged, "tot_converged": num_converged, @@ -1014,14 +934,22 @@ def _persist_coefficients_to_config( def _append_iteration_records( - state: workflow.State, records: list[dict[str, Any]] + state: workflow.State, component_name: str, records: list[dict[str, Any]] ) -> None: """Append per-coefficient calibration iteration records.""" if not records: return - path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) df = pd.DataFrame(records) - _append_csv(df, path) + + # Save a global iteration history file + global_path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) + _append_csv(df, global_path) + + # Also write component-local iteration history + component_path = _component_output_dir(state, component_name) / Path( + CALIBRATION_ITERATION_FILE + ).name + _append_csv(df, component_path) def _append_summary_records( @@ -1042,6 +970,122 @@ def _append_csv(df: pd.DataFrame, path: Path) -> None: df.to_csv(path, mode="a", index=False, header=write_header) +def _component_output_dir(state: workflow.State, component_name: str) -> Path: + """Return output/calibration/ and ensure it exists.""" + component_dir = state.get_output_file_path( + f"calibration/{component_name}" + ) + os.makedirs(component_dir, exist_ok=True) + return component_dir + + +def _write_component_plots(state: workflow.State, component_name: str) -> None: + """Write/update all standard plots for one calibrated component.""" + recs = _read_component_iteration_records(state, component_name) + if recs is None or recs.empty: + return + + # Segment coefficients into manageable sets for plotting + coefs = sorted(recs.index.get_level_values("coefficient").unique()) + n_sets = math.ceil(len(coefs) / MAX_COEFFS_IN_GRAPH) + for coef_set in range(n_sets): + set_coefs = coefs[ + coef_set * MAX_COEFFS_IN_GRAPH : min( + len(coefs), (coef_set + 1) * MAX_COEFFS_IN_GRAPH + ) + ] + _plot_coefficient_progress(state, component_name, recs, set_coefs, coef_set) + last_records = _component_last_records(recs, set_coefs) + _plot_component_values(state, component_name, last_records, coef_set) + _plot_component_pct_change(state, component_name, last_records, coef_set) + + +def _read_component_iteration_records( + state: workflow.State, component_name: str +) -> pd.DataFrame | None: + """Read all iteration records for a single component.""" + path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) + if not path.exists(): + return None + + iteration_records = ( + pd.read_csv(path) + .set_index(["global_iter", "component_iter", "coefficient"]) + .sort_index() + ) + return iteration_records.loc[iteration_records.component == component_name] + + +def _plot_coefficient_progress( + state: workflow.State, + component_name: str, + recs: pd.DataFrame, + set_coefs: list[str], + coef_set: int, +) -> None: + """Plot coefficient value progression for one coefficient subset.""" + component_dir = _component_output_dir(state, component_name) + ax = ( + recs[recs.index.get_level_values("coefficient").isin(set_coefs)] + .next_coefficient.unstack("coefficient") + .plot(figsize=(10,5)) + ) + ax.xaxis.set_label_text("Component iteration") + ax.yaxis.set_label_text("Coefficient value") + ax.legend(title="Coefficient label", loc="center left", bbox_to_anchor=(1.02, 0.5)) + plt.tight_layout() + ax.figure.savefig( + component_dir / f"coefficient_progress_set_{coef_set}.png", + bbox_inches="tight", + ) + plt.close(ax.figure) + + +def _component_last_records(recs: pd.DataFrame, set_coefs: list[str]) -> pd.DataFrame: + """Select target/model values for the latest iteration and coefficient subset.""" + filtered = recs[recs.index.get_level_values("coefficient").isin(set_coefs)] + last_global = filtered.index.get_level_values("global_iter")[-1] + last_comp = filtered.loc[last_global].index.get_level_values("component_iter")[-1] + return filtered.xs((last_global, last_comp), level=("global_iter", "component_iter"))[ + ["target_value", "model_value"] + ] + + +def _plot_component_values( + state: workflow.State, + component_name: str, + last_records: pd.DataFrame, + coef_set: int, +) -> None: + """Plot final target/model component values for one coefficient subset.""" + component_dir = _component_output_dir(state, component_name) + ax = last_records.plot.bar(figsize=(10,5)) + ax.xaxis.set_tick_params(rotation=45) + ax.xaxis.set_label_text("Component value") + plt.tight_layout() + ax.figure.savefig(component_dir / f"final_components_set_{coef_set}.png") + plt.close(ax.figure) + + +def _plot_component_pct_change( + state: workflow.State, + component_name: str, + last_records: pd.DataFrame, + coef_set: int, +) -> None: + """Plot final percent difference for one coefficient subset.""" + component_dir = _component_output_dir(state, component_name) + fig, ax = plt.subplots(figsize=(10,5)) + pct_diff = last_records.diff(axis=1).model_value / last_records.target_value + ax = pct_diff.plot.bar(ax=ax) + ax.xaxis.set_tick_params(rotation=45) + ax.xaxis.set_label_text("Coefficient") + ax.yaxis.set_label_text("% Difference between Model and Target") + plt.tight_layout() + ax.figure.savefig(component_dir / f"final_pct_change_set_{coef_set}.png") + plt.close(ax.figure) + + def _write_generic_report( state: workflow.State, component_name: str, @@ -1068,9 +1112,7 @@ def _write_generic_report( .sort_values(["global_iter", "component_iter", "description"]) ) - path = state.get_output_file_path( - f"calibration/{component_name}_generic_report.csv" - ) + path = _component_output_dir(state, component_name) / "generic_report.csv" _append_csv(report, path) From 9b026082731c4f22023272e3fa826930806ef7d8 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 7 Jul 2026 11:54:48 -0400 Subject: [PATCH 13/90] add component settings --- activitysim/core/calibration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 2fd2888f99..e9dd40cc31 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -400,7 +400,7 @@ def _calibrate_component( ) state.run(models=[run_model_name], resume_after=prior_step) - eval_context = _build_expression_context(state, helper_symbols, component_name) + eval_context = _build_expression_context(state, helper_symbols, component_name, component_settings) _bind_context_to_helper_module_globals(helper_module, eval_context) ( @@ -614,6 +614,7 @@ def _build_expression_context( state: workflow.State, helper_symbols: dict[str, Any], component_name: str, + component_settings: CalibrationComponentSettings, ) -> dict[str, Any]: """Create the evaluation context for model_value and target_value expressions.""" context: dict[str, Any] = { @@ -621,6 +622,7 @@ def _build_expression_context( "np": np, "pd": pd, "component_output_dir": _component_output_dir(state, component_name), + "component_settings": component_settings, } # Load active tables into context for direct expression access. From 4475471e1a4057e5e66af6712f7d445c0bbebc90 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 7 Jul 2026 12:17:59 -0400 Subject: [PATCH 14/90] Add calibration configs --- .../auto_ownership_calib_helper.py | 36 ++++++++ .../auto_ownership_calibration.csv | 5 ++ .../configs_calibration/calibration.yaml | 36 ++++++++ .../tour_mode_choice_calib_helper.py | 36 ++++++++ .../tour_mode_choice_calibration.csv | 61 +++++++++++++ .../workplace_location_calib_helper.py | 89 +++++++++++++++++++ .../workplace_location_calibration.csv | 7 ++ 7 files changed, 270 insertions(+) create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calibration.csv create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calibration.csv create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv diff --git a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py new file mode 100644 index 0000000000..516f5cd85f --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py @@ -0,0 +1,36 @@ +import pandas as pd +import os + +SURVEY_DATA_FOLDER = r"U:\projects\clients\ASIM\autocalibration\calibration_test_mtc\data" + +def report_auto_ownership(): + model_hhs = households + survey_hhs = None + try: + survey_hhs = pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_households.csv")) + except FileNotFoundError: + raise FileNotFoundError(f"No survey file override_households.csv found in {SURVEY_DATA_FOLDER}!") + + model_summary = model_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) + # survey_summary = survey_hhs.groupby("auto_ownership").household_weight.sum() + # survey_summary = survey_summary / survey_hhs.household_weight.sum() + survey_summary = survey_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) + + summary_df = ( + pd.DataFrame({"model": model_summary, "survey": survey_summary}) + .reset_index() + .rename(columns={"index": "num_autos"}) + ) + + print(summary_df) + + # plot comparing model and survey distributions + import matplotlib.pyplot as plt + + summary_df.plot(x="auto_ownership", y=["model", "survey"], kind="bar") + plt.title("Auto Ownership Distribution: Model vs Survey") + plt.xlabel("Number of Autos") + plt.ylabel("Proportion of Households") + plt.legend(title="Data Source") + plt.savefig(os.path.join(component_output_dir, "auto_ownership_comparison.png")) + plt.close() \ No newline at end of file diff --git a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calibration.csv b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calibration.csv new file mode 100644 index 0000000000..b320b379b2 --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calibration.csv @@ -0,0 +1,5 @@ +description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance +0 auto ownership share,coef_calib_auto_0,len(households[households.auto_ownership ==0]) / len(households),0.06812,FALSE,-5,5,1,log_ratio,0.02 +2 auto ownership share,coef_calib_auto_2,len(households[households.auto_ownership ==2]) / len(households),0.348413,FALSE,-5,5,1,log_ratio,0.01 +3 auto ownership share,coef_calib_auto_3,len(households[households.auto_ownership ==3]) / len(households),0.13718,FALSE,-5,5,1,log_ratio,0.01 +4 auto ownership share,coef_calib_auto_4,len(households[households.auto_ownership ==4]) / len(households),0.057501,FALSE,-5,5,1,log_ratio,0.01 diff --git a/activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml b/activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml new file mode 100644 index 0000000000..c4949c0718 --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml @@ -0,0 +1,36 @@ +enable: True +run: + calibrate_models: + - workplace_location + - auto_ownership_simulate + - tour_mode_choice_simulate + global_iterations: 3 + +model_settings: + workplace_location: + calibration_spec: workplace_location_calibration.csv + helper_module: workplace_location_calib_helper.py + submodel_max_iterations: 3 + reports: + generic: true + bespoke: report_workplace_location + survey_file: survey_persons.csv + + auto_ownership_simulate: + calibration_spec: auto_ownership_calibration.csv + helper_module: auto_ownership_calib_helper.py + submodel_max_iterations: 3 + reports: + generic: true + bespoke: report_auto_ownership + survey_file: survey_households.csv + + tour_mode_choice_simulate: + calibration_spec: tour_mode_choice_calibration.csv + helper_module: tour_mode_choice_calib_helper.py + submodel_max_iterations: 3 + reports: + generic: true + bespoke: report_tour_mode_choice + survey_file: survey_tours.csv + diff --git a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py new file mode 100644 index 0000000000..a87708a795 --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py @@ -0,0 +1,36 @@ +import pandas as pd +import os + +def report_tour_mode_choice(): + model_tours = state.get_table("tours") + survey_tours = None + for data_dir in state.filesystem.data_dir: + try: + survey_tours = pd.read_csv(os.path.join(data_dir, component_settings.survey_file)) + break + except FileNotFoundError: + pass + assert survey_tours is not None, f"No survey file {component_settings.survey_file} found in data dirs!" + + model_summary = model_tours.tour_mode.value_counts(normalize=True).sort_index().fillna(0) + survey_summary = survey_tours.groupby("tour_mode").tour_weight.sum() + survey_summary = survey_summary / survey_tours.tour_weight.sum() + + summary_df = ( + pd.DataFrame({"model": model_summary, "survey": survey_summary}) + .reset_index() + .rename(columns={"index": "tour_mode"}) + ) + + print(summary_df) + + # plot comparing model and survey distributions + import matplotlib.pyplot as plt + + summary_df.plot(x="tour_mode", y=["model", "survey"], kind="bar") + plt.title("Tour Mode Choice Distribution: Model vs Survey") + plt.xlabel("Tour Mode") + plt.ylabel("Proportion of Tours") + plt.legend(title="Data Source") + plt.savefig("tour_mode_choice_comparison.png") + plt.close() \ No newline at end of file diff --git a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calibration.csv b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calibration.csv new file mode 100644 index 0000000000..a8c1c50af0 --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calibration.csv @@ -0,0 +1,61 @@ +description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance,default_increment +DRIVEALONEPAY share for zero auto HHs,coef_calib_DRIVEALONEPAY_zero_auto,"len(tours[(tours.tour_mode == ""DRIVEALONEPAY"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.036379517,FALSE,-5,5,1,log_ratio,0.02,2 +SHARED2FREE share for zero auto HHs,coef_calib_SHARED2FREE_zero_auto,"len(tours[(tours.tour_mode == ""SHARED2FREE"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.098670942,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED2PAY share for zero auto HHs,coef_calib_SHARED2PAY_zero_auto,"len(tours[(tours.tour_mode == ""SHARED2PAY"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.043195465,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED3FREE share for zero auto HHs,coef_calib_SHARED3FREE_zero_auto,"len(tours[(tours.tour_mode == ""SHARED3FREE"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.054018294,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED3PAY share for zero auto HHs,coef_calib_SHARED3PAY_zero_auto,"len(tours[(tours.tour_mode == ""SHARED3PAY"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.027328167,FALSE,-5,5,1,log_ratio,0.01,2 +WALK share for zero auto HHs,coef_calib_WALK_zero_auto,"len(tours[(tours.tour_mode == ""WALK"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.059569409,FALSE,-5,5,1,log_ratio,0.01,2 +BIKE share for zero auto HHs,coef_calib_BIKE_zero_auto,"len(tours[(tours.tour_mode == ""BIKE"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.064270507,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_LOC share for zero auto HHs,coef_calib_WALK_LOC_zero_auto,"len(tours[(tours.tour_mode == ""WALK_LOC"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.031069537,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_LRF share for zero auto HHs,coef_calib_WALK_LRF_zero_auto,"len(tours[(tours.tour_mode == ""WALK_LRF"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.055759405,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_EXP share for zero auto HHs,coef_calib_WALK_EXP_zero_auto,"len(tours[(tours.tour_mode == ""WALK_EXP"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.012576768,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_HVY share for zero auto HHs,coef_calib_WALK_HVY_zero_auto,"len(tours[(tours.tour_mode == ""WALK_HVY"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.004320335,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_COM share for zero auto HHs,coef_calib_WALK_COM_zero_auto,"len(tours[(tours.tour_mode == ""WALK_COM"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.055175324,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_LOC share for zero auto HHs,coef_calib_DRIVE_LOC_zero_auto,"len(tours[(tours.tour_mode == ""DRIVE_LOC"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.086416787,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_LRF share for zero auto HHs,coef_calib_DRIVE_LRF_zero_auto,"len(tours[(tours.tour_mode == ""DRIVE_LRF"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.067921667,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_EXP share for zero auto HHs,coef_calib_DRIVE_EXP_zero_auto,"len(tours[(tours.tour_mode == ""DRIVE_EXP"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.044918204,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_HVY share for zero auto HHs,coef_calib_DRIVE_HVY_zero_auto,"len(tours[(tours.tour_mode == ""DRIVE_HVY"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.033260131,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_COM share for zero auto HHs,coef_calib_DRIVE_COM_zero_auto,"len(tours[(tours.tour_mode == ""DRIVE_COM"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.021919061,FALSE,-5,5,1,log_ratio,0.01,2 +TAXI share for zero auto HHs,coef_calib_TAXI_zero_auto,"len(tours[(tours.tour_mode == ""TAXI"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.076918756,FALSE,-5,5,1,log_ratio,0.01,2 +TNC_SINGLE share for zero auto HHs,coef_calib_TNC_SINGLE_zero_auto,"len(tours[(tours.tour_mode == ""TNC_SINGLE"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.07478863,FALSE,-5,5,1,log_ratio,0.01,2 +TNC_SHARED share for zero auto HHs,coef_calib_TNC_SHARED_zero_auto,"len(tours[(tours.tour_mode == ""TNC_SHARED"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.051523091,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVEALONEPAY share for auto insufficient HHs,coef_calib_DRIVEALONEPAY_auto_insuff,"len(tours[(tours.tour_mode == ""DRIVEALONEPAY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.036379517,FALSE,-5,5,1,log_ratio,0.02,2 +SHARED2FREE share for auto insufficient HHs,coef_calib_SHARED2FREE_auto_insuff,"len(tours[(tours.tour_mode == ""SHARED2FREE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.098670942,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED2PAY share for auto insufficient HHs,coef_calib_SHARED2PAY_auto_insuff,"len(tours[(tours.tour_mode == ""SHARED2PAY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.043195465,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED3FREE share for auto insufficient HHs,coef_calib_SHARED3FREE_auto_insuff,"len(tours[(tours.tour_mode == ""SHARED3FREE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.054018294,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED3PAY share for auto insufficient HHs,coef_calib_SHARED3PAY_auto_insuff,"len(tours[(tours.tour_mode == ""SHARED3PAY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.027328167,FALSE,-5,5,1,log_ratio,0.01,2 +WALK share for auto insufficient HHs,coef_calib_WALK_auto_insuff,"len(tours[(tours.tour_mode == ""WALK"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.059569409,FALSE,-5,5,1,log_ratio,0.01,2 +BIKE share for auto insufficient HHs,coef_calib_BIKE_auto_insuff,"len(tours[(tours.tour_mode == ""BIKE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.064270507,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_LOC share for auto insufficient HHs,coef_calib_WALK_LOC_auto_insuff,"len(tours[(tours.tour_mode == ""WALK_LOC"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.031069537,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_LRF share for auto insufficient HHs,coef_calib_WALK_LRF_auto_insuff,"len(tours[(tours.tour_mode == ""WALK_LRF"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.055759405,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_EXP share for auto insufficient HHs,coef_calib_WALK_EXP_auto_insuff,"len(tours[(tours.tour_mode == ""WALK_EXP"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.012576768,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_HVY share for auto insufficient HHs,coef_calib_WALK_HVY_auto_insuff,"len(tours[(tours.tour_mode == ""WALK_HVY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.004320335,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_COM share for auto insufficient HHs,coef_calib_WALK_COM_auto_insuff,"len(tours[(tours.tour_mode == ""WALK_COM"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.055175324,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_LOC share for auto insufficient HHs,coef_calib_DRIVE_LOC_auto_insuff,"len(tours[(tours.tour_mode == ""DRIVE_LOC"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.086416787,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_LRF share for auto insufficient HHs,coef_calib_DRIVE_LRF_auto_insuff,"len(tours[(tours.tour_mode == ""DRIVE_LRF"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.067921667,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_EXP share for auto insufficient HHs,coef_calib_DRIVE_EXP_auto_insuff,"len(tours[(tours.tour_mode == ""DRIVE_EXP"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.044918204,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_HVY share for auto insufficient HHs,coef_calib_DRIVE_HVY_auto_insuff,"len(tours[(tours.tour_mode == ""DRIVE_HVY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.033260131,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_COM share for auto insufficient HHs,coef_calib_DRIVE_COM_auto_insuff,"len(tours[(tours.tour_mode == ""DRIVE_COM"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.021919061,FALSE,-5,5,1,log_ratio,0.01,2 +TAXI share for auto insufficient HHs,coef_calib_TAXI_auto_insuff,"len(tours[(tours.tour_mode == ""TAXI"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.076918756,FALSE,-5,5,1,log_ratio,0.01,2 +TNC_SINGLE share for auto insufficient HHs,coef_calib_TNC_SINGLE_auto_insuff,"len(tours[(tours.tour_mode == ""TNC_SINGLE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.07478863,FALSE,-5,5,1,log_ratio,0.01,2 +TNC_SHARED share for auto insufficient HHs,coef_calib_TNC_SHARED_auto_insuff,"len(tours[(tours.tour_mode == ""TNC_SHARED"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.051523091,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVEALONEPAY share for auto sufficient HHs,coef_calib_DRIVEALONEPAY_auto_suff,"len(tours[(tours.tour_mode == ""DRIVEALONEPAY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.036379517,FALSE,-5,5,1,log_ratio,0.02,2 +SHARED2FREE share for auto sufficient HHs,coef_calib_SHARED2FREE_auto_suff,"len(tours[(tours.tour_mode == ""SHARED2FREE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.098670942,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED2PAY share for auto sufficient HHs,coef_calib_SHARED2PAY_auto_suff,"len(tours[(tours.tour_mode == ""SHARED2PAY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.043195465,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED3FREE share for auto sufficient HHs,coef_calib_SHARED3FREE_auto_suff,"len(tours[(tours.tour_mode == ""SHARED3FREE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.054018294,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED3PAY share for auto sufficient HHs,coef_calib_SHARED3PAY_auto_suff,"len(tours[(tours.tour_mode == ""SHARED3PAY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.027328167,FALSE,-5,5,1,log_ratio,0.01,2 +WALK share for auto sufficient HHs,coef_calib_WALK_auto_suff,"len(tours[(tours.tour_mode == ""WALK"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.059569409,FALSE,-5,5,1,log_ratio,0.01,2 +BIKE share for auto sufficient HHs,coef_calib_BIKE_auto_suff,"len(tours[(tours.tour_mode == ""BIKE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.064270507,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_LOC share for auto sufficient HHs,coef_calib_WALK_LOC_auto_suff,"len(tours[(tours.tour_mode == ""WALK_LOC"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.031069537,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_LRF share for auto sufficient HHs,coef_calib_WALK_LRF_auto_suff,"len(tours[(tours.tour_mode == ""WALK_LRF"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.055759405,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_EXP share for auto sufficient HHs,coef_calib_WALK_EXP_auto_suff,"len(tours[(tours.tour_mode == ""WALK_EXP"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.012576768,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_HVY share for auto sufficient HHs,coef_calib_WALK_HVY_auto_suff,"len(tours[(tours.tour_mode == ""WALK_HVY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.004320335,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_COM share for auto sufficient HHs,coef_calib_WALK_COM_auto_suff,"len(tours[(tours.tour_mode == ""WALK_COM"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.055175324,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_LOC share for auto sufficient HHs,coef_calib_DRIVE_LOC_auto_suff,"len(tours[(tours.tour_mode == ""DRIVE_LOC"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.086416787,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_LRF share for auto sufficient HHs,coef_calib_DRIVE_LRF_auto_suff,"len(tours[(tours.tour_mode == ""DRIVE_LRF"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.067921667,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_EXP share for auto sufficient HHs,coef_calib_DRIVE_EXP_auto_suff,"len(tours[(tours.tour_mode == ""DRIVE_EXP"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.044918204,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_HVY share for auto sufficient HHs,coef_calib_DRIVE_HVY_auto_suff,"len(tours[(tours.tour_mode == ""DRIVE_HVY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.033260131,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_COM share for auto sufficient HHs,coef_calib_DRIVE_COM_auto_suff,"len(tours[(tours.tour_mode == ""DRIVE_COM"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.021919061,FALSE,-5,5,1,log_ratio,0.01,2 +TAXI share for auto sufficient HHs,coef_calib_TAXI_auto_suff,"len(tours[(tours.tour_mode == ""TAXI"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.076918756,FALSE,-5,5,1,log_ratio,0.01,2 +TNC_SINGLE share for auto sufficient HHs,coef_calib_TNC_SINGLE_auto_suff,"len(tours[(tours.tour_mode == ""TNC_SINGLE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.07478863,FALSE,-5,5,1,log_ratio,0.01,2 +TNC_SHARED share for auto sufficient HHs,coef_calib_TNC_SHARED_auto_suff,"len(tours[(tours.tour_mode == ""TNC_SHARED"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.051523091,FALSE,-5,5,1,log_ratio,0.01,2 diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py new file mode 100644 index 0000000000..ed7555c427 --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py @@ -0,0 +1,89 @@ +import matplotlib.pyplot as plt +import pandas as pd +import os +from functools import lru_cache + +SURVEY_DATA_FOLDER = r"U:\projects\clients\ASIM\autocalibration\calibration_test_mtc\data" + +def compute_distances(origins, destinations): + # Compute distances between origins and destinations using the network level of service + # using non-time-dependent DIST skim + distances = skim_dict.lookup(origins, destinations, 'DIST') + # time dependent example + # distances = skim_dict.lookup_3d(origins, destinations, 'AM', 'SOV_DIST') + return distances + + +@lru_cache(maxsize=1) +def _survey_persons() -> pd.DataFrame: + """Load survey persons once per Python process.""" + return pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_persons.csv")) + + +@lru_cache(maxsize=1) +def _survey_households() -> pd.DataFrame: + """Load survey households once per Python process.""" + return pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_households.csv")) + + +@lru_cache(maxsize=1) +def _survey_worker_distances(): + """Compute survey worker distances once and reuse across calibration rows.""" + survey_persons = _survey_persons() + survey_workers = survey_persons[survey_persons["workplace_zone_id"] > 0] + survey_home_zone_ids = _survey_households().set_index("household_id")["home_zone_id"] + survey_home_zone_ids = survey_workers["household_id"].map(survey_home_zone_ids) + survey_workplace_zone_ids = survey_workers["workplace_zone_id"] + return compute_distances(survey_home_zone_ids, survey_workplace_zone_ids) + + +def summarize_model(min_dist=1, max_dist=2): + """Summarize the model results for workplaces within the specified distance range.""" + workers = persons[persons['workplace_zone_id'] > 0] + home_zone_ids = workers['home_zone_id'] + workplace_zone_ids = workers['workplace_zone_id'] + + distances = compute_distances(home_zone_ids, workplace_zone_ids) + + # Filter distances within the specified range + mask = (distances >= min_dist) & (distances < max_dist) + filtered_distances = distances[mask] + + share = len(filtered_distances) / len(distances) if len(distances) > 0 else 0 + return share + +def summarize_survey(min_dist=1, max_dist=2): + """Summarize the survey results for workplaces within the specified distance range.""" + + distances = _survey_worker_distances() + + # Filter distances within the specified range + mask = (distances >= min_dist) & (distances < max_dist) + filtered_distances = distances[mask] + + share = len(filtered_distances) / len(distances) if len(distances) > 0 else 0 + return share + +def report_workplace_location(): + """Workplace location distance frequency plot comparing model results with observed data.""" + print("summarizing workplace location model") + model_persons = persons + model_workers = model_persons[model_persons['workplace_zone_id'] > 0] + model_home_zone_ids = model_workers['home_zone_id'] + model_workplace_zone_ids = model_workers['workplace_zone_id'] + + model_distances = compute_distances(model_home_zone_ids, model_workplace_zone_ids) + + # survey_distances = _survey_worker_distances() + survey_distances = model_distances + + # Here you can add code to compare model_distances and survey_distances, + # for example by plotting histograms or computing summary statistics. + plt.hist(model_distances, bins=20, density=True, alpha=0.5, label='Model') + plt.hist(survey_distances, bins=20, density=True, alpha=0.5, label='Survey') + plt.xlabel('Distance') + plt.ylabel('Frequency') + plt.legend() + # component_output_dir set in the evaluation context + plt.savefig(os.path.join(component_output_dir, 'workplace_location_comparison.png')) + plt.close() \ No newline at end of file diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv new file mode 100644 index 0000000000..4a7193acd2 --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv @@ -0,0 +1,7 @@ +description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance +#Distance 1 to 2 mi share,coef_calib_dist_1_2,"summarize_model(min_dist=.1, max_dist=.2)","summarize_survey(min_dist=.1, max_dist=.2)",FALSE,-5,5,1,log_ratio,0.02 +#Distance 5 to 15 mi share,coef_calib_dist_5_15,"summarize_model(min_dist=.15, max_dist=.5)","summarize_survey(min_dist=.15, max_dist=.5)",FALSE,-5,5,1,log_ratio,0.01 +#Distance 15+ mi share,coef_calib_dist_15_up,"summarize_model(min_dist=.5, max_dist=999)","summarize_survey(min_dist=.5, max_dist=999)",FALSE,-5,5,1,log_ratio,0.01 +Distance 1 to 2 mi share,coef_calib_dist_1_2,"summarize_model(min_dist=.1, max_dist=.2)",0.1,FALSE,-5,5,1,log_ratio,0.02 +Distance 5 to 15 mi share,coef_calib_dist_5_15,"summarize_model(min_dist=.15, max_dist=.5)",0.65,FALSE,-5,5,1,log_ratio,0.01 +Distance 15+ mi share,coef_calib_dist_15_up,"summarize_model(min_dist=.5, max_dist=999)",0.15,FALSE,-5,5,1,log_ratio,0.01 From 1bf010c54926b780739ee3be84060aa909542ba3 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:03:56 -0700 Subject: [PATCH 15/90] passing context dict --- activitysim/core/calibration.py | 36 ++------------- .../prototype_mtc/configs/auto_ownership.csv | 4 ++ .../auto_ownership_calib_helper.py | 22 +++------- .../tour_mode_choice_calib_helper.py | 21 +++------ .../workplace_location_calib_helper.py | 44 +++++++++++-------- 5 files changed, 46 insertions(+), 81 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index e9dd40cc31..e60e05dc0e 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -401,7 +401,6 @@ def _calibrate_component( state.run(models=[run_model_name], resume_after=prior_step) eval_context = _build_expression_context(state, helper_symbols, component_name, component_settings) - _bind_context_to_helper_module_globals(helper_module, eval_context) ( row_records, @@ -427,9 +426,7 @@ def _calibrate_component( _write_generic_report(state, component_name, row_records) if bespoke_callable is not None: - # Helper module globals were already updated from eval_context, - # so bespoke functions can access tables/injectables directly. - bespoke_callable() + bespoke_callable(eval_context) if component_converged: break @@ -641,38 +638,11 @@ def _build_expression_context( pass context.update(helper_symbols) + # Explicit function-call context used by calibration expressions. + context["context"] = context return context -def _bind_context_to_helper_module_globals( - helper_module: Any | None, - eval_context: dict[str, Any], -) -> None: - """Bind eval context names into helper module globals. - - Helper functions referenced by calibration expressions resolve free names - from their module globals, not from eval locals. This syncs the current - evaluation context into the helper module globals each component iteration. - """ - if helper_module is None: - return - - excluded = { - "__builtins__", - "__name__", - "__package__", - "__loader__", - "__spec__", - "__file__", - "__cached__", - } - - filtered_context = { - key: value for key, value in eval_context.items() if key not in excluded - } - helper_module.__dict__.update(filtered_context) - - def _evaluate_and_update( component_name: str, calibration_spec_df: pd.DataFrame, diff --git a/activitysim/examples/prototype_mtc/configs/auto_ownership.csv b/activitysim/examples/prototype_mtc/configs/auto_ownership.csv index aa30bdb4db..9f00def376 100644 --- a/activitysim/examples/prototype_mtc/configs/auto_ownership.csv +++ b/activitysim/examples/prototype_mtc/configs/auto_ownership.csv @@ -28,3 +28,7 @@ util_retail_transit_workers,"Retail accessibility (0.66*PK + 0.34*OP) by transit util_retail_non_motor_no_workers,"Retail accessibility by non-motorized, if 0 workers",(num_workers==0)*nmRetail,,coef_retail_non_motor,coef_retail_non_motor,coef_retail_non_motor,coef_retail_non_motor util_retail_non_motor_workers,"Retail accessibility by non-motorized, if 1+ workers",(num_workers>0)*nmRetail,,coef_retail_non_motor,coef_retail_non_motor,coef_retail_non_motor,coef_retail_non_motor util_auto_time_saving_per_worker,Auto time savings per worker to work,"@np.where(df.num_workers > 0, df.hh_work_auto_savings_ratio / df.num_workers, 0)",,coef_cars1_auto_time_saving_per_worker,coef_cars2_auto_time_saving_per_worker,coef_cars3_auto_time_saving_per_worker,coef_cars4_auto_time_saving_per_worker +util_calib_auto_0,Calibration utility for 0 autos,"@df.auto_ownership == 0",,coef_calib_auto_0 +util_calib_auto_2,Calibration utility for 2 autos,"@df.auto_ownership == 2",,coef_calib_auto_2 +util_calib_auto_3,Calibration utility for 3 autos,"@df.auto_ownership == 3",,coef_calib_auto_3 +util_calib_auto_4,Calibration utility for 4 autos,"@df.auto_ownership == 4",,coef_calib_auto_4 \ No newline at end of file diff --git a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py index 516f5cd85f..ee6f319ce7 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py @@ -1,36 +1,26 @@ +import matplotlib.pyplot as plt import pandas as pd import os -SURVEY_DATA_FOLDER = r"U:\projects\clients\ASIM\autocalibration\calibration_test_mtc\data" +SURVEY_DATA_FOLDER = r"C:\Users\david.hensle\OneDrive - Resource Systems Group, Inc\Documents\projects\activitysim\rsg_activitysim\activitysim\examples\example_estimation\data_test\survey_data" -def report_auto_ownership(): - model_hhs = households - survey_hhs = None - try: - survey_hhs = pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_households.csv")) - except FileNotFoundError: - raise FileNotFoundError(f"No survey file override_households.csv found in {SURVEY_DATA_FOLDER}!") +def report_auto_ownership(context): + model_hhs = context["households"] + survey_hhs = pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_households.csv")) model_summary = model_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) - # survey_summary = survey_hhs.groupby("auto_ownership").household_weight.sum() - # survey_summary = survey_summary / survey_hhs.household_weight.sum() survey_summary = survey_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) - summary_df = ( pd.DataFrame({"model": model_summary, "survey": survey_summary}) .reset_index() .rename(columns={"index": "num_autos"}) ) - print(summary_df) - # plot comparing model and survey distributions - import matplotlib.pyplot as plt - summary_df.plot(x="auto_ownership", y=["model", "survey"], kind="bar") plt.title("Auto Ownership Distribution: Model vs Survey") plt.xlabel("Number of Autos") plt.ylabel("Proportion of Households") plt.legend(title="Data Source") - plt.savefig(os.path.join(component_output_dir, "auto_ownership_comparison.png")) + plt.savefig(os.path.join(context["component_output_dir"], "auto_ownership_comparison.png")) plt.close() \ No newline at end of file diff --git a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py index a87708a795..bbec248ca7 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py @@ -1,16 +1,13 @@ import pandas as pd +import matplotlib.pyplot as plt import os -def report_tour_mode_choice(): - model_tours = state.get_table("tours") +SURVEY_DATA_FOLDER = r"C:\Users\david.hensle\OneDrive - Resource Systems Group, Inc\Documents\projects\activitysim\rsg_activitysim\activitysim\examples\example_estimation\data_test\survey_data" + +def report_tour_mode_choice(context): + model_tours = context["tours"] survey_tours = None - for data_dir in state.filesystem.data_dir: - try: - survey_tours = pd.read_csv(os.path.join(data_dir, component_settings.survey_file)) - break - except FileNotFoundError: - pass - assert survey_tours is not None, f"No survey file {component_settings.survey_file} found in data dirs!" + survey_tours = pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_tours.csv")) model_summary = model_tours.tour_mode.value_counts(normalize=True).sort_index().fillna(0) survey_summary = survey_tours.groupby("tour_mode").tour_weight.sum() @@ -22,15 +19,11 @@ def report_tour_mode_choice(): .rename(columns={"index": "tour_mode"}) ) - print(summary_df) - # plot comparing model and survey distributions - import matplotlib.pyplot as plt - summary_df.plot(x="tour_mode", y=["model", "survey"], kind="bar") plt.title("Tour Mode Choice Distribution: Model vs Survey") plt.xlabel("Tour Mode") plt.ylabel("Proportion of Tours") plt.legend(title="Data Source") - plt.savefig("tour_mode_choice_comparison.png") + plt.savefig(os.path.join(context["component_output_dir"], "tour_mode_choice_comparison.png")) plt.close() \ No newline at end of file diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py index ed7555c427..3c2496667f 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py @@ -1,49 +1,58 @@ +""" +Workplace location calibration helper functions. + +Notes: + The lru_cache decorator to ensure that survey data is loaded and computed only once per Python process. + Context is developed in _build_expression_context() in activitysim.core.calibration. +""" + import matplotlib.pyplot as plt import pandas as pd import os from functools import lru_cache -SURVEY_DATA_FOLDER = r"U:\projects\clients\ASIM\autocalibration\calibration_test_mtc\data" +SURVEY_DATA_FOLDER = r"C:\Users\david.hensle\OneDrive - Resource Systems Group, Inc\Documents\projects\activitysim\rsg_activitysim\activitysim\examples\example_estimation\data_test\survey_data" -def compute_distances(origins, destinations): +def compute_distances(context, origins, destinations): # Compute distances between origins and destinations using the network level of service # using non-time-dependent DIST skim - distances = skim_dict.lookup(origins, destinations, 'DIST') + distances = context["skim_dict"].lookup(origins.clip(upper=24), destinations.clip(upper=24), 'DIST') # time dependent example # distances = skim_dict.lookup_3d(origins, destinations, 'AM', 'SOV_DIST') return distances -@lru_cache(maxsize=1) +# @lru_cache(maxsize=1) def _survey_persons() -> pd.DataFrame: """Load survey persons once per Python process.""" return pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_persons.csv")) -@lru_cache(maxsize=1) +# @lru_cache(maxsize=1) def _survey_households() -> pd.DataFrame: """Load survey households once per Python process.""" return pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_households.csv")) -@lru_cache(maxsize=1) -def _survey_worker_distances(): +# @lru_cache(maxsize=1) +def _survey_worker_distances(context): """Compute survey worker distances once and reuse across calibration rows.""" survey_persons = _survey_persons() survey_workers = survey_persons[survey_persons["workplace_zone_id"] > 0] survey_home_zone_ids = _survey_households().set_index("household_id")["home_zone_id"] survey_home_zone_ids = survey_workers["household_id"].map(survey_home_zone_ids) survey_workplace_zone_ids = survey_workers["workplace_zone_id"] - return compute_distances(survey_home_zone_ids, survey_workplace_zone_ids) + return compute_distances(context, survey_home_zone_ids, survey_workplace_zone_ids) -def summarize_model(min_dist=1, max_dist=2): +def summarize_model(context, min_dist=1, max_dist=2): """Summarize the model results for workplaces within the specified distance range.""" + persons = context['persons'] workers = persons[persons['workplace_zone_id'] > 0] home_zone_ids = workers['home_zone_id'] workplace_zone_ids = workers['workplace_zone_id'] - distances = compute_distances(home_zone_ids, workplace_zone_ids) + distances = compute_distances(context, home_zone_ids, workplace_zone_ids) # Filter distances within the specified range mask = (distances >= min_dist) & (distances < max_dist) @@ -52,10 +61,10 @@ def summarize_model(min_dist=1, max_dist=2): share = len(filtered_distances) / len(distances) if len(distances) > 0 else 0 return share -def summarize_survey(min_dist=1, max_dist=2): +def summarize_survey(context, min_dist=1, max_dist=2): """Summarize the survey results for workplaces within the specified distance range.""" - distances = _survey_worker_distances() + distances = _survey_worker_distances(context) # Filter distances within the specified range mask = (distances >= min_dist) & (distances < max_dist) @@ -64,18 +73,17 @@ def summarize_survey(min_dist=1, max_dist=2): share = len(filtered_distances) / len(distances) if len(distances) > 0 else 0 return share -def report_workplace_location(): +def report_workplace_location(context): """Workplace location distance frequency plot comparing model results with observed data.""" print("summarizing workplace location model") - model_persons = persons + model_persons = context["persons"] model_workers = model_persons[model_persons['workplace_zone_id'] > 0] model_home_zone_ids = model_workers['home_zone_id'] model_workplace_zone_ids = model_workers['workplace_zone_id'] - model_distances = compute_distances(model_home_zone_ids, model_workplace_zone_ids) + model_distances = compute_distances(context, model_home_zone_ids, model_workplace_zone_ids) - # survey_distances = _survey_worker_distances() - survey_distances = model_distances + survey_distances = _survey_worker_distances(context) # Here you can add code to compare model_distances and survey_distances, # for example by plotting histograms or computing summary statistics. @@ -85,5 +93,5 @@ def report_workplace_location(): plt.ylabel('Frequency') plt.legend() # component_output_dir set in the evaluation context - plt.savefig(os.path.join(component_output_dir, 'workplace_location_comparison.png')) + plt.savefig(os.path.join(context["component_output_dir"], 'workplace_location_comparison.png')) plt.close() \ No newline at end of file From dc731d0df8bf2d1e4cfd2b9bfd7b4acf7c4837fc Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:04:11 -0700 Subject: [PATCH 16/90] updated specs for calibration --- .../configs/auto_ownership_coefficients.csv | 20 +++++++++++-------- .../configs/workplace_location.csv | 6 +++++- .../workplace_location_coefficients.csv | 4 ++++ .../workplace_location_calibration.csv | 12 +++++------ 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv b/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv index b9d7fd07b0..4ade078423 100644 --- a/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv @@ -1,12 +1,12 @@ coefficient_name,value,constrain -coef_cars1_drivers_2,0,T -coef_cars1_drivers_3,0,T -coef_cars1_persons_16_17,0,T -coef_cars234_asc_marin,0,T -coef_cars1_persons_25_34,0,T -coef_cars1_num_workers_clip_3,0,T -coef_cars1_hh_income_30_up,0,T -coef_cars1_density_0_10_no_workers,0,T +coef_cars1_drivers_2,0.0,T +coef_cars1_drivers_3,0.0,T +coef_cars1_persons_16_17,0.0,T +coef_cars234_asc_marin,0.0,T +coef_cars1_persons_25_34,0.0,T +coef_cars1_num_workers_clip_3,0.0,T +coef_cars1_hh_income_30_up,0.0,T +coef_cars1_density_0_10_no_workers,0.0,T coef_cars1_density_10_up_workers,-0.0152,F coef_retail_non_motor,-0.03,T coef_cars4_asc,-5.313,F @@ -66,3 +66,7 @@ coef_cars3_drivers_3,5.5131,F coef_cars2_drivers_4_up,6.3662,F coef_cars3_drivers_4_up,8.5148,F coef_cars4_drivers_4_up,9.5807,F +coef_calib_auto_0,-5.0,T +coef_calib_auto_2,5.0,T +coef_calib_auto_3,5.0,T +coef_calib_auto_4,5.0,T diff --git a/activitysim/examples/prototype_mtc/configs/workplace_location.csv b/activitysim/examples/prototype_mtc/configs/workplace_location.csv index 3ac8b59e96..67ca4a22b8 100644 --- a/activitysim/examples/prototype_mtc/configs/workplace_location.csv +++ b/activitysim/examples/prototype_mtc/configs/workplace_location.csv @@ -11,4 +11,8 @@ util_size_variable,Size variable,@(df['size_term'] * df['shadow_price_size_term_ util_utility_adjustment,utility adjustment,@df['shadow_price_utility_adjustment'],1 util_no_attractions,No attractions,@df['size_term']==0,-999 util_mode_logsum,Mode choice logsum,mode_choice_logsum,coef_mode_logsum -util_sample_of_corrections_factor,Sample of alternatives correction factor,"@np.minimum(np.log(df.pick_count/df.prob), 60)",1 \ No newline at end of file +util_sample_of_corrections_factor,Sample of alternatives correction factor,"@np.minimum(np.log(df.pick_count/df.prob), 60)",1 +util_calib_dist_0_2,Calibration utility 0 to 2 mi,"@(_DIST >= 0) & (_DIST < 2)",coef_calib_dist_0_2 +util_calib_dist_2_5,Calibration utility 2 to 5 mi,"@(_DIST >= 2) & (_DIST < 5)",coef_calib_dist_2_5 +util_calib_dist_5_15,Calibration utility 5 to 15 mi,"@(_DIST >= 5) & (_DIST < 15)",coef_calib_dist_5_15 +util_calib_dist_15_up,Calibration utility 15+ mi,"@(_DIST >= 15)",coef_calib_dist_15_up diff --git a/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv b/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv index b3ac834103..7b1b52e4ac 100644 --- a/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv @@ -7,3 +7,7 @@ coef_dist_15_up,-0.0917,F coef_dist_0_5_high,0.15,F coef_dist_5_up_high,0.02,F coef_mode_logsum,0.3,F +coef_calib_dist_0_2,-5.0,T +coef_calib_dist_2_5,0.0,T +coef_calib_dist_5_15,-2.771779906602003,T +coef_calib_dist_15_up,0.749738309309575,T diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv index 4a7193acd2..c12ada8045 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv @@ -1,7 +1,7 @@ description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance -#Distance 1 to 2 mi share,coef_calib_dist_1_2,"summarize_model(min_dist=.1, max_dist=.2)","summarize_survey(min_dist=.1, max_dist=.2)",FALSE,-5,5,1,log_ratio,0.02 -#Distance 5 to 15 mi share,coef_calib_dist_5_15,"summarize_model(min_dist=.15, max_dist=.5)","summarize_survey(min_dist=.15, max_dist=.5)",FALSE,-5,5,1,log_ratio,0.01 -#Distance 15+ mi share,coef_calib_dist_15_up,"summarize_model(min_dist=.5, max_dist=999)","summarize_survey(min_dist=.5, max_dist=999)",FALSE,-5,5,1,log_ratio,0.01 -Distance 1 to 2 mi share,coef_calib_dist_1_2,"summarize_model(min_dist=.1, max_dist=.2)",0.1,FALSE,-5,5,1,log_ratio,0.02 -Distance 5 to 15 mi share,coef_calib_dist_5_15,"summarize_model(min_dist=.15, max_dist=.5)",0.65,FALSE,-5,5,1,log_ratio,0.01 -Distance 15+ mi share,coef_calib_dist_15_up,"summarize_model(min_dist=.5, max_dist=999)",0.15,FALSE,-5,5,1,log_ratio,0.01 +#Distance 1 to 2 mi share,coef_calib_dist_1_2,"summarize_model(context, min_dist=.1, max_dist=.2)","summarize_survey(context, min_dist=.1, max_dist=.2)",FALSE,-5,5,1,log_ratio,0.02 +#Distance 5 to 15 mi share,coef_calib_dist_5_15,"summarize_model(context, min_dist=.15, max_dist=.5)","summarize_survey(context, min_dist=.15, max_dist=.5)",FALSE,-5,5,1,log_ratio,0.01 +#Distance 15+ mi share,coef_calib_dist_15_up,"summarize_model(context, min_dist=.5, max_dist=999)","summarize_survey(context, min_dist=.5, max_dist=999)",FALSE,-5,5,1,log_ratio,0.01 +Distance 0 to 2 mi share,coef_calib_dist_0_2,"summarize_model(context, min_dist=.0, max_dist=.5)",0.05,FALSE,-5,5,1,log_ratio,0.02 +Distance 5 to 15 mi share,coef_calib_dist_5_15,"summarize_model(context, min_dist=.5, max_dist=1)",0.25,FALSE,-5,5,1,log_ratio,0.01 +Distance 15+ mi share,coef_calib_dist_15_up,"summarize_model(context, min_dist=2, max_dist=999)",0.1,FALSE,-5,5,1,log_ratio,0.01 From 034545cae892b603572f28b869187ce5ca88f5be Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 7 Jul 2026 13:09:38 -0400 Subject: [PATCH 17/90] Add tour mode choice calibration coefficients --- .../configs/tour_mode_choice.csv | 3 + .../configs/tour_mode_choice_coefficients.csv | 104 ++++++++++++++---- ...tour_mode_choice_coefficients_template.csv | 60 ++++++++++ 3 files changed, 145 insertions(+), 22 deletions(-) diff --git a/activitysim/examples/prototype_mtc/configs/tour_mode_choice.csv b/activitysim/examples/prototype_mtc/configs/tour_mode_choice.csv index 5bd4898e03..80a900d7b0 100644 --- a/activitysim/examples/prototype_mtc/configs/tour_mode_choice.csv +++ b/activitysim/examples/prototype_mtc/configs/tour_mode_choice.csv @@ -342,3 +342,6 @@ util_Walk_not_available_for_long_distances,Walk not available for long distances util_Bike_not_available_for_long_distances,Bike not available for long distances,@od_skims.max('DISTBIKE') > 8,,,,,,,,-999,,,,,,,,,,,,, util_Drive_alone_not_available_for_escort_tours,Drive alone not available for escort tours,is_escort,-999,-999,,,,,,,,,,,,,,,,,,, #, max(c_densityIndexOrigin*originDensityIndex,originDensityIndexMax),,,,,,,,,1,1,1,1,1,1,1,,,,,, +util_calib_zero_autos,Calibration utility for zero-auto households,1,,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_WALK_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SHARED_zero_auto +util_calib_auto_insufficient,Calibration utility for auto-insufficient households,1,,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SHARED_auto_insuff +util_calib_auto_sufficient,Calibration utility for auto-sufficient hou8seholds,1,,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_WALK_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SHARED_auto_suff diff --git a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv index 9693953808..a6d5c07487 100644 --- a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv @@ -1,6 +1,6 @@ coefficient_name,value,constrain -coef_one,1,T -coef_nest_root,1.00,T +coef_one,1.0,T +coef_nest_root,1.0,T coef_nest_AUTO,0.72,T coef_nest_AUTO_DRIVEALONE,0.35,T coef_nest_AUTO_SHAREDRIDE2,0.35,T @@ -14,23 +14,23 @@ coef_ivt_eatout_escort_othdiscr_othmaint_shopping_social,-0.0175,F coef_ivt_school_univ,-0.0224,F coef_ivt_work,-0.0134,F coef_ivt_atwork,-0.0188,F -coef_topology_walk_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,15,F +coef_topology_walk_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,15.0,F coef_topology_walk_multiplier_atwork,7.5,F -coef_topology_bike_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,20,F -coef_topology_bike_multiplier_atwork,10,F +coef_topology_bike_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,20.0,F +coef_topology_bike_multiplier_atwork,10.0,F coef_topology_trn_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,2.2,F -coef_topology_trn_multiplier_atwork,2,F -coef_age1619_da_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0,F +coef_topology_trn_multiplier_atwork,2.0,F +coef_age1619_da_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0.0,F coef_age1619_da_multiplier_school_univ,-1.3813,F coef_age1619_da_multiplier_atwork,0.0032336,F -coef_age010_trn_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0,F +coef_age010_trn_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0.0,F coef_age010_trn_multiplier_school_univ,-1.5548,F coef_age010_trn_multiplier_atwork,0.000722,F coef_age16p_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social,-1.366,F -coef_age16p_sr_multiplier_school_univ_work_atwork,0,F -coef_hhsize1_sr_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_atwork,0,F +coef_age16p_sr_multiplier_school_univ_work_atwork,0.0,F +coef_hhsize1_sr_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_atwork,0.0,F coef_hhsize1_sr_multiplier_work,-0.734588,F -coef_hhsize2_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work_atwork,0,F +coef_hhsize2_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work_atwork,0.0,F coef_hhsize2_sr_multiplier_school_univ,-0.6359,F walk_ASC_no_auto_eatout,5.1251173,F walk_ASC_no_auto_escort,2.8012068,F @@ -92,9 +92,9 @@ bike_ASC_auto_sufficient_social,-1.368071,F bike_ASC_auto_sufficient_univ,-1.9397832,F bike_ASC_auto_sufficient_work,-1.5800232,F bike_ASC_auto_sufficient_atwork,15.72017,F -sr2_ASC_no_auto_all,0,F +sr2_ASC_no_auto_all,0.0,F sr2_ASC_auto_deficient_eatout,0.5882345,F -sr2_ASC_auto_deficient_escort,0,F +sr2_ASC_auto_deficient_escort,0.0,F sr2_ASC_auto_deficient_othdiscr,0.6601513,F sr2_ASC_auto_deficient_othmaint,0.2621527,F sr2_ASC_auto_deficient_school,0.12474365,F @@ -104,7 +104,7 @@ sr2_ASC_auto_deficient_univ,-1.6922346,F sr2_ASC_auto_deficient_work,-0.33803123,F sr2_ASC_auto_deficient_atwork,-2.1102421,F sr2_ASC_auto_sufficient_eatout,0.86280555,F -sr2_ASC_auto_sufficient_escort,0,F +sr2_ASC_auto_sufficient_escort,0.0,F sr2_ASC_auto_sufficient_othdiscr,0.49684617,F sr2_ASC_auto_sufficient_othmaint,0.25817883,F sr2_ASC_auto_sufficient_school,-1.6062657,F @@ -173,7 +173,7 @@ walk_transit_ASC_auto_sufficient_social,-0.3453759,F walk_transit_ASC_auto_sufficient_univ,0.4731163,F walk_transit_ASC_auto_sufficient_work,-0.8916507,F walk_transit_ASC_auto_sufficient_atwork,-3.401027,F -drive_transit_ASC_no_auto_all,0,F +drive_transit_ASC_no_auto_all,0.0,F drive_transit_ASC_auto_deficient_eatout,0.5998061,F drive_transit_ASC_auto_deficient_escort,-1.1537067,F drive_transit_ASC_auto_deficient_othdiscr,0.3199308,F @@ -196,7 +196,7 @@ drive_transit_ASC_auto_sufficient_work,-1.0045459,F drive_transit_ASC_auto_sufficient_atwork,-999.21466,F taxi_ASC_no_auto_eatout_othdiscr_social,0.9923,F taxi_ASC_no_auto_escort_othmaint_shopping,1.8939,F -taxi_ASC_no_auto_school_univ,-7,T +taxi_ASC_no_auto_school_univ,-7.0,T taxi_ASC_no_auto_work,4.7291,F taxi_ASC_no_auto_atwork,4.1021,F taxi_ASC_auto_deficient_eatout_othdiscr_social,-3.1317,F @@ -213,7 +213,7 @@ taxi_ASC_auto_sufficient_work,-4.8509,F taxi_ASC_auto_sufficient_atwork,-2.8804,F tnc_single_ASC_no_auto_eatout_othdiscr_social,1.6852,F tnc_single_ASC_no_auto_escort_othmaint_shopping,1.8605,F -tnc_single_ASC_no_auto_school,-7,T +tnc_single_ASC_no_auto_school,-7.0,T tnc_single_ASC_no_auto_univ,-2.519,F tnc_single_ASC_no_auto_work,5.7855,F tnc_single_ASC_no_auto_atwork,4.4982,F @@ -231,7 +231,7 @@ tnc_single_ASC_auto_sufficient_work,-4.1946,F tnc_single_ASC_auto_sufficient_atwork,-2.7988,F tnc_shared_ASC_no_auto_eatout_othdiscr_social,0.6464,F tnc_shared_ASC_no_auto_escort_othmaint_shopping,0.9361,F -tnc_shared_ASC_no_auto_school,-7,T +tnc_shared_ASC_no_auto_school,-7.0,T tnc_shared_ASC_no_auto_univ,-5.8116,F tnc_shared_ASC_no_auto_work,3.2429,F tnc_shared_ASC_no_auto_atwork,3.3672,F @@ -253,16 +253,16 @@ joint_walk_ASC_auto_sufficient_all,-3.2352157,F joint_bike_ASC_no_auto_all,-2.8671598,F joint_bike_ASC_auto_deficient_all,-6.076415,F joint_bike_ASC_auto_sufficient_all,-6.3760657,F -joint_sr2_ASC_no_auto_all,0,T -joint_sr2_ASC_auto_deficient_all,0,T -joint_sr2_ASC_auto_sufficient_all,0,T +joint_sr2_ASC_no_auto_all,0.0,T +joint_sr2_ASC_auto_deficient_all,0.0,T +joint_sr2_ASC_auto_sufficient_all,0.0,T joint_sr3p_ASC_no_auto_all,0.5630671,F joint_sr3p_ASC_auto_deficient_all,-1.8841692,F joint_sr3p_ASC_auto_sufficient_all,-2.234826,F joint_walk_transit_ASC_no_auto_all,0.62292415,F joint_walk_transit_ASC_auto_deficient_all,-5.1634483,F joint_walk_transit_ASC_auto_sufficient_all,-18.264534,F -joint_drive_transit_ASC_no_auto_all,0,T +joint_drive_transit_ASC_no_auto_all,0.0,T joint_drive_transit_ASC_auto_deficient_all,-5.9632215,F joint_drive_transit_ASC_auto_sufficient_all,-8.045285,F joint_taxi_ASC_no_auto_all,-4.5792,F @@ -306,3 +306,63 @@ drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,0.525,F drive_transit_CBD_ASC_school_univ,0.672,F drive_transit_CBD_ASC_work,1.1,F drive_transit_CBD_ASC_atwork,0.564,F +coef_calib_DRIVEALONEPAY_zero_auto,5.0,T +coef_calib_SHARED2FREE_zero_auto,5.0,T +coef_calib_SHARED2PAY_zero_auto,5.0,T +coef_calib_SHARED3FREE_zero_auto,5.0,T +coef_calib_SHARED3PAY_zero_auto,5.0,T +coef_calib_WALK_zero_auto,1.19369549873279,T +coef_calib_BIKE_zero_auto,5.0,T +coef_calib_WALK_LOC_zero_auto,0.29206606912731364,T +coef_calib_WALK_LRF_zero_auto,1.2520906218382721,T +coef_calib_WALK_EXP_zero_auto,5.0,T +coef_calib_WALK_HVY_zero_auto,2.427455130545431,T +coef_calib_WALK_COM_zero_auto,5.0,T +coef_calib_DRIVE_LOC_zero_auto,5.0,T +coef_calib_DRIVE_LRF_zero_auto,5.0,T +coef_calib_DRIVE_EXP_zero_auto,5.0,T +coef_calib_DRIVE_HVY_zero_auto,5.0,T +coef_calib_DRIVE_COM_zero_auto,5.0,T +coef_calib_TAXI_zero_auto,5.0,T +coef_calib_TNC_SINGLE_zero_auto,3.2347512774613625,T +coef_calib_TNC_SHARED_zero_auto,5.0,T +coef_calib_DRIVEALONEPAY_auto_insuff,5.0,T +coef_calib_SHARED2FREE_auto_insuff,4.655634851986755,T +coef_calib_SHARED2PAY_auto_insuff,5.0,T +coef_calib_SHARED3FREE_auto_insuff,5.0,T +coef_calib_SHARED3PAY_auto_insuff,5.0,T +coef_calib_WALK_auto_insuff,3.1931900192458924,T +coef_calib_BIKE_auto_insuff,4.992944512095712,T +coef_calib_WALK_LOC_auto_insuff,5.0,T +coef_calib_WALK_LRF_auto_insuff,3.9302088455457014,T +coef_calib_WALK_EXP_auto_insuff,5.0,T +coef_calib_WALK_HVY_auto_insuff,1.3844675036448821,T +coef_calib_WALK_COM_auto_insuff,5.0,T +coef_calib_DRIVE_LOC_auto_insuff,5.0,T +coef_calib_DRIVE_LRF_auto_insuff,5.0,T +coef_calib_DRIVE_EXP_auto_insuff,5.0,T +coef_calib_DRIVE_HVY_auto_insuff,5.0,T +coef_calib_DRIVE_COM_auto_insuff,5.0,T +coef_calib_TAXI_auto_insuff,5.0,T +coef_calib_TNC_SINGLE_auto_insuff,3.9140984082873835,T +coef_calib_TNC_SHARED_auto_insuff,5.0,T +coef_calib_DRIVEALONEPAY_auto_suff,5.0,T +coef_calib_SHARED2FREE_auto_suff,2.396741245612774,T +coef_calib_SHARED2PAY_auto_suff,5.0,T +coef_calib_SHARED3FREE_auto_suff,4.928377149112955,T +coef_calib_SHARED3PAY_auto_suff,5.0,T +coef_calib_WALK_auto_suff,3.077985217800374,T +coef_calib_BIKE_auto_suff,5.0,T +coef_calib_WALK_LOC_auto_suff,5.0,T +coef_calib_WALK_LRF_auto_suff,4.399523089022793,T +coef_calib_WALK_EXP_auto_suff,5.0,T +coef_calib_WALK_HVY_auto_suff,2.8217459359394326,T +coef_calib_WALK_COM_auto_suff,5.0,T +coef_calib_DRIVE_LOC_auto_suff,5.0,T +coef_calib_DRIVE_LRF_auto_suff,5.0,T +coef_calib_DRIVE_EXP_auto_suff,5.0,T +coef_calib_DRIVE_HVY_auto_suff,5.0,T +coef_calib_DRIVE_COM_auto_suff,5.0,T +coef_calib_TAXI_auto_suff,5.0,T +coef_calib_TNC_SINGLE_auto_suff,4.29485267200215,T +coef_calib_TNC_SHARED_auto_suff,5.0,T diff --git a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients_template.csv b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients_template.csv index b1b009a3f0..212e02d0ea 100644 --- a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients_template.csv +++ b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients_template.csv @@ -85,3 +85,63 @@ heavy_rail_ASC,heavy_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_at commuter_rail_ASC,commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,commuter_rail_ASC_school_univ,commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,commuter_rail_ASC_school_univ,commuter_rail_ASC_work,commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork walk_transit_CBD_ASC,walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,walk_transit_CBD_ASC_school_univ,walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,walk_transit_CBD_ASC_school_univ,walk_transit_CBD_ASC_work,walk_transit_CBD_ASC_atwork drive_transit_CBD_ASC,drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,drive_transit_CBD_ASC_school_univ,drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,drive_transit_CBD_ASC_school_univ,drive_transit_CBD_ASC_work,drive_transit_CBD_ASC_atwork +coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto +coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto +coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto +coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto +coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto +coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto +coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto +coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto +coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto +coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto +coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto +coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto +coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto +coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto +coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto +coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto +coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto +coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto +coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto +coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto +coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff +coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff +coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff +coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff +coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff +coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff +coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff +coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff +coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff +coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff +coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff +coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff +coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff +coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff +coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff +coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff +coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff +coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff +coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff +coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff +coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff +coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff +coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff +coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff +coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff +coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff +coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff +coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff +coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff +coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff +coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff +coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff +coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff +coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff +coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff +coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff +coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff +coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff +coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff +coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff From fbf24d8289c2861c54ecc8b9227ff3e9aeedf423 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 7 Jul 2026 13:24:05 -0400 Subject: [PATCH 18/90] Update workplace location survey file directory --- .../configs/workplace_location_coefficients.csv | 8 ++++---- .../workplace_location_calib_helper.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv b/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv index 7b1b52e4ac..28be9b2e5a 100644 --- a/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv @@ -7,7 +7,7 @@ coef_dist_15_up,-0.0917,F coef_dist_0_5_high,0.15,F coef_dist_5_up_high,0.02,F coef_mode_logsum,0.3,F -coef_calib_dist_0_2,-5.0,T -coef_calib_dist_2_5,0.0,T -coef_calib_dist_5_15,-2.771779906602003,T -coef_calib_dist_15_up,0.749738309309575,T +coef_calib_dist_0_2,1.0,T +coef_calib_dist_2_5,1.0,T +coef_calib_dist_5_15,1.0,T +coef_calib_dist_15_up,1.0,T diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py index 3c2496667f..207af38bc5 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py @@ -11,7 +11,7 @@ import os from functools import lru_cache -SURVEY_DATA_FOLDER = r"C:\Users\david.hensle\OneDrive - Resource Systems Group, Inc\Documents\projects\activitysim\rsg_activitysim\activitysim\examples\example_estimation\data_test\survey_data" +SURVEY_DATA_FOLDER = "activitysim/examples/example_estimation/data_sf/survey_data" def compute_distances(context, origins, destinations): # Compute distances between origins and destinations using the network level of service From 33ee99393a194ae1a204702ecf94f880e52b5e6a Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 7 Jul 2026 13:57:53 -0400 Subject: [PATCH 19/90] auto ownership & tour mc relative paths --- .../configs/auto_ownership_coefficients.csv | 24 +-- .../configs/tour_mode_choice_coefficients.csv | 164 +++++++++--------- .../auto_ownership_calib_helper.py | 2 +- .../tour_mode_choice_calib_helper.py | 2 +- 4 files changed, 96 insertions(+), 96 deletions(-) diff --git a/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv b/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv index 4ade078423..d8c072de3b 100644 --- a/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv @@ -1,12 +1,12 @@ coefficient_name,value,constrain -coef_cars1_drivers_2,0.0,T -coef_cars1_drivers_3,0.0,T -coef_cars1_persons_16_17,0.0,T -coef_cars234_asc_marin,0.0,T -coef_cars1_persons_25_34,0.0,T -coef_cars1_num_workers_clip_3,0.0,T -coef_cars1_hh_income_30_up,0.0,T -coef_cars1_density_0_10_no_workers,0.0,T +coef_cars1_drivers_2,0,T +coef_cars1_drivers_3,0,T +coef_cars1_persons_16_17,0,T +coef_cars234_asc_marin,0,T +coef_cars1_persons_25_34,0,T +coef_cars1_num_workers_clip_3,0,T +coef_cars1_hh_income_30_up,0,T +coef_cars1_density_0_10_no_workers,0,T coef_cars1_density_10_up_workers,-0.0152,F coef_retail_non_motor,-0.03,T coef_cars4_asc,-5.313,F @@ -66,7 +66,7 @@ coef_cars3_drivers_3,5.5131,F coef_cars2_drivers_4_up,6.3662,F coef_cars3_drivers_4_up,8.5148,F coef_cars4_drivers_4_up,9.5807,F -coef_calib_auto_0,-5.0,T -coef_calib_auto_2,5.0,T -coef_calib_auto_3,5.0,T -coef_calib_auto_4,5.0,T +coef_calib_auto_0,1,T +coef_calib_auto_2,1,T +coef_calib_auto_3,1,T +coef_calib_auto_4,1,T diff --git a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv index a6d5c07487..afe19ff500 100644 --- a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv @@ -1,6 +1,6 @@ coefficient_name,value,constrain -coef_one,1.0,T -coef_nest_root,1.0,T +coef_one,1,T +coef_nest_root,1,T coef_nest_AUTO,0.72,T coef_nest_AUTO_DRIVEALONE,0.35,T coef_nest_AUTO_SHAREDRIDE2,0.35,T @@ -14,23 +14,23 @@ coef_ivt_eatout_escort_othdiscr_othmaint_shopping_social,-0.0175,F coef_ivt_school_univ,-0.0224,F coef_ivt_work,-0.0134,F coef_ivt_atwork,-0.0188,F -coef_topology_walk_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,15.0,F +coef_topology_walk_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,15,F coef_topology_walk_multiplier_atwork,7.5,F -coef_topology_bike_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,20.0,F -coef_topology_bike_multiplier_atwork,10.0,F +coef_topology_bike_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,20,F +coef_topology_bike_multiplier_atwork,10,F coef_topology_trn_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,2.2,F -coef_topology_trn_multiplier_atwork,2.0,F -coef_age1619_da_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0.0,F +coef_topology_trn_multiplier_atwork,2,F +coef_age1619_da_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0,F coef_age1619_da_multiplier_school_univ,-1.3813,F coef_age1619_da_multiplier_atwork,0.0032336,F -coef_age010_trn_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0.0,F +coef_age010_trn_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0,F coef_age010_trn_multiplier_school_univ,-1.5548,F coef_age010_trn_multiplier_atwork,0.000722,F coef_age16p_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social,-1.366,F -coef_age16p_sr_multiplier_school_univ_work_atwork,0.0,F -coef_hhsize1_sr_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_atwork,0.0,F +coef_age16p_sr_multiplier_school_univ_work_atwork,0,F +coef_hhsize1_sr_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_atwork,0,F coef_hhsize1_sr_multiplier_work,-0.734588,F -coef_hhsize2_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work_atwork,0.0,F +coef_hhsize2_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work_atwork,0,F coef_hhsize2_sr_multiplier_school_univ,-0.6359,F walk_ASC_no_auto_eatout,5.1251173,F walk_ASC_no_auto_escort,2.8012068,F @@ -92,9 +92,9 @@ bike_ASC_auto_sufficient_social,-1.368071,F bike_ASC_auto_sufficient_univ,-1.9397832,F bike_ASC_auto_sufficient_work,-1.5800232,F bike_ASC_auto_sufficient_atwork,15.72017,F -sr2_ASC_no_auto_all,0.0,F +sr2_ASC_no_auto_all,0,F sr2_ASC_auto_deficient_eatout,0.5882345,F -sr2_ASC_auto_deficient_escort,0.0,F +sr2_ASC_auto_deficient_escort,0,F sr2_ASC_auto_deficient_othdiscr,0.6601513,F sr2_ASC_auto_deficient_othmaint,0.2621527,F sr2_ASC_auto_deficient_school,0.12474365,F @@ -104,7 +104,7 @@ sr2_ASC_auto_deficient_univ,-1.6922346,F sr2_ASC_auto_deficient_work,-0.33803123,F sr2_ASC_auto_deficient_atwork,-2.1102421,F sr2_ASC_auto_sufficient_eatout,0.86280555,F -sr2_ASC_auto_sufficient_escort,0.0,F +sr2_ASC_auto_sufficient_escort,0,F sr2_ASC_auto_sufficient_othdiscr,0.49684617,F sr2_ASC_auto_sufficient_othmaint,0.25817883,F sr2_ASC_auto_sufficient_school,-1.6062657,F @@ -173,7 +173,7 @@ walk_transit_ASC_auto_sufficient_social,-0.3453759,F walk_transit_ASC_auto_sufficient_univ,0.4731163,F walk_transit_ASC_auto_sufficient_work,-0.8916507,F walk_transit_ASC_auto_sufficient_atwork,-3.401027,F -drive_transit_ASC_no_auto_all,0.0,F +drive_transit_ASC_no_auto_all,0,F drive_transit_ASC_auto_deficient_eatout,0.5998061,F drive_transit_ASC_auto_deficient_escort,-1.1537067,F drive_transit_ASC_auto_deficient_othdiscr,0.3199308,F @@ -196,7 +196,7 @@ drive_transit_ASC_auto_sufficient_work,-1.0045459,F drive_transit_ASC_auto_sufficient_atwork,-999.21466,F taxi_ASC_no_auto_eatout_othdiscr_social,0.9923,F taxi_ASC_no_auto_escort_othmaint_shopping,1.8939,F -taxi_ASC_no_auto_school_univ,-7.0,T +taxi_ASC_no_auto_school_univ,-7,T taxi_ASC_no_auto_work,4.7291,F taxi_ASC_no_auto_atwork,4.1021,F taxi_ASC_auto_deficient_eatout_othdiscr_social,-3.1317,F @@ -213,7 +213,7 @@ taxi_ASC_auto_sufficient_work,-4.8509,F taxi_ASC_auto_sufficient_atwork,-2.8804,F tnc_single_ASC_no_auto_eatout_othdiscr_social,1.6852,F tnc_single_ASC_no_auto_escort_othmaint_shopping,1.8605,F -tnc_single_ASC_no_auto_school,-7.0,T +tnc_single_ASC_no_auto_school,-7,T tnc_single_ASC_no_auto_univ,-2.519,F tnc_single_ASC_no_auto_work,5.7855,F tnc_single_ASC_no_auto_atwork,4.4982,F @@ -231,7 +231,7 @@ tnc_single_ASC_auto_sufficient_work,-4.1946,F tnc_single_ASC_auto_sufficient_atwork,-2.7988,F tnc_shared_ASC_no_auto_eatout_othdiscr_social,0.6464,F tnc_shared_ASC_no_auto_escort_othmaint_shopping,0.9361,F -tnc_shared_ASC_no_auto_school,-7.0,T +tnc_shared_ASC_no_auto_school,-7,T tnc_shared_ASC_no_auto_univ,-5.8116,F tnc_shared_ASC_no_auto_work,3.2429,F tnc_shared_ASC_no_auto_atwork,3.3672,F @@ -253,16 +253,16 @@ joint_walk_ASC_auto_sufficient_all,-3.2352157,F joint_bike_ASC_no_auto_all,-2.8671598,F joint_bike_ASC_auto_deficient_all,-6.076415,F joint_bike_ASC_auto_sufficient_all,-6.3760657,F -joint_sr2_ASC_no_auto_all,0.0,T -joint_sr2_ASC_auto_deficient_all,0.0,T -joint_sr2_ASC_auto_sufficient_all,0.0,T +joint_sr2_ASC_no_auto_all,0,T +joint_sr2_ASC_auto_deficient_all,0,T +joint_sr2_ASC_auto_sufficient_all,0,T joint_sr3p_ASC_no_auto_all,0.5630671,F joint_sr3p_ASC_auto_deficient_all,-1.8841692,F joint_sr3p_ASC_auto_sufficient_all,-2.234826,F joint_walk_transit_ASC_no_auto_all,0.62292415,F joint_walk_transit_ASC_auto_deficient_all,-5.1634483,F joint_walk_transit_ASC_auto_sufficient_all,-18.264534,F -joint_drive_transit_ASC_no_auto_all,0.0,T +joint_drive_transit_ASC_no_auto_all,0,T joint_drive_transit_ASC_auto_deficient_all,-5.9632215,F joint_drive_transit_ASC_auto_sufficient_all,-8.045285,F joint_taxi_ASC_no_auto_all,-4.5792,F @@ -306,63 +306,63 @@ drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,0.525,F drive_transit_CBD_ASC_school_univ,0.672,F drive_transit_CBD_ASC_work,1.1,F drive_transit_CBD_ASC_atwork,0.564,F -coef_calib_DRIVEALONEPAY_zero_auto,5.0,T -coef_calib_SHARED2FREE_zero_auto,5.0,T -coef_calib_SHARED2PAY_zero_auto,5.0,T -coef_calib_SHARED3FREE_zero_auto,5.0,T -coef_calib_SHARED3PAY_zero_auto,5.0,T -coef_calib_WALK_zero_auto,1.19369549873279,T -coef_calib_BIKE_zero_auto,5.0,T -coef_calib_WALK_LOC_zero_auto,0.29206606912731364,T -coef_calib_WALK_LRF_zero_auto,1.2520906218382721,T -coef_calib_WALK_EXP_zero_auto,5.0,T -coef_calib_WALK_HVY_zero_auto,2.427455130545431,T -coef_calib_WALK_COM_zero_auto,5.0,T -coef_calib_DRIVE_LOC_zero_auto,5.0,T -coef_calib_DRIVE_LRF_zero_auto,5.0,T -coef_calib_DRIVE_EXP_zero_auto,5.0,T -coef_calib_DRIVE_HVY_zero_auto,5.0,T -coef_calib_DRIVE_COM_zero_auto,5.0,T -coef_calib_TAXI_zero_auto,5.0,T -coef_calib_TNC_SINGLE_zero_auto,3.2347512774613625,T -coef_calib_TNC_SHARED_zero_auto,5.0,T -coef_calib_DRIVEALONEPAY_auto_insuff,5.0,T -coef_calib_SHARED2FREE_auto_insuff,4.655634851986755,T -coef_calib_SHARED2PAY_auto_insuff,5.0,T -coef_calib_SHARED3FREE_auto_insuff,5.0,T -coef_calib_SHARED3PAY_auto_insuff,5.0,T -coef_calib_WALK_auto_insuff,3.1931900192458924,T -coef_calib_BIKE_auto_insuff,4.992944512095712,T -coef_calib_WALK_LOC_auto_insuff,5.0,T -coef_calib_WALK_LRF_auto_insuff,3.9302088455457014,T -coef_calib_WALK_EXP_auto_insuff,5.0,T -coef_calib_WALK_HVY_auto_insuff,1.3844675036448821,T -coef_calib_WALK_COM_auto_insuff,5.0,T -coef_calib_DRIVE_LOC_auto_insuff,5.0,T -coef_calib_DRIVE_LRF_auto_insuff,5.0,T -coef_calib_DRIVE_EXP_auto_insuff,5.0,T -coef_calib_DRIVE_HVY_auto_insuff,5.0,T -coef_calib_DRIVE_COM_auto_insuff,5.0,T -coef_calib_TAXI_auto_insuff,5.0,T -coef_calib_TNC_SINGLE_auto_insuff,3.9140984082873835,T -coef_calib_TNC_SHARED_auto_insuff,5.0,T -coef_calib_DRIVEALONEPAY_auto_suff,5.0,T -coef_calib_SHARED2FREE_auto_suff,2.396741245612774,T -coef_calib_SHARED2PAY_auto_suff,5.0,T -coef_calib_SHARED3FREE_auto_suff,4.928377149112955,T -coef_calib_SHARED3PAY_auto_suff,5.0,T -coef_calib_WALK_auto_suff,3.077985217800374,T -coef_calib_BIKE_auto_suff,5.0,T -coef_calib_WALK_LOC_auto_suff,5.0,T -coef_calib_WALK_LRF_auto_suff,4.399523089022793,T -coef_calib_WALK_EXP_auto_suff,5.0,T -coef_calib_WALK_HVY_auto_suff,2.8217459359394326,T -coef_calib_WALK_COM_auto_suff,5.0,T -coef_calib_DRIVE_LOC_auto_suff,5.0,T -coef_calib_DRIVE_LRF_auto_suff,5.0,T -coef_calib_DRIVE_EXP_auto_suff,5.0,T -coef_calib_DRIVE_HVY_auto_suff,5.0,T -coef_calib_DRIVE_COM_auto_suff,5.0,T -coef_calib_TAXI_auto_suff,5.0,T -coef_calib_TNC_SINGLE_auto_suff,4.29485267200215,T -coef_calib_TNC_SHARED_auto_suff,5.0,T +coef_calib_DRIVEALONEPAY_zero_auto,1,T +coef_calib_SHARED2FREE_zero_auto,1,T +coef_calib_SHARED2PAY_zero_auto,1,T +coef_calib_SHARED3FREE_zero_auto,1,T +coef_calib_SHARED3PAY_zero_auto,1,T +coef_calib_WALK_zero_auto,1,T +coef_calib_BIKE_zero_auto,1,T +coef_calib_WALK_LOC_zero_auto,1,T +coef_calib_WALK_LRF_zero_auto,1,T +coef_calib_WALK_EXP_zero_auto,1,T +coef_calib_WALK_HVY_zero_auto,1,T +coef_calib_WALK_COM_zero_auto,1,T +coef_calib_DRIVE_LOC_zero_auto,1,T +coef_calib_DRIVE_LRF_zero_auto,1,T +coef_calib_DRIVE_EXP_zero_auto,1,T +coef_calib_DRIVE_HVY_zero_auto,1,T +coef_calib_DRIVE_COM_zero_auto,1,T +coef_calib_TAXI_zero_auto,1,T +coef_calib_TNC_SINGLE_zero_auto,1,T +coef_calib_TNC_SHARED_zero_auto,1,T +coef_calib_DRIVEALONEPAY_auto_insuff,1,T +coef_calib_SHARED2FREE_auto_insuff,1,T +coef_calib_SHARED2PAY_auto_insuff,1,T +coef_calib_SHARED3FREE_auto_insuff,1,T +coef_calib_SHARED3PAY_auto_insuff,1,T +coef_calib_WALK_auto_insuff,1,T +coef_calib_BIKE_auto_insuff,1,T +coef_calib_WALK_LOC_auto_insuff,1,T +coef_calib_WALK_LRF_auto_insuff,1,T +coef_calib_WALK_EXP_auto_insuff,1,T +coef_calib_WALK_HVY_auto_insuff,1,T +coef_calib_WALK_COM_auto_insuff,1,T +coef_calib_DRIVE_LOC_auto_insuff,1,T +coef_calib_DRIVE_LRF_auto_insuff,1,T +coef_calib_DRIVE_EXP_auto_insuff,1,T +coef_calib_DRIVE_HVY_auto_insuff,1,T +coef_calib_DRIVE_COM_auto_insuff,1,T +coef_calib_TAXI_auto_insuff,1,T +coef_calib_TNC_SINGLE_auto_insuff,1,T +coef_calib_TNC_SHARED_auto_insuff,1,T +coef_calib_DRIVEALONEPAY_auto_suff,1,T +coef_calib_SHARED2FREE_auto_suff,1,T +coef_calib_SHARED2PAY_auto_suff,1,T +coef_calib_SHARED3FREE_auto_suff,1,T +coef_calib_SHARED3PAY_auto_suff,1,T +coef_calib_WALK_auto_suff,1,T +coef_calib_BIKE_auto_suff,1,T +coef_calib_WALK_LOC_auto_suff,1,T +coef_calib_WALK_LRF_auto_suff,1,T +coef_calib_WALK_EXP_auto_suff,1,T +coef_calib_WALK_HVY_auto_suff,1,T +coef_calib_WALK_COM_auto_suff,1,T +coef_calib_DRIVE_LOC_auto_suff,1,T +coef_calib_DRIVE_LRF_auto_suff,1,T +coef_calib_DRIVE_EXP_auto_suff,1,T +coef_calib_DRIVE_HVY_auto_suff,1,T +coef_calib_DRIVE_COM_auto_suff,1,T +coef_calib_TAXI_auto_suff,1,T +coef_calib_TNC_SINGLE_auto_suff,1,T +coef_calib_TNC_SHARED_auto_suff,1,T diff --git a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py index ee6f319ce7..e5640fb6f3 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py @@ -2,7 +2,7 @@ import pandas as pd import os -SURVEY_DATA_FOLDER = r"C:\Users\david.hensle\OneDrive - Resource Systems Group, Inc\Documents\projects\activitysim\rsg_activitysim\activitysim\examples\example_estimation\data_test\survey_data" +SURVEY_DATA_FOLDER = "activitysim/examples/example_estimation/data_sf/survey_data" def report_auto_ownership(context): model_hhs = context["households"] diff --git a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py index bbec248ca7..9b29b2deb5 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py @@ -2,7 +2,7 @@ import matplotlib.pyplot as plt import os -SURVEY_DATA_FOLDER = r"C:\Users\david.hensle\OneDrive - Resource Systems Group, Inc\Documents\projects\activitysim\rsg_activitysim\activitysim\examples\example_estimation\data_test\survey_data" +SURVEY_DATA_FOLDER = "activitysim/examples/example_estimation/data_sf/survey_data" def report_tour_mode_choice(context): model_tours = context["tours"] From ff71c98c29c639d1dc4630d3b6289ef14bc1a85a Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 7 Jul 2026 17:50:56 -0400 Subject: [PATCH 20/90] Allow no weights in tour mc summary --- .../configs_calibration/tour_mode_choice_calib_helper.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py index 9b29b2deb5..4fd8b31ca7 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py @@ -10,8 +10,11 @@ def report_tour_mode_choice(context): survey_tours = pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_tours.csv")) model_summary = model_tours.tour_mode.value_counts(normalize=True).sort_index().fillna(0) - survey_summary = survey_tours.groupby("tour_mode").tour_weight.sum() - survey_summary = survey_summary / survey_tours.tour_weight.sum() + if 'tour_weight' in survey_tours.columns: + survey_summary = survey_tours.groupby("tour_mode").tour_weight.sum() + survey_summary = survey_summary / survey_tours.tour_weight.sum() + else: + survey_summary = survey_tours.groupby("tour_mode").size() summary_df = ( pd.DataFrame({"model": model_summary, "survey": survey_summary}) From 217b3e7c8dbaaea6fb9e1c2d27783b20065d56b5 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Mon, 13 Jul 2026 11:17:05 -0400 Subject: [PATCH 21/90] Remove duplicated checkpoint directories --- activitysim/core/calibration.py | 14 ++++++++------ activitysim/core/mp_tasks.py | 8 ++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index e60e05dc0e..c1090b9ca1 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -382,7 +382,10 @@ def _calibrate_component( for component_iter in range(1, component_settings.submodel_max_iterations + 1): component_iterations = component_iter - + run_model_name = ( + f"{component_name}.component_i{component_iter};" + f"global_i{global_iter}" + ) # Re-run only this component from its prior checkpoint so model values # reflect the current candidate coefficients for this component. if state.settings.multiprocess: @@ -390,14 +393,10 @@ def _calibrate_component( # so table coalescing semantics match the initial global run path. _run_in_configured_mode( state, - models=state.settings.models, + models=state.settings.models[:state.settings.models.index(component_name) + 1], resume_after=prior_step, ) else: - run_model_name = ( - f"{component_name}.calibration_component_iter={component_iter};" - f"calibration_global_iter={global_iter}" - ) state.run(models=[run_model_name], resume_after=prior_step) eval_context = _build_expression_context(state, helper_symbols, component_name, component_settings) @@ -1251,9 +1250,11 @@ def _run_multiprocess_with_overrides( from activitysim.core import mp_tasks original_models = state.settings.models + original_mp_steps = state.settings.multiprocess_steps original_resume_after = state.settings.resume_after state.settings.models = models + state.settings.multiprocess_steps = [step for step in original_mp_steps if step.begin in models] state.settings.resume_after = resume_after try: @@ -1268,3 +1269,4 @@ def _run_multiprocess_with_overrides( finally: state.settings.models = original_models state.settings.resume_after = original_resume_after + state.settings.multiprocess_steps = original_mp_steps diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index c8507138f4..70cd70ebb9 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -738,7 +738,7 @@ def apportion_pipeline(state: workflow.State, sub_proc_names, step_info): ) # - write table to pipeline - pipeline_path.joinpath(table_name).mkdir(parents=True, exist_ok=True) + # pipeline_path.joinpath(table_name).mkdir(parents=True, exist_ok=True) ParquetStore(pipeline_path).put( table_name=table_name, @@ -755,9 +755,9 @@ def apportion_pipeline(state: workflow.State, sub_proc_names, step_info): f"writing checkpoints ({checkpoints_df.shape}) " f"to {CHECKPOINT_TABLE_NAME} in {pipeline_path}", ) - pipeline_path.joinpath(CHECKPOINT_TABLE_NAME).mkdir( - parents=True, exist_ok=True - ) + # pipeline_path.joinpath(CHECKPOINT_TABLE_NAME).mkdir( + # parents=True, exist_ok=True + # ) ParquetStore(pipeline_path).put( table_name=CHECKPOINT_TABLE_NAME, df=checkpoints_df, From aadd5293778fefe0312a923d8eebb45bcb5b7c30 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 14 Jul 2026 14:39:28 -0500 Subject: [PATCH 22/90] Minor bugfixes --- activitysim/core/calibration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index c1090b9ca1..05be6d762e 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -61,6 +61,7 @@ "imported_extensions", "run_timestamp", "run_id", + "pipeline_file_name", ] @@ -311,6 +312,8 @@ def _run_intermediate_components( resume_after: str, memory_sidecar_process=None, ) -> None: + if len(models) == 0: + return # don't modify the pipeline, just run the models needed _run_in_configured_mode( state, From 67fd29fcd986bf71ed503bc1b8e0d366f375302b Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 14 Jul 2026 15:33:10 -0500 Subject: [PATCH 23/90] Separate Claude attempt at MP calibration --- activitysim/core/calibration.py | 360 ++++++++++++++++++++++++++++++-- activitysim/core/mp_tasks.py | 131 ++++++------ 2 files changed, 410 insertions(+), 81 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 05be6d762e..de21667436 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -7,6 +7,7 @@ import json import logging import math +import multiprocessing import os import re from dataclasses import dataclass @@ -22,6 +23,7 @@ from activitysim.core import workflow from activitysim.core.configuration import PydanticReadable from activitysim.core.configuration.base import PydanticBase +from activitysim.core.configuration.top import MultiprocessStep logger = logging.getLogger("calibration") @@ -189,6 +191,12 @@ def run_calibration_loop( original_pipeline_name = state.filesystem.pipeline_file_name + # Initialize shared resources for multiprocess mode (skims, shadow pricing). + # These are allocated once and reused across all calibration iterations. + shared_data_buffers = None + if state.settings.multiprocess: + shared_data_buffers = _initialize_mp_shared_resources(state) + try: for global_iter in range( start_global_iter, @@ -211,6 +219,7 @@ def run_calibration_loop( ), global_iter=global_iter, memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, ) all_converged = True @@ -232,6 +241,7 @@ def run_calibration_loop( ], resume_after=last_calibrated_component, memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, ) component_result = _calibrate_component( @@ -240,6 +250,7 @@ def run_calibration_loop( component_settings=component_settings, prior_step=prior_step, global_iter=global_iter, + shared_data_buffers=shared_data_buffers, ) _write_component_plots(state, component) @@ -257,6 +268,7 @@ def run_calibration_loop( models=models[models.index(last_calibrated_component) + 1 :], resume_after=last_calibrated_component, memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, ) _write_progress( @@ -283,12 +295,10 @@ def _run_precursor_components( resume_after: str, global_iter: int, memory_sidecar_process=None, + shared_data_buffers: dict | None = None, ) -> None: """Run the normal ActivitySim model flow for one global calibration iteration.""" - assert (resume_after is None) or ( - resume_after in models - ), f"resume_after step {resume_after} not in models preceding calibration models" if global_iter > 1: # Seed a fresh pipeline from the configured resume checkpoint to avoid # duplicate checkpoint-name collisions across global calibration loops. @@ -297,12 +307,12 @@ def _run_precursor_components( state.filesystem.pipeline_file_name = f"pipeline_calibration_iter_{global_iter}" state.checkpoint.restore_from(prior_pipeline, checkpoint_name=resume_after) else: - _run_in_configured_mode( state, models=models, resume_after=resume_after, memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, ) @@ -311,15 +321,16 @@ def _run_intermediate_components( models: list[str], resume_after: str, memory_sidecar_process=None, + shared_data_buffers: dict | None = None, ) -> None: if len(models) == 0: return - # don't modify the pipeline, just run the models needed _run_in_configured_mode( state, models=models, resume_after=resume_after, memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, ) @@ -328,13 +339,14 @@ def _run_subsequent_components( models: list[str], resume_after: str, memory_sidecar_process=None, + shared_data_buffers: dict | None = None, ) -> None: - # don't modify the pipeline, just run the models needed _run_in_configured_mode( state, models=models, resume_after=resume_after, memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, ) @@ -344,6 +356,7 @@ def _calibrate_component( component_settings: CalibrationComponentSettings, prior_step: str, global_iter: int, + shared_data_buffers: dict | None = None, ) -> CalibrationComponentResult: """Run iterative coefficient calibration for one component.""" model_settings_file = _infer_model_settings_file(component_name) @@ -383,6 +396,22 @@ def _calibrate_component( component_converged = False component_iterations = 0 + # Determine the checkpoint name to restore from for component re-runs. + # In MP mode, the checkpoint that represents prior_step's completed state + # is the last checkpoint in the pipeline before we run the component. + # We capture it once and reuse across component iterations. + mp_restore_checkpoint = None + if state.settings.multiprocess and shared_data_buffers is not None: + # The pipeline should already be open from _restore_parent_state_from_pipeline + # called after precursor/intermediate models ran. The last checkpoint + # in the pipeline represents the state at prior_step. + if state.checkpoint.checkpoints: + mp_restore_checkpoint = state.checkpoint.last_checkpoint.get( + "checkpoint_name", "_" + ) + else: + mp_restore_checkpoint = "_" + for component_iter in range(1, component_settings.submodel_max_iterations + 1): component_iterations = component_iter run_model_name = ( @@ -391,13 +420,15 @@ def _calibrate_component( ) # Re-run only this component from its prior checkpoint so model values # reflect the current candidate coefficients for this component. - if state.settings.multiprocess: - # In multiprocess mode, preserve the standard multiprocess orchestration - # so table coalescing semantics match the initial global run path. - _run_in_configured_mode( + if state.settings.multiprocess and shared_data_buffers is not None: + # Use direct MP orchestration with explicit checkpoint control. + # This ensures we always apportion from prior_step's state, + # even after multiple component iterations. + _run_mp_single_component( state, - models=state.settings.models[:state.settings.models.index(component_name) + 1], - resume_after=prior_step, + component_name=component_name, + restore_checkpoint=mp_restore_checkpoint, + shared_data_buffers=shared_data_buffers, ) else: state.run(models=[run_model_name], resume_after=prior_step) @@ -1222,19 +1253,158 @@ def _write_progress(state: workflow.State, payload: dict[str, Any]) -> None: json.dump(payload, f, indent=2) +def _run_mp_single_component( + state: workflow.State, + component_name: str, + restore_checkpoint: str, + shared_data_buffers: dict, +) -> None: + """Run a single component in multiprocess mode with explicit checkpoint control. + + This directly orchestrates the apportion → simulate → coalesce flow + without going through run_multiprocess/get_run_list, giving us precise + control over which checkpoint to apportion from. This is essential for + calibration component re-runs where we must always restart from prior_step's + state regardless of what other checkpoints exist in the pipeline. + + Parameters + ---------- + state : workflow.State + component_name : str + The model component to run. + restore_checkpoint : str + The checkpoint name to restore from before apportioning. + This should be the checkpoint representing prior_step's state. + shared_data_buffers : dict + Pre-allocated shared memory buffers for skims/shadow pricing. + """ + from activitysim.core import mp_tasks + + # Determine slice info from original settings + original_steps = state.settings.multiprocess_steps + all_models = state.settings.models + + slice_info = None + num_processes = state.settings.num_processes or 2 + chunk_size = state.settings.chunk_size or 0 + + # Find which original step this component belongs to + step_boundaries = [] + for i, step in enumerate(original_steps): + step_boundaries.append(all_models.index(step.begin)) + step_boundaries.append(len(all_models)) + + component_idx = all_models.index(component_name) + for i, step in enumerate(original_steps): + if step_boundaries[i] <= component_idx < step_boundaries[i + 1]: + if step.slice: + slice_info = step.slice.model_dump() + if step.num_processes: + num_processes = step.num_processes + if step.chunk_size: + chunk_size = step.chunk_size + break + + step_name = f"calibration_{component_name}" + + # Build step_info dict matching what mp_tasks functions expect + step_info = { + "name": step_name, + "models": [component_name], + "num_processes": num_processes, + "chunk_size": chunk_size, + "step_num": 0, + "slice": slice_info, + "last_checkpoint_in_previous_multiprocess_step": restore_checkpoint, + } + + injectables = _build_calibration_injectables(state) + + if num_processes == 1: + sub_proc_names = [step_name] + else: + sub_proc_names = [f"{step_name}_{i}" for i in range(num_processes)] + + fail_fast = state.settings.fail_fast + + # Apportion pipeline (split tables across sub-processes) + if num_processes > 1 and slice_info is not None: + mp_tasks.run_sub_task( + state, + multiprocessing.Process( + target=mp_tasks.mp_apportion_pipeline, + name=f"{step_name}_apportion", + args=(injectables, sub_proc_names, step_info), + ), + ) + + # For multi-process runs, subprocesses must restore from the apportioned + # pipeline (which has one checkpoint). Use LAST_CHECKPOINT so they don't + # overwrite the apportioned data with a fresh pipeline. + # For single-process runs (no apportion), use restore_checkpoint to resume + # from the correct point in the main pipeline. + if num_processes > 1: + sim_resume_after = "_" # LAST_CHECKPOINT in apportioned sub-pipeline + else: + sim_resume_after = restore_checkpoint + + # Run simulations in sub-processes + completed = mp_tasks.run_sub_simulations( + state, + injectables, + shared_data_buffers, + step_info, + sub_proc_names, + sim_resume_after, + [], # previously_completed + fail_fast, + ) + + if len(completed) != num_processes: + from activitysim.core.exceptions import SubprocessError + + raise SubprocessError( + f"{num_processes - len(completed)} processes failed in " + f"calibration step {step_name}" + ) + + # Coalesce sub-process pipelines back into main pipeline + if num_processes > 1 and slice_info is not None: + mp_tasks.run_sub_task( + state, + multiprocessing.Process( + target=mp_tasks.mp_coalesce_pipelines, + name=f"{step_name}_coalesce", + args=(injectables, sub_proc_names, slice_info), + ), + ) + + # Restore coalesced results into parent state + _restore_parent_state_from_pipeline(state) + + def _run_in_configured_mode( state: workflow.State, models: list[str], resume_after: str | None, memory_sidecar_process=None, + shared_data_buffers: dict | None = None, ) -> None: """Run models using the same single/multiprocess mode as the parent run.""" + if not models: + return + if state.settings.multiprocess: _run_multiprocess_with_overrides( state, models=models, resume_after=resume_after, + shared_data_buffers=shared_data_buffers, ) + # After multiprocess completes, the coalesced pipeline exists on disk. + # Restore it into the parent process state so tables are accessible + # for calibration expression evaluation. + _restore_parent_state_from_pipeline(state) return state.run( @@ -1248,6 +1418,7 @@ def _run_multiprocess_with_overrides( state: workflow.State, models: list[str], resume_after: str | None, + shared_data_buffers: dict | None = None, ) -> None: """Run multiprocess with temporary settings overrides for calibration passes.""" from activitysim.core import mp_tasks @@ -1256,20 +1427,167 @@ def _run_multiprocess_with_overrides( original_mp_steps = state.settings.multiprocess_steps original_resume_after = state.settings.resume_after + # Build valid multiprocess_steps for the requested model subset. + calibration_mp_steps = _build_calibration_mp_steps( + models=models, + original_steps=original_mp_steps, + all_models=original_models, + ) + state.settings.models = models - state.settings.multiprocess_steps = [step for step in original_mp_steps if step.begin in models] + state.settings.multiprocess_steps = calibration_mp_steps state.settings.resume_after = resume_after try: - injectables = {} - for key in MP_INJECTABLES: - try: - injectables[key] = state.get_injectable(key) - except KeyError: - pass - injectables["settings"] = state.settings - mp_tasks.run_multiprocess(state, injectables) + injectables = _build_calibration_injectables(state) + mp_tasks.run_multiprocess( + state, + injectables, + shared_data_buffers=shared_data_buffers, + skip_final_checkpoint=True, + ) finally: state.settings.models = original_models state.settings.resume_after = original_resume_after state.settings.multiprocess_steps = original_mp_steps + + +def _restore_parent_state_from_pipeline(state: workflow.State) -> None: + """Restore coalesced pipeline tables into the parent process state. + + After a multiprocess run, the parent's in-memory state is stale. + This loads the latest checkpoint from the pipeline store so that + calibration expressions can evaluate against model outputs. + """ + if state.checkpoint.store_is_open(): + state.checkpoint.close_store() + state.checkpoint.restore(resume_after="_") + + +def _initialize_mp_shared_resources(state: workflow.State) -> dict: + """Allocate shared data buffers (skims, shadow pricing) once for reuse. + + This mirrors the allocation logic in mp_tasks.run_multiprocess but + is called once at calibration start rather than on every sub-run. + """ + from activitysim.core import mp_tasks, tracing + + shared_data_buffers = {} + sharrow_enabled = state.settings.sharrow + + t0 = tracing.print_elapsed_time() + if not sharrow_enabled: + shared_data_buffers.update(mp_tasks.allocate_shared_skim_buffers(state)) + t0 = tracing.print_elapsed_time("calibration: allocate shared skim buffer", t0) + + shared_data_buffers.update(mp_tasks.allocate_shared_shadow_pricing_buffers(state)) + t0 = tracing.print_elapsed_time( + "calibration: allocate shared shadow_pricing buffer", t0 + ) + + shared_data_buffers.update( + mp_tasks.allocate_shared_shadow_pricing_buffers_choice(state) + ) + t0 = tracing.print_elapsed_time( + "calibration: allocate shared shadow_pricing choice buffer", t0 + ) + + # Load skim data into the shared buffers. + if sharrow_enabled: + shared_data_buffers["skim_dataset"] = "sh.Dataset:skim_dataset" + from activitysim.core import flow, skim_dataset # noqa: F401 + + state.get_injectable("skim_dataset") + else: + if len(shared_data_buffers) > 0: + injectables = _build_calibration_injectables(state) + mp_tasks.run_sub_task( + state, + multiprocessing.Process( + target=mp_tasks.mp_setup_skims, + name="mp_setup_skims_calibration", + args=(injectables,), + kwargs=shared_data_buffers, + ), + ) + + # Make skims available in the parent process for expression evaluation. + state.add_injectable("data_buffers", shared_data_buffers) + try: + state.get_injectable("network_los") + except Exception: + logger.warning( + "calibration: could not resolve network_los in parent process; " + "skim-dependent expressions may fail" + ) + + return shared_data_buffers + + +def _build_calibration_mp_steps( + models: list[str], + original_steps: list[MultiprocessStep], + all_models: list[str], +) -> list[MultiprocessStep]: + """Build valid MultiprocessStep objects for a calibration model subset. + + The key challenge is that get_run_list() in mp_tasks requires: + - The first step's begin == models[0] + - Steps are ordered and non-overlapping + - Each step's begin is in the models list + + We intersect the original multiprocess_steps with the requested model + subset and construct new steps that satisfy these constraints. + """ + if not models: + return [] + + # Determine which original step each model in the full list belongs to. + # Build a mapping: model_name -> original step index + model_to_step: dict[str, int] = {} + step_boundaries = [] + for i, step in enumerate(original_steps): + begin_idx = all_models.index(step.begin) + step_boundaries.append(begin_idx) + step_boundaries.append(len(all_models)) + + for i, step in enumerate(original_steps): + for model_idx in range(step_boundaries[i], step_boundaries[i + 1]): + model_to_step[all_models[model_idx]] = i + + # Group the requested models by their original step + from collections import OrderedDict + + step_model_groups: OrderedDict[int, list[str]] = OrderedDict() + for model in models: + step_idx = model_to_step.get(model) + if step_idx is None: + continue + step_model_groups.setdefault(step_idx, []).append(model) + + # Build new MultiprocessStep for each group + new_steps = [] + for step_idx, step_models in step_model_groups.items(): + orig_step = original_steps[step_idx] + new_step = MultiprocessStep( + name=f"calibration_{orig_step.name}", + begin=step_models[0], + num_processes=orig_step.num_processes, + slice=orig_step.slice, + chunk_size=orig_step.chunk_size, + ) + new_steps.append(new_step) + + return new_steps + + +def _build_calibration_injectables(state: workflow.State) -> dict: + """Build the injectables dict for multiprocess sub-processes.""" + injectables = {} + for key in MP_INJECTABLES: + try: + injectables[key] = state.get_injectable(key) + except KeyError: + pass + injectables["settings"] = state.settings + return injectables diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index 70cd70ebb9..6adf85f853 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -558,12 +558,10 @@ def apportion_pipeline(state: workflow.State, sub_proc_names, step_info): # ensure that if we are resuming, we don't apportion any tables from future model steps last_checkpoint_in_previous_multiprocess_step = step_info.get( - "last_checkpoint_in_previous_multiprocess_step", None + "last_checkpoint_in_previous_multiprocess_step", LAST_CHECKPOINT ) if last_checkpoint_in_previous_multiprocess_step is None: - raise CheckpointNameNotFoundError( - "missing last_checkpoint_in_previous_multiprocess_step" - ) + last_checkpoint_in_previous_multiprocess_step = LAST_CHECKPOINT state.checkpoint.restore(resume_after=last_checkpoint_in_previous_multiprocess_step) # ensure all tables are in the pipeline @@ -1588,7 +1586,12 @@ def drop_breadcrumb(state: workflow.State, step_name, crumb, value=True): write_breadcrumbs(state, breadcrumbs) -def run_multiprocess(state: workflow.State, injectables): +def run_multiprocess( + state: workflow.State, + injectables, + shared_data_buffers=None, + skip_final_checkpoint=False, +): """ run the steps in run_list, possibly resuming after checkpoint specified by resume_after @@ -1616,6 +1619,13 @@ def run_multiprocess(state: workflow.State, injectables): annotated run_list (including prior run breadcrumbs if resuming) injectables : dict dict of values to inject in sub-processes + shared_data_buffers : dict, optional + Pre-allocated shared data buffers (skims, shadow pricing). If provided, + allocation and skim loading are skipped (useful for calibration loops + that call run_multiprocess repeatedly). + skip_final_checkpoint : bool, default False + If True, skip writing the final checkpoint at the end of the run. + Useful when the caller manages checkpoints externally. """ state.trace_memory_info("run_multiprocess.start") @@ -1645,60 +1655,61 @@ def find_breadcrumb(crumb, default=None): sharrow_enabled = state.settings.sharrow - # - allocate shared data - shared_data_buffers = {} - - state.trace_memory_info("allocate_shared_skim_buffer.before") - - t0 = tracing.print_elapsed_time() - if not sharrow_enabled: - shared_data_buffers.update(allocate_shared_skim_buffers(state)) - t0 = tracing.print_elapsed_time("allocate shared skim buffer", t0) - state.trace_memory_info("allocate_shared_skim_buffer.completed") - - # combine shared_skim_buffer and shared_shadow_pricing_buffer in shared_data_buffer - t0 = tracing.print_elapsed_time() - shared_data_buffers.update(allocate_shared_shadow_pricing_buffers(state)) - t0 = tracing.print_elapsed_time("allocate shared shadow_pricing buffer", t0) - state.trace_memory_info("allocate_shared_shadow_pricing_buffers.completed") - - # combine shared_shadow_pricing_buffers to pool choices across all processes - t0 = tracing.print_elapsed_time() - shared_data_buffers.update(allocate_shared_shadow_pricing_buffers_choice(state)) - t0 = tracing.print_elapsed_time("allocate shared shadow_pricing choice buffer", t0) - state.trace_memory_info("allocate_shared_shadow_pricing_buffers_choice.completed") - - start_time = time.time() - if sharrow_enabled: - shared_data_buffers["skim_dataset"] = "sh.Dataset:skim_dataset" - - # Loading skim_dataset must be done in the main process, not a subprocess, - # so that this min process can hold on to the shared memory and then cleanly - # release it on exit. - from . import flow, skim_dataset # make injectables known # noqa: F401 - - state.get_injectable("skim_dataset") - - tracing.print_elapsed_time("setup skim_dataset", t0) - state.trace_memory_info("skim_dataset.completed") - - # - mp_setup_skims - else: # not sharrow_enabled - if len(shared_data_buffers) > 0: - start_time = time.time() - run_sub_task( - state, - multiprocessing.Process( - target=mp_setup_skims, - name="mp_setup_skims", - args=(injectables,), - kwargs=shared_data_buffers, - ), - ) + # - allocate shared data (skip if pre-allocated buffers were provided) + if shared_data_buffers is None: + shared_data_buffers = {} + + state.trace_memory_info("allocate_shared_skim_buffer.before") + + t0 = tracing.print_elapsed_time() + if not sharrow_enabled: + shared_data_buffers.update(allocate_shared_skim_buffers(state)) + t0 = tracing.print_elapsed_time("allocate shared skim buffer", t0) + state.trace_memory_info("allocate_shared_skim_buffer.completed") + + # combine shared_skim_buffer and shared_shadow_pricing_buffer in shared_data_buffer + t0 = tracing.print_elapsed_time() + shared_data_buffers.update(allocate_shared_shadow_pricing_buffers(state)) + t0 = tracing.print_elapsed_time("allocate shared shadow_pricing buffer", t0) + state.trace_memory_info("allocate_shared_shadow_pricing_buffers.completed") + + # combine shared_shadow_pricing_buffers to pool choices across all processes + t0 = tracing.print_elapsed_time() + shared_data_buffers.update(allocate_shared_shadow_pricing_buffers_choice(state)) + t0 = tracing.print_elapsed_time("allocate shared shadow_pricing choice buffer", t0) + state.trace_memory_info("allocate_shared_shadow_pricing_buffers_choice.completed") + + start_time = time.time() + if sharrow_enabled: + shared_data_buffers["skim_dataset"] = "sh.Dataset:skim_dataset" + + # Loading skim_dataset must be done in the main process, not a subprocess, + # so that this min process can hold on to the shared memory and then cleanly + # release it on exit. + from . import flow, skim_dataset # make injectables known # noqa: F401 + + state.get_injectable("skim_dataset") + + tracing.print_elapsed_time("setup skim_dataset", t0) + state.trace_memory_info("skim_dataset.completed") + + # - mp_setup_skims + else: # not sharrow_enabled + if len(shared_data_buffers) > 0: + start_time = time.time() + run_sub_task( + state, + multiprocessing.Process( + target=mp_setup_skims, + name="mp_setup_skims", + args=(injectables,), + kwargs=shared_data_buffers, + ), + ) - tracing.print_elapsed_time("setup shared_data_buffers", t0) - state.trace_memory_info("mp_setup_skims.completed") - state.run.log_runtime("mp_setup_skims", start_time=start_time, force=True) + tracing.print_elapsed_time("setup shared_data_buffers", t0) + state.trace_memory_info("mp_setup_skims.completed") + state.run.log_runtime("mp_setup_skims", start_time=start_time, force=True) # - for each step in run list for step_info in run_list["multiprocess_steps"]: @@ -1769,7 +1780,7 @@ def find_breadcrumb(crumb, default=None): drop_breadcrumb(state, step_name, "coalesce") # add checkpoint with final tables even if not intermediate checkpointing - if not state.should_save_checkpoint(): + if not skip_final_checkpoint and not state.should_save_checkpoint(): state.checkpoint.restore(resume_after="_") state.checkpoint.add(FINAL_CHECKPOINT_NAME) state.checkpoint.close_store() @@ -2069,7 +2080,7 @@ def get_run_list(state: workflow.State): # remember there should always be a final checkpoint with same name as multiprocess_step name multiprocess_steps[istep][ "last_checkpoint_in_previous_multiprocess_step" - ] = (multiprocess_steps[istep - 1].get("name") if istep > 0 else None) + ] = (multiprocess_steps[istep - 1].get("name") if istep > 0 else LAST_CHECKPOINT) # - build individual step model lists based on starts starts.append(len(models)) # so last step gets remaining models in list From 6de9e4850a6ea8c0fe30a6ee63670a4398fc79f1 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 14 Jul 2026 15:54:58 -0500 Subject: [PATCH 24/90] MultiprocessStep creation fix --- activitysim/core/calibration.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index de21667436..1826f1dc82 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1565,18 +1565,24 @@ def _build_calibration_mp_steps( continue step_model_groups.setdefault(step_idx, []).append(model) - # Build new MultiprocessStep for each group + # Build new MultiprocessStep for each group. + # Some original steps (e.g. mp_initialize) omit num_processes, slice, + # and chunk_size — these default to None on MultiprocessStep and + # get_run_list() applies global defaults when they are absent. new_steps = [] for step_idx, step_models in step_model_groups.items(): orig_step = original_steps[step_idx] - new_step = MultiprocessStep( - name=f"calibration_{orig_step.name}", - begin=step_models[0], - num_processes=orig_step.num_processes, - slice=orig_step.slice, - chunk_size=orig_step.chunk_size, - ) - new_steps.append(new_step) + kwargs: dict[str, Any] = { + "name": f"calibration_{orig_step.name}", + "begin": step_models[0], + } + if orig_step.num_processes is not None: + kwargs["num_processes"] = orig_step.num_processes + if orig_step.slice is not None: + kwargs["slice"] = orig_step.slice + if orig_step.chunk_size is not None: + kwargs["chunk_size"] = orig_step.chunk_size + new_steps.append(MultiprocessStep(**kwargs)) return new_steps From de894c24021ea251ddbcaae016d7a3a74dad6fd7 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 14 Jul 2026 16:13:07 -0500 Subject: [PATCH 25/90] Alleged checkpoint fix --- activitysim/core/mp_tasks.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index 6adf85f853..258b95450f 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -575,6 +575,18 @@ def apportion_pipeline(state: workflow.State, sub_proc_names, step_info): # for the subprocess pipelines, keep only the last row of checkpoints and patch the last checkpoint name checkpoints_df = checkpoints_df.tail(1).copy() + # Drop table columns not present at the restored checkpoint. This is + # necessary because get_inventory() reads from the on-disk store which + # may contain later checkpoints that added tables not yet created at the + # restore point (e.g. during calibration iteration re-runs). + extra_cols = [ + c + for c in checkpoints_df.columns + if c not in NON_TABLE_COLUMNS and c not in checkpointed_tables + ] + if extra_cols: + checkpoints_df = checkpoints_df.drop(columns=extra_cols) + # load all tables from pipeline checkpoint_name = multiprocess_step_name tables = {} From 3c7870094f0b14570045d6e83342d968ff588e2c Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 14 Jul 2026 16:24:00 -0500 Subject: [PATCH 26/90] Deduplicate checkpoints --- activitysim/core/workflow/checkpoint.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/activitysim/core/workflow/checkpoint.py b/activitysim/core/workflow/checkpoint.py index 7391e1c9b9..ec721a4790 100644 --- a/activitysim/core/workflow/checkpoint.py +++ b/activitysim/core/workflow/checkpoint.py @@ -181,7 +181,14 @@ def _get_store_checkpoint_from_named_checkpoint( if checkpoint_name == LAST_CHECKPOINT: checkpoint_name = cp_df.index[-1] try: - return cp_df.loc[checkpoint_name, table_name] + result = cp_df.loc[checkpoint_name, table_name] + # If checkpoint_name appears multiple times in the index (e.g. when + # run_simulation adds a final checkpoint with the same name as the + # apportion checkpoint), loc returns a Series. Take the last value + # which represents the most recent state. + if isinstance(result, pd.Series): + result = result.iloc[-1] + return result except KeyError: if checkpoint_name not in cp_df.index: raise CheckpointNameNotFoundError(checkpoint_name) From 9856ee65926ca56ab271b465a4c7ed5b9547f5e5 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 14 Jul 2026 16:45:24 -0500 Subject: [PATCH 27/90] Prevent resume_after setting --- activitysim/core/calibration.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 1826f1dc82..91a18a6175 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1436,7 +1436,11 @@ def _run_multiprocess_with_overrides( state.settings.models = models state.settings.multiprocess_steps = calibration_mp_steps - state.settings.resume_after = resume_after + # Always None: calibration manages pipeline state externally via + # _restore_parent_state_from_pipeline and checkpoint.add, so the MP + # system's breadcrumb-based resume logic must not be triggered. + # Apportion uses LAST_CHECKPOINT to read from the current pipeline state. + state.settings.resume_after = None try: injectables = _build_calibration_injectables(state) From fd7950ab69eaeda0f037d3adc3d6567f28e95a31 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 15 Jul 2026 10:10:35 -0500 Subject: [PATCH 28/90] More resume_after fixes --- activitysim/core/calibration.py | 1 + activitysim/core/mp_tasks.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 91a18a6175..5ebdfa12c0 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1449,6 +1449,7 @@ def _run_multiprocess_with_overrides( injectables, shared_data_buffers=shared_data_buffers, skip_final_checkpoint=True, + force_resume=True, ) finally: state.settings.models = original_models diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index 258b95450f..a96a317b9d 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -1603,6 +1603,7 @@ def run_multiprocess( injectables, shared_data_buffers=None, skip_final_checkpoint=False, + force_resume=False, ): """ run the steps in run_list, possibly resuming after checkpoint specified by resume_after @@ -1638,6 +1639,10 @@ def run_multiprocess( skip_final_checkpoint : bool, default False If True, skip writing the final checkpoint at the end of the run. Useful when the caller manages checkpoints externally. + force_resume : bool, default False + If True, all subprocess steps resume from LAST_CHECKPOINT regardless + of step_num. Use when the pipeline already has data that must be + preserved (e.g. calibration sub-runs). """ state.trace_memory_info("run_multiprocess.start") @@ -1755,6 +1760,12 @@ def find_breadcrumb(crumb, default=None): if not skip_phase("simulate"): resume_after = step_info.get("resume_after", None) + # When force_resume is set (e.g. calibration sub-runs), always + # resume from the last checkpoint so subprocesses don't discard + # existing pipeline data by starting fresh. + if resume_after is None and force_resume: + resume_after = LAST_CHECKPOINT + previously_completed = find_breadcrumb("completed", default=[]) completed = run_sub_simulations( From 97a29d0dd3848418f1068d4338ffa59885e0e9aa Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 09:59:13 -0500 Subject: [PATCH 29/90] update existing table status --- activitysim/core/calibration.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 5ebdfa12c0..f980bbca2a 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1463,11 +1463,22 @@ def _restore_parent_state_from_pipeline(state: workflow.State) -> None: After a multiprocess run, the parent's in-memory state is stale. This loads the latest checkpoint from the pipeline store so that calibration expressions can evaluate against model outputs. + + All tables are explicitly re-checkpointed so that subsequent apportion + subprocesses can load them from a direct file path without relying on + checkpoint backtracking through potentially ambiguous checkpoint history. """ if state.checkpoint.store_is_open(): state.checkpoint.close_store() state.checkpoint.restore(resume_after="_") + # After restore, all tables are clean (status=False). Mark them dirty so + # the next checkpoint.add() writes them to disk at a known checkpoint name. + # This ensures apportion subprocesses find table files at a single, + # unambiguous checkpoint rather than needing to backtrack through history. + for table_name in list(state.existing_table_names): + state.existing_table_status[table_name] = True + def _initialize_mp_shared_resources(state: workflow.State) -> dict: """Allocate shared data buffers (skims, shadow pricing) once for reuse. From dbdff183c8daf9c798633980bbedabaa6a3c58ef Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 10:26:17 -0500 Subject: [PATCH 30/90] Checkpoint rename --- activitysim/core/calibration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index f980bbca2a..b79e5ab8e4 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1585,11 +1585,13 @@ def _build_calibration_mp_steps( # Some original steps (e.g. mp_initialize) omit num_processes, slice, # and chunk_size — these default to None on MultiprocessStep and # get_run_list() applies global defaults when they are absent. + # Step names include the first model to ensure uniqueness across multiple + # intermediate runs that draw from the same original step. new_steps = [] for step_idx, step_models in step_model_groups.items(): orig_step = original_steps[step_idx] kwargs: dict[str, Any] = { - "name": f"calibration_{orig_step.name}", + "name": f"calibration_{orig_step.name}_{step_models[0]}", "begin": step_models[0], } if orig_step.num_processes is not None: From d2dd7ae49d8adc4e49314acd815c3b18654206b9 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 10:47:40 -0500 Subject: [PATCH 31/90] Store last checkpoint --- activitysim/core/calibration.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index b79e5ab8e4..747a0b7edc 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1405,6 +1405,11 @@ def _run_in_configured_mode( # Restore it into the parent process state so tables are accessible # for calibration expression evaluation. _restore_parent_state_from_pipeline(state) + # Add a checkpoint named after the last model so that model-name + # references (e.g. _prior_step_name, resume_after on global_iter > 1) + # resolve correctly. Without this, only the step-level coalesce name + # exists in the pipeline. + state.checkpoint.add(models[-1]) return state.run( From 510b5d49aff8e6a203607f9c4660808621d5d2a7 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 12:15:51 -0500 Subject: [PATCH 32/90] Blacken! It works! --- activitysim/core/calibration.py | 35 ++++++------- activitysim/core/mp_tasks.py | 14 +++-- .../auto_ownership_calib_helper.py | 19 +++++-- .../tour_mode_choice_calib_helper.py | 13 +++-- .../workplace_location_calib_helper.py | 51 ++++++++++++------- 5 files changed, 84 insertions(+), 48 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 747a0b7edc..a0c1651eb0 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -28,7 +28,7 @@ logger = logging.getLogger("calibration") plt.style.use("seaborn-v0_8-darkgrid") -matplotlib.use('Agg') # Forces non-interactive background rendering +matplotlib.use("Agg") # Forces non-interactive background rendering CALIBRATION_SETTINGS_FILE_NAME = "calibration.yaml" CALIBRATION_OUTPUT_DIR = "calibration" @@ -415,8 +415,7 @@ def _calibrate_component( for component_iter in range(1, component_settings.submodel_max_iterations + 1): component_iterations = component_iter run_model_name = ( - f"{component_name}.component_i{component_iter};" - f"global_i{global_iter}" + f"{component_name}.component_i{component_iter};" f"global_i{global_iter}" ) # Re-run only this component from its prior checkpoint so model values # reflect the current candidate coefficients for this component. @@ -433,7 +432,9 @@ def _calibrate_component( else: state.run(models=[run_model_name], resume_after=prior_step) - eval_context = _build_expression_context(state, helper_symbols, component_name, component_settings) + eval_context = _build_expression_context( + state, helper_symbols, component_name, component_settings + ) ( row_records, @@ -951,9 +952,10 @@ def _append_iteration_records( _append_csv(df, global_path) # Also write component-local iteration history - component_path = _component_output_dir(state, component_name) / Path( - CALIBRATION_ITERATION_FILE - ).name + component_path = ( + _component_output_dir(state, component_name) + / Path(CALIBRATION_ITERATION_FILE).name + ) _append_csv(df, component_path) @@ -977,9 +979,7 @@ def _append_csv(df: pd.DataFrame, path: Path) -> None: def _component_output_dir(state: workflow.State, component_name: str) -> Path: """Return output/calibration/ and ensure it exists.""" - component_dir = state.get_output_file_path( - f"calibration/{component_name}" - ) + component_dir = state.get_output_file_path(f"calibration/{component_name}") os.makedirs(component_dir, exist_ok=True) return component_dir @@ -995,7 +995,8 @@ def _write_component_plots(state: workflow.State, component_name: str) -> None: n_sets = math.ceil(len(coefs) / MAX_COEFFS_IN_GRAPH) for coef_set in range(n_sets): set_coefs = coefs[ - coef_set * MAX_COEFFS_IN_GRAPH : min( + coef_set + * MAX_COEFFS_IN_GRAPH : min( len(coefs), (coef_set + 1) * MAX_COEFFS_IN_GRAPH ) ] @@ -1033,7 +1034,7 @@ def _plot_coefficient_progress( ax = ( recs[recs.index.get_level_values("coefficient").isin(set_coefs)] .next_coefficient.unstack("coefficient") - .plot(figsize=(10,5)) + .plot(figsize=(10, 5)) ) ax.xaxis.set_label_text("Component iteration") ax.yaxis.set_label_text("Coefficient value") @@ -1051,9 +1052,9 @@ def _component_last_records(recs: pd.DataFrame, set_coefs: list[str]) -> pd.Data filtered = recs[recs.index.get_level_values("coefficient").isin(set_coefs)] last_global = filtered.index.get_level_values("global_iter")[-1] last_comp = filtered.loc[last_global].index.get_level_values("component_iter")[-1] - return filtered.xs((last_global, last_comp), level=("global_iter", "component_iter"))[ - ["target_value", "model_value"] - ] + return filtered.xs( + (last_global, last_comp), level=("global_iter", "component_iter") + )[["target_value", "model_value"]] def _plot_component_values( @@ -1064,7 +1065,7 @@ def _plot_component_values( ) -> None: """Plot final target/model component values for one coefficient subset.""" component_dir = _component_output_dir(state, component_name) - ax = last_records.plot.bar(figsize=(10,5)) + ax = last_records.plot.bar(figsize=(10, 5)) ax.xaxis.set_tick_params(rotation=45) ax.xaxis.set_label_text("Component value") plt.tight_layout() @@ -1080,7 +1081,7 @@ def _plot_component_pct_change( ) -> None: """Plot final percent difference for one coefficient subset.""" component_dir = _component_output_dir(state, component_name) - fig, ax = plt.subplots(figsize=(10,5)) + fig, ax = plt.subplots(figsize=(10, 5)) pct_diff = last_records.diff(axis=1).model_value / last_records.target_value ax = pct_diff.plot.bar(ax=ax) ax.xaxis.set_tick_params(rotation=45) diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index a96a317b9d..4e881211f2 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -1693,8 +1693,12 @@ def find_breadcrumb(crumb, default=None): # combine shared_shadow_pricing_buffers to pool choices across all processes t0 = tracing.print_elapsed_time() shared_data_buffers.update(allocate_shared_shadow_pricing_buffers_choice(state)) - t0 = tracing.print_elapsed_time("allocate shared shadow_pricing choice buffer", t0) - state.trace_memory_info("allocate_shared_shadow_pricing_buffers_choice.completed") + t0 = tracing.print_elapsed_time( + "allocate shared shadow_pricing choice buffer", t0 + ) + state.trace_memory_info( + "allocate_shared_shadow_pricing_buffers_choice.completed" + ) start_time = time.time() if sharrow_enabled: @@ -2103,7 +2107,11 @@ def get_run_list(state: workflow.State): # remember there should always be a final checkpoint with same name as multiprocess_step name multiprocess_steps[istep][ "last_checkpoint_in_previous_multiprocess_step" - ] = (multiprocess_steps[istep - 1].get("name") if istep > 0 else LAST_CHECKPOINT) + ] = ( + multiprocess_steps[istep - 1].get("name") + if istep > 0 + else LAST_CHECKPOINT + ) # - build individual step model lists based on starts starts.append(len(models)) # so last step gets remaining models in list diff --git a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py index e5640fb6f3..3e6a882fb8 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py @@ -4,12 +4,19 @@ SURVEY_DATA_FOLDER = "activitysim/examples/example_estimation/data_sf/survey_data" + def report_auto_ownership(context): model_hhs = context["households"] - survey_hhs = pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_households.csv")) + survey_hhs = pd.read_csv( + os.path.join(SURVEY_DATA_FOLDER, "override_households.csv") + ) - model_summary = model_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) - survey_summary = survey_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) + model_summary = ( + model_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) + ) + survey_summary = ( + survey_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) + ) summary_df = ( pd.DataFrame({"model": model_summary, "survey": survey_summary}) .reset_index() @@ -22,5 +29,7 @@ def report_auto_ownership(context): plt.xlabel("Number of Autos") plt.ylabel("Proportion of Households") plt.legend(title="Data Source") - plt.savefig(os.path.join(context["component_output_dir"], "auto_ownership_comparison.png")) - plt.close() \ No newline at end of file + plt.savefig( + os.path.join(context["component_output_dir"], "auto_ownership_comparison.png") + ) + plt.close() diff --git a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py index 4fd8b31ca7..b647ef85bf 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py @@ -4,13 +4,16 @@ SURVEY_DATA_FOLDER = "activitysim/examples/example_estimation/data_sf/survey_data" + def report_tour_mode_choice(context): model_tours = context["tours"] survey_tours = None survey_tours = pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_tours.csv")) - model_summary = model_tours.tour_mode.value_counts(normalize=True).sort_index().fillna(0) - if 'tour_weight' in survey_tours.columns: + model_summary = ( + model_tours.tour_mode.value_counts(normalize=True).sort_index().fillna(0) + ) + if "tour_weight" in survey_tours.columns: survey_summary = survey_tours.groupby("tour_mode").tour_weight.sum() survey_summary = survey_summary / survey_tours.tour_weight.sum() else: @@ -28,5 +31,7 @@ def report_tour_mode_choice(context): plt.xlabel("Tour Mode") plt.ylabel("Proportion of Tours") plt.legend(title="Data Source") - plt.savefig(os.path.join(context["component_output_dir"], "tour_mode_choice_comparison.png")) - plt.close() \ No newline at end of file + plt.savefig( + os.path.join(context["component_output_dir"], "tour_mode_choice_comparison.png") + ) + plt.close() diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py index 207af38bc5..b9926eb0a1 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py @@ -13,10 +13,13 @@ SURVEY_DATA_FOLDER = "activitysim/examples/example_estimation/data_sf/survey_data" + def compute_distances(context, origins, destinations): # Compute distances between origins and destinations using the network level of service # using non-time-dependent DIST skim - distances = context["skim_dict"].lookup(origins.clip(upper=24), destinations.clip(upper=24), 'DIST') + distances = context["skim_dict"].lookup( + origins.clip(upper=24), destinations.clip(upper=24), "DIST" + ) # time dependent example # distances = skim_dict.lookup_3d(origins, destinations, 'AM', 'SOV_DIST') return distances @@ -39,7 +42,9 @@ def _survey_worker_distances(context): """Compute survey worker distances once and reuse across calibration rows.""" survey_persons = _survey_persons() survey_workers = survey_persons[survey_persons["workplace_zone_id"] > 0] - survey_home_zone_ids = _survey_households().set_index("household_id")["home_zone_id"] + survey_home_zone_ids = _survey_households().set_index("household_id")[ + "home_zone_id" + ] survey_home_zone_ids = survey_workers["household_id"].map(survey_home_zone_ids) survey_workplace_zone_ids = survey_workers["workplace_zone_id"] return compute_distances(context, survey_home_zone_ids, survey_workplace_zone_ids) @@ -47,11 +52,11 @@ def _survey_worker_distances(context): def summarize_model(context, min_dist=1, max_dist=2): """Summarize the model results for workplaces within the specified distance range.""" - persons = context['persons'] - workers = persons[persons['workplace_zone_id'] > 0] - home_zone_ids = workers['home_zone_id'] - workplace_zone_ids = workers['workplace_zone_id'] - + persons = context["persons"] + workers = persons[persons["workplace_zone_id"] > 0] + home_zone_ids = workers["home_zone_id"] + workplace_zone_ids = workers["workplace_zone_id"] + distances = compute_distances(context, home_zone_ids, workplace_zone_ids) # Filter distances within the specified range @@ -61,6 +66,7 @@ def summarize_model(context, min_dist=1, max_dist=2): share = len(filtered_distances) / len(distances) if len(distances) > 0 else 0 return share + def summarize_survey(context, min_dist=1, max_dist=2): """Summarize the survey results for workplaces within the specified distance range.""" @@ -73,25 +79,32 @@ def summarize_survey(context, min_dist=1, max_dist=2): share = len(filtered_distances) / len(distances) if len(distances) > 0 else 0 return share + def report_workplace_location(context): """Workplace location distance frequency plot comparing model results with observed data.""" print("summarizing workplace location model") model_persons = context["persons"] - model_workers = model_persons[model_persons['workplace_zone_id'] > 0] - model_home_zone_ids = model_workers['home_zone_id'] - model_workplace_zone_ids = model_workers['workplace_zone_id'] + model_workers = model_persons[model_persons["workplace_zone_id"] > 0] + model_home_zone_ids = model_workers["home_zone_id"] + model_workplace_zone_ids = model_workers["workplace_zone_id"] - model_distances = compute_distances(context, model_home_zone_ids, model_workplace_zone_ids) + model_distances = compute_distances( + context, model_home_zone_ids, model_workplace_zone_ids + ) survey_distances = _survey_worker_distances(context) - - # Here you can add code to compare model_distances and survey_distances, + + # Here you can add code to compare model_distances and survey_distances, # for example by plotting histograms or computing summary statistics. - plt.hist(model_distances, bins=20, density=True, alpha=0.5, label='Model') - plt.hist(survey_distances, bins=20, density=True, alpha=0.5, label='Survey') - plt.xlabel('Distance') - plt.ylabel('Frequency') + plt.hist(model_distances, bins=20, density=True, alpha=0.5, label="Model") + plt.hist(survey_distances, bins=20, density=True, alpha=0.5, label="Survey") + plt.xlabel("Distance") + plt.ylabel("Frequency") plt.legend() # component_output_dir set in the evaluation context - plt.savefig(os.path.join(context["component_output_dir"], 'workplace_location_comparison.png')) - plt.close() \ No newline at end of file + plt.savefig( + os.path.join( + context["component_output_dir"], "workplace_location_comparison.png" + ) + ) + plt.close() From 0ee33d374201c4e16aadd8d510b42d274e0059bc Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 14:53:35 -0500 Subject: [PATCH 33/90] Set calibration coefficients to zero --- .../configs/auto_ownership_coefficients.csv | 8 +- .../configs/tour_mode_choice_coefficients.csv | 120 +++++++++--------- .../workplace_location_coefficients.csv | 8 +- 3 files changed, 68 insertions(+), 68 deletions(-) diff --git a/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv b/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv index d8c072de3b..b285538d7f 100644 --- a/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv @@ -66,7 +66,7 @@ coef_cars3_drivers_3,5.5131,F coef_cars2_drivers_4_up,6.3662,F coef_cars3_drivers_4_up,8.5148,F coef_cars4_drivers_4_up,9.5807,F -coef_calib_auto_0,1,T -coef_calib_auto_2,1,T -coef_calib_auto_3,1,T -coef_calib_auto_4,1,T +coef_calib_auto_0,0,T +coef_calib_auto_2,0,T +coef_calib_auto_3,0,T +coef_calib_auto_4,0,T diff --git a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv index afe19ff500..394fd090d8 100644 --- a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv @@ -306,63 +306,63 @@ drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,0.525,F drive_transit_CBD_ASC_school_univ,0.672,F drive_transit_CBD_ASC_work,1.1,F drive_transit_CBD_ASC_atwork,0.564,F -coef_calib_DRIVEALONEPAY_zero_auto,1,T -coef_calib_SHARED2FREE_zero_auto,1,T -coef_calib_SHARED2PAY_zero_auto,1,T -coef_calib_SHARED3FREE_zero_auto,1,T -coef_calib_SHARED3PAY_zero_auto,1,T -coef_calib_WALK_zero_auto,1,T -coef_calib_BIKE_zero_auto,1,T -coef_calib_WALK_LOC_zero_auto,1,T -coef_calib_WALK_LRF_zero_auto,1,T -coef_calib_WALK_EXP_zero_auto,1,T -coef_calib_WALK_HVY_zero_auto,1,T -coef_calib_WALK_COM_zero_auto,1,T -coef_calib_DRIVE_LOC_zero_auto,1,T -coef_calib_DRIVE_LRF_zero_auto,1,T -coef_calib_DRIVE_EXP_zero_auto,1,T -coef_calib_DRIVE_HVY_zero_auto,1,T -coef_calib_DRIVE_COM_zero_auto,1,T -coef_calib_TAXI_zero_auto,1,T -coef_calib_TNC_SINGLE_zero_auto,1,T -coef_calib_TNC_SHARED_zero_auto,1,T -coef_calib_DRIVEALONEPAY_auto_insuff,1,T -coef_calib_SHARED2FREE_auto_insuff,1,T -coef_calib_SHARED2PAY_auto_insuff,1,T -coef_calib_SHARED3FREE_auto_insuff,1,T -coef_calib_SHARED3PAY_auto_insuff,1,T -coef_calib_WALK_auto_insuff,1,T -coef_calib_BIKE_auto_insuff,1,T -coef_calib_WALK_LOC_auto_insuff,1,T -coef_calib_WALK_LRF_auto_insuff,1,T -coef_calib_WALK_EXP_auto_insuff,1,T -coef_calib_WALK_HVY_auto_insuff,1,T -coef_calib_WALK_COM_auto_insuff,1,T -coef_calib_DRIVE_LOC_auto_insuff,1,T -coef_calib_DRIVE_LRF_auto_insuff,1,T -coef_calib_DRIVE_EXP_auto_insuff,1,T -coef_calib_DRIVE_HVY_auto_insuff,1,T -coef_calib_DRIVE_COM_auto_insuff,1,T -coef_calib_TAXI_auto_insuff,1,T -coef_calib_TNC_SINGLE_auto_insuff,1,T -coef_calib_TNC_SHARED_auto_insuff,1,T -coef_calib_DRIVEALONEPAY_auto_suff,1,T -coef_calib_SHARED2FREE_auto_suff,1,T -coef_calib_SHARED2PAY_auto_suff,1,T -coef_calib_SHARED3FREE_auto_suff,1,T -coef_calib_SHARED3PAY_auto_suff,1,T -coef_calib_WALK_auto_suff,1,T -coef_calib_BIKE_auto_suff,1,T -coef_calib_WALK_LOC_auto_suff,1,T -coef_calib_WALK_LRF_auto_suff,1,T -coef_calib_WALK_EXP_auto_suff,1,T -coef_calib_WALK_HVY_auto_suff,1,T -coef_calib_WALK_COM_auto_suff,1,T -coef_calib_DRIVE_LOC_auto_suff,1,T -coef_calib_DRIVE_LRF_auto_suff,1,T -coef_calib_DRIVE_EXP_auto_suff,1,T -coef_calib_DRIVE_HVY_auto_suff,1,T -coef_calib_DRIVE_COM_auto_suff,1,T -coef_calib_TAXI_auto_suff,1,T -coef_calib_TNC_SINGLE_auto_suff,1,T -coef_calib_TNC_SHARED_auto_suff,1,T +coef_calib_DRIVEALONEPAY_zero_auto,0,T +coef_calib_SHARED2FREE_zero_auto,0,T +coef_calib_SHARED2PAY_zero_auto,0,T +coef_calib_SHARED3FREE_zero_auto,0,T +coef_calib_SHARED3PAY_zero_auto,0,T +coef_calib_WALK_zero_auto,0,T +coef_calib_BIKE_zero_auto,0,T +coef_calib_WALK_LOC_zero_auto,0,T +coef_calib_WALK_LRF_zero_auto,0,T +coef_calib_WALK_EXP_zero_auto,0,T +coef_calib_WALK_HVY_zero_auto,0,T +coef_calib_WALK_COM_zero_auto,0,T +coef_calib_DRIVE_LOC_zero_auto,0,T +coef_calib_DRIVE_LRF_zero_auto,0,T +coef_calib_DRIVE_EXP_zero_auto,0,T +coef_calib_DRIVE_HVY_zero_auto,0,T +coef_calib_DRIVE_COM_zero_auto,0,T +coef_calib_TAXI_zero_auto,0,T +coef_calib_TNC_SINGLE_zero_auto,0,T +coef_calib_TNC_SHARED_zero_auto,0,T +coef_calib_DRIVEALONEPAY_auto_insuff,0,T +coef_calib_SHARED2FREE_auto_insuff,0,T +coef_calib_SHARED2PAY_auto_insuff,0,T +coef_calib_SHARED3FREE_auto_insuff,0,T +coef_calib_SHARED3PAY_auto_insuff,0,T +coef_calib_WALK_auto_insuff,0,T +coef_calib_BIKE_auto_insuff,0,T +coef_calib_WALK_LOC_auto_insuff,0,T +coef_calib_WALK_LRF_auto_insuff,0,T +coef_calib_WALK_EXP_auto_insuff,0,T +coef_calib_WALK_HVY_auto_insuff,0,T +coef_calib_WALK_COM_auto_insuff,0,T +coef_calib_DRIVE_LOC_auto_insuff,0,T +coef_calib_DRIVE_LRF_auto_insuff,0,T +coef_calib_DRIVE_EXP_auto_insuff,0,T +coef_calib_DRIVE_HVY_auto_insuff,0,T +coef_calib_DRIVE_COM_auto_insuff,0,T +coef_calib_TAXI_auto_insuff,0,T +coef_calib_TNC_SINGLE_auto_insuff,0,T +coef_calib_TNC_SHARED_auto_insuff,0,T +coef_calib_DRIVEALONEPAY_auto_suff,0,T +coef_calib_SHARED2FREE_auto_suff,0,T +coef_calib_SHARED2PAY_auto_suff,0,T +coef_calib_SHARED3FREE_auto_suff,0,T +coef_calib_SHARED3PAY_auto_suff,0,T +coef_calib_WALK_auto_suff,0,T +coef_calib_BIKE_auto_suff,0,T +coef_calib_WALK_LOC_auto_suff,0,T +coef_calib_WALK_LRF_auto_suff,0,T +coef_calib_WALK_EXP_auto_suff,0,T +coef_calib_WALK_HVY_auto_suff,0,T +coef_calib_WALK_COM_auto_suff,0,T +coef_calib_DRIVE_LOC_auto_suff,0,T +coef_calib_DRIVE_LRF_auto_suff,0,T +coef_calib_DRIVE_EXP_auto_suff,0,T +coef_calib_DRIVE_HVY_auto_suff,0,T +coef_calib_DRIVE_COM_auto_suff,0,T +coef_calib_TAXI_auto_suff,0,T +coef_calib_TNC_SINGLE_auto_suff,0,T +coef_calib_TNC_SHARED_auto_suff,0,T diff --git a/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv b/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv index 28be9b2e5a..2c3f64c5ce 100644 --- a/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv @@ -7,7 +7,7 @@ coef_dist_15_up,-0.0917,F coef_dist_0_5_high,0.15,F coef_dist_5_up_high,0.02,F coef_mode_logsum,0.3,F -coef_calib_dist_0_2,1.0,T -coef_calib_dist_2_5,1.0,T -coef_calib_dist_5_15,1.0,T -coef_calib_dist_15_up,1.0,T +coef_calib_dist_0_2,0,T +coef_calib_dist_2_5,0,T +coef_calib_dist_5_15,0,T +coef_calib_dist_15_up,0,T From fcfcb3c9a68507182f8fe27a73cc969a6a3be31d Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 15:01:27 -0500 Subject: [PATCH 34/90] Add override CI test coefficients for calibration --- .../auto_ownership_coefficients.csv | 72 ++++ .../tour_mode_choice_coefficients.csv | 368 ++++++++++++++++++ .../workplace_location_coefficients.csv | 13 + 3 files changed, 453 insertions(+) create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_coefficients.csv create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_coefficients.csv create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/workplace_location_coefficients.csv diff --git a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_coefficients.csv b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_coefficients.csv new file mode 100644 index 0000000000..b285538d7f --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_coefficients.csv @@ -0,0 +1,72 @@ +coefficient_name,value,constrain +coef_cars1_drivers_2,0,T +coef_cars1_drivers_3,0,T +coef_cars1_persons_16_17,0,T +coef_cars234_asc_marin,0,T +coef_cars1_persons_25_34,0,T +coef_cars1_num_workers_clip_3,0,T +coef_cars1_hh_income_30_up,0,T +coef_cars1_density_0_10_no_workers,0,T +coef_cars1_density_10_up_workers,-0.0152,F +coef_retail_non_motor,-0.03,T +coef_cars4_asc,-5.313,F +coef_cars3_asc,-3.2502,F +coef_cars34_persons_16_17,-1.7313,F +coef_cars2_asc,-1.0846,F +coef_cars34_persons_18_24,-1.0107,F +coef_cars2_persons_18_24,-1.0095,F +coef_cars2_persons_16_17,-0.881,F +coef_cars34_persons_25_34,-0.8596,F +coef_cars1_asc_county,-0.566,F +coef_retail_transit_workers,-0.5117,F +coef_cars2_persons_25_34,-0.4849,F +coef_cars2_asc_county,-0.4429,F +coef_cars1_persons_18_24,-0.4087,F +coef_cars34_density_0_10_no_workers,-0.3654,F +coef_retail_transit_no_workers,-0.3053,F +coef_cars1_asc_marin,-0.2434,F +coef_cars34_asc_county,-0.2372,F +coef_cars2_density_0_10_no_workers,-0.2028,F +coef_cars34_density_10_up_no_workers,-0.1766,F +coef_cars2_density_10_up_no_workers,-0.1106,F +coef_cars2_density_10_up_workers,-0.1106,F +coef_cars1_density_10_up_no_workers,-0.0152,F +coef_cars2_hh_income_30_up,0.0083,F +coef_cars3_hh_income_30_up,0.011,F +coef_cars4_hh_income_30_up,0.0147,F +coef_cars1_presence_children_5_17,0.0158,F +coef_cars1_hh_income_0_30k,0.0383,F +coef_cars2_hh_income_0_30k,0.054,F +coef_cars3_hh_income_0_30k,0.0559,F +coef_cars4_hh_income_0_30k,0.0619,F +coef_retail_auto_no_workers,0.0626,F +coef_cars34_asc_san_francisco,0.1458,F +coef_retail_auto_workers,0.1646,F +coef_cars2_presence_children_5_17,0.2936,F +coef_cars2_num_workers_clip_3,0.2936,F +coef_cars1_presence_children_0_4,0.3669,F +coef_cars1_asc_san_francisco,0.4259,F +coef_cars2_asc_san_francisco,0.4683,F +coef_cars1_auto_time_saving_per_worker,0.4707,F +coef_cars34_presence_children_5_17,0.4769,F +coef_cars3_auto_time_saving_per_worker,0.5705,F +coef_cars2_auto_time_saving_per_worker,0.6142,F +coef_cars3_num_workers_clip_3,0.6389,F +coef_cars234_presence_children_0_4,0.7627,F +coef_cars4_auto_time_saving_per_worker,0.7693,F +coef_cars4_num_workers_clip_3,0.8797,F +coef_cars1_asc,1.1865,F +coef_cars1_drivers_4_up,2.0107,F +coef_cars4_drivers_2,2.6616,F +coef_cars2_drivers_2,3.0773,F +coef_cars3_drivers_2,3.1962,F +coef_cars2_drivers_3,3.5401,F +coef_cars4_drivers_3,5.208,F +coef_cars3_drivers_3,5.5131,F +coef_cars2_drivers_4_up,6.3662,F +coef_cars3_drivers_4_up,8.5148,F +coef_cars4_drivers_4_up,9.5807,F +coef_calib_auto_0,0,T +coef_calib_auto_2,0,T +coef_calib_auto_3,0,T +coef_calib_auto_4,0,T diff --git a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_coefficients.csv b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_coefficients.csv new file mode 100644 index 0000000000..394fd090d8 --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_coefficients.csv @@ -0,0 +1,368 @@ +coefficient_name,value,constrain +coef_one,1,T +coef_nest_root,1,T +coef_nest_AUTO,0.72,T +coef_nest_AUTO_DRIVEALONE,0.35,T +coef_nest_AUTO_SHAREDRIDE2,0.35,T +coef_nest_AUTO_SHAREDRIDE3,0.35,T +coef_nest_NONMOTORIZED,0.72,T +coef_nest_TRANSIT,0.72,T +coef_nest_TRANSIT_WALKACCESS,0.5,T +coef_nest_TRANSIT_DRIVEACCESS,0.5,T +coef_nest_RIDEHAIL,0.36,T +coef_ivt_eatout_escort_othdiscr_othmaint_shopping_social,-0.0175,F +coef_ivt_school_univ,-0.0224,F +coef_ivt_work,-0.0134,F +coef_ivt_atwork,-0.0188,F +coef_topology_walk_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,15,F +coef_topology_walk_multiplier_atwork,7.5,F +coef_topology_bike_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,20,F +coef_topology_bike_multiplier_atwork,10,F +coef_topology_trn_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,2.2,F +coef_topology_trn_multiplier_atwork,2,F +coef_age1619_da_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0,F +coef_age1619_da_multiplier_school_univ,-1.3813,F +coef_age1619_da_multiplier_atwork,0.0032336,F +coef_age010_trn_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0,F +coef_age010_trn_multiplier_school_univ,-1.5548,F +coef_age010_trn_multiplier_atwork,0.000722,F +coef_age16p_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social,-1.366,F +coef_age16p_sr_multiplier_school_univ_work_atwork,0,F +coef_hhsize1_sr_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_atwork,0,F +coef_hhsize1_sr_multiplier_work,-0.734588,F +coef_hhsize2_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work_atwork,0,F +coef_hhsize2_sr_multiplier_school_univ,-0.6359,F +walk_ASC_no_auto_eatout,5.1251173,F +walk_ASC_no_auto_escort,2.8012068,F +walk_ASC_no_auto_othdiscr,3.2665946,F +walk_ASC_no_auto_othmaint,1.287299,F +walk_ASC_no_auto_school,18.414557,F +walk_ASC_no_auto_shopping,2.3768773,F +walk_ASC_no_auto_social,1.8680915,F +walk_ASC_no_auto_univ,6.408967,F +walk_ASC_no_auto_work,5.7672157,F +walk_ASC_no_auto_atwork,6.669213,F +walk_ASC_auto_deficient_eatout,3.274605,F +walk_ASC_auto_deficient_escort,-0.90204656,F +walk_ASC_auto_deficient_othdiscr,2.2494075,F +walk_ASC_auto_deficient_othmaint,1.3690404,F +walk_ASC_auto_deficient_school,3.2573624,F +walk_ASC_auto_deficient_shopping,2.2701733,F +walk_ASC_auto_deficient_social,2.870184,F +walk_ASC_auto_deficient_univ,4.50591,F +walk_ASC_auto_deficient_work,2.4010417,F +walk_ASC_auto_deficient_atwork,0.92546093,F +walk_ASC_auto_sufficient_eatout,1.5516903,F +walk_ASC_auto_sufficient_escort,-0.8116066,F +walk_ASC_auto_sufficient_othdiscr,1.2633476,F +walk_ASC_auto_sufficient_othmaint,0.7999634,F +walk_ASC_auto_sufficient_school,0.6476856,F +walk_ASC_auto_sufficient_shopping,0.7312663,F +walk_ASC_auto_sufficient_social,1.7072186,F +walk_ASC_auto_sufficient_univ,1.0607665,F +walk_ASC_auto_sufficient_work,0.053265337,F +walk_ASC_auto_sufficient_atwork,0.677216,F +bike_ASC_no_auto_eatout,0.86807096,F +bike_ASC_no_auto_escort,-0.716212,F +bike_ASC_no_auto_othdiscr,-0.3764232,F +bike_ASC_no_auto_othmaint,1.5394334,F +bike_ASC_no_auto_school,12.098735,F +bike_ASC_no_auto_shopping,0.8341555,F +bike_ASC_no_auto_social,0.02058321,F +bike_ASC_no_auto_univ,4.2945156,F +bike_ASC_no_auto_work,3.1940088,F +bike_ASC_no_auto_atwork,-0.90725845,F +bike_ASC_auto_deficient_eatout,-1.5691106,F +bike_ASC_auto_deficient_escort,-4.527928,F +bike_ASC_auto_deficient_othdiscr,-0.09246834,F +bike_ASC_auto_deficient_othmaint,-1.5184649,F +bike_ASC_auto_deficient_school,-0.5280678,F +bike_ASC_auto_deficient_shopping,-0.87584466,F +bike_ASC_auto_deficient_social,0.6345214,F +bike_ASC_auto_deficient_univ,-0.669235,F +bike_ASC_auto_deficient_work,0.25318968,F +bike_ASC_auto_deficient_atwork,-0.8074083,F +bike_ASC_auto_sufficient_eatout,-1.2003471,F +bike_ASC_auto_sufficient_escort,-5.0631084,F +bike_ASC_auto_sufficient_othdiscr,-1.0714597,F +bike_ASC_auto_sufficient_othmaint,-2.8083024,F +bike_ASC_auto_sufficient_school,-2.1134686,F +bike_ASC_auto_sufficient_shopping,-2.5662103,F +bike_ASC_auto_sufficient_social,-1.368071,F +bike_ASC_auto_sufficient_univ,-1.9397832,F +bike_ASC_auto_sufficient_work,-1.5800232,F +bike_ASC_auto_sufficient_atwork,15.72017,F +sr2_ASC_no_auto_all,0,F +sr2_ASC_auto_deficient_eatout,0.5882345,F +sr2_ASC_auto_deficient_escort,0,F +sr2_ASC_auto_deficient_othdiscr,0.6601513,F +sr2_ASC_auto_deficient_othmaint,0.2621527,F +sr2_ASC_auto_deficient_school,0.12474365,F +sr2_ASC_auto_deficient_shopping,0.24409756,F +sr2_ASC_auto_deficient_social,1.8558528,F +sr2_ASC_auto_deficient_univ,-1.6922346,F +sr2_ASC_auto_deficient_work,-0.33803123,F +sr2_ASC_auto_deficient_atwork,-2.1102421,F +sr2_ASC_auto_sufficient_eatout,0.86280555,F +sr2_ASC_auto_sufficient_escort,0,F +sr2_ASC_auto_sufficient_othdiscr,0.49684617,F +sr2_ASC_auto_sufficient_othmaint,0.25817883,F +sr2_ASC_auto_sufficient_school,-1.6062657,F +sr2_ASC_auto_sufficient_shopping,0.19770707,F +sr2_ASC_auto_sufficient_social,0.5236025,F +sr2_ASC_auto_sufficient_univ,-1.859427,F +sr2_ASC_auto_sufficient_work,-1.0857458,F +sr2_ASC_auto_sufficient_atwork,-1.4450618,F +sr3p_ASC_no_auto_eatout,0.3219998,F +sr3p_ASC_no_auto_escort,-1.8129267,F +sr3p_ASC_no_auto_othdiscr,0.27216902,F +sr3p_ASC_no_auto_othmaint,-0.8031854,F +sr3p_ASC_no_auto_school,-6.0240827,F +sr3p_ASC_no_auto_shopping,-0.27978948,F +sr3p_ASC_no_auto_social,-1.4036902,F +sr3p_ASC_no_auto_univ,-6.056001,F +sr3p_ASC_no_auto_work,-0.5831269,F +sr3p_ASC_no_auto_atwork,0.5826626,F +sr3p_ASC_auto_deficient_eatout,0.04605236,F +sr3p_ASC_auto_deficient_escort,-0.40818766,F +sr3p_ASC_auto_deficient_othdiscr,1.0470966,F +sr3p_ASC_auto_deficient_othmaint,-1.3493925,F +sr3p_ASC_auto_deficient_school,0.7149571,F +sr3p_ASC_auto_deficient_shopping,-0.073370166,F +sr3p_ASC_auto_deficient_social,1.5007243,F +sr3p_ASC_auto_deficient_univ,-1.7277422,F +sr3p_ASC_auto_deficient_work,-0.8527042,F +sr3p_ASC_auto_deficient_atwork,-2.514658,F +sr3p_ASC_auto_sufficient_eatout,0.8468596,F +sr3p_ASC_auto_sufficient_escort,-0.05741253,F +sr3p_ASC_auto_sufficient_othdiscr,0.58850205,F +sr3p_ASC_auto_sufficient_othmaint,-0.07549867,F +sr3p_ASC_auto_sufficient_school,-1.0201935,F +sr3p_ASC_auto_sufficient_shopping,-0.077571295,F +sr3p_ASC_auto_sufficient_social,0.50617886,F +sr3p_ASC_auto_sufficient_univ,-1.9047098,F +sr3p_ASC_auto_sufficient_work,-1.4699702,F +sr3p_ASC_auto_sufficient_atwork,-1.652174,F +walk_transit_ASC_no_auto_eatout,2.5936368,F +walk_transit_ASC_no_auto_escort,-2.2172081,F +walk_transit_ASC_no_auto_othdiscr,2.2437785,F +walk_transit_ASC_no_auto_othmaint,2.5643456,F +walk_transit_ASC_no_auto_school,21.383749,F +walk_transit_ASC_no_auto_shopping,2.1067476,F +walk_transit_ASC_no_auto_social,1.3814651,F +walk_transit_ASC_no_auto_univ,8.786037,F +walk_transit_ASC_no_auto_work,5.0354166,F +walk_transit_ASC_no_auto_atwork,2.7041876,F +walk_transit_ASC_auto_deficient_eatout,-0.03896324,F +walk_transit_ASC_auto_deficient_escort,-4.960704,F +walk_transit_ASC_auto_deficient_othdiscr,0.9530884,F +walk_transit_ASC_auto_deficient_othmaint,-3.0597258,F +walk_transit_ASC_auto_deficient_school,4.120708,F +walk_transit_ASC_auto_deficient_shopping,-0.8476569,F +walk_transit_ASC_auto_deficient_social,0.97444487,F +walk_transit_ASC_auto_deficient_univ,3.1362555,F +walk_transit_ASC_auto_deficient_work,0.65302855,F +walk_transit_ASC_auto_deficient_atwork,-2.9988291,F +walk_transit_ASC_auto_sufficient_eatout,-1.1126906,F +walk_transit_ASC_auto_sufficient_escort,-4.934847,F +walk_transit_ASC_auto_sufficient_othdiscr,-0.80636793,F +walk_transit_ASC_auto_sufficient_othmaint,-1.5471172,F +walk_transit_ASC_auto_sufficient_school,0.74590874,F +walk_transit_ASC_auto_sufficient_shopping,-2.2036798,F +walk_transit_ASC_auto_sufficient_social,-0.3453759,F +walk_transit_ASC_auto_sufficient_univ,0.4731163,F +walk_transit_ASC_auto_sufficient_work,-0.8916507,F +walk_transit_ASC_auto_sufficient_atwork,-3.401027,F +drive_transit_ASC_no_auto_all,0,F +drive_transit_ASC_auto_deficient_eatout,0.5998061,F +drive_transit_ASC_auto_deficient_escort,-1.1537067,F +drive_transit_ASC_auto_deficient_othdiscr,0.3199308,F +drive_transit_ASC_auto_deficient_othmaint,-0.29943228,F +drive_transit_ASC_auto_deficient_school,5.3252654,F +drive_transit_ASC_auto_deficient_shopping,-0.41849178,F +drive_transit_ASC_auto_deficient_social,1.5627195,F +drive_transit_ASC_auto_deficient_univ,1.8501176,F +drive_transit_ASC_auto_deficient_work,0.10081567,F +drive_transit_ASC_auto_deficient_atwork,-998.8196,F +drive_transit_ASC_auto_sufficient_eatout,-0.96951586,F +drive_transit_ASC_auto_sufficient_escort,-4.6014247,F +drive_transit_ASC_auto_sufficient_othdiscr,-0.3785917,F +drive_transit_ASC_auto_sufficient_othmaint,-2.6249478,F +drive_transit_ASC_auto_sufficient_school,1.40135,F +drive_transit_ASC_auto_sufficient_shopping,-2.1718938,F +drive_transit_ASC_auto_sufficient_social,-0.61585575,F +drive_transit_ASC_auto_sufficient_univ,1.3587753,F +drive_transit_ASC_auto_sufficient_work,-1.0045459,F +drive_transit_ASC_auto_sufficient_atwork,-999.21466,F +taxi_ASC_no_auto_eatout_othdiscr_social,0.9923,F +taxi_ASC_no_auto_escort_othmaint_shopping,1.8939,F +taxi_ASC_no_auto_school_univ,-7,T +taxi_ASC_no_auto_work,4.7291,F +taxi_ASC_no_auto_atwork,4.1021,F +taxi_ASC_auto_deficient_eatout_othdiscr_social,-3.1317,F +taxi_ASC_auto_deficient_escort_othmaint_shopping,0.1766,F +taxi_ASC_auto_deficient_school,-0.3338,F +taxi_ASC_auto_deficient_univ,4.2492,F +taxi_ASC_auto_deficient_work,-1.4766,F +taxi_ASC_auto_deficient_atwork,-4.4046,F +taxi_ASC_auto_sufficient_eatout_othdiscr_social,-3.0374,F +taxi_ASC_auto_sufficient_escort_othmaint_shopping,-1.8055,F +taxi_ASC_auto_sufficient_school,-2.4294,F +taxi_ASC_auto_sufficient_univ,-0.3131,F +taxi_ASC_auto_sufficient_work,-4.8509,F +taxi_ASC_auto_sufficient_atwork,-2.8804,F +tnc_single_ASC_no_auto_eatout_othdiscr_social,1.6852,F +tnc_single_ASC_no_auto_escort_othmaint_shopping,1.8605,F +tnc_single_ASC_no_auto_school,-7,T +tnc_single_ASC_no_auto_univ,-2.519,F +tnc_single_ASC_no_auto_work,5.7855,F +tnc_single_ASC_no_auto_atwork,4.4982,F +tnc_single_ASC_auto_deficient_eatout_othdiscr_social,-2.9623,F +tnc_single_ASC_auto_deficient_escort_othmaint_shopping,0.6748,F +tnc_single_ASC_auto_deficient_school,-0.5524,F +tnc_single_ASC_auto_deficient_univ,1.0221,F +tnc_single_ASC_auto_deficient_work,-0.8013,F +tnc_single_ASC_auto_deficient_atwork,-3.7626,F +tnc_single_ASC_auto_sufficient_eatout_othdiscr_social,-2.3239,F +tnc_single_ASC_auto_sufficient_escort_othmaint_shopping,-1.45,F +tnc_single_ASC_auto_sufficient_school,-2.8375,F +tnc_single_ASC_auto_sufficient_univ,0.2088,F +tnc_single_ASC_auto_sufficient_work,-4.1946,F +tnc_single_ASC_auto_sufficient_atwork,-2.7988,F +tnc_shared_ASC_no_auto_eatout_othdiscr_social,0.6464,F +tnc_shared_ASC_no_auto_escort_othmaint_shopping,0.9361,F +tnc_shared_ASC_no_auto_school,-7,T +tnc_shared_ASC_no_auto_univ,-5.8116,F +tnc_shared_ASC_no_auto_work,3.2429,F +tnc_shared_ASC_no_auto_atwork,3.3672,F +tnc_shared_ASC_auto_deficient_eatout_othdiscr_social,-4.3576,F +tnc_shared_ASC_auto_deficient_escort_othmaint_shopping,-0.3863,F +tnc_shared_ASC_auto_deficient_school,-1.4746,F +tnc_shared_ASC_auto_deficient_univ,3.25,F +tnc_shared_ASC_auto_deficient_work,-2.1435,F +tnc_shared_ASC_auto_deficient_atwork,-4.5089,F +tnc_shared_ASC_auto_sufficient_eatout_othdiscr_social,-3.6638,F +tnc_shared_ASC_auto_sufficient_escort_othmaint_shopping,-2.4365,F +tnc_shared_ASC_auto_sufficient_school,-3.7219,F +tnc_shared_ASC_auto_sufficient_univ,-0.9068,F +tnc_shared_ASC_auto_sufficient_work,-5.3575,F +tnc_shared_ASC_auto_sufficient_atwork,-3.5397,F +joint_walk_ASC_no_auto_all,-0.21274701,F +joint_walk_ASC_auto_deficient_all,-1.9607706,F +joint_walk_ASC_auto_sufficient_all,-3.2352157,F +joint_bike_ASC_no_auto_all,-2.8671598,F +joint_bike_ASC_auto_deficient_all,-6.076415,F +joint_bike_ASC_auto_sufficient_all,-6.3760657,F +joint_sr2_ASC_no_auto_all,0,T +joint_sr2_ASC_auto_deficient_all,0,T +joint_sr2_ASC_auto_sufficient_all,0,T +joint_sr3p_ASC_no_auto_all,0.5630671,F +joint_sr3p_ASC_auto_deficient_all,-1.8841692,F +joint_sr3p_ASC_auto_sufficient_all,-2.234826,F +joint_walk_transit_ASC_no_auto_all,0.62292415,F +joint_walk_transit_ASC_auto_deficient_all,-5.1634483,F +joint_walk_transit_ASC_auto_sufficient_all,-18.264534,F +joint_drive_transit_ASC_no_auto_all,0,T +joint_drive_transit_ASC_auto_deficient_all,-5.9632215,F +joint_drive_transit_ASC_auto_sufficient_all,-8.045285,F +joint_taxi_ASC_no_auto_all,-4.5792,F +joint_taxi_ASC_auto_deficient_all,-9.8157,F +joint_taxi_ASC_auto_sufficient_all,-11.7099,T +joint_tnc_single_ASC_no_auto_all,-4.4917,F +joint_tnc_single_ASC_auto_deficient_all,-9.8961,F +joint_tnc_single_ASC_auto_sufficient_all,-14.0159,T +joint_tnc_shared_ASC_no_auto_all,-4.3002,F +joint_tnc_shared_ASC_auto_deficient_all,-11.1572,F +joint_tnc_shared_ASC_auto_sufficient_all,-13.205,T +local_bus_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,-0.090703264,F +local_bus_ASC_school_univ,-0.06508621,F +local_bus_ASC_work,0.06689507,F +walk_light_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,0.76895475,F +walk_light_rail_ASC_school_univ,1.6814003,F +walk_light_rail_ASC_work,0.8255567,F +drive_light_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,0.76895475,F +drive_light_rail_ASC_school_univ,1.6814003,F +drive_light_rail_ASC_work,0.8255567,F +walk_ferry_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,0.9401238,F +walk_ferry_ASC_school_univ,2.0202317,F +walk_ferry_ASC_work,0.93322605,F +drive_ferry_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,0.9401238,F +drive_ferry_ASC_school_univ,2.0202317,F +drive_ferry_ASC_work,0.93322605,F +express_bus_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,0.9692316,F +express_bus_ASC_school_univ,0.32496938,F +express_bus_ASC_work,-0.5165474,F +heavy_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,0.7706121,F +heavy_rail_ASC_school_univ,0.96200377,F +heavy_rail_ASC_work,0.64772975,F +commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,0.7270185,F +commuter_rail_ASC_school_univ,1.0336206,F +commuter_rail_ASC_work,0.725503,F +walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,0.525,F +walk_transit_CBD_ASC_school_univ,0.672,F +walk_transit_CBD_ASC_work,0.804,F +walk_transit_CBD_ASC_atwork,0.564,F +drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,0.525,F +drive_transit_CBD_ASC_school_univ,0.672,F +drive_transit_CBD_ASC_work,1.1,F +drive_transit_CBD_ASC_atwork,0.564,F +coef_calib_DRIVEALONEPAY_zero_auto,0,T +coef_calib_SHARED2FREE_zero_auto,0,T +coef_calib_SHARED2PAY_zero_auto,0,T +coef_calib_SHARED3FREE_zero_auto,0,T +coef_calib_SHARED3PAY_zero_auto,0,T +coef_calib_WALK_zero_auto,0,T +coef_calib_BIKE_zero_auto,0,T +coef_calib_WALK_LOC_zero_auto,0,T +coef_calib_WALK_LRF_zero_auto,0,T +coef_calib_WALK_EXP_zero_auto,0,T +coef_calib_WALK_HVY_zero_auto,0,T +coef_calib_WALK_COM_zero_auto,0,T +coef_calib_DRIVE_LOC_zero_auto,0,T +coef_calib_DRIVE_LRF_zero_auto,0,T +coef_calib_DRIVE_EXP_zero_auto,0,T +coef_calib_DRIVE_HVY_zero_auto,0,T +coef_calib_DRIVE_COM_zero_auto,0,T +coef_calib_TAXI_zero_auto,0,T +coef_calib_TNC_SINGLE_zero_auto,0,T +coef_calib_TNC_SHARED_zero_auto,0,T +coef_calib_DRIVEALONEPAY_auto_insuff,0,T +coef_calib_SHARED2FREE_auto_insuff,0,T +coef_calib_SHARED2PAY_auto_insuff,0,T +coef_calib_SHARED3FREE_auto_insuff,0,T +coef_calib_SHARED3PAY_auto_insuff,0,T +coef_calib_WALK_auto_insuff,0,T +coef_calib_BIKE_auto_insuff,0,T +coef_calib_WALK_LOC_auto_insuff,0,T +coef_calib_WALK_LRF_auto_insuff,0,T +coef_calib_WALK_EXP_auto_insuff,0,T +coef_calib_WALK_HVY_auto_insuff,0,T +coef_calib_WALK_COM_auto_insuff,0,T +coef_calib_DRIVE_LOC_auto_insuff,0,T +coef_calib_DRIVE_LRF_auto_insuff,0,T +coef_calib_DRIVE_EXP_auto_insuff,0,T +coef_calib_DRIVE_HVY_auto_insuff,0,T +coef_calib_DRIVE_COM_auto_insuff,0,T +coef_calib_TAXI_auto_insuff,0,T +coef_calib_TNC_SINGLE_auto_insuff,0,T +coef_calib_TNC_SHARED_auto_insuff,0,T +coef_calib_DRIVEALONEPAY_auto_suff,0,T +coef_calib_SHARED2FREE_auto_suff,0,T +coef_calib_SHARED2PAY_auto_suff,0,T +coef_calib_SHARED3FREE_auto_suff,0,T +coef_calib_SHARED3PAY_auto_suff,0,T +coef_calib_WALK_auto_suff,0,T +coef_calib_BIKE_auto_suff,0,T +coef_calib_WALK_LOC_auto_suff,0,T +coef_calib_WALK_LRF_auto_suff,0,T +coef_calib_WALK_EXP_auto_suff,0,T +coef_calib_WALK_HVY_auto_suff,0,T +coef_calib_WALK_COM_auto_suff,0,T +coef_calib_DRIVE_LOC_auto_suff,0,T +coef_calib_DRIVE_LRF_auto_suff,0,T +coef_calib_DRIVE_EXP_auto_suff,0,T +coef_calib_DRIVE_HVY_auto_suff,0,T +coef_calib_DRIVE_COM_auto_suff,0,T +coef_calib_TAXI_auto_suff,0,T +coef_calib_TNC_SINGLE_auto_suff,0,T +coef_calib_TNC_SHARED_auto_suff,0,T diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_coefficients.csv b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_coefficients.csv new file mode 100644 index 0000000000..2c3f64c5ce --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_coefficients.csv @@ -0,0 +1,13 @@ +coefficient_name,value,constrain +coef_dist_0_1,-0.8428,F +coef_dist_1_2,-0.3104,F +coef_dist_2_5,-0.3783,F +coef_dist_5_15,-0.1285,F +coef_dist_15_up,-0.0917,F +coef_dist_0_5_high,0.15,F +coef_dist_5_up_high,0.02,F +coef_mode_logsum,0.3,F +coef_calib_dist_0_2,0,T +coef_calib_dist_2_5,0,T +coef_calib_dist_5_15,0,T +coef_calib_dist_15_up,0,T From 9d9f00615ef06825073298b81f84f45973e64ef4 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 16:23:05 -0500 Subject: [PATCH 35/90] Add estimation mode regression test coeffs --- .../test_auto_ownership.csv | 4 ++ .../test_tour_mode_choice.csv | 60 +++++++++++++++++++ .../test_workplace_location.csv | 4 ++ 3 files changed, 68 insertions(+) diff --git a/activitysim/estimation/test/test_larch_estimation/test_auto_ownership.csv b/activitysim/estimation/test/test_larch_estimation/test_auto_ownership.csv index 2a39f593d9..dcd75d90d9 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_auto_ownership.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_auto_ownership.csv @@ -65,3 +65,7 @@ coef_retail_auto_workers,-0.53111245140175878,0,0,,,0,-0.53111245140175878 coef_retail_non_motor,-0.029999999999999999,0,0,,,1,-0.029999999999999999 coef_retail_transit_no_workers,-0.33344704884648813,0,0,,,0,-0.33344704884648813 coef_retail_transit_workers,-0.46438215846637165,0,0,,,0,-0.46438215846637165 +coef_calib_auto_0,0,0,0,,,1,0 +coef_calib_auto_2,0,0,0,,,1,0 +coef_calib_auto_3,0,0,0,,,1,0 +coef_calib_auto_4,0,0,0,,,1,0 diff --git a/activitysim/estimation/test/test_larch_estimation/test_tour_mode_choice.csv b/activitysim/estimation/test/test_larch_estimation/test_tour_mode_choice.csv index 2b0c82486a..dbc03dc56a 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_tour_mode_choice.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_tour_mode_choice.csv @@ -300,3 +300,63 @@ walk_transit_ASC_auto_deficient_atwork,-2.9988291,0,0,,,0,-2.9988291 walk_transit_ASC_auto_sufficient_atwork,-3.401027,0,0,,,0,-3.401027 walk_transit_ASC_no_auto_atwork,2.7041876,0,0,,,0,2.7041876 walk_transit_CBD_ASC_atwork,0.56399999999999995,0,0,,,0,0.56399999999999995 +coef_calib_DRIVEALONEPAY_zero_auto,0,0,0,,,1,0 +coef_calib_SHARED2FREE_zero_auto,0,0,0,,,1,0 +coef_calib_SHARED2PAY_zero_auto,0,0,0,,,1,0 +coef_calib_SHARED3FREE_zero_auto,0,0,0,,,1,0 +coef_calib_SHARED3PAY_zero_auto,0,0,0,,,1,0 +coef_calib_WALK_zero_auto,0,0,0,,,1,0 +coef_calib_BIKE_zero_auto,0,0,0,,,1,0 +coef_calib_WALK_LOC_zero_auto,0,0,0,,,1,0 +coef_calib_WALK_LRF_zero_auto,0,0,0,,,1,0 +coef_calib_WALK_EXP_zero_auto,0,0,0,,,1,0 +coef_calib_WALK_HVY_zero_auto,0,0,0,,,1,0 +coef_calib_WALK_COM_zero_auto,0,0,0,,,1,0 +coef_calib_DRIVE_LOC_zero_auto,0,0,0,,,1,0 +coef_calib_DRIVE_LRF_zero_auto,0,0,0,,,1,0 +coef_calib_DRIVE_EXP_zero_auto,0,0,0,,,1,0 +coef_calib_DRIVE_HVY_zero_auto,0,0,0,,,1,0 +coef_calib_DRIVE_COM_zero_auto,0,0,0,,,1,0 +coef_calib_TAXI_zero_auto,0,0,0,,,1,0 +coef_calib_TNC_SINGLE_zero_auto,0,0,0,,,1,0 +coef_calib_TNC_SHARED_zero_auto,0,0,0,,,1,0 +coef_calib_DRIVEALONEPAY_auto_insuff,0,0,0,,,1,0 +coef_calib_SHARED2FREE_auto_insuff,0,0,0,,,1,0 +coef_calib_SHARED2PAY_auto_insuff,0,0,0,,,1,0 +coef_calib_SHARED3FREE_auto_insuff,0,0,0,,,1,0 +coef_calib_SHARED3PAY_auto_insuff,0,0,0,,,1,0 +coef_calib_WALK_auto_insuff,0,0,0,,,1,0 +coef_calib_BIKE_auto_insuff,0,0,0,,,1,0 +coef_calib_WALK_LOC_auto_insuff,0,0,0,,,1,0 +coef_calib_WALK_LRF_auto_insuff,0,0,0,,,1,0 +coef_calib_WALK_EXP_auto_insuff,0,0,0,,,1,0 +coef_calib_WALK_HVY_auto_insuff,0,0,0,,,1,0 +coef_calib_WALK_COM_auto_insuff,0,0,0,,,1,0 +coef_calib_DRIVE_LOC_auto_insuff,0,0,0,,,1,0 +coef_calib_DRIVE_LRF_auto_insuff,0,0,0,,,1,0 +coef_calib_DRIVE_EXP_auto_insuff,0,0,0,,,1,0 +coef_calib_DRIVE_HVY_auto_insuff,0,0,0,,,1,0 +coef_calib_DRIVE_COM_auto_insuff,0,0,0,,,1,0 +coef_calib_TAXI_auto_insuff,0,0,0,,,1,0 +coef_calib_TNC_SINGLE_auto_insuff,0,0,0,,,1,0 +coef_calib_TNC_SHARED_auto_insuff,0,0,0,,,1,0 +coef_calib_DRIVEALONEPAY_auto_suff,0,0,0,,,1,0 +coef_calib_SHARED2FREE_auto_suff,0,0,0,,,1,0 +coef_calib_SHARED2PAY_auto_suff,0,0,0,,,1,0 +coef_calib_SHARED3FREE_auto_suff,0,0,0,,,1,0 +coef_calib_SHARED3PAY_auto_suff,0,0,0,,,1,0 +coef_calib_WALK_auto_suff,0,0,0,,,1,0 +coef_calib_BIKE_auto_suff,0,0,0,,,1,0 +coef_calib_WALK_LOC_auto_suff,0,0,0,,,1,0 +coef_calib_WALK_LRF_auto_suff,0,0,0,,,1,0 +coef_calib_WALK_EXP_auto_suff,0,0,0,,,1,0 +coef_calib_WALK_HVY_auto_suff,0,0,0,,,1,0 +coef_calib_WALK_COM_auto_suff,0,0,0,,,1,0 +coef_calib_DRIVE_LOC_auto_suff,0,0,0,,,1,0 +coef_calib_DRIVE_LRF_auto_suff,0,0,0,,,1,0 +coef_calib_DRIVE_EXP_auto_suff,0,0,0,,,1,0 +coef_calib_DRIVE_HVY_auto_suff,0,0,0,,,1,0 +coef_calib_DRIVE_COM_auto_suff,0,0,0,,,1,0 +coef_calib_TAXI_auto_suff,0,0,0,,,1,0 +coef_calib_TNC_SINGLE_auto_suff,0,0,0,,,1,0 +coef_calib_TNC_SHARED_auto_suff,0,0,0,,,1,0 diff --git a/activitysim/estimation/test/test_larch_estimation/test_workplace_location.csv b/activitysim/estimation/test/test_larch_estimation/test_workplace_location.csv index 528d77a9ba..30c16478da 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_workplace_location.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_workplace_location.csv @@ -33,3 +33,7 @@ work_veryhigh_HEREMPN,-5.6530572756846489,-5.6530572756846489,-1.422958374023437 work_veryhigh_MWTEMPN,-6,-6,-1.4024237394332886,0 work_veryhigh_OTHEMPN,-5.8240005358464035,-5.8240005358464035,-1.9241486787796021,0 work_veryhigh_RETEMPN,-2.3751556873321533,-2.3751556873321533,-2.3751556873321533,0 +coef_calib_dist_0_2,0,0,0,0 +coef_calib_dist_2_5,0,0,0,0 +coef_calib_dist_5_15,0,0,0,0 +coef_calib_dist_15_up,0,0,0,0 From afebfe3e0bbb2077c17349c22fa664e6bcef5463 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 17:04:46 -0500 Subject: [PATCH 36/90] More coefficient adds --- ...n_model_workplace_location_SLSQP_None_.csv | 4 ++ ...t_simple_simulate_auto_ownership_BHHH_.csv | 4 ++ .../test_tour_and_subtour_mode_choice.csv | 60 +++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/activitysim/estimation/test/test_larch_estimation/test_location_model_workplace_location_SLSQP_None_.csv b/activitysim/estimation/test/test_larch_estimation/test_location_model_workplace_location_SLSQP_None_.csv index d4b8d728d6..72d28ac897 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_location_model_workplace_location_SLSQP_None_.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_location_model_workplace_location_SLSQP_None_.csv @@ -33,3 +33,7 @@ work_veryhigh_HEREMPN,-5.6489927637825792,-5.6489927637825792,-1.422958374023437 work_veryhigh_MWTEMPN,-6,-6,-1.4024237394332886,0 work_veryhigh_OTHEMPN,-5.8313629173672439,-5.8313629173672439,-1.9241486787796021,0 work_veryhigh_RETEMPN,-2.3751556873321533,-2.3751556873321533,-2.3751556873321533,0 +coef_calib_dist_0_2,0,0,0,0 +coef_calib_dist_2_5,0,0,0,0 +coef_calib_dist_5_15,0,0,0,0 +coef_calib_dist_15_up,0,0,0,0 diff --git a/activitysim/estimation/test/test_larch_estimation/test_simple_simulate_auto_ownership_BHHH_.csv b/activitysim/estimation/test/test_larch_estimation/test_simple_simulate_auto_ownership_BHHH_.csv index a45f944823..7047821b45 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_simple_simulate_auto_ownership_BHHH_.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_simple_simulate_auto_ownership_BHHH_.csv @@ -65,3 +65,7 @@ coef_retail_auto_workers,-0.53111240158828621,0.1646,0,,,-0.53111240158828621 coef_retail_non_motor,-0.029999999999999999,-0.029999999999999999,0,-0.029999999999999999,-0.029999999999999999,-0.029999999999999999 coef_retail_transit_no_workers,-0.33344704526636021,-0.30530000000000002,0,,,-0.33344704526636021 coef_retail_transit_workers,-0.46438215677534161,-0.51170000000000004,0,,,-0.46438215677534161 +coef_calib_auto_0,0,0,0,0,0,0 +coef_calib_auto_2,0,0,0,0,0,0 +coef_calib_auto_3,0,0,0,0,0,0 +coef_calib_auto_4,0,0,0,0,0,0 diff --git a/activitysim/estimation/test/test_larch_estimation/test_tour_and_subtour_mode_choice.csv b/activitysim/estimation/test/test_larch_estimation/test_tour_and_subtour_mode_choice.csv index 8d8064cbb4..dee933e942 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_tour_and_subtour_mode_choice.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_tour_and_subtour_mode_choice.csv @@ -300,3 +300,63 @@ walk_transit_CBD_ASC_atwork,0.34851829905878767,0.34851829905878767,0.5640000104 walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,0.69144903771302224,0.69144903771302224,0.52499997615814209,0 walk_transit_CBD_ASC_school_univ,0.78893465707209709,0.78893465707209709,0.67199999094009399,0 walk_transit_CBD_ASC_work,0.97702352207787502,0.97702352207787502,0.80400002002716064,0 +coef_calib_DRIVEALONEPAY_zero_auto,0,0,0,0 +coef_calib_SHARED2FREE_zero_auto,0,0,0,0 +coef_calib_SHARED2PAY_zero_auto,0,0,0,0 +coef_calib_SHARED3FREE_zero_auto,0,0,0,0 +coef_calib_SHARED3PAY_zero_auto,0,0,0,0 +coef_calib_WALK_zero_auto,0,0,0,0 +coef_calib_BIKE_zero_auto,0,0,0,0 +coef_calib_WALK_LOC_zero_auto,0,0,0,0 +coef_calib_WALK_LRF_zero_auto,0,0,0,0 +coef_calib_WALK_EXP_zero_auto,0,0,0,0 +coef_calib_WALK_HVY_zero_auto,0,0,0,0 +coef_calib_WALK_COM_zero_auto,0,0,0,0 +coef_calib_DRIVE_LOC_zero_auto,0,0,0,0 +coef_calib_DRIVE_LRF_zero_auto,0,0,0,0 +coef_calib_DRIVE_EXP_zero_auto,0,0,0,0 +coef_calib_DRIVE_HVY_zero_auto,0,0,0,0 +coef_calib_DRIVE_COM_zero_auto,0,0,0,0 +coef_calib_TAXI_zero_auto,0,0,0,0 +coef_calib_TNC_SINGLE_zero_auto,0,0,0,0 +coef_calib_TNC_SHARED_zero_auto,0,0,0,0 +coef_calib_DRIVEALONEPAY_auto_insuff,0,0,0,0 +coef_calib_SHARED2FREE_auto_insuff,0,0,0,0 +coef_calib_SHARED2PAY_auto_insuff,0,0,0,0 +coef_calib_SHARED3FREE_auto_insuff,0,0,0,0 +coef_calib_SHARED3PAY_auto_insuff,0,0,0,0 +coef_calib_WALK_auto_insuff,0,0,0,0 +coef_calib_BIKE_auto_insuff,0,0,0,0 +coef_calib_WALK_LOC_auto_insuff,0,0,0,0 +coef_calib_WALK_LRF_auto_insuff,0,0,0,0 +coef_calib_WALK_EXP_auto_insuff,0,0,0,0 +coef_calib_WALK_HVY_auto_insuff,0,0,0,0 +coef_calib_WALK_COM_auto_insuff,0,0,0,0 +coef_calib_DRIVE_LOC_auto_insuff,0,0,0,0 +coef_calib_DRIVE_LRF_auto_insuff,0,0,0,0 +coef_calib_DRIVE_EXP_auto_insuff,0,0,0,0 +coef_calib_DRIVE_HVY_auto_insuff,0,0,0,0 +coef_calib_DRIVE_COM_auto_insuff,0,0,0,0 +coef_calib_TAXI_auto_insuff,0,0,0,0 +coef_calib_TNC_SINGLE_auto_insuff,0,0,0,0 +coef_calib_TNC_SHARED_auto_insuff,0,0,0,0 +coef_calib_DRIVEALONEPAY_auto_suff,0,0,0,0 +coef_calib_SHARED2FREE_auto_suff,0,0,0,0 +coef_calib_SHARED2PAY_auto_suff,0,0,0,0 +coef_calib_SHARED3FREE_auto_suff,0,0,0,0 +coef_calib_SHARED3PAY_auto_suff,0,0,0,0 +coef_calib_WALK_auto_suff,0,0,0,0 +coef_calib_BIKE_auto_suff,0,0,0,0 +coef_calib_WALK_LOC_auto_suff,0,0,0,0 +coef_calib_WALK_LRF_auto_suff,0,0,0,0 +coef_calib_WALK_EXP_auto_suff,0,0,0,0 +coef_calib_WALK_HVY_auto_suff,0,0,0,0 +coef_calib_WALK_COM_auto_suff,0,0,0,0 +coef_calib_DRIVE_LOC_auto_suff,0,0,0,0 +coef_calib_DRIVE_LRF_auto_suff,0,0,0,0 +coef_calib_DRIVE_EXP_auto_suff,0,0,0,0 +coef_calib_DRIVE_HVY_auto_suff,0,0,0,0 +coef_calib_DRIVE_COM_auto_suff,0,0,0,0 +coef_calib_TAXI_auto_suff,0,0,0,0 +coef_calib_TNC_SINGLE_auto_suff,0,0,0,0 +coef_calib_TNC_SHARED_auto_suff,0,0,0,0 From 4cda8e6b89706b60398a6f172f2cac05ed6de62d Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Fri, 17 Jul 2026 16:24:58 -0500 Subject: [PATCH 37/90] Add calibration documentation RST --- docs/calibration.rst | 713 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 713 insertions(+) create mode 100644 docs/calibration.rst diff --git a/docs/calibration.rst b/docs/calibration.rst new file mode 100644 index 0000000000..e64f1afb0f --- /dev/null +++ b/docs/calibration.rst @@ -0,0 +1,713 @@ +.. _calibration: + +=========================== +ActivitySim Auto-Calibration +=========================== + +ActivitySim includes an automated calibration framework that iteratively adjusts +model coefficients to match observed survey targets. Calibration wraps around +ActivitySim's normal model execution — it reuses the standard model runner and +only adds orchestration logic around it. + +Overview +======== + +The calibration loop works as follows: + +1. Run all model steps preceding the first calibrated component (the "precursor" models). +2. For each calibrated component, iteratively: + + - Run the component from its prior checkpoint. + - Evaluate model output shares and compare against survey targets. + - Compute coefficient adjustments using a damped log-ratio or odds-ratio method. + - Write updated coefficients back to the config file on disk. + - Check convergence; stop early if all coefficients are within tolerance. + +3. Run any intermediate (non-calibrated) model steps between calibrated components. +4. Optionally run the remaining model steps after the last calibrated component. +5. Repeat the entire process for a configurable number of **global iterations**. +6. Write a final snapshot of all calibrated coefficients. + +Both single-process and multiprocess execution modes are fully supported. Shared +resources (skim buffers, shadow pricing) are allocated once and reused across all +calibration iterations. + +Quick Start +=========== + +1. Prepare Your Configs Directory +---------------------------------- + +Add the following files to your ActivitySim configs directory (or a +calibration-specific overlay directory): + +- ``calibration.yaml`` — top-level calibration configuration +- One **calibration spec CSV** per calibrated component +- One **coefficients CSV** per calibrated component (may already exist from estimation) +- Optionally, one **helper module** (``.py``) per component for custom expressions or reports + +2. Create ``calibration.yaml`` +------------------------------- + +.. code-block:: yaml + + enable: True + + run: + calibrate_models: + - workplace_location + - auto_ownership_simulate + - tour_mode_choice_simulate + resume_after: null # checkpoint to resume from on global iteration 1 + restart_after: [] # components after which to restart (advanced) + global_iterations: 3 # number of full calibration passes + complete_steps: false # run model steps after the last calibrated component + + model_settings: + workplace_location: + calibration_spec: workplace_location_calibration.csv + helper_module: workplace_location_calib_helper.py + submodel_max_iterations: 3 + survey_file: survey_persons.csv + reports: + generic: true + bespoke: report_workplace_location + + auto_ownership_simulate: + calibration_spec: auto_ownership_calibration.csv + helper_module: auto_ownership_calib_helper.py + submodel_max_iterations: 3 + survey_file: survey_households.csv + reports: + generic: true + bespoke: report_auto_ownership + + tour_mode_choice_simulate: + calibration_spec: tour_mode_choice_calibration.csv + helper_module: tour_mode_choice_calib_helper.py + submodel_max_iterations: 3 + survey_file: survey_tours.csv + reports: + generic: true + bespoke: report_tour_mode_choice + +3. Run Calibration +------------------- + +Calibration is triggered through the standard ``activitysim run`` command. No +special subcommand is needed — when ``calibration.yaml`` exists in the config +directory with ``enable: True``, the calibration loop runs automatically instead +of the normal model flow. + +.. code-block:: bash + + activitysim run -c configs_calibration -c configs -d data -o output + +Use multiple ``-c`` flags to layer a calibration-specific config directory on top +of your base configs. The calibration configs override base configs via +ActivitySim's standard config resolution order (earlier directories take priority). + + +Configuration Reference +======================= + +``calibration.yaml`` — Top-Level Settings +------------------------------------------- + +.. list-table:: + :header-rows: 1 + :widths: 20 15 15 50 + + * - Field + - Type + - Default + - Description + * - ``enable`` + - ``bool`` + - ``False`` + - Master switch. Set to ``True`` to activate calibration. + * - ``run`` + - object + - *required* + - Run-control settings (see below). + * - ``model_settings`` + - dict + - ``{}`` + - Per-component calibration config, keyed by component name. + +**Validation rules:** + +- Every component listed in ``run.calibrate_models`` must have a corresponding + entry in ``model_settings``. +- Every component in ``run.restart_after`` must also appear in ``run.calibrate_models``. +- ``run.global_iterations`` must be ≥ 1. + +``run`` — Run Control Settings +------------------------------- + +.. list-table:: + :header-rows: 1 + :widths: 20 15 10 55 + + * - Field + - Type + - Default + - Description + * - ``calibrate_models`` + - ``list[str]`` + - *required* + - Model component names to calibrate. Must match names in ``settings.yaml`` + ``models`` list. + * - ``resume_after`` + - ``str`` or ``null`` + - ``null`` + - Checkpoint to resume from on the first global iteration. Equivalent to + ``resume_after`` in ``settings.yaml``. Use this to skip expensive + initialization steps that do not change across calibration iterations. + * - ``restart_after`` + - ``list[str]`` + - ``[]`` + - Components after which to restart. + * - ``global_iterations`` + - ``int`` + - ``1`` + - Number of full outer-loop calibration passes over all components. Each + global iteration re-runs all precursor models and re-calibrates all + components from scratch using the latest coefficients. + * - ``complete_steps`` + - ``bool`` + - ``False`` + - Whether to run all model steps after the last calibrated component. + When ``False``, only runs them on the final global iteration. Set to + ``True`` if downstream model outputs are needed every iteration (e.g., + for dashboarding). + +``model_settings.`` — Per-Component Settings +------------------------------------------------------------- + +.. list-table:: + :header-rows: 1 + :widths: 25 15 10 50 + + * - Field + - Type + - Default + - Description + * - ``calibration_spec`` + - ``str`` + - *required* + - Filename of the calibration spec CSV (must be in a configs directory). + * - ``helper_module`` + - ``str`` or ``null`` + - ``null`` + - Python file path (e.g., ``my_helper.py``) or importable module path. + Functions defined in this module are available as evaluation context for + ``model_value`` and ``target_value`` expressions. + * - ``submodel_max_iterations`` + - ``int`` + - ``1`` + - Maximum number of inner-loop iterations per component per global + iteration. The component re-runs from its prior checkpoint each iteration. + * - ``survey_file`` + - ``str`` + - *required* + - Survey data CSV filename. Made available via + ``component_settings.survey_file`` in the expression context. + * - ``reports.generic`` + - ``bool`` + - ``True`` + - Write a generic CSV report each iteration. + * - ``reports.bespoke`` + - ``str`` or ``null`` + - ``null`` + - Name of a function in the helper module to call for custom reporting. The + function receives the full evaluation context dict. + + +Calibration Spec CSV +==================== + +The calibration spec is a CSV file that defines — for each coefficient — how to +compute model and target values, the adjustment method, convergence tolerance, +and bounds. Each row represents one coefficient to calibrate. + +Required Columns +----------------- + +.. list-table:: + :header-rows: 1 + :widths: 20 15 65 + + * - Column + - Type + - Description + * - ``description`` + - ``str`` + - Human-readable label for the coefficient (used in reports and plots). + * - ``coefficient`` + - ``str`` + - Name of the coefficient in the component's coefficients CSV file. Must + also appear in the component's utility specification. + * - ``model_value`` + - numeric or expression + - The model's current output for the target metric. Can be a literal number + or a Python expression evaluated against the expression context. + * - ``target_value`` + - numeric or expression + - The observed/survey target. Can be a literal number or a Python expression. + * - ``hold_fast`` + - ``bool`` + - If ``True``, the coefficient is evaluated but never updated. Useful for + monitoring convergence of a held coefficient. + * - ``min`` + - numeric + - Lower bound for the coefficient value. Leave blank for no bound. + * - ``max`` + - numeric + - Upper bound for the coefficient value. Leave blank for no bound. + * - ``damping`` + - numeric + - Damping factor (≥ 0) applied to the computed delta. ``1.0`` means no + damping; values < 1 slow convergence for stability. + * - ``method`` + - ``str`` + - Adjustment method: ``log_ratio`` or ``odds_ratio``. + * - ``tolerance`` + - numeric + - Absolute difference threshold. A coefficient is "converged" when + ``|target_value - model_value| <= tolerance``. + +Optional Columns +----------------- + +.. list-table:: + :header-rows: 1 + :widths: 20 15 10 55 + + * - Column + - Type + - Default + - Description + * - ``default_increment`` + - numeric + - ``2.0`` + - Fallback delta when the chosen method encounters invalid inputs (e.g., + zero or negative values for ``log_ratio``). + +Comment Rows +------------- + +Lines beginning with ``#`` are treated as comments and ignored. This is useful +for temporarily disabling individual coefficients without removing them. + +Example: Auto Ownership +------------------------- + +.. code-block:: text + + description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance + 0 auto ownership share,coef_calib_auto_0,len(households[households.auto_ownership==0]) / len(households),0.06812,FALSE,-5,5,1,log_ratio,0.02 + 2 auto ownership share,coef_calib_auto_2,len(households[households.auto_ownership==2]) / len(households),0.348413,FALSE,-5,5,1,log_ratio,0.01 + 3 auto ownership share,coef_calib_auto_3,len(households[households.auto_ownership==3]) / len(households),0.13718,FALSE,-5,5,1,log_ratio,0.01 + 4 auto ownership share,coef_calib_auto_4,len(households[households.auto_ownership==4]) / len(households),0.057501,FALSE,-5,5,1,log_ratio,0.01 + +In this example: + +- ``model_value`` is a Python expression that computes the share of households + with a given auto ownership level from the pipeline ``households`` table. +- ``target_value`` is a fixed numeric target derived from observed survey data. +- Each coefficient is bounded to ``[-5, 5]`` with no damping (``1.0``) and uses + the ``log_ratio`` method. + +Example: Workplace Location (Using Helper Functions) +----------------------------------------------------- + +.. code-block:: text + + description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance + Distance 0 to 2 mi share,coef_calib_dist_0_2,"summarize_model(context, min_dist=.0, max_dist=.5)",0.05,FALSE,-5,5,1,log_ratio,0.02 + Distance 5 to 15 mi share,coef_calib_dist_5_15,"summarize_model(context, min_dist=.5, max_dist=1)",0.25,FALSE,-5,5,1,log_ratio,0.01 + Distance 15+ mi share,coef_calib_dist_15_up,"summarize_model(context, min_dist=2, max_dist=999)",0.1,FALSE,-5,5,1,log_ratio,0.01 + +Here ``summarize_model()`` is a function defined in the helper module. Both +``model_value`` and ``target_value`` can call helper functions via ``context``. + + +Adjustment Methods +================== + +``log_ratio`` +-------------- + +Computes the coefficient delta as: + +.. math:: + + \Delta = \ln\!\left(\frac{\text{target\_value}}{\text{model\_value}}\right) \times \text{damping} + +Requires both ``model_value`` and ``target_value`` to be positive. If either is +zero or negative, falls back to ``default_increment`` (positive or negative +depending on direction). + +Best suited for **share-based targets** where both model and target represent +proportions. + +``odds_ratio`` +--------------- + +Computes the coefficient delta as: + +.. math:: + + \Delta = \ln\!\left(\frac{T \cdot M - T}{T \cdot M - M}\right) \times \text{damping} + +where :math:`T` = ``target_value`` and :math:`M` = ``model_value``. + +Falls back to ``default_increment`` when the numerator or denominator is +non-positive. Appropriate for **logit-based models** where coefficients operate +in utility space. + +Damping +-------- + +The ``damping`` factor multiplies the computed delta. Use values less than 1.0 +(e.g., ``0.5``) to slow convergence and improve stability when coefficients +oscillate. A value of ``1.0`` applies the full computed adjustment. + +Bounds +------- + +When ``min`` and/or ``max`` are specified, the candidate coefficient value is +clamped after the delta is applied. The iteration record tracks whether clamping +occurred via ``at_min`` and ``at_max`` flags. + + +Expression Context +================== + +The ``model_value`` and ``target_value`` fields in the calibration spec can +contain Python expressions. These expressions are evaluated with access to the +following context variables: + +.. list-table:: + :header-rows: 1 + :widths: 25 20 55 + + * - Variable + - Type + - Description + * - ``state`` + - ``workflow.State`` + - The ActivitySim state object. + * - ``np`` + - module + - NumPy. + * - ``pd`` + - module + - pandas. + * - ``households`` + - ``DataFrame`` + - The ``households`` pipeline table (if it exists). + * - ``persons`` + - ``DataFrame`` + - The ``persons`` pipeline table (if it exists). + * - ``tours`` + - ``DataFrame`` + - The ``tours`` pipeline table (if it exists). + * - ``trips`` + - ``DataFrame`` + - The ``trips`` pipeline table (if it exists). + * - *(other tables)* + - ``DataFrame`` + - Any other registered pipeline table. + * - ``network_los`` + - object + - Network level of service (if available). + * - ``skim_dict`` + - ``SkimDict`` + - Default skim dictionary (if available). + * - ``component_output_dir`` + - ``Path`` + - Output directory for the current component (``output/calibration//``). + * - ``component_settings`` + - object + - The ``CalibrationComponentSettings`` for the current component. + * - ``context`` + - ``dict`` + - Self-reference to the full context dict, for passing to helper functions. + * - *(helper symbols)* + - varies + - All public names from the helper module (functions, variables, classes). + +Writing Expressions +-------------------- + +**Inline expressions** work well for simple share computations: + +.. code-block:: python + + len(households[households.auto_ownership == 0]) / len(households) + +**Helper function calls** are preferred for complex logic: + +.. code-block:: python + + summarize_model(context, min_dist=0.5, max_dist=1.0) + +The ``context`` variable gives helper functions access to everything: pipeline +tables, skims, pandas, numpy, and other helpers. + + +Helper Modules +============== + +A helper module is a Python file placed in the configs directory (or an +importable Python module) that provides functions for use in calibration spec +expressions and custom reports. + +Loading +-------- + +Specify the helper module in ``calibration.yaml``: + +.. code-block:: yaml + + model_settings: + workplace_location: + helper_module: workplace_location_calib_helper.py + +File paths ending in ``.py`` are loaded from the config directory. Other values +are treated as Python import paths (e.g., ``mypackage.calibration_helpers``). + +All public names (functions, variables, classes) from the module are injected +into the expression context. + +Example Helper Module +---------------------- + +.. code-block:: python + + """Workplace location calibration helpers.""" + + import pandas as pd + + + def compute_distances(context, origins, destinations): + """Compute distances between zones using the skim dictionary.""" + return context["skim_dict"].lookup(origins, destinations, "DIST") + + + def summarize_model(context, min_dist=1, max_dist=2): + """Compute the share of workers with workplace distance in [min_dist, max_dist).""" + persons = context["persons"] + workers = persons[persons["workplace_zone_id"] > 0] + distances = compute_distances( + context, workers["home_zone_id"], workers["workplace_zone_id"] + ) + mask = (distances >= min_dist) & (distances < max_dist) + return len(distances[mask]) / len(distances) if len(distances) > 0 else 0 + + + def report_workplace_location(context): + """Custom bespoke report called after each iteration.""" + import matplotlib.pyplot as plt + import os + + # Custom plotting logic here... + plt.savefig(os.path.join(context["component_output_dir"], "custom_report.png")) + plt.close() + +Bespoke Reports +---------------- + +If ``reports.bespoke`` is set to a function name (e.g., +``report_workplace_location``), that function is called after each component +iteration with the full expression context dict. Use this for custom plots, +tables, or dashboards beyond what the generic report provides. + + +Output Files +============ + +All calibration output is written under ``output/calibration/``. + +Global Files +------------- + +.. list-table:: + :header-rows: 1 + :widths: 40 60 + + * - File + - Description + * - ``calibration_progress.json`` + - Tracks ``next_global_iteration`` for crash recovery. If a run is + interrupted, restarting will resume from the last completed global + iteration. + * - ``calibration_iteration_records.csv`` + - Appended per-coefficient detail for every iteration across all + components. Columns include ``global_iter``, ``component_iter``, + ``coefficient``, ``target_value``, ``model_value``, ``difference``, + ``pct_difference``, ``prev_coefficient``, ``next_coefficient``, + ``converged``, ``at_min``, ``at_max``. + * - ``calibration_iteration_summary.csv`` + - One row per component iteration with summary statistics: + ``max_difference``, ``max_change``, ``num_converged``, + ``num_unconverged``. + * - ``final_calibrated_coefficients.csv`` + - Combined snapshot of all calibrated coefficients at the end of the run, + with ``component`` and ``coefficient_name`` columns. + +Per-Component Files (``calibration//``) +-------------------------------------------------------- + +.. list-table:: + :header-rows: 1 + :widths: 40 60 + + * - File + - Description + * - ``calibration_iteration_records.csv`` + - Component-specific iteration records (same schema as the global file). + * - ``generic_report.csv`` + - Simplified iteration report with ``description``, ``difference``, + ``pct_difference``, and ``converged``. + * - ``coefficient_progress_set_N.png`` + - Line plot of coefficient values across iterations (up to 10 coefficients + per plot). + * - ``final_components_set_N.png`` + - Bar chart comparing final ``target_value`` vs ``model_value``. + * - ``final_pct_change_set_N.png`` + - Bar chart of final percent difference between model and target. + +Updated Config Files +--------------------- + +Coefficient CSV files in the configs directory are **updated in-place** after +each iteration. This means: + +- The calibrated coefficients persist across runs. +- You can inspect intermediate coefficient values at any time. +- To reset, restore the original coefficient files from version control. + + +Crash Recovery +============== + +Calibration progress is persisted to ``calibration_progress.json`` after each +completed global iteration. If a run is interrupted: + +1. The coefficient files on disk reflect the state at the last completed iteration. +2. Restarting ``activitysim run`` with the same configuration will resume from + the ``next_global_iteration`` recorded in the progress file. + +To force a fresh start, delete ``output/calibration/calibration_progress.json`` +and restore original coefficient files. + + +Multiprocess Mode +================= + +Calibration works with ActivitySim's multiprocess execution mode. When +``multiprocess: True`` is set in ``settings.yaml``: + +- Shared resources (skim buffers, shadow pricing buffers) are allocated **once** + at calibration start and reused across all iterations, avoiding repeated + expensive allocations. +- Precursor, intermediate, and subsequent model steps use ActivitySim's normal + multiprocess orchestration. +- Calibrated component re-runs use direct apportion → simulate → coalesce + orchestration with explicit checkpoint control, ensuring correct state + restoration on each iteration. +- After each multiprocess run, the coalesced pipeline is loaded back into the + parent process so calibration expressions can evaluate model outputs. + +No special configuration is needed — the calibration framework automatically +respects ``num_processes``, ``multiprocess_steps``, ``slice``, and +``chunk_size`` from the existing ``settings.yaml``. + + +Convergence +=========== + +A coefficient is considered **converged** when: + +.. math:: + + |\text{target\_value} - \text{model\_value}| \leq \text{tolerance} + +A component is converged when **all** of its coefficients are converged. The +component inner loop stops early upon convergence. + +The overall calibration run completes after all ``global_iterations`` have +executed. Global convergence is tracked but does not currently trigger early +termination of the outer loop — use ``global_iterations`` to control the total +number of passes. + + +Coefficient Requirements +======================== + +Calibration coefficients must satisfy these requirements: + +1. **Present in utility specification**: Every ``coefficient`` in the calibration + spec must appear as a token in the component's utility expression CSV (the + files referenced by settings keys ending in ``SPEC``). A validation error is + raised at startup if any are missing. + +2. **Present in coefficients file**: If a calibration coefficient is not found in + the component's coefficients CSV, it is automatically added with an initial + value of ``0.0`` and a warning is logged. + +3. **Numeric values**: All coefficient values must be numeric. Non-numeric values + raise an error. + + +Working Example +=============== + +A complete working example is included in the repository at:: + + activitysim/examples/prototype_mtc/configs_calibration/ + +This example calibrates three components — ``workplace_location``, +``auto_ownership_simulate``, and ``tour_mode_choice_simulate`` — and includes: + +- ``calibration.yaml`` — top-level configuration +- ``*_calibration.csv`` — calibration specs with inline expressions and helper + function calls +- ``*_calib_helper.py`` — helper modules with custom summary functions and + bespoke reports +- ``*_coefficients.csv`` — initial coefficient values + +To run the example: + +.. code-block:: bash + + activitysim run \ + -c activitysim/examples/prototype_mtc/configs_calibration \ + -c activitysim/examples/prototype_mtc/configs \ + -d activitysim/examples/prototype_mtc/data \ + -o output + + +Tips +==== + +- **Start with** ``submodel_max_iterations: 1`` and a small number of + ``global_iterations`` to verify your setup before committing to a long + calibration run. +- **Use** ``damping < 1.0`` if coefficients oscillate between iterations. +- **Set** ``hold_fast: True`` on reference-category coefficients that should be + fixed (e.g., the base alternative in a logit model). +- **Comment out rows** in the calibration spec with ``#`` to temporarily exclude + coefficients without modifying the file structure. +- **Check** ``calibration_iteration_records.csv`` to diagnose convergence issues + — look for coefficients hitting bounds (``at_min``/``at_max``) or oscillating. +- **Use** ``complete_steps: true`` if you need full model outputs each iteration + (e.g., for trip generation downstream of mode choice). +- **Version control your coefficient files** so you can diff changes and reset to + initial values. +- **Use the** ``resume_after`` **setting** to skip expensive upstream steps (like + skims loading or accessibility computation) that don't change across + calibration iterations. From 15df92cc348836e459b2f81843efff7fdfbc0987 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 20 Jul 2026 21:47:31 -0700 Subject: [PATCH 38/90] move calib documentation into users guide --- docs/{ => users-guide}/calibration.rst | 4 ++-- docs/users-guide/index.rst | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) rename docs/{ => users-guide}/calibration.rst (99%) diff --git a/docs/calibration.rst b/docs/users-guide/calibration.rst similarity index 99% rename from docs/calibration.rst rename to docs/users-guide/calibration.rst index e64f1afb0f..db0cf719d0 100644 --- a/docs/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -343,7 +343,7 @@ Computes the coefficient delta as: .. math:: - \Delta = \ln\!\left(\frac{\text{target\_value}}{\text{model\_value}}\right) \times \text{damping} + \Delta = \ln\!\left(\frac{\text{target_value}}{\text{model_value}}\right) \times \text{damping} Requires both ``model_value`` and ``target_value`` to be positive. If either is zero or negative, falls back to ``default_increment`` (positive or negative @@ -634,7 +634,7 @@ A coefficient is considered **converged** when: .. math:: - |\text{target\_value} - \text{model\_value}| \leq \text{tolerance} + \text{target_value} - \text{model_value} \leq \text{tolerance} A component is converged when **all** of its coefficients are converged. The component inner loop stops early upon convergence. diff --git a/docs/users-guide/index.rst b/docs/users-guide/index.rst index d464a6cd0d..3dc88a373e 100644 --- a/docs/users-guide/index.rst +++ b/docs/users-guide/index.rst @@ -45,6 +45,7 @@ Contents example_models example_performance estimation-mode/index + calibration .. toctree:: :maxdepth: 1 other_examples From 4e38277ba94ffd142318ffb51eb18d984fac8b24 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 20 Jul 2026 21:48:16 -0700 Subject: [PATCH 39/90] all converged fix and dead restart_after --- activitysim/core/calibration.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index a0c1651eb0..547f6499c3 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -72,7 +72,6 @@ class CalibrationRunSettings(PydanticBase): resume_after: Optional[str] = None calibrate_models: list[str] - restart_after: list[str] = [] global_iterations: int = 1 complete_steps: bool = False @@ -110,12 +109,6 @@ def validate_model_settings(self): f"calibration model '{component}' is not in model_settings" ) - for component in self.run.restart_after: - if component not in self.run.calibrate_models: - raise ValueError( - f"restart_after component '{component}' is not in calibrate_models" - ) - if self.run.global_iterations < 1: raise ValueError("max_iterations must be >= 1") @@ -279,10 +272,18 @@ def run_calibration_loop( }, ) + if all_converged: + logger.info( + "calibration converged after global iteration %s/%s", + global_iter - start_global_iter, + calibration_settings.run.global_iterations, + ) + break + _write_final_coefficients_snapshot(state, calibration_settings) return CalibrationRunResult( - converged=False, + converged=all_converged, completed_global_iterations=calibration_settings.run.global_iterations, ) finally: @@ -415,7 +416,7 @@ def _calibrate_component( for component_iter in range(1, component_settings.submodel_max_iterations + 1): component_iterations = component_iter run_model_name = ( - f"{component_name}.component_i{component_iter};" f"global_i{global_iter}" + f"{component_name}.c_i{component_iter};" f"g_i{global_iter}" ) # Re-run only this component from its prior checkpoint so model values # reflect the current candidate coefficients for this component. From 391d9a1371449c7fe00daa03001f1c57482bba42 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 20 Jul 2026 21:51:27 -0700 Subject: [PATCH 40/90] blacken --- activitysim/core/calibration.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 547f6499c3..fa554043d9 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -415,9 +415,7 @@ def _calibrate_component( for component_iter in range(1, component_settings.submodel_max_iterations + 1): component_iterations = component_iter - run_model_name = ( - f"{component_name}.c_i{component_iter};" f"g_i{global_iter}" - ) + run_model_name = f"{component_name}.c_i{component_iter};" f"g_i{global_iter}" # Re-run only this component from its prior checkpoint so model values # reflect the current candidate coefficients for this component. if state.settings.multiprocess and shared_data_buffers is not None: From eb4e6bad79fc116cd4a77e54020edfee26ab2347 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 21 Jul 2026 12:29:57 -0400 Subject: [PATCH 41/90] Fix resume_after bug --- activitysim/core/calibration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index fa554043d9..faf09dd8fc 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -205,7 +205,7 @@ def run_calibration_loop( _run_precursor_components( state, models=models[:first_calib_model_idx], - resume_after=calibration_settings.run.resume_after + resume_after=state.settings.resume_after if global_iter == 1 else _prior_step_name( models, calibration_settings.run.calibrate_models[0] From de119ed12c35190fde86d83f16a05b746203aecf Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 22 Jul 2026 16:57:16 -0400 Subject: [PATCH 42/90] partial MP resume_after fix --- activitysim/core/calibration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index faf09dd8fc..6e01bcb159 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -206,7 +206,7 @@ def run_calibration_loop( state, models=models[:first_calib_model_idx], resume_after=state.settings.resume_after - if global_iter == 1 + if global_iter == start_global_iter else _prior_step_name( models, calibration_settings.run.calibrate_models[0] ), @@ -300,7 +300,7 @@ def _run_precursor_components( ) -> None: """Run the normal ActivitySim model flow for one global calibration iteration.""" - if global_iter > 1: + if global_iter > 1 and resume_after is not None: # Seed a fresh pipeline from the configured resume checkpoint to avoid # duplicate checkpoint-name collisions across global calibration loops. prior_pipeline = state.checkpoint.store.filename @@ -1454,7 +1454,7 @@ def _run_multiprocess_with_overrides( injectables, shared_data_buffers=shared_data_buffers, skip_final_checkpoint=True, - force_resume=True, + force_resume=resume_after is not None, ) finally: state.settings.models = original_models From f0e9ccb7411ed1168bde40bfdf189d69830c2dfc Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 29 Jul 2026 19:42:14 -0400 Subject: [PATCH 43/90] More checkpointing work --- activitysim/core/calibration.py | 181 ++++++++++++++++++------ activitysim/core/workflow/checkpoint.py | 10 +- 2 files changed, 141 insertions(+), 50 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 6e01bcb159..11df42f26a 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -163,6 +163,12 @@ def run_calibration_loop( if not calibration_settings or not calibration_settings.enable: raise RuntimeError("calibration loop called while calibration is disabled") + if state.settings.duplicate_step_execution != "allow": + state.settings.duplicate_step_execution = "allow" + logger.warning( + "Overriding duplicate_step_execution setting: must be enabled for calibration" + ) + assert all( [c in models for c in calibration_settings.run.calibrate_models] ), f"settings.yaml steps list does not include calibration model{'s' if len([c for c in calibration_settings.run.calibrate_models if c not in models]) != 1 else ''} {[c for c in calibration_settings.run.calibrate_models if c not in models]}" @@ -172,6 +178,12 @@ def run_calibration_loop( calibration_settings.run.calibrate_models, key=lambda x: models.index(x) ) first_calib_model_idx = models.index(calibration_settings.run.calibrate_models[0]) + last_calib_model_idx = models.index(calibration_settings.run.calibrate_models[-1]) + first_model_idx = ( + models.index(state.settings.resume_after) + 1 + if state.settings.resume_after + else None + ) _ensure_calibration_output_dir(state) @@ -191,6 +203,23 @@ def run_calibration_loop( shared_data_buffers = _initialize_mp_shared_resources(state) try: + # skip precursors if, on first iter, resume_after exists and is >= first_calib_model_idx + if ( + state.settings.resume_after is None + or first_model_idx < first_calib_model_idx + ): + # Run ActivitySim normally from resume_after through production model steps. + _run_precursor_components( + state, + models=models[:first_calib_model_idx] + if first_model_idx is None + else models[first_model_idx:first_calib_model_idx], + resume_after=state.settings.resume_after, + global_iter=start_global_iter, + memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, + ) + for global_iter in range( start_global_iter, start_global_iter + calibration_settings.run.global_iterations, @@ -201,24 +230,20 @@ def run_calibration_loop( calibration_settings.run.global_iterations, ) - # Run ActivitySim normally from resume_after through production model steps. - _run_precursor_components( - state, - models=models[:first_calib_model_idx], - resume_after=state.settings.resume_after - if global_iter == start_global_iter - else _prior_step_name( - models, calibration_settings.run.calibrate_models[0] - ), - global_iter=global_iter, - memory_sidecar_process=memory_sidecar_process, - shared_data_buffers=shared_data_buffers, - ) - - all_converged = True + # suppress early termination on first iteration if resume_after is after all calibrated models + all_converged = ( + first_model_idx is not None and first_model_idx <= last_calib_model_idx + ) or global_iter > start_global_iter last_calibrated_component = None for component in calibration_settings.run.calibrate_models: + # on the first global iter, skip model if it's before or == resume_after + if ( + global_iter == start_global_iter + and first_model_idx is not None + and first_model_idx > models.index(component) + ): + continue component_settings = calibration_settings.model_settings[component] prior_step = _prior_step_name(models, component) @@ -251,15 +276,32 @@ def run_calibration_loop( last_calibrated_component = component - if calibration_settings.run.complete_steps or ( - start_global_iter + calibration_settings.run.global_iterations - == global_iter + 1 + if ( + calibration_settings.run.complete_steps + or ( + start_global_iter + calibration_settings.run.global_iterations + == global_iter + 1 + ) + or ( + global_iter == start_global_iter + and state.settings.resume_after is not None + and first_model_idx > last_calib_model_idx + ) ): + subsequent_components = ( + models[first_model_idx:] + if global_iter == start_global_iter + and first_model_idx > last_calib_model_idx + else models[models.index(last_calibrated_component) + 1 :] + ) # finish the full model chain _run_subsequent_components( state, - models=models[models.index(last_calibrated_component) + 1 :], - resume_after=last_calibrated_component, + models=subsequent_components, + resume_after=state.settings.resume_after + if global_iter == start_global_iter + and first_model_idx > last_calib_model_idx + else last_calibrated_component, memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) @@ -300,21 +342,22 @@ def _run_precursor_components( ) -> None: """Run the normal ActivitySim model flow for one global calibration iteration.""" - if global_iter > 1 and resume_after is not None: - # Seed a fresh pipeline from the configured resume checkpoint to avoid - # duplicate checkpoint-name collisions across global calibration loops. - prior_pipeline = state.checkpoint.store.filename - state.checkpoint.close_store() - state.filesystem.pipeline_file_name = f"pipeline_calibration_iter_{global_iter}" - state.checkpoint.restore_from(prior_pipeline, checkpoint_name=resume_after) - else: - _run_in_configured_mode( - state, - models=models, - resume_after=resume_after, - memory_sidecar_process=memory_sidecar_process, - shared_data_buffers=shared_data_buffers, - ) + # if global_iter > 1 and resume_after is not None: + # # Seed a fresh pipeline from the configured resume checkpoint to avoid + # # duplicate checkpoint-name collisions across global calibration loops. + # prior_pipeline = state.checkpoint.store.filename + # state.checkpoint.close_store() + # state.filesystem.pipeline_file_name = f"pipeline_calibration_iter_{global_iter}" + # state.checkpoint.restore_from(prior_pipeline, checkpoint_name=resume_after) + # else: + + _run_in_configured_mode( + state, + models=models, + resume_after=resume_after, + memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, + ) def _run_intermediate_components( @@ -425,7 +468,9 @@ def _calibrate_component( _run_mp_single_component( state, component_name=component_name, - restore_checkpoint=mp_restore_checkpoint, + restore_checkpoint=mp_restore_checkpoint + if component_iter == 1 + else "_", shared_data_buffers=shared_data_buffers, ) else: @@ -1305,11 +1350,9 @@ def _run_mp_single_component( chunk_size = step.chunk_size break - step_name = f"calibration_{component_name}" - # Build step_info dict matching what mp_tasks functions expect step_info = { - "name": step_name, + "name": component_name, "models": [component_name], "num_processes": num_processes, "chunk_size": chunk_size, @@ -1321,9 +1364,9 @@ def _run_mp_single_component( injectables = _build_calibration_injectables(state) if num_processes == 1: - sub_proc_names = [step_name] + sub_proc_names = [component_name] else: - sub_proc_names = [f"{step_name}_{i}" for i in range(num_processes)] + sub_proc_names = [f"{component_name}_{i}" for i in range(num_processes)] fail_fast = state.settings.fail_fast @@ -1333,7 +1376,7 @@ def _run_mp_single_component( state, multiprocessing.Process( target=mp_tasks.mp_apportion_pipeline, - name=f"{step_name}_apportion", + name=f"{component_name}_apportion", args=(injectables, sub_proc_names, step_info), ), ) @@ -1365,7 +1408,7 @@ def _run_mp_single_component( raise SubprocessError( f"{num_processes - len(completed)} processes failed in " - f"calibration step {step_name}" + f"calibration step {component_name}" ) # Coalesce sub-process pipelines back into main pipeline @@ -1374,7 +1417,7 @@ def _run_mp_single_component( state, multiprocessing.Process( target=mp_tasks.mp_coalesce_pipelines, - name=f"{step_name}_coalesce", + name=f"{component_name}_coalesce", args=(injectables, sub_proc_names, slice_info), ), ) @@ -1394,6 +1437,8 @@ def _run_in_configured_mode( if not models: return + _prep_model_data(state, models) + if state.settings.multiprocess: _run_multiprocess_with_overrides( state, @@ -1419,6 +1464,52 @@ def _run_in_configured_mode( ) +def _prep_model_data(state, models): + checkpoint = False + _restore_parent_state_from_pipeline(state) + if "compute_accessibility" in models and state.is_table("accessibility"): + state.add_table( + "accessibility", pd.DataFrame(index=state.get_table("accessibility").index) + ) + checkpoint = True + if "mandatory_tour_frequency" in models and state.is_table("tours"): + state.add_table( + "tours", + state.get_table("tours")[ + state.get_table("tours").tour_category != "mandatory" + ], + ) + checkpoint = True + + if "non_mandatory_tour_frequency" in models and state.is_table("tours"): + state.add_table( + "tours", + state.get_table("tours")[ + state.get_table("tours").tour_category != "non_mandatory" + ], + ) + checkpoint = True + + if "joint_tour_frequency" in models and state.is_table("tours"): + state.add_table( + "tours", + state.get_table("tours")[state.get_table("tours").tour_category != "joint"], + ) + checkpoint = True + + if "atwork_subtour_frequency" in models and state.is_table("tours"): + state.add_table( + "tours", + state.get_table("tours")[ + state.get_table("tours").tour_category != "atwork" + ], + ) + checkpoint = True + + if checkpoint: + state.checkpoint.add(state.checkpoint.checkpoints[-1]["checkpoint_name"]) + + def _run_multiprocess_with_overrides( state: workflow.State, models: list[str], @@ -1596,7 +1687,7 @@ def _build_calibration_mp_steps( for step_idx, step_models in step_model_groups.items(): orig_step = original_steps[step_idx] kwargs: dict[str, Any] = { - "name": f"calibration_{orig_step.name}_{step_models[0]}", + "name": orig_step.name, "begin": step_models[0], } if orig_step.num_processes is not None: diff --git a/activitysim/core/workflow/checkpoint.py b/activitysim/core/workflow/checkpoint.py index ec721a4790..63b5583254 100644 --- a/activitysim/core/workflow/checkpoint.py +++ b/activitysim/core/workflow/checkpoint.py @@ -767,7 +767,7 @@ def load(self, checkpoint_name: str, store=None): try: # truncate rows after target checkpoint - i = checkpoints[checkpoints[CHECKPOINT_NAME] == checkpoint_name].index[0] + i = checkpoints[checkpoints[CHECKPOINT_NAME] == checkpoint_name].index[-1] checkpoints = checkpoints.loc[:i] # if the store is not open in read-only mode, @@ -1231,10 +1231,10 @@ def load_dataframe(self, table_name, checkpoint_name=None): return self._obj.get_dataframe(table_name) # find the requested checkpoint - checkpoint = next( - (x for x in self.checkpoints if x["checkpoint_name"] == checkpoint_name), - None, - ) + matching_checkpoints = [ + x for x in self.checkpoints if x["checkpoint_name"] == checkpoint_name + ] + checkpoint = matching_checkpoints[-1] if matching_checkpoints else None if checkpoint is None: raise CheckpointNameNotFoundError( "checkpoint '%s' not in checkpoints." % checkpoint_name From c9afece562f8cdbb890cf94cfe564e97ea8f9450 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 29 Jul 2026 19:45:17 -0400 Subject: [PATCH 44/90] Add tour mode choice handler --- activitysim/core/calibration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 11df42f26a..7aae3cbb3e 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1497,7 +1497,9 @@ def _prep_model_data(state, models): ) checkpoint = True - if "atwork_subtour_frequency" in models and state.is_table("tours"): + if ( + "atwork_subtour_frequency" in models or "tour_mode_choice_simulate" in models + ) and state.is_table("tours"): state.add_table( "tours", state.get_table("tours")[ From ba8a14287f1e8993c273ba99ae11cb78c40e17f4 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 30 Jul 2026 11:32:17 -0700 Subject: [PATCH 45/90] Clean checkpointing --- activitysim/core/calibration.py | 131 ++++++++++++++++++++------------ 1 file changed, 82 insertions(+), 49 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 7aae3cbb3e..2c01f277b2 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1437,7 +1437,7 @@ def _run_in_configured_mode( if not models: return - _prep_model_data(state, models) + _prep_model_data(state, models, resume_after=resume_after) if state.settings.multiprocess: _run_multiprocess_with_overrides( @@ -1464,52 +1464,73 @@ def _run_in_configured_mode( ) -def _prep_model_data(state, models): - checkpoint = False - _restore_parent_state_from_pipeline(state) - if "compute_accessibility" in models and state.is_table("accessibility"): - state.add_table( - "accessibility", pd.DataFrame(index=state.get_table("accessibility").index) - ) - checkpoint = True - if "mandatory_tour_frequency" in models and state.is_table("tours"): - state.add_table( - "tours", - state.get_table("tours")[ - state.get_table("tours").tour_category != "mandatory" - ], - ) - checkpoint = True - - if "non_mandatory_tour_frequency" in models and state.is_table("tours"): - state.add_table( - "tours", - state.get_table("tours")[ - state.get_table("tours").tour_category != "non_mandatory" - ], - ) - checkpoint = True +def _prep_model_data(state, models, resume_after=None): + """Restore the pipeline to the correct state before running models. - if "joint_tour_frequency" in models and state.is_table("tours"): - state.add_table( - "tours", - state.get_table("tours")[state.get_table("tours").tour_category != "joint"], - ) - checkpoint = True - - if ( - "atwork_subtour_frequency" in models or "tour_mode_choice_simulate" in models - ) and state.is_table("tours"): - state.add_table( - "tours", - state.get_table("tours")[ - state.get_table("tours").tour_category != "atwork" - ], - ) - checkpoint = True + When a specific ``resume_after`` checkpoint exists in the pipeline, we + restore from it directly. This gives the exact state as it was when that + model completed — without downstream data that later models may have added + (e.g. tours from mandatory_tour_frequency polluting the state for a re-run + of that same model). + + If ``resume_after`` is not available as a named checkpoint (e.g. first + calibration run from a multiprocess pipeline with only step-level + checkpoints), fall back to LAST_CHECKPOINT which is the most recently + written state. + """ + if resume_after: + # Try to restore from the exact resume_after checkpoint + try: + if state.checkpoint.store_is_open(): + checkpoint_names = [ + cp.get("checkpoint_name", "") for cp in state.checkpoint.checkpoints + ] + else: + from activitysim.core.workflow.checkpoint import HdfStore, ParquetStore + + pipeline_path = Path(state.checkpoint.default_pipeline_file_path()) + if state.settings.checkpoint_format == "hdf": + store = HdfStore(pipeline_path, mode="r") + else: + store = ParquetStore(pipeline_path, mode="r") + try: + checkpoint_names = store.list_checkpoint_names() + finally: + store.close() + + if resume_after in checkpoint_names: + _restore_parent_state_from_pipeline(state, checkpoint_name=resume_after) + return + + # Resolve to step-level checkpoint if model-level not found. + # Find the multiprocess step that contains resume_after and use + # its step name as the checkpoint. + resolved = None + all_models = state.settings.models + mp_steps = state.settings.multiprocess_steps + if mp_steps and resume_after in all_models: + resume_idx = all_models.index(resume_after) + step_boundaries = [all_models.index(s.begin) for s in mp_steps] + step_boundaries.append(len(all_models)) + for i, step in enumerate(mp_steps): + if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: + if step.name in checkpoint_names: + resolved = step.name + break + + if resolved: + _restore_parent_state_from_pipeline(state, checkpoint_name=resolved) + return + except Exception: + logger.warning( + "calibration: could not restore from checkpoint %r, " + "falling back to LAST_CHECKPOINT", + resume_after, + ) - if checkpoint: - state.checkpoint.add(state.checkpoint.checkpoints[-1]["checkpoint_name"]) + # Fallback: load LAST_CHECKPOINT (appropriate after a coalesce that + # only ran the desired models) + _restore_parent_state_from_pipeline(state) def _run_multiprocess_with_overrides( @@ -1555,20 +1576,32 @@ def _run_multiprocess_with_overrides( state.settings.multiprocess_steps = original_mp_steps -def _restore_parent_state_from_pipeline(state: workflow.State) -> None: - """Restore coalesced pipeline tables into the parent process state. +def _restore_parent_state_from_pipeline( + state: workflow.State, checkpoint_name: str = "_" +) -> None: + """Restore pipeline tables into the parent process state. After a multiprocess run, the parent's in-memory state is stale. - This loads the latest checkpoint from the pipeline store so that + This loads a specific checkpoint from the pipeline store so that calibration expressions can evaluate against model outputs. + Parameters + ---------- + checkpoint_name : str, default "_" + The checkpoint to restore from. Use a model-level checkpoint name + (e.g. the prior step name) to get the exact state at that point, + avoiding pollution from downstream models that may have added rows + to shared tables like ``tours``. The default ``"_"`` loads the + last checkpoint, which is appropriate immediately after a coalesce + that only ran the desired models. + All tables are explicitly re-checkpointed so that subsequent apportion subprocesses can load them from a direct file path without relying on checkpoint backtracking through potentially ambiguous checkpoint history. """ if state.checkpoint.store_is_open(): state.checkpoint.close_store() - state.checkpoint.restore(resume_after="_") + state.checkpoint.restore(resume_after=checkpoint_name) # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. From c7f3c7756c8587b9524e55c9952765e1675c9553 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 30 Jul 2026 12:24:34 -0700 Subject: [PATCH 46/90] More checkpointing fixes --- activitysim/core/calibration.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 2c01f277b2..b9fb437e4c 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -219,6 +219,14 @@ def run_calibration_loop( memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) + else: + # Precursors skipped — but the pipeline must still be initialized + # at the resume_after point so that _calibrate_component (and its + # apportion subprocess) starts from the correct state without + # downstream model data. + _prep_model_data(state, [], resume_after=state.settings.resume_after) + state.checkpoint.add(state.settings.resume_after) + state.checkpoint.close_store() for global_iter in range( start_global_iter, @@ -1440,6 +1448,14 @@ def _run_in_configured_mode( _prep_model_data(state, models, resume_after=resume_after) if state.settings.multiprocess: + # Write the restored state as a checkpoint so LAST_CHECKPOINT on disk + # reflects the correct (clean) state for the apportion subprocess. + # Without this, Parquet stores retain a stale LAST_CHECKPOINT from a + # prior run that may include downstream data (e.g. non-mandatory tours + # polluting a re-run of non_mandatory_tour_frequency). + state.checkpoint.add(resume_after or models[0]) + state.checkpoint.close_store() + _run_multiprocess_with_overrides( state, models=models, From c9aca4575828640de5c8359afa1c44bf03970e77 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 30 Jul 2026 12:58:42 -0700 Subject: [PATCH 47/90] Restore from subprocess checkpoints --- activitysim/core/calibration.py | 256 ++++++++++++++++++++++++++++---- 1 file changed, 230 insertions(+), 26 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index b9fb437e4c..ebe5fe4668 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -224,9 +224,23 @@ def run_calibration_loop( # at the resume_after point so that _calibrate_component (and its # apportion subprocess) starts from the correct state without # downstream model data. - _prep_model_data(state, [], resume_after=state.settings.resume_after) - state.checkpoint.add(state.settings.resume_after) - state.checkpoint.close_store() + extra_models = _prep_model_data( + state, [], resume_after=state.settings.resume_after + ) + if extra_models: + # No model-level checkpoint exists for resume_after; we must + # run models from the prior step through resume_after to + # recreate the correct intermediate state. + _run_in_configured_mode( + state, + models=extra_models, + resume_after=None, + memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, + ) + else: + state.checkpoint.add(state.settings.resume_after) + state.checkpoint.close_store() for global_iter in range( start_global_iter, @@ -1445,7 +1459,12 @@ def _run_in_configured_mode( if not models: return - _prep_model_data(state, models, resume_after=resume_after) + extra_models = _prep_model_data(state, models, resume_after=resume_after) + if extra_models: + # Models from the step beginning through resume_after must run first + # to recreate the correct intermediate state (since no model-level + # checkpoint existed for resume_after). + models = extra_models + models if state.settings.multiprocess: # Write the restored state as a checkpoint so LAST_CHECKPOINT on disk @@ -1483,19 +1502,22 @@ def _run_in_configured_mode( def _prep_model_data(state, models, resume_after=None): """Restore the pipeline to the correct state before running models. - When a specific ``resume_after`` checkpoint exists in the pipeline, we - restore from it directly. This gives the exact state as it was when that - model completed — without downstream data that later models may have added - (e.g. tours from mandatory_tour_frequency polluting the state for a re-run - of that same model). - - If ``resume_after`` is not available as a named checkpoint (e.g. first - calibration run from a multiprocess pipeline with only step-level - checkpoints), fall back to LAST_CHECKPOINT which is the most recently - written state. + Resolution priority: + 1. Direct model-level checkpoint in the main pipeline (fastest, exact). + 2. Subprocess pipelines from a prior multiprocess run — performs a + "coalesce at specific checkpoint" to recover the exact intermediate + state without re-running anything. + 3. Previous step checkpoint + re-run models from step begin through + resume_after (slowest but always works). + + Returns + ------- + list[str] + Models that must be prepended to the caller's models list to reach + the correct state at ``resume_after``. Empty when the exact + checkpoint was found and restored directly (paths 1 or 2). """ if resume_after: - # Try to restore from the exact resume_after checkpoint try: if state.checkpoint.store_is_open(): checkpoint_names = [ @@ -1514,14 +1536,16 @@ def _prep_model_data(state, models, resume_after=None): finally: store.close() + # Path 1: direct model-level checkpoint in main pipeline if resume_after in checkpoint_names: _restore_parent_state_from_pipeline(state, checkpoint_name=resume_after) - return + return [] + + # Path 2: subprocess pipelines (model-level checkpoints preserved) + if _restore_from_subprocess_pipelines(state, resume_after): + return [] - # Resolve to step-level checkpoint if model-level not found. - # Find the multiprocess step that contains resume_after and use - # its step name as the checkpoint. - resolved = None + # Path 3: restore from previous step and re-run all_models = state.settings.models mp_steps = state.settings.multiprocess_steps if mp_steps and resume_after in all_models: @@ -1530,13 +1554,20 @@ def _prep_model_data(state, models, resume_after=None): step_boundaries.append(len(all_models)) for i, step in enumerate(mp_steps): if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: - if step.name in checkpoint_names: - resolved = step.name + if i > 0 and mp_steps[i - 1].name in checkpoint_names: + _restore_parent_state_from_pipeline( + state, checkpoint_name=mp_steps[i - 1].name + ) + step_begin_idx = step_boundaries[i] + extra_models = all_models[step_begin_idx : resume_idx + 1] + return extra_models + elif i == 0: + _restore_parent_state_from_pipeline( + state, checkpoint_name="_" + ) + extra_models = all_models[: resume_idx + 1] + return extra_models break - - if resolved: - _restore_parent_state_from_pipeline(state, checkpoint_name=resolved) - return except Exception: logger.warning( "calibration: could not restore from checkpoint %r, " @@ -1547,6 +1578,179 @@ def _prep_model_data(state, models, resume_after=None): # Fallback: load LAST_CHECKPOINT (appropriate after a coalesce that # only ran the desired models) _restore_parent_state_from_pipeline(state) + return [] + + +def _restore_from_subprocess_pipelines( + state: workflow.State, resume_after: str +) -> bool: + """Restore state from subprocess pipelines at a specific model checkpoint. + + Subprocess pipelines retain model-level checkpoints that don't exist in + the main pipeline. This performs a "coalesce at checkpoint" — reading + mirrored tables from one subprocess and concatenating sliced tables from + all subprocesses at the specified checkpoint name. + + Parameters + ---------- + state : workflow.State + resume_after : str + Model-level checkpoint name to restore from. + + Returns + ------- + bool + True if the restore succeeded; False if subprocess pipelines don't + exist or don't contain the requested checkpoint. + """ + from activitysim.core.workflow.checkpoint import ( + CHECKPOINT_NAME, + CHECKPOINT_TABLE_NAME, + HdfStore, + NON_TABLE_COLUMNS, + ParquetStore, + ) + + all_models = state.settings.models + mp_steps = state.settings.multiprocess_steps + if not mp_steps or resume_after not in all_models: + return False + + # Find the multiprocess step containing resume_after + resume_idx = all_models.index(resume_after) + step_boundaries = [all_models.index(s.begin) for s in mp_steps] + step_boundaries.append(len(all_models)) + + enclosing_step = None + num_processes = state.settings.num_processes or 2 + slice_info = None + for i, step in enumerate(mp_steps): + if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: + enclosing_step = step + if step.num_processes: + num_processes = step.num_processes + if step.slice: + slice_info = ( + step.slice.model_dump() + if hasattr(step.slice, "model_dump") + else step.slice + ) + break + + if enclosing_step is None or num_processes <= 1: + return False + + # Build subprocess pipeline file paths + step_name = enclosing_step.name + pipeline_file_name = state.filesystem.pipeline_file_name + sub_proc_names = [f"{step_name}_{i}" for i in range(num_processes)] + + def _subprocess_path(proc_name): + base = state.get_output_file_path(pipeline_file_name, prefix=proc_name) + if state.settings.checkpoint_format == "hdf": + return base + pq = Path(str(base)).with_suffix(ParquetStore.extension) + return pq if pq.exists() else base + + first_path = _subprocess_path(sub_proc_names[0]) + if not first_path.exists(): + return False + + # Open first subprocess pipeline and verify checkpoint exists + if state.settings.checkpoint_format == "hdf": + first_store = HdfStore(first_path, mode="r") + else: + first_store = ParquetStore(first_path, mode="r") + + try: + cp_names = first_store.list_checkpoint_names() + if resume_after not in cp_names: + return False + + # Read checkpoint row to get table→checkpoint mapping + cp_df = first_store.get_dataframe(CHECKPOINT_TABLE_NAME) + cp_row = cp_df[cp_df[CHECKPOINT_NAME] == resume_after].iloc[-1] + + table_map = {} + for col in cp_row.index: + if col not in NON_TABLE_COLUMNS and cp_row[col]: + table_map[col] = cp_row[col] + + # Read all tables from first subprocess at this checkpoint + tables = {} + for table_name, cp_for_table in table_map.items(): + try: + tables[table_name] = first_store.get_dataframe(table_name, cp_for_table) + except (FileNotFoundError, KeyError): + logger.warning( + f"calibration: subprocess pipeline missing table " + f"{table_name} at {cp_for_table}" + ) + finally: + first_store.close() + + if not tables: + return False + + # Determine sliced tables that need concatenation across processes + sliced_table_names = set(slice_info.get("tables", [])) if slice_info else set() + + # Read sliced tables from remaining subprocesses and concatenate + if num_processes > 1 and sliced_table_names: + omnibus = {t: [tables[t]] for t in sliced_table_names if t in tables} + + for proc_name in sub_proc_names[1:]: + proc_path = _subprocess_path(proc_name) + if not proc_path.exists(): + logger.warning( + f"calibration: subprocess pipeline not found: {proc_path}" + ) + return False + + if state.settings.checkpoint_format == "hdf": + proc_store = HdfStore(proc_path, mode="r") + else: + proc_store = ParquetStore(proc_path, mode="r") + + try: + proc_cp_df = proc_store.get_dataframe(CHECKPOINT_TABLE_NAME) + proc_row = proc_cp_df[proc_cp_df[CHECKPOINT_NAME] == resume_after].iloc[ + -1 + ] + + for table_name in list(omnibus.keys()): + cp_for_table = proc_row.get(table_name, "") + if cp_for_table: + omnibus[table_name].append( + proc_store.get_dataframe(table_name, cp_for_table) + ) + finally: + proc_store.close() + + # Replace sliced tables with concatenated versions + for table_name, dfs in omnibus.items(): + tables[table_name] = pd.concat(dfs, sort=False) + + # Load into parent state + state.init_state() + if state.checkpoint.store_is_open(): + state.checkpoint.close_store() + state.checkpoint.open_store(overwrite=False) + + for table_name, df in tables.items(): + state.add_table(table_name, df) + + # Mark all tables dirty for subsequent checkpoint.add + for table_name in list(state.existing_table_names): + state.existing_table_status[table_name] = True + + logger.info( + "calibration: restored %d tables from subprocess pipelines at " + "checkpoint '%s'", + len(tables), + resume_after, + ) + return True def _run_multiprocess_with_overrides( From a19638bab8e6def343f5945da339dad3cb3c9953 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Fri, 31 Jul 2026 10:15:52 -0700 Subject: [PATCH 48/90] resume_after bugfix --- activitysim/core/calibration.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index ebe5fe4668..469e8b929b 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1469,10 +1469,14 @@ def _run_in_configured_mode( if state.settings.multiprocess: # Write the restored state as a checkpoint so LAST_CHECKPOINT on disk # reflects the correct (clean) state for the apportion subprocess. - # Without this, Parquet stores retain a stale LAST_CHECKPOINT from a - # prior run that may include downstream data (e.g. non-mandatory tours - # polluting a re-run of non_mandatory_tour_frequency). - state.checkpoint.add(resume_after or models[0]) + # When extra_models were prepended, the state is from a PRIOR step + # (not the actual resume_after point) — use a non-conflicting name + # so it won't be mistakenly loaded as the resume_after state on a + # subsequent restart. + if extra_models: + state.checkpoint.add(f"_calibration_staging") + else: + state.checkpoint.add(resume_after or models[0]) state.checkpoint.close_store() _run_multiprocess_with_overrides( From 2fd3db779c153ba0f7513100def557c372e5d46f Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 4 Aug 2026 17:42:51 -0500 Subject: [PATCH 49/90] add subproc reuse --- activitysim/core/calibration.py | 71 ++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 10 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 469e8b929b..411e063ac9 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -225,7 +225,7 @@ def run_calibration_loop( # apportion subprocess) starts from the correct state without # downstream model data. extra_models = _prep_model_data( - state, [], resume_after=state.settings.resume_after + state, resume_after=state.settings.resume_after ) if extra_models: # No model-level checkpoint exists for resume_after; we must @@ -1459,7 +1459,7 @@ def _run_in_configured_mode( if not models: return - extra_models = _prep_model_data(state, models, resume_after=resume_after) + extra_models = _prep_model_data(state, resume_after=resume_after) if extra_models: # Models from the step beginning through resume_after must run first # to recreate the correct intermediate state (since no model-level @@ -1479,11 +1479,19 @@ def _run_in_configured_mode( state.checkpoint.add(resume_after or models[0]) state.checkpoint.close_store() + # When subprocess pipelines from a prior run already have the + # resume_after checkpoint (Path 2 in _prep_model_data), subprocesses + # can skip models before resume_after by reusing those pipelines + # instead of freshly apportioning. Signal this by passing + # can_reuse_subprocs=True. + can_reuse = not extra_models and resume_after is not None + _run_multiprocess_with_overrides( state, models=models, resume_after=resume_after, shared_data_buffers=shared_data_buffers, + can_reuse_subprocs=can_reuse, ) # After multiprocess completes, the coalesced pipeline exists on disk. # Restore it into the parent process state so tables are accessible @@ -1503,7 +1511,7 @@ def _run_in_configured_mode( ) -def _prep_model_data(state, models, resume_after=None): +def _prep_model_data(state, resume_after=None): """Restore the pipeline to the correct state before running models. Resolution priority: @@ -1762,8 +1770,22 @@ def _run_multiprocess_with_overrides( models: list[str], resume_after: str | None, shared_data_buffers: dict | None = None, + can_reuse_subprocs: bool = False, ) -> None: - """Run multiprocess with temporary settings overrides for calibration passes.""" + """Run multiprocess with temporary settings overrides for calibration passes. + + Parameters + ---------- + can_reuse_subprocs : bool, default False + When True, subprocess pipelines from a prior run are assumed to exist + and contain the ``resume_after`` checkpoint. Breadcrumbs are written + so that ``get_run_list`` populates ``step_info["resume_after"]``, + apportion is skipped (reusing existing subprocess pipelines), and + subprocesses resume from their model-level checkpoint — skipping + already-completed models. + """ + from collections import OrderedDict + from activitysim.core import mp_tasks original_models = state.settings.models @@ -1779,11 +1801,40 @@ def _run_multiprocess_with_overrides( state.settings.models = models state.settings.multiprocess_steps = calibration_mp_steps - # Always None: calibration manages pipeline state externally via - # _restore_parent_state_from_pipeline and checkpoint.add, so the MP - # system's breadcrumb-based resume logic must not be triggered. - # Apportion uses LAST_CHECKPOINT to read from the current pipeline state. - state.settings.resume_after = None + + if can_reuse_subprocs and resume_after: + # Enable the normal resume mechanism: set resume_after in settings + # and write breadcrumbs so get_run_list can properly populate + # step_info["resume_after"]. Apportion will be skipped (prior + # subprocess pipelines reused), and subprocesses will restore from + # their existing model-level checkpoint. + state.settings.resume_after = resume_after + + # Write minimal breadcrumbs indicating the step containing + # resume_after has completed apportion (so it's skipped) but + # simulate/coalesce need re-running. + breadcrumbs = OrderedDict() + for step in calibration_mp_steps: + step_dict = {"name": step.name, "apportion": True} + breadcrumbs[step.name] = step_dict + # Find the step containing resume_after + all_models = state.settings.models + if resume_after in all_models: + step_begin = all_models.index(step.begin) + step_models_in_step = [ + m for m in all_models[step_begin:] if m in models + ] + if resume_after in step_models_in_step: + # This step contains resume_after — stop here. + # get_breadcrumbs will mark simulate/coalesce for re-run. + break + + mp_tasks.write_breadcrumbs(state, breadcrumbs) + else: + # No reuse: calibration manages pipeline state externally via + # _restore_parent_state_from_pipeline and checkpoint.add, so the MP + # system's breadcrumb-based resume logic must not be triggered. + state.settings.resume_after = None try: injectables = _build_calibration_injectables(state) @@ -1792,7 +1843,7 @@ def _run_multiprocess_with_overrides( injectables, shared_data_buffers=shared_data_buffers, skip_final_checkpoint=True, - force_resume=resume_after is not None, + force_resume=resume_after is not None and not can_reuse_subprocs, ) finally: state.settings.models = original_models From cc344c7c78f0fec28f3b1f085fd1dc867f94227a Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 5 Aug 2026 13:41:01 -0500 Subject: [PATCH 50/90] Modify breadcrumb logic --- activitysim/core/calibration.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 411e063ac9..b75171febe 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1803,11 +1803,19 @@ def _run_multiprocess_with_overrides( state.settings.multiprocess_steps = calibration_mp_steps if can_reuse_subprocs and resume_after: - # Enable the normal resume mechanism: set resume_after in settings - # and write breadcrumbs so get_run_list can properly populate - # step_info["resume_after"]. Apportion will be skipped (prior - # subprocess pipelines reused), and subprocesses will restore from - # their existing model-level checkpoint. + # Include resume_after in the models list so get_breadcrumbs can + # locate the step containing it. Subprocesses will skip this model + # (it's already checkpointed in their pipeline) and run the rest. + models = [resume_after] + models + + # Rebuild steps with resume_after included. + calibration_mp_steps = _build_calibration_mp_steps( + models=models, + original_steps=original_mp_steps, + all_models=original_models, + ) + state.settings.models = models + state.settings.multiprocess_steps = calibration_mp_steps state.settings.resume_after = resume_after # Write minimal breadcrumbs indicating the step containing From fcc65672d3387e97e5fa245c70ded4ce89249038 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 08:59:27 -0500 Subject: [PATCH 51/90] Add RNG channel reloading --- activitysim/core/calibration.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index b75171febe..201c0abd28 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1744,6 +1744,8 @@ def _subprocess_path(proc_name): tables[table_name] = pd.concat(dfs, sort=False) # Load into parent state + prior_rng_channels = list(state.get_injectable("rng_channels", [])) + state.init_state() if state.checkpoint.store_is_open(): state.checkpoint.close_store() @@ -1752,6 +1754,8 @@ def _subprocess_path(proc_name): for table_name, df in tables.items(): state.add_table(table_name, df) + _reregister_rng_channels(state, prior_rng_channels) + # Mark all tables dirty for subsequent checkpoint.add for table_name in list(state.existing_table_names): state.existing_table_status[table_name] = True @@ -1859,6 +1863,19 @@ def _run_multiprocess_with_overrides( state.settings.multiprocess_steps = original_mp_steps +def _reregister_rng_channels(state: workflow.State, prior_channels: list[str]) -> None: + """Re-register RNG channels that were lost during init_state().""" + current_channels = set(state.get_injectable("rng_channels", [])) + for channel_name in prior_channels: + if channel_name not in current_channels and state.is_table(channel_name): + try: + state.rng().add_channel(channel_name, state.get_dataframe(channel_name)) + current_channels.add(channel_name) + except Exception: + pass + state.add_injectable("rng_channels", list(current_channels)) + + def _restore_parent_state_from_pipeline( state: workflow.State, checkpoint_name: str = "_" ) -> None: @@ -1882,10 +1899,17 @@ def _restore_parent_state_from_pipeline( subprocesses can load them from a direct file path without relying on checkpoint backtracking through potentially ambiguous checkpoint history. """ + # Capture RNG channels before restore — models may have dynamically + # added channels (e.g. "vehicles") that aren't in the default + # rng_channels injectable and would be lost by init_state(). + prior_rng_channels = list(state.get_injectable("rng_channels", [])) + if state.checkpoint.store_is_open(): state.checkpoint.close_store() state.checkpoint.restore(resume_after=checkpoint_name) + _reregister_rng_channels(state, prior_rng_channels) + # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. # This ensures apportion subprocesses find table files at a single, From a854b67560445c09880e617eb3dcba25e4e847ad Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 10:01:17 -0400 Subject: [PATCH 52/90] Make survey file optional --- activitysim/core/calibration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 201c0abd28..9048fb7b40 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -90,7 +90,7 @@ class CalibrationComponentSettings(PydanticBase): helper_module: str | None = None submodel_max_iterations: int = 1 reports: CalibrationReportsSettings = CalibrationReportsSettings() - survey_file: str + survey_file: Optional[str] = None class CalibrationConfig(PydanticReadable): From 1b4eb8bcdffed4438c89193aa19c155437f72434 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 10:15:17 -0500 Subject: [PATCH 53/90] Add single-process channel restoration --- activitysim/core/calibration.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 9048fb7b40..8c67ace215 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1504,12 +1504,18 @@ def _run_in_configured_mode( state.checkpoint.add(models[-1]) return + # Single-process: checkpoint.restore inside state.run calls init_state() + # which loses dynamically-added RNG channels (e.g. vehicles). + prior_rng_channels = list(state.get_injectable("rng_channels", [])) + state.run( models=models, resume_after=resume_after, memory_sidecar_process=memory_sidecar_process, ) + _reregister_rng_channels(state, prior_rng_channels) + def _prep_model_data(state, resume_after=None): """Restore the pipeline to the correct state before running models. From 7b66d4f98eabd5589f5911ed890f802f145a47e8 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 10:51:23 -0500 Subject: [PATCH 54/90] Update rng_channels --- activitysim/core/calibration.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 8c67ace215..f75993e2c4 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1504,9 +1504,11 @@ def _run_in_configured_mode( state.checkpoint.add(models[-1]) return - # Single-process: checkpoint.restore inside state.run calls init_state() - # which loses dynamically-added RNG channels (e.g. vehicles). - prior_rng_channels = list(state.get_injectable("rng_channels", [])) + # Ensure rng_channels injectable includes all currently-registered + # channels (not just the defaults). checkpoint.load reads this injectable + # after init_state() to re-register channels; without this, dynamically- + # added channels like "vehicles" are lost mid-run. + _sync_rng_channels_injectable(state) state.run( models=models, @@ -1514,8 +1516,6 @@ def _run_in_configured_mode( memory_sidecar_process=memory_sidecar_process, ) - _reregister_rng_channels(state, prior_rng_channels) - def _prep_model_data(state, resume_after=None): """Restore the pipeline to the correct state before running models. @@ -1869,6 +1869,16 @@ def _run_multiprocess_with_overrides( state.settings.multiprocess_steps = original_mp_steps +def _sync_rng_channels_injectable(state: workflow.State) -> None: + """Update rng_channels injectable to include all registered channels.""" + rng = state.rng() + if hasattr(rng, "channels"): + all_channels = list( + set(state.get_injectable("rng_channels", [])) | set(rng.channels.keys()) + ) + state.add_injectable("rng_channels", all_channels) + + def _reregister_rng_channels(state: workflow.State, prior_channels: list[str]) -> None: """Re-register RNG channels that were lost during init_state().""" current_channels = set(state.get_injectable("rng_channels", [])) From c0c937daced2b68f85f2b8a72de635012b825095 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 11:24:37 -0500 Subject: [PATCH 55/90] Modify single-process run approach --- activitysim/core/calibration.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index f75993e2c4..0c80e7cd7d 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -496,7 +496,17 @@ def _calibrate_component( shared_data_buffers=shared_data_buffers, ) else: - state.run(models=[run_model_name], resume_after=prior_step) + # Restore to prior_step ourselves then run the model directly. + # state.run(resume_after=prior_step) would trigger + # checkpoint.restore → init_state which creates a fresh RNG. + # If prior_step is before the calibrated model created its table + # (e.g. vehicles), the table won't be in that checkpoint and the + # RNG channel won't be registered — causing a crash when the + # model tries to use it. By restoring here and calling by_name, + # we keep the RNG channels from _prep_model_data intact. + _prep_model_data(state, resume_after=prior_step) + state.checkpoint.add(prior_step) + state.run.by_name(run_model_name) eval_context = _build_expression_context( state, helper_symbols, component_name, component_settings From cf2c743fd662fc673098e9f587ca978dca37b9b9 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 12:24:15 -0500 Subject: [PATCH 56/90] handle index_to_channel --- activitysim/core/calibration.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 0c80e7cd7d..3ca1aa972e 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1761,6 +1761,7 @@ def _subprocess_path(proc_name): # Load into parent state prior_rng_channels = list(state.get_injectable("rng_channels", [])) + prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} state.init_state() if state.checkpoint.store_is_open(): @@ -1770,7 +1771,7 @@ def _subprocess_path(proc_name): for table_name, df in tables.items(): state.add_table(table_name, df) - _reregister_rng_channels(state, prior_rng_channels) + _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) # Mark all tables dirty for subsequent checkpoint.add for table_name in list(state.existing_table_names): @@ -1880,16 +1881,18 @@ def _run_multiprocess_with_overrides( def _sync_rng_channels_injectable(state: workflow.State) -> None: - """Update rng_channels injectable to include all registered channels.""" + """Update rng_channels injectable and preserve index_to_channel mapping.""" rng = state.rng() if hasattr(rng, "channels"): all_channels = list( set(state.get_injectable("rng_channels", [])) | set(rng.channels.keys()) ) state.add_injectable("rng_channels", all_channels) + if hasattr(rng, "index_to_channel"): + state.add_injectable("_prior_index_to_channel", dict(rng.index_to_channel)) -def _reregister_rng_channels(state: workflow.State, prior_channels: list[str]) -> None: +def _reregister_rng_channels(state: workflow.State, prior_channels: list[str], prior_index_to_channel: dict[str, str] = None) -> None: """Re-register RNG channels that were lost during init_state().""" current_channels = set(state.get_injectable("rng_channels", [])) for channel_name in prior_channels: @@ -1899,6 +1902,20 @@ def _reregister_rng_channels(state: workflow.State, prior_channels: list[str]) - current_channels.add(channel_name) except Exception: pass + # Pre-register empty channels for index_to_channel mappings that were + # lost but whose table doesn't exist yet (e.g. vehicles before vehicle + # type choice runs). The model will extend the domain via add_channel. + if prior_index_to_channel: + for index_name, channel_name in prior_index_to_channel.items(): + if index_name not in state.rng().index_to_channel: + if channel_name not in state.rng().channels: + empty_df = pd.DataFrame( + index=pd.Index([], dtype="int64", name=index_name) + ) + state.rng().add_channel(channel_name, empty_df) + else: + state.rng().index_to_channel[index_name] = channel_name + current_channels.add(channel_name) state.add_injectable("rng_channels", list(current_channels)) @@ -1925,16 +1942,17 @@ def _restore_parent_state_from_pipeline( subprocesses can load them from a direct file path without relying on checkpoint backtracking through potentially ambiguous checkpoint history. """ - # Capture RNG channels before restore — models may have dynamically + # Capture RNG state before restore — models may have dynamically # added channels (e.g. "vehicles") that aren't in the default # rng_channels injectable and would be lost by init_state(). prior_rng_channels = list(state.get_injectable("rng_channels", [])) + prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} if state.checkpoint.store_is_open(): state.checkpoint.close_store() state.checkpoint.restore(resume_after=checkpoint_name) - _reregister_rng_channels(state, prior_rng_channels) + _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. From ec6e2695554b38ad4df4caaf24970de51b7be905 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 13:55:45 -0500 Subject: [PATCH 57/90] Modify configured mode to use same path --- activitysim/core/calibration.py | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 3ca1aa972e..64a4e0fa4b 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1514,17 +1514,13 @@ def _run_in_configured_mode( state.checkpoint.add(models[-1]) return - # Ensure rng_channels injectable includes all currently-registered - # channels (not just the defaults). checkpoint.load reads this injectable - # after init_state() to re-register channels; without this, dynamically- - # added channels like "vehicles" are lost mid-run. - _sync_rng_channels_injectable(state) - - state.run( - models=models, - resume_after=resume_after, - memory_sidecar_process=memory_sidecar_process, - ) + # Run models individually via by_name, avoiding state.run()'s internal + # checkpoint.restore which would create a fresh RNG and lose channels + # for tables not yet created at the resume_after checkpoint (e.g. vehicles). + _prep_model_data(state, resume_after=resume_after) + state.checkpoint.add(resume_after or models[0]) + for model in models: + state.run.by_name(model) def _prep_model_data(state, resume_after=None): @@ -1880,18 +1876,6 @@ def _run_multiprocess_with_overrides( state.settings.multiprocess_steps = original_mp_steps -def _sync_rng_channels_injectable(state: workflow.State) -> None: - """Update rng_channels injectable and preserve index_to_channel mapping.""" - rng = state.rng() - if hasattr(rng, "channels"): - all_channels = list( - set(state.get_injectable("rng_channels", [])) | set(rng.channels.keys()) - ) - state.add_injectable("rng_channels", all_channels) - if hasattr(rng, "index_to_channel"): - state.add_injectable("_prior_index_to_channel", dict(rng.index_to_channel)) - - def _reregister_rng_channels(state: workflow.State, prior_channels: list[str], prior_index_to_channel: dict[str, str] = None) -> None: """Re-register RNG channels that were lost during init_state().""" current_channels = set(state.get_injectable("rng_channels", [])) From c287efb904fafe8fbe703eae1f9c712a6db2c44b Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 14:04:40 -0500 Subject: [PATCH 58/90] Remove memory sidecar process from calib --- activitysim/cli/run.py | 1 - activitysim/core/calibration.py | 12 ------------ 2 files changed, 13 deletions(-) diff --git a/activitysim/cli/run.py b/activitysim/cli/run.py index 289a27914f..4d7a1929f4 100644 --- a/activitysim/cli/run.py +++ b/activitysim/cli/run.py @@ -424,7 +424,6 @@ def run(args): calibration_result = calibration.run_calibration_loop( state=state, models=state.settings.models, - memory_sidecar_process=memory_sidecar_process, ) logger.info( diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 64a4e0fa4b..2fcb5998b8 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -150,7 +150,6 @@ def calibration_enabled(state: workflow.State) -> bool: def run_calibration_loop( state: workflow.State, models: list[str], - memory_sidecar_process=None, ) -> CalibrationRunResult: """ Run the global calibration workflow. @@ -216,7 +215,6 @@ def run_calibration_loop( else models[first_model_idx:first_calib_model_idx], resume_after=state.settings.resume_after, global_iter=start_global_iter, - memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) else: @@ -235,7 +233,6 @@ def run_calibration_loop( state, models=extra_models, resume_after=None, - memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) else: @@ -280,7 +277,6 @@ def run_calibration_loop( + 1 : models.index(component) ], resume_after=last_calibrated_component, - memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) @@ -324,7 +320,6 @@ def run_calibration_loop( if global_iter == start_global_iter and first_model_idx > last_calib_model_idx else last_calibrated_component, - memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) @@ -359,7 +354,6 @@ def _run_precursor_components( models: list[str], resume_after: str, global_iter: int, - memory_sidecar_process=None, shared_data_buffers: dict | None = None, ) -> None: """Run the normal ActivitySim model flow for one global calibration iteration.""" @@ -377,7 +371,6 @@ def _run_precursor_components( state, models=models, resume_after=resume_after, - memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) @@ -386,7 +379,6 @@ def _run_intermediate_components( state: workflow.State, models: list[str], resume_after: str, - memory_sidecar_process=None, shared_data_buffers: dict | None = None, ) -> None: if len(models) == 0: @@ -395,7 +387,6 @@ def _run_intermediate_components( state, models=models, resume_after=resume_after, - memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) @@ -404,14 +395,12 @@ def _run_subsequent_components( state: workflow.State, models: list[str], resume_after: str, - memory_sidecar_process=None, shared_data_buffers: dict | None = None, ) -> None: _run_in_configured_mode( state, models=models, resume_after=resume_after, - memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) @@ -1462,7 +1451,6 @@ def _run_in_configured_mode( state: workflow.State, models: list[str], resume_after: str | None, - memory_sidecar_process=None, shared_data_buffers: dict | None = None, ) -> None: """Run models using the same single/multiprocess mode as the parent run.""" From 52ee5cd3b9ccc1de75efe12e9e3b8e3a4211bad7 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 15:01:00 -0500 Subject: [PATCH 59/90] Add reloading of tables in reregister_rng --- activitysim/core/calibration.py | 62 ++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 2fcb5998b8..65008ab9e0 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1864,7 +1864,12 @@ def _run_multiprocess_with_overrides( state.settings.multiprocess_steps = original_mp_steps -def _reregister_rng_channels(state: workflow.State, prior_channels: list[str], prior_index_to_channel: dict[str, str] = None) -> None: +def _reregister_rng_channels( + state: workflow.State, + prior_channels: list[str], + prior_index_to_channel: dict[str, str] = None, + table_checkpoint_map: dict[str, str] = None, +) -> None: """Re-register RNG channels that were lost during init_state().""" current_channels = set(state.get_injectable("rng_channels", [])) for channel_name in prior_channels: @@ -1874,19 +1879,31 @@ def _reregister_rng_channels(state: workflow.State, prior_channels: list[str], p current_channels.add(channel_name) except Exception: pass - # Pre-register empty channels for index_to_channel mappings that were - # lost but whose table doesn't exist yet (e.g. vehicles before vehicle - # type choice runs). The model will extend the domain via add_channel. + # Re-register channels whose tables don't exist at the restored checkpoint + # but DO exist elsewhere in the pipeline store (from a prior iteration). if prior_index_to_channel: for index_name, channel_name in prior_index_to_channel.items(): if index_name not in state.rng().index_to_channel: - if channel_name not in state.rng().channels: - empty_df = pd.DataFrame( - index=pd.Index([], dtype="int64", name=index_name) - ) - state.rng().add_channel(channel_name, empty_df) - else: - state.rng().index_to_channel[index_name] = channel_name + # Try loading the actual table from the store at its last-written checkpoint + loaded = False + if table_checkpoint_map and channel_name in table_checkpoint_map: + cp_name = table_checkpoint_map[channel_name] + if cp_name and state.checkpoint.store_is_open(): + try: + df = state.checkpoint._read_df(channel_name, checkpoint_name=cp_name) + state.rng().add_channel(channel_name, df) + loaded = True + except Exception: + pass + if not loaded: + # Fallback: register with empty domain; model must call add_channel + if channel_name not in state.rng().channels: + empty_df = pd.DataFrame( + index=pd.Index([], dtype="int64", name=index_name) + ) + state.rng().add_channel(channel_name, empty_df) + else: + state.rng().index_to_channel[index_name] = channel_name current_channels.add(channel_name) state.add_injectable("rng_channels", list(current_channels)) @@ -1920,11 +1937,32 @@ def _restore_parent_state_from_pipeline( prior_rng_channels = list(state.get_injectable("rng_channels", [])) prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} + # Capture where each table was last written BEFORE the truncating restore, + # so we can load actual table data for RNG channels that won't exist at + # the restored checkpoint (e.g. vehicles before vehicle_type_choice runs). + table_checkpoint_map = {} + try: + if not state.checkpoint.store_is_open(): + state.checkpoint.open_store(overwrite=False, mode="r") + _opened = True + else: + _opened = False + from activitysim.core.workflow.checkpoint import CHECKPOINT_TABLE_NAME, CHECKPOINT_NAME, NON_TABLE_COLUMNS + full_cp_df = state.checkpoint.store.get_dataframe(CHECKPOINT_TABLE_NAME) + last_row = full_cp_df.iloc[-1] + for col in last_row.index: + if col not in NON_TABLE_COLUMNS and last_row[col]: + table_checkpoint_map[col] = last_row[col] + if _opened: + state.checkpoint.close_store() + except Exception: + pass + if state.checkpoint.store_is_open(): state.checkpoint.close_store() state.checkpoint.restore(resume_after=checkpoint_name) - _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) + _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel, table_checkpoint_map) # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. From af641ed1c4def316ca944ffed7fb1f5b266a187e Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 16:07:57 -0500 Subject: [PATCH 60/90] Maybe a different approach to checkpoint restoration --- activitysim/core/calibration.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 65008ab9e0..32b1699c05 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1937,26 +1937,15 @@ def _restore_parent_state_from_pipeline( prior_rng_channels = list(state.get_injectable("rng_channels", [])) prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} - # Capture where each table was last written BEFORE the truncating restore, - # so we can load actual table data for RNG channels that won't exist at - # the restored checkpoint (e.g. vehicles before vehicle_type_choice runs). + # Build map of table_name → checkpoint_name from the in-memory checkpoint + # history (which still has entries from the prior iteration, before truncation). table_checkpoint_map = {} - try: - if not state.checkpoint.store_is_open(): - state.checkpoint.open_store(overwrite=False, mode="r") - _opened = True - else: - _opened = False - from activitysim.core.workflow.checkpoint import CHECKPOINT_TABLE_NAME, CHECKPOINT_NAME, NON_TABLE_COLUMNS - full_cp_df = state.checkpoint.store.get_dataframe(CHECKPOINT_TABLE_NAME) - last_row = full_cp_df.iloc[-1] - for col in last_row.index: - if col not in NON_TABLE_COLUMNS and last_row[col]: - table_checkpoint_map[col] = last_row[col] - if _opened: - state.checkpoint.close_store() - except Exception: - pass + from activitysim.core.workflow.checkpoint import NON_TABLE_COLUMNS + if state.checkpoint.checkpoints: + last_entry = state.checkpoint.checkpoints[-1] + for key, val in last_entry.items(): + if key not in NON_TABLE_COLUMNS and val: + table_checkpoint_map[key] = val if state.checkpoint.store_is_open(): state.checkpoint.close_store() From b38b329d81d8ad321cc6064d4f642a17b8e35ebe Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 17:38:12 -0500 Subject: [PATCH 61/90] Restore correct checkpoint --- activitysim/core/calibration.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 32b1699c05..eeb43955e5 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1937,13 +1937,12 @@ def _restore_parent_state_from_pipeline( prior_rng_channels = list(state.get_injectable("rng_channels", [])) prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} - # Build map of table_name → checkpoint_name from the in-memory checkpoint - # history (which still has entries from the prior iteration, before truncation). + # Build map of table_name → checkpoint_name by scanning the full in-memory + # checkpoint history for the last non-empty value for each table. table_checkpoint_map = {} from activitysim.core.workflow.checkpoint import NON_TABLE_COLUMNS - if state.checkpoint.checkpoints: - last_entry = state.checkpoint.checkpoints[-1] - for key, val in last_entry.items(): + for entry in state.checkpoint.checkpoints: + for key, val in entry.items(): if key not in NON_TABLE_COLUMNS and val: table_checkpoint_map[key] = val From b4520562d39c2540d2baa341e8c8ce69229a57cb Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 10:44:29 -0500 Subject: [PATCH 62/90] Fix add_table bug --- activitysim/core/calibration.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index eeb43955e5..194e23a97a 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1502,10 +1502,10 @@ def _run_in_configured_mode( state.checkpoint.add(models[-1]) return - # Run models individually via by_name, avoiding state.run()'s internal - # checkpoint.restore which would create a fresh RNG and lose channels - # for tables not yet created at the resume_after checkpoint (e.g. vehicles). - _prep_model_data(state, resume_after=resume_after) + # State is already at the correct point from _prep_model_data above. + # Do NOT call _prep_model_data again — the second call would build its + # table_checkpoint_map from the now-truncated in-memory checkpoint history, + # losing references to tables created after resume_after (e.g. vehicles). state.checkpoint.add(resume_after or models[0]) for model in models: state.run.by_name(model) @@ -1892,6 +1892,11 @@ def _reregister_rng_channels( try: df = state.checkpoint._read_df(channel_name, checkpoint_name=cp_name) state.rng().add_channel(channel_name, df) + # Also add the table to state so that @workflow.table + # factories are not re-triggered. Without this, the + # factory would call add_channel again with the same + # indices, hitting the disjoint-index assertion. + state.add_table(channel_name, df) loaded = True except Exception: pass From d169bd488703196ffe98dc1df589a6cfcfbe469c Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 11:11:54 -0500 Subject: [PATCH 63/90] Different approach to checkpoint/channel restoration --- activitysim/core/calibration.py | 52 +++++++++------------------------ 1 file changed, 14 insertions(+), 38 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 194e23a97a..1b4021f072 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1868,7 +1868,6 @@ def _reregister_rng_channels( state: workflow.State, prior_channels: list[str], prior_index_to_channel: dict[str, str] = None, - table_checkpoint_map: dict[str, str] = None, ) -> None: """Re-register RNG channels that were lost during init_state().""" current_channels = set(state.get_injectable("rng_channels", [])) @@ -1879,36 +1878,22 @@ def _reregister_rng_channels( current_channels.add(channel_name) except Exception: pass - # Re-register channels whose tables don't exist at the restored checkpoint - # but DO exist elsewhere in the pipeline store (from a prior iteration). + # For channels whose tables don't exist at the restored checkpoint, + # register an empty channel. Do NOT pre-load from a later checkpoint + # in the store — that data may include modifications from downstream + # models and would pollute the pre-model state. The empty channel + # allows the model's normal table factory to create the table fresh + # and extend the channel without hitting the disjoint-index assertion. if prior_index_to_channel: for index_name, channel_name in prior_index_to_channel.items(): if index_name not in state.rng().index_to_channel: - # Try loading the actual table from the store at its last-written checkpoint - loaded = False - if table_checkpoint_map and channel_name in table_checkpoint_map: - cp_name = table_checkpoint_map[channel_name] - if cp_name and state.checkpoint.store_is_open(): - try: - df = state.checkpoint._read_df(channel_name, checkpoint_name=cp_name) - state.rng().add_channel(channel_name, df) - # Also add the table to state so that @workflow.table - # factories are not re-triggered. Without this, the - # factory would call add_channel again with the same - # indices, hitting the disjoint-index assertion. - state.add_table(channel_name, df) - loaded = True - except Exception: - pass - if not loaded: - # Fallback: register with empty domain; model must call add_channel - if channel_name not in state.rng().channels: - empty_df = pd.DataFrame( - index=pd.Index([], dtype="int64", name=index_name) - ) - state.rng().add_channel(channel_name, empty_df) - else: - state.rng().index_to_channel[index_name] = channel_name + if channel_name not in state.rng().channels: + empty_df = pd.DataFrame( + index=pd.Index([], dtype="int64", name=index_name) + ) + state.rng().add_channel(channel_name, empty_df) + else: + state.rng().index_to_channel[index_name] = channel_name current_channels.add(channel_name) state.add_injectable("rng_channels", list(current_channels)) @@ -1942,20 +1927,11 @@ def _restore_parent_state_from_pipeline( prior_rng_channels = list(state.get_injectable("rng_channels", [])) prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} - # Build map of table_name → checkpoint_name by scanning the full in-memory - # checkpoint history for the last non-empty value for each table. - table_checkpoint_map = {} - from activitysim.core.workflow.checkpoint import NON_TABLE_COLUMNS - for entry in state.checkpoint.checkpoints: - for key, val in entry.items(): - if key not in NON_TABLE_COLUMNS and val: - table_checkpoint_map[key] = val - if state.checkpoint.store_is_open(): state.checkpoint.close_store() state.checkpoint.restore(resume_after=checkpoint_name) - _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel, table_checkpoint_map) + _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. From 2eb889dfed8fbad4ebf1d342212f32cfa54a7e66 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 11:59:46 -0500 Subject: [PATCH 64/90] Add table invalidation --- activitysim/core/calibration.py | 60 +++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 1b4021f072..94fd2fb6f7 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -74,6 +74,11 @@ class CalibrationRunSettings(PydanticBase): calibrate_models: list[str] global_iterations: int = 1 complete_steps: bool = False + invalidate_tables: list[str] | None = None + """Tables to drop after each restore so their @workflow.table factories + regenerate from current data. If None (default), automatically detects + factory tables that depend on other tables and have RNG channels (e.g. + vehicles). Set to an explicit list to override.""" class CalibrationReportsSettings(PydanticBase): @@ -1898,6 +1903,56 @@ def _reregister_rng_channels( state.add_injectable("rng_channels", list(current_channels)) +def _invalidate_derived_tables(state: workflow.State) -> None: + """Drop factory-produced tables that may be stale after a calibration restore. + + When a calibrated model (e.g. auto_ownership) changes a table that a + @workflow.table factory depends on (e.g. vehicles depends on households), + the checkpoint may contain a stale version of that factory table. Dropping + it forces the factory to regenerate from current data on next access. + + Auto-detection rule: invalidate any table that is (a) registered as a + @workflow.table factory, (b) has DataFrame parameters (= table dependencies), + and (c) is in RANDOM_CHANNELS. This currently matches only 'vehicles' but + will automatically cover future factory tables with the same pattern. + """ + settings = read_calibration_settings(state) + if not settings: + return + + tables_to_invalidate = settings.run.invalidate_tables + if not tables_to_invalidate: + tables_to_invalidate = _detect_derived_rng_tables(state) + + for table_name in tables_to_invalidate: + if state.is_table(table_name): + state.drop_table(table_name) + state.rng().drop_channel(table_name) + logger.debug("calibration: invalidated derived table '%s'", table_name) + + +def _detect_derived_rng_tables(state: workflow.State) -> list[str]: + """Identify factory tables with table dependencies and RNG channels.""" + import inspect + + from activitysim.abm.models.util.canonical_ids import RANDOM_CHANNELS + + result = [] + for table_name, factory_func in state._LOADABLE_TABLES.items(): + if table_name not in RANDOM_CHANNELS: + continue + sig = inspect.signature(factory_func) + has_table_dep = any( + p.annotation is pd.DataFrame + or (p.annotation is inspect.Parameter.empty and p.name != "state") + for p in sig.parameters.values() + if p.name != "state" + ) + if has_table_dep: + result.append(table_name) + return result + + def _restore_parent_state_from_pipeline( state: workflow.State, checkpoint_name: str = "_" ) -> None: @@ -1933,6 +1988,11 @@ def _restore_parent_state_from_pipeline( _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) + # Drop derived tables so their factories regenerate from current data. + # Without this, a stale vehicles table (based on old auto_ownership values) + # would be loaded from the checkpoint and never refreshed. + _invalidate_derived_tables(state) + # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. # This ensures apportion subprocesses find table files at a single, From 67b87a421abcce285e8da0763af6e5c4751b05de Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 13:08:21 -0500 Subject: [PATCH 65/90] Additional precursor guard --- activitysim/core/calibration.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 94fd2fb6f7..5ab0badf2b 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -240,6 +240,25 @@ def run_calibration_loop( resume_after=None, shared_data_buffers=shared_data_buffers, ) + elif not any( + cp.get("checkpoint_name") == state.settings.resume_after + for cp in state.checkpoint.checkpoints + ): + # _prep_model_data took its fallback path — the pipeline either + # doesn't exist or doesn't contain resume_after's checkpoint. + # The restored state is incomplete (precursor models never ran). + logger.warning( + "calibration: resume_after=%r not found in restored pipeline; " + "running precursor models", + state.settings.resume_after, + ) + _run_precursor_components( + state, + models=models[:first_calib_model_idx], + resume_after=None, + global_iter=start_global_iter, + shared_data_buffers=shared_data_buffers, + ) else: state.checkpoint.add(state.settings.resume_after) state.checkpoint.close_store() From 8a19f6c817e4826888800430cfb3754f415c8c9d Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 13:10:16 -0500 Subject: [PATCH 66/90] Blacken --- activitysim/core/calibration.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 5ab0badf2b..82d19f8caf 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1769,7 +1769,11 @@ def _subprocess_path(proc_name): # Load into parent state prior_rng_channels = list(state.get_injectable("rng_channels", [])) - prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} + prior_index_to_channel = ( + dict(state.rng().index_to_channel) + if hasattr(state.rng(), "index_to_channel") + else {} + ) state.init_state() if state.checkpoint.store_is_open(): @@ -1999,7 +2003,11 @@ def _restore_parent_state_from_pipeline( # added channels (e.g. "vehicles") that aren't in the default # rng_channels injectable and would be lost by init_state(). prior_rng_channels = list(state.get_injectable("rng_channels", [])) - prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} + prior_index_to_channel = ( + dict(state.rng().index_to_channel) + if hasattr(state.rng(), "index_to_channel") + else {} + ) if state.checkpoint.store_is_open(): state.checkpoint.close_store() From 21a0d7854b867ecc103d166cc67d713fe8d65211 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 13:42:27 -0500 Subject: [PATCH 67/90] Fix calibrate_component precursor calls --- activitysim/core/calibration.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 82d19f8caf..cfd69dac9d 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -517,7 +517,12 @@ def _calibrate_component( # RNG channel won't be registered — causing a crash when the # model tries to use it. By restoring here and calling by_name, # we keep the RNG channels from _prep_model_data intact. - _prep_model_data(state, resume_after=prior_step) + extra_models = _prep_model_data(state, resume_after=prior_step) + if extra_models: + # prior_step checkpoint not found directly; run intermediate + # models (e.g. annotators) to recreate the correct state. + for m in extra_models: + state.run.by_name(m) state.checkpoint.add(prior_step) state.run.by_name(run_model_name) From 0437b7081c74ebf5898f611d8efd6afd3b691a1f Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 14:33:00 -0500 Subject: [PATCH 68/90] Add temporary diagnostic tracing --- activitysim/core/calibration.py | 67 +++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index cfd69dac9d..8d768a35f6 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -524,6 +524,34 @@ def _calibrate_component( for m in extra_models: state.run.by_name(m) state.checkpoint.add(prior_step) + + # Diagnostic: verify households state before running calibrated model + if state.is_table("households"): + _hh = state.get_dataframe("households") + logger.debug( + "calibration: households before %s has %d rows, columns: %s", + run_model_name, + len(_hh), + list(_hh.columns), + ) + if "num_drivers" not in _hh.columns: + logger.error( + "calibration: households missing 'num_drivers' before %s. " + "prior_step=%r, extra_models=%r, checkpoint_names=%s", + run_model_name, + prior_step, + extra_models, + [ + cp.get("checkpoint_name", "") + for cp in state.checkpoint.checkpoints + ], + ) + else: + logger.error( + "calibration: households table not in state before %s", + run_model_name, + ) + state.run.by_name(run_model_name) eval_context = _build_expression_context( @@ -2018,13 +2046,52 @@ def _restore_parent_state_from_pipeline( state.checkpoint.close_store() state.checkpoint.restore(resume_after=checkpoint_name) + # DEBUG: trace where num_drivers disappears + _trace_col = "num_drivers" + if state.is_table("households"): + _hh = state.get_dataframe("households") + logger.debug( + "calibration TRACE [after restore]: households has %d cols, " + "%s present=%s, checkpoint=%r", + len(_hh.columns), + _trace_col, + _trace_col in _hh.columns, + checkpoint_name, + ) + else: + logger.debug( + "calibration TRACE [after restore]: households NOT in state, checkpoint=%r", + checkpoint_name, + ) + _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) + if state.is_table("households"): + _hh = state.get_dataframe("households") + logger.debug( + "calibration TRACE [after _reregister_rng_channels]: %s present=%s", + _trace_col, + _trace_col in _hh.columns, + ) + # Drop derived tables so their factories regenerate from current data. # Without this, a stale vehicles table (based on old auto_ownership values) # would be loaded from the checkpoint and never refreshed. _invalidate_derived_tables(state) + if state.is_table("households"): + _hh = state.get_dataframe("households") + logger.debug( + "calibration TRACE [after _invalidate_derived_tables]: %s present=%s", + _trace_col, + _trace_col in _hh.columns, + ) + else: + logger.debug( + "calibration TRACE [after _invalidate_derived_tables]: " + "households NOT in state (was it invalidated?)" + ) + # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. # This ensures apportion subprocesses find table files at a single, From 2f588283e934cf2156b6e3dbea0b3d71882394f2 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 14:58:49 -0500 Subject: [PATCH 69/90] More diagnostic logging --- activitysim/core/calibration.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 8d768a35f6..8ef1abad96 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1980,12 +1980,24 @@ def _invalidate_derived_tables(state: workflow.State) -> None: if not tables_to_invalidate: tables_to_invalidate = _detect_derived_rng_tables(state) + logger.debug( + "calibration: tables detected for invalidation: %s", tables_to_invalidate + ) + tables_before = set(state.existing_table_names) + for table_name in tables_to_invalidate: if state.is_table(table_name): state.drop_table(table_name) state.rng().drop_channel(table_name) logger.debug("calibration: invalidated derived table '%s'", table_name) + tables_after = set(state.existing_table_names) + lost = tables_before - tables_after - set(tables_to_invalidate) + if lost: + logger.error( + "calibration: tables unexpectedly removed during invalidation: %s", lost + ) + def _detect_derived_rng_tables(state: workflow.State) -> list[str]: """Identify factory tables with table dependencies and RNG channels.""" @@ -1998,9 +2010,15 @@ def _detect_derived_rng_tables(state: workflow.State) -> list[str]: if table_name not in RANDOM_CHANNELS: continue sig = inspect.signature(factory_func) + # Only match parameters that are actual table dependencies: + # annotated as pd.DataFrame, or positional without a default value. has_table_dep = any( p.annotation is pd.DataFrame - or (p.annotation is inspect.Parameter.empty and p.name != "state") + or ( + p.annotation is inspect.Parameter.empty + and p.default is inspect.Parameter.empty + and p.name != "state" + ) for p in sig.parameters.values() if p.name != "state" ) From 656352522fba18b98b2e65566cb946c5c305c3cb Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 15:19:10 -0500 Subject: [PATCH 70/90] Change to default tables_to_invalidate --- activitysim/core/calibration.py | 34 ++++----------------------------- 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 8ef1abad96..3bbaa45b38 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1977,8 +1977,10 @@ def _invalidate_derived_tables(state: workflow.State) -> None: return tables_to_invalidate = settings.run.invalidate_tables - if not tables_to_invalidate: - tables_to_invalidate = _detect_derived_rng_tables(state) + if tables_to_invalidate is None: + # Default: vehicles is the only RNG-channel table whose row identity + # depends on another table's values (households.auto_ownership). + tables_to_invalidate = ["vehicles"] logger.debug( "calibration: tables detected for invalidation: %s", tables_to_invalidate @@ -1999,34 +2001,6 @@ def _invalidate_derived_tables(state: workflow.State) -> None: ) -def _detect_derived_rng_tables(state: workflow.State) -> list[str]: - """Identify factory tables with table dependencies and RNG channels.""" - import inspect - - from activitysim.abm.models.util.canonical_ids import RANDOM_CHANNELS - - result = [] - for table_name, factory_func in state._LOADABLE_TABLES.items(): - if table_name not in RANDOM_CHANNELS: - continue - sig = inspect.signature(factory_func) - # Only match parameters that are actual table dependencies: - # annotated as pd.DataFrame, or positional without a default value. - has_table_dep = any( - p.annotation is pd.DataFrame - or ( - p.annotation is inspect.Parameter.empty - and p.default is inspect.Parameter.empty - and p.name != "state" - ) - for p in sig.parameters.values() - if p.name != "state" - ) - if has_table_dep: - result.append(table_name) - return result - - def _restore_parent_state_from_pipeline( state: workflow.State, checkpoint_name: str = "_" ) -> None: From 210454e553d62e5cd261ac8922462a26209da8ce Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 15:28:02 -0500 Subject: [PATCH 71/90] Change when table invalidation occurs --- activitysim/core/calibration.py | 106 +++++++++----------------------- 1 file changed, 30 insertions(+), 76 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 3bbaa45b38..86d10142f1 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -75,10 +75,33 @@ class CalibrationRunSettings(PydanticBase): global_iterations: int = 1 complete_steps: bool = False invalidate_tables: list[str] | None = None - """Tables to drop after each restore so their @workflow.table factories - regenerate from current data. If None (default), automatically detects - factory tables that depend on other tables and have RNG channels (e.g. - vehicles). Set to an explicit list to override.""" + """Tables to drop from state after each calibration restore so their + ``@workflow.table`` factories regenerate from current data. + + Default (None): invalidates ``["vehicles"]``. Set to ``[]`` to disable. + + A table should be listed here when ALL of the following are true: + + 1. It is created by a ``@workflow.table`` factory from another table's + values (not just from input data files). + 2. That source table is modified by a calibrated model or by a model + whose outputs change when calibrated coefficients change. + 3. The factory uses source-table values to determine **row identity** + (index values) or **row count**, not just column values. + + The canonical example is ``vehicles``: its factory repeats household + rows by ``households["auto_ownership"]`` and derives ``vehicle_id`` + from ``household_id``. When ``auto_ownership_simulate`` is calibrated, + different coefficients produce different ownership counts, so the + stale vehicles table loaded from a prior checkpoint would have the + wrong number of rows and wrong vehicle IDs. Dropping it forces the + factory to regenerate vehicles consistent with the current households. + + Tables that only read *column values* from upstream tables (without + affecting row identity) generally do NOT need invalidation — their + content will be correct as long as the upstream table is correct at + the restored checkpoint. + """ class CalibrationReportsSettings(PydanticBase): @@ -413,6 +436,7 @@ def _run_intermediate_components( resume_after=resume_after, shared_data_buffers=shared_data_buffers, ) + _invalidate_derived_tables(state) def _run_subsequent_components( @@ -427,6 +451,7 @@ def _run_subsequent_components( resume_after=resume_after, shared_data_buffers=shared_data_buffers, ) + _invalidate_derived_tables(state) def _calibrate_component( @@ -523,35 +548,8 @@ def _calibrate_component( # models (e.g. annotators) to recreate the correct state. for m in extra_models: state.run.by_name(m) + _invalidate_derived_tables(state) state.checkpoint.add(prior_step) - - # Diagnostic: verify households state before running calibrated model - if state.is_table("households"): - _hh = state.get_dataframe("households") - logger.debug( - "calibration: households before %s has %d rows, columns: %s", - run_model_name, - len(_hh), - list(_hh.columns), - ) - if "num_drivers" not in _hh.columns: - logger.error( - "calibration: households missing 'num_drivers' before %s. " - "prior_step=%r, extra_models=%r, checkpoint_names=%s", - run_model_name, - prior_step, - extra_models, - [ - cp.get("checkpoint_name", "") - for cp in state.checkpoint.checkpoints - ], - ) - else: - logger.error( - "calibration: households table not in state before %s", - run_model_name, - ) - state.run.by_name(run_model_name) eval_context = _build_expression_context( @@ -2038,52 +2036,8 @@ def _restore_parent_state_from_pipeline( state.checkpoint.close_store() state.checkpoint.restore(resume_after=checkpoint_name) - # DEBUG: trace where num_drivers disappears - _trace_col = "num_drivers" - if state.is_table("households"): - _hh = state.get_dataframe("households") - logger.debug( - "calibration TRACE [after restore]: households has %d cols, " - "%s present=%s, checkpoint=%r", - len(_hh.columns), - _trace_col, - _trace_col in _hh.columns, - checkpoint_name, - ) - else: - logger.debug( - "calibration TRACE [after restore]: households NOT in state, checkpoint=%r", - checkpoint_name, - ) - _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) - if state.is_table("households"): - _hh = state.get_dataframe("households") - logger.debug( - "calibration TRACE [after _reregister_rng_channels]: %s present=%s", - _trace_col, - _trace_col in _hh.columns, - ) - - # Drop derived tables so their factories regenerate from current data. - # Without this, a stale vehicles table (based on old auto_ownership values) - # would be loaded from the checkpoint and never refreshed. - _invalidate_derived_tables(state) - - if state.is_table("households"): - _hh = state.get_dataframe("households") - logger.debug( - "calibration TRACE [after _invalidate_derived_tables]: %s present=%s", - _trace_col, - _trace_col in _hh.columns, - ) - else: - logger.debug( - "calibration TRACE [after _invalidate_derived_tables]: " - "households NOT in state (was it invalidated?)" - ) - # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. # This ensures apportion subprocesses find table files at a single, From e8f6a33a6094133a62d1a219d45f271c74e6f4d5 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 16:10:27 -0500 Subject: [PATCH 72/90] Fix SP checkpointing --- activitysim/core/calibration.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 86d10142f1..e05d502c87 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1564,6 +1564,10 @@ def _run_in_configured_mode( state.checkpoint.add(resume_after or models[0]) for model in models: state.run.by_name(model) + # Ensure final model's state is persisted even if should_save_checkpoint + # returned False for it — _calibrate_component needs to restore to it. + if models: + state.checkpoint.add(models[-1]) def _prep_model_data(state, resume_after=None): From fb923bb4e52a063d7cb9639343568210cc59e362 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:01:25 -0700 Subject: [PATCH 73/90] fixing resume after and rng calib invariance --- activitysim/core/calibration.py | 90 +++++++++++++++++++++++++---- activitysim/core/workflow/runner.py | 14 ++++- 2 files changed, 91 insertions(+), 13 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index e05d502c87..a0d0ec1ce1 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -75,6 +75,15 @@ class CalibrationRunSettings(PydanticBase): global_iterations: int = 1 complete_steps: bool = False invalidate_tables: list[str] | None = None + + @model_validator(mode="after") + def validate_run_settings(self): + if not self.calibrate_models: + raise ValueError( + "calibration.run.calibrate_models must contain at least one model name" + ) + return self + """Tables to drop from state after each calibration restore so their ``@workflow.table`` factories regenerate from current data. @@ -196,9 +205,29 @@ def run_calibration_loop( "Overriding duplicate_step_execution setting: must be enabled for calibration" ) - assert all( - [c in models for c in calibration_settings.run.calibrate_models] - ), f"settings.yaml steps list does not include calibration model{'s' if len([c for c in calibration_settings.run.calibrate_models if c not in models]) != 1 else ''} {[c for c in calibration_settings.run.calibrate_models if c not in models]}" + if not calibration_settings.run.calibrate_models: + raise ValueError( + "calibration.run.calibrate_models must contain at least one model name" + ) + + missing_calibration_models = [ + component + for component in calibration_settings.run.calibrate_models + if component not in models + ] + if missing_calibration_models: + raise ValueError( + "settings.yaml models list does not include configured calibration " + f"model(s): {missing_calibration_models}" + ) + + resume_after = state.settings.resume_after + if resume_after is not None and resume_after not in models: + raise ValueError( + f"settings.yaml resume_after={resume_after!r} is not present in the " + "settings.yaml models list. Calibration requires resume_after to be " + "a model-level checkpoint name." + ) # sort calibration models into main model order calibration_settings.run.calibrate_models = sorted( @@ -206,12 +235,27 @@ def run_calibration_loop( ) first_calib_model_idx = models.index(calibration_settings.run.calibrate_models[0]) last_calib_model_idx = models.index(calibration_settings.run.calibrate_models[-1]) - first_model_idx = ( - models.index(state.settings.resume_after) + 1 - if state.settings.resume_after - else None + first_model_idx = models.index(resume_after) + 1 if resume_after else None + first_calibration_restart_step = _prior_step_name( + models, calibration_settings.run.calibrate_models[0] ) + if resume_after is not None: + skipped_calibration_models = [ + component + for component in calibration_settings.run.calibrate_models + if models.index(component) <= models.index(resume_after) + ] + if skipped_calibration_models: + logger.warning( + "Calibration is honoring settings.yaml resume_after=%r using strict " + "ActivitySim semantics. The following calibrated model(s) occur at " + "or before resume_after and will be skipped during the first global " + "iteration: %s", + resume_after, + skipped_calibration_models, + ) + _ensure_calibration_output_dir(state) # If there is recoverable calibration progress from a prior interrupted run, @@ -290,6 +334,31 @@ def run_calibration_loop( start_global_iter, start_global_iter + calibration_settings.run.global_iterations, ): + # Every global iteration after the first begins from the immutable + # checkpoint directly before the first calibrated model. This makes + # global reruns independent of the state left by the final calibrated + # model in the preceding iteration. + if global_iter > start_global_iter: + logger.info( + "Restarting global calibration iteration %s from checkpoint %r", + global_iter, + first_calibration_restart_step, + ) + extra_models = _prep_model_data( + state, resume_after=first_calibration_restart_step + ) + if extra_models: + _run_in_configured_mode( + state, + models=extra_models, + resume_after=None, + shared_data_buffers=shared_data_buffers, + ) + _invalidate_derived_tables(state) + if first_calibration_restart_step is not None: + state.checkpoint.add(first_calibration_restart_step) + state.checkpoint.close_store() + logger.info( "calibration global iteration %s/%s", global_iter - start_global_iter, @@ -528,9 +597,10 @@ def _calibrate_component( _run_mp_single_component( state, component_name=component_name, - restore_checkpoint=mp_restore_checkpoint - if component_iter == 1 - else "_", + # Always restore from the same immutable pre-component + # checkpoint. LAST_CHECKPOINT may point to the prior iteration's + # coalesced component output and is therefore not a safe baseline. + restore_checkpoint=mp_restore_checkpoint, shared_data_buffers=shared_data_buffers, ) else: diff --git a/activitysim/core/workflow/runner.py b/activitysim/core/workflow/runner.py index 79ecd0ed4f..9596e6c47c 100644 --- a/activitysim/core/workflow/runner.py +++ b/activitysim/core/workflow/runner.py @@ -270,7 +270,10 @@ def _pre_run_step(self, model_name: str) -> bool | None: f"Cannot run model '{model_name}' more than once" ) - self._obj.rng().begin_step(model_name) + # Parse the canonical workflow step before initializing the RNG. + # Arguments appended to a model invocation (for example calibration + # iteration labels) may affect logging and checkpoint names, but must + # never affect the deterministic random stream for the model itself. # check for args if "." in model_name: @@ -285,6 +288,11 @@ def _pre_run_step(self, model_name: str) -> bool | None: step_name = model_name args = {} + self.rng_step_name = ( + step_name[1:] if step_name.startswith(NO_CHECKPOINT_PREFIX) else step_name + ) + self._obj.rng().begin_step(self.rng_step_name) + # check for no_checkpoint prefix if step_name[0] == NO_CHECKPOINT_PREFIX: step_name = step_name[1:] @@ -351,7 +359,7 @@ def by_name(self, model_name, **kwargs): except Exception: self.t0 = self._log_elapsed_time(f"run.{model_name} UNTIL ERROR", self.t0) self._obj.add_injectable("step_args", None) - self._obj.rng().end_step(model_name) + self._obj.rng().end_step(self.rng_step_name) raise else: @@ -361,7 +369,7 @@ def by_name(self, model_name, **kwargs): self._obj.add_injectable("step_args", None) - self._obj.rng().end_step(model_name) + self._obj.rng().end_step(self.rng_step_name) if self.checkpoint: self._obj.checkpoint.add(model_name) else: From 92bbc84c763961d2fe8e72b53aff425f8c03b099 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:43:41 -0700 Subject: [PATCH 74/90] fixing pipeline and rng restarting issues --- activitysim/abm/models/initialize.py | 11 +++++++++-- activitysim/core/calibration.py | 8 ++++++++ activitysim/core/random.py | 5 +++++ activitysim/core/test/test_random.py | 14 +++++++++++++- activitysim/core/workflow/runner.py | 4 +++- 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/activitysim/abm/models/initialize.py b/activitysim/abm/models/initialize.py index 84fe001e90..d986e111d7 100644 --- a/activitysim/abm/models/initialize.py +++ b/activitysim/abm/models/initialize.py @@ -154,12 +154,19 @@ def initialize_households( households = state.get_dataframe("households") assert not households._is_view chunk_sizer.log_df(trace_label, "households", households) - del households - chunk_sizer.log_df(trace_label, "households", None) persons = state.get_dataframe("persons") assert not persons._is_view chunk_sizer.log_df(trace_label, "persons", persons) + + rng = state.get_rn_generator() + if "households" not in rng.channels: + rng.add_channel("households", households) + if "persons" not in rng.channels: + rng.add_channel("persons", persons) + + del households + chunk_sizer.log_df(trace_label, "households", None) del persons chunk_sizer.log_df(trace_label, "persons", None) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index a0d0ec1ce1..825cb16c51 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -258,6 +258,14 @@ def run_calibration_loop( _ensure_calibration_output_dir(state) + if state.settings.resume_after is None: + # compute_accessibility requires its accessibility table to be empty; + # unlike most model steps, it will not overwrite a prior result. + # Remove a cached result before restore clears table-status metadata, + # so the table factory recreates its empty placeholder for the replay. + state.drop_table("accessibility") + state.checkpoint.restore() + # If there is recoverable calibration progress from a prior interrupted run, # continue from that iteration. Coefficient updates are persisted in config # coefficient files, so restarting from a later global iteration is compatible diff --git a/activitysim/core/random.py b/activitysim/core/random.py index 37b1976403..1c95683b26 100644 --- a/activitysim/core/random.py +++ b/activitysim/core/random.py @@ -503,6 +503,11 @@ def drop_channel(self, channel_name): if channel_name in self.channels: logger.debug("Dropping channel '%s'" % (channel_name,)) del self.channels[channel_name] + self.index_to_channel = { + index_name: mapped_channel_name + for index_name, mapped_channel_name in self.index_to_channel.items() + if mapped_channel_name != channel_name + } else: logger.error( "drop_channel called with unknown channel '%s'" % (channel_name,) diff --git a/activitysim/core/test/test_random.py b/activitysim/core/test/test_random.py index bcbc602685..5e1a66bda4 100644 --- a/activitysim/core/test/test_random.py +++ b/activitysim/core/test/test_random.py @@ -8,7 +8,7 @@ import pytest from activitysim.core import random -from activitysim.core.exceptions import DuplicateLoadableObjectError +from activitysim.core.exceptions import DuplicateLoadableObjectError, TableIndexError def test_basic(): @@ -126,3 +126,15 @@ def test_channel(): npt.assert_almost_equal(np.asanyarray(rands).flatten(), test1_expected_rands2) rng.end_step("test_step") + + +def test_drop_channel_removes_index_mapping(): + rng = random.Random() + persons = pd.DataFrame(index=pd.Index([1], name="person_id")) + + rng.add_channel("persons", persons) + rng.drop_channel("persons") + + assert "person_id" not in rng.index_to_channel + with pytest.raises(TableIndexError, match="No channel with index name 'person_id'"): + rng.get_channel_for_df(persons) diff --git a/activitysim/core/workflow/runner.py b/activitysim/core/workflow/runner.py index 9596e6c47c..14ef972049 100644 --- a/activitysim/core/workflow/runner.py +++ b/activitysim/core/workflow/runner.py @@ -322,6 +322,7 @@ def by_name(self, model_name, **kwargs): model_name is assumed to be the name of a registered workflow step """ self.t0 = time.time() + self.rng_step_name = None try: should_skip = self._pre_run_step(model_name) if should_skip: @@ -359,7 +360,8 @@ def by_name(self, model_name, **kwargs): except Exception: self.t0 = self._log_elapsed_time(f"run.{model_name} UNTIL ERROR", self.t0) self._obj.add_injectable("step_args", None) - self._obj.rng().end_step(self.rng_step_name) + if self.rng_step_name is not None: + self._obj.rng().end_step(self.rng_step_name) raise else: From af1e21bec767f23a4bc0ca43a67a84491e789959 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:46:59 -0700 Subject: [PATCH 75/90] fixing more pipeline restart issues --- activitysim/core/calibration.py | 69 ++++++++++++++++++++------ activitysim/core/test/test_pipeline.py | 16 ++++++ activitysim/core/workflow/state.py | 3 ++ 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 825cb16c51..61785a9e09 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -20,7 +20,7 @@ import matplotlib.pyplot as plt from pydantic import model_validator -from activitysim.core import workflow +from activitysim.core import simulate, workflow from activitysim.core.configuration import PydanticReadable from activitysim.core.configuration.base import PydanticBase from activitysim.core.configuration.top import MultiprocessStep @@ -38,7 +38,7 @@ CALIBRATION_FINAL_COEFFICIENTS_FILE = "calibration/final_calibrated_coefficients.csv" DEFAULT_INCREMENT = 2.0 -MAX_COEFFS_IN_GRAPH = 10 +MAX_COEFFS_IN_GRAPH = 15 CALIBRATION_REQUIRED_COLUMNS = [ "description", @@ -513,7 +513,6 @@ def _run_intermediate_components( resume_after=resume_after, shared_data_buffers=shared_data_buffers, ) - _invalidate_derived_tables(state) def _run_subsequent_components( @@ -528,7 +527,6 @@ def _run_subsequent_components( resume_after=resume_after, shared_data_buffers=shared_data_buffers, ) - _invalidate_derived_tables(state) def _calibrate_component( @@ -721,11 +719,18 @@ def _extract_utility_coefficient_names( model_settings: dict[str, Any] | Any, ) -> set[str]: """ - Extract coefficient tokens from configured utility spec files. + Extract coefficient names used by the configured utility specifications. - The extraction scans all settings keys ending with "SPEC" and parses - tokens from utility columns (all non-description/expression columns). + Templated logit models map utility-spec row labels to actual coefficient + names by segment, so their template cell values are the source of truth. + Other models use coefficient tokens in utility-spec columns. """ + if _setting_value(model_settings, "COEFFICIENT_TEMPLATE"): + template = simulate.read_model_coefficient_template( + state.filesystem, model_settings + ) + return {str(name) for name in template.to_numpy().ravel()} + names: set[str] = set() model_settings_dict = _settings_to_dict(model_settings) @@ -1147,14 +1152,22 @@ def _append_iteration_records( # Save a global iteration history file global_path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) - _append_csv(df, global_path) + _append_csv( + df, + global_path, + unique_on=["global_iter", "component_iter", "component", "coefficient"], + ) # Also write component-local iteration history component_path = ( _component_output_dir(state, component_name) / Path(CALIBRATION_ITERATION_FILE).name ) - _append_csv(df, component_path) + _append_csv( + df, + component_path, + unique_on=["global_iter", "component_iter", "component", "coefficient"], + ) def _append_summary_records( @@ -1165,12 +1178,26 @@ def _append_summary_records( return path = state.get_output_file_path(CALIBRATION_SUMMARY_FILE) df = pd.DataFrame(records) - _append_csv(df, path) + _append_csv( + df, + path, + unique_on=["global_iter", "component_iter", "component"], + ) -def _append_csv(df: pd.DataFrame, path: Path) -> None: - """Append a dataframe to a CSV file with header-once behavior.""" +def _append_csv( + df: pd.DataFrame, path: Path, unique_on: list[str] | None = None +) -> None: + """Append a dataframe to a CSV file, replacing rows with matching keys.""" os.makedirs(path.parent, exist_ok=True) + if unique_on and path.exists(): + existing = pd.read_csv(path) + df = pd.concat([existing, df], ignore_index=True).drop_duplicates( + subset=unique_on, keep="last" + ) + df.to_csv(path, index=False) + return + write_header = not path.exists() df.to_csv(path, mode="a", index=False, header=write_header) @@ -1317,7 +1344,11 @@ def _write_generic_report( ) path = _component_output_dir(state, component_name) / "generic_report.csv" - _append_csv(report, path) + _append_csv( + report, + path, + unique_on=["global_iter", "component_iter", "component", "description"], + ) def _load_helper_symbols( @@ -2058,9 +2089,14 @@ def _invalidate_derived_tables(state: workflow.State) -> None: tables_to_invalidate = settings.run.invalidate_tables if tables_to_invalidate is None: - # Default: vehicles is the only RNG-channel table whose row identity - # depends on another table's values (households.auto_ownership). - tables_to_invalidate = ["vehicles"] + # Vehicles needs regeneration only when calibration changes + # households.auto_ownership. Downstream calibration components must + # retain vehicle_type_choice's vehicle attributes. + tables_to_invalidate = ( + ["vehicles"] + if "auto_ownership_simulate" in settings.run.calibrate_models + else [] + ) logger.debug( "calibration: tables detected for invalidation: %s", tables_to_invalidate @@ -2071,6 +2107,7 @@ def _invalidate_derived_tables(state: workflow.State) -> None: if state.is_table(table_name): state.drop_table(table_name) state.rng().drop_channel(table_name) + state.get_dataframe(table_name, as_copy=False) logger.debug("calibration: invalidated derived table '%s'", table_name) tables_after = set(state.existing_table_names) diff --git a/activitysim/core/test/test_pipeline.py b/activitysim/core/test/test_pipeline.py index 12f31dbc66..95dca6e436 100644 --- a/activitysim/core/test/test_pipeline.py +++ b/activitysim/core/test/test_pipeline.py @@ -5,6 +5,7 @@ import logging import os +import pandas as pd import pytest import tables @@ -128,6 +129,21 @@ def test_pipeline_checkpoint_drop(state): close_handlers() +def test_get_table_returns_current_table_after_recreation(state): + original = pd.DataFrame({"value": [1]}, index=pd.Index([1], name="id")) + recreated = pd.DataFrame({"value": [2]}, index=pd.Index([1], name="id")) + + state.add_table("recreated_table", original) + state.checkpoint.add("before_drop") + state.drop_table("recreated_table") + state.add_table("recreated_table", recreated) + + pd.testing.assert_frame_equal(state.get_table("recreated_table"), recreated) + + state.checkpoint.close_store() + close_handlers() + + # if __name__ == "__main__": # # print "\n\ntest_pipeline_run" diff --git a/activitysim/core/workflow/state.py b/activitysim/core/workflow/state.py index de22b0687d..fbf85addfc 100644 --- a/activitysim/core/workflow/state.py +++ b/activitysim/core/workflow/state.py @@ -1185,6 +1185,9 @@ def get_table(self, table_name, checkpoint_name=None): df : pandas.DataFrame """ + if checkpoint_name is None and table_name in self._context: + return self._context[table_name] + if table_name not in self.checkpoint.last_checkpoint and self.is_table( table_name ): From 022b4fb83111454b5c5c1900fcea1bd89d7ed8a1 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:34:08 -0700 Subject: [PATCH 76/90] appropriately handling crashing in middle of iteration --- activitysim/core/calibration.py | 210 +++++++++++++++++++++++++++---- docs/users-guide/calibration.rst | 30 +++-- 2 files changed, 207 insertions(+), 33 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 61785a9e09..cdeaa32e17 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -10,6 +10,7 @@ import multiprocessing import os import re +import shutil from dataclasses import dataclass from pathlib import Path from typing import Any, Optional @@ -36,6 +37,7 @@ CALIBRATION_ITERATION_FILE = "calibration/calibration_iteration_records.csv" CALIBRATION_SUMMARY_FILE = "calibration/calibration_iteration_summary.csv" CALIBRATION_FINAL_COEFFICIENTS_FILE = "calibration/final_calibrated_coefficients.csv" +CALIBRATION_RECOVERY_DIR = "calibration/recovery" DEFAULT_INCREMENT = 2.0 MAX_COEFFS_IN_GRAPH = 15 @@ -258,6 +260,62 @@ def run_calibration_loop( _ensure_calibration_output_dir(state) + progress = _read_progress(state) + if progress and progress.get("complete"): + logger.info( + "calibration progress is already complete; remove %s to start a " + "fresh calibration run", + CALIBRATION_PROGRESS_FILE, + ) + return CalibrationRunResult( + converged=bool(progress.get("converged", False)), + completed_global_iterations=int( + progress.get( + "last_completed_global_iteration", + calibration_settings.run.global_iterations, + ) + ), + ) + + interrupted_iteration = ( + progress.get("in_progress_iteration") if progress else None + ) + if interrupted_iteration is not None: + interrupted_iteration = int(interrupted_iteration) + logger.warning( + "recovering interrupted calibration global iteration %s", + interrupted_iteration, + ) + _restore_coefficient_backups(state, calibration_settings) + progress = { + "in_progress_iteration": None, + "next_global_iteration": interrupted_iteration, + "last_completed_global_iteration": interrupted_iteration - 1, + } + _write_progress(state, progress) + + # Progress files from earlier versions contain next_global_iteration, so + # they remain compatible with the corrected total-count semantics. + start_global_iter = int(progress.get("next_global_iteration", 1)) if progress else 1 + completed_global_iterations = start_global_iter - 1 + + if start_global_iter > calibration_settings.run.global_iterations: + logger.info( + "calibration progress already reached configured global_iterations=%s", + calibration_settings.run.global_iterations, + ) + converged = bool(progress.get("converged", False)) if progress else False + _write_final_coefficients_snapshot(state, calibration_settings) + _write_completed_progress( + state, + completed_global_iterations, + converged, + ) + return CalibrationRunResult( + converged=converged, + completed_global_iterations=completed_global_iterations, + ) + if state.settings.resume_after is None: # compute_accessibility requires its accessibility table to be empty; # unlike most model steps, it will not overwrite a prior result. @@ -266,13 +324,6 @@ def run_calibration_loop( state.drop_table("accessibility") state.checkpoint.restore() - # If there is recoverable calibration progress from a prior interrupted run, - # continue from that iteration. Coefficient updates are persisted in config - # coefficient files, so restarting from a later global iteration is compatible - # with current checkpoint semantics. - progress = _read_progress(state) - start_global_iter = int(progress.get("next_global_iteration", 1)) if progress else 1 - original_pipeline_name = state.filesystem.pipeline_file_name # Initialize shared resources for multiprocess mode (skims, shadow pricing). @@ -340,8 +391,14 @@ def run_calibration_loop( for global_iter in range( start_global_iter, - start_global_iter + calibration_settings.run.global_iterations, + calibration_settings.run.global_iterations + 1, ): + _begin_global_iteration_transaction( + state, + calibration_settings, + global_iter, + ) + # Every global iteration after the first begins from the immutable # checkpoint directly before the first calibrated model. This makes # global reruns independent of the state left by the final calibrated @@ -369,7 +426,7 @@ def run_calibration_loop( logger.info( "calibration global iteration %s/%s", - global_iter - start_global_iter, + global_iter, calibration_settings.run.global_iterations, ) @@ -420,10 +477,7 @@ def run_calibration_loop( if ( calibration_settings.run.complete_steps - or ( - start_global_iter + calibration_settings.run.global_iterations - == global_iter + 1 - ) + or global_iter == calibration_settings.run.global_iterations or ( global_iter == start_global_iter and state.settings.resume_after is not None @@ -447,27 +501,40 @@ def run_calibration_loop( shared_data_buffers=shared_data_buffers, ) - _write_progress( - state, - { - "next_global_iteration": global_iter + 1, - "last_completed_global_iteration": global_iter, - }, + completed_global_iterations = global_iter + iteration_is_complete = ( + all_converged + or global_iter == calibration_settings.run.global_iterations ) + if not iteration_is_complete: + _write_progress( + state, + { + "in_progress_iteration": None, + "next_global_iteration": global_iter + 1, + "last_completed_global_iteration": global_iter, + "converged": all_converged, + }, + ) if all_converged: logger.info( "calibration converged after global iteration %s/%s", - global_iter - start_global_iter, + global_iter, calibration_settings.run.global_iterations, ) break _write_final_coefficients_snapshot(state, calibration_settings) + _write_completed_progress( + state, + completed_global_iterations, + all_converged, + ) return CalibrationRunResult( converged=all_converged, - completed_global_iterations=calibration_settings.run.global_iterations, + completed_global_iterations=completed_global_iterations, ) finally: state.filesystem.pipeline_file_name = original_pipeline_name @@ -1466,6 +1533,83 @@ def _ensure_calibration_output_dir(state: workflow.State) -> None: os.makedirs(path, exist_ok=True) +def _calibration_coefficient_paths( + state: workflow.State, + calibration_settings: CalibrationConfig, +) -> list[Path]: + """Return the unique coefficient files modified by this calibration run.""" + paths: list[Path] = [] + seen: set[str] = set() + + for component_name in calibration_settings.run.calibrate_models: + model_settings_file = _infer_model_settings_file(component_name) + model_settings = state.filesystem.read_model_settings( + model_settings_file, mandatory=True + ) + coefficient_file = _setting_value(model_settings, "COEFFICIENTS") + if not coefficient_file: + raise RuntimeError( + f"component {component_name} model settings missing COEFFICIENTS" + ) + + path = Path(state.filesystem.get_config_file_path(coefficient_file)).resolve() + key = os.path.normcase(str(path)) + if key not in seen: + paths.append(path) + seen.add(key) + + return paths + + +def _begin_global_iteration_transaction( + state: workflow.State, + calibration_settings: CalibrationConfig, + global_iteration: int, +) -> None: + """Snapshot coefficients and durably mark a global iteration in progress.""" + recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) + os.makedirs(recovery_dir, exist_ok=True) + + for file_number, coefficient_path in enumerate( + _calibration_coefficient_paths(state, calibration_settings) + ): + if not coefficient_path.exists(): + raise FileNotFoundError( + f"calibration coefficient file not found: {coefficient_path}" + ) + + backup_name = f"{file_number:03d}_{coefficient_path.name}" + shutil.copyfile(coefficient_path, recovery_dir / backup_name) + + # Write the marker only after all backups exist. If backup creation is + # interrupted, the previous between-iteration progress remains valid. + _write_progress( + state, + { + "in_progress_iteration": global_iteration, + "next_global_iteration": global_iteration, + "last_completed_global_iteration": global_iteration - 1, + }, + ) + + +def _restore_coefficient_backups( + state: workflow.State, + calibration_settings: CalibrationConfig, +) -> None: + """Restore the coefficient backups for an interrupted global iteration.""" + recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) + for file_number, coefficient_path in enumerate( + _calibration_coefficient_paths(state, calibration_settings) + ): + backup_path = recovery_dir / f"{file_number:03d}_{coefficient_path.name}" + if not backup_path.exists(): + raise RuntimeError( + f"cannot recover interrupted calibration iteration: missing {backup_path}" + ) + shutil.copyfile(backup_path, coefficient_path) + + def _read_progress(state: workflow.State) -> dict[str, Any] | None: """Read persisted calibration progress metadata if it exists.""" path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) @@ -1476,11 +1620,31 @@ def _read_progress(state: workflow.State) -> dict[str, Any] | None: def _write_progress(state: workflow.State, payload: dict[str, Any]) -> None: - """Write calibration progress metadata.""" + """Atomically write calibration progress metadata.""" path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) os.makedirs(path.parent, exist_ok=True) - with open(path, "w", encoding="utf-8") as f: + temporary_path = path.with_name(f".{path.name}.tmp") + with open(temporary_path, "w", encoding="utf-8") as f: json.dump(payload, f, indent=2) + os.replace(temporary_path, path) + + +def _write_completed_progress( + state: workflow.State, + completed_global_iterations: int, + converged: bool, +) -> None: + """Mark calibration complete after all final output has been written.""" + _write_progress( + state, + { + "complete": True, + "in_progress_iteration": None, + "next_global_iteration": completed_global_iterations + 1, + "last_completed_global_iteration": completed_global_iterations, + "converged": converged, + }, + ) def _run_mp_single_component( diff --git a/docs/users-guide/calibration.rst b/docs/users-guide/calibration.rst index db0cf719d0..2d817f32fe 100644 --- a/docs/users-guide/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -541,9 +541,11 @@ Global Files * - File - Description * - ``calibration_progress.json`` - - Tracks ``next_global_iteration`` for crash recovery. If a run is - interrupted, restarting will resume from the last completed global - iteration. + - Tracks the active and next global iterations for crash recovery, and + whether calibration is complete. + * - ``recovery/`` + - Reusable start-of-iteration coefficient backups used to roll back and + replay an interrupted global iteration. * - ``calibration_iteration_records.csv`` - Appended per-coefficient detail for every iteration across all components. Columns include ``global_iter``, ``component_iter``, @@ -584,7 +586,7 @@ Updated Config Files --------------------- Coefficient CSV files in the configs directory are **updated in-place** after -each iteration. This means: +each component iteration. This means: - The calibrated coefficients persist across runs. - You can inspect intermediate coefficient values at any time. @@ -594,15 +596,23 @@ each iteration. This means: Crash Recovery ============== -Calibration progress is persisted to ``calibration_progress.json`` after each -completed global iteration. If a run is interrupted: +Before each global iteration, calibration replaces the files in its recovery +directory with a copy of every coefficient file that it may modify, then records +the active iteration in ``calibration_progress.json``. If a run is interrupted: -1. The coefficient files on disk reflect the state at the last completed iteration. -2. Restarting ``activitysim run`` with the same configuration will resume from - the ``next_global_iteration`` recorded in the progress file. +1. Restarting ``activitysim run`` restores all coefficient files from the + start-of-iteration recovery snapshot. +2. The interrupted global iteration is replayed from that consistent boundary. +3. Remaining iterations run only until the configured total + ``global_iterations`` is reached. + +The progress file is written using atomic replacement. Once progress is marked +complete, rerunning with the same output directory does not apply additional +calibration iterations. To force a fresh start, delete ``output/calibration/calibration_progress.json`` -and restore original coefficient files. +and restore original coefficient files. Recovery snapshots can also be removed +after a completed run if they are no longer needed. Multiprocess Mode From d79517e12be980fb6edfc0bfa250ce2937206e7b Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:18:03 -0700 Subject: [PATCH 77/90] refactor into separate files --- activitysim/core/calibration.py | 2466 ----------------- activitysim/core/calibration/README.md | 27 + activitysim/core/calibration/__init__.py | 70 + activitysim/core/calibration/coefficients.py | 85 + activitysim/core/calibration/component.py | 514 ++++ activitysim/core/calibration/execution.py | 262 ++ activitysim/core/calibration/expressions.py | 199 ++ activitysim/core/calibration/multiprocess.py | 604 ++++ activitysim/core/calibration/orchestrator.py | 455 +++ activitysim/core/calibration/recovery.py | 103 + activitysim/core/calibration/reporting.py | 268 ++ activitysim/core/calibration/settings.py | 132 + .../workplace_location_calib_helper.py | 3 +- 13 files changed, 2721 insertions(+), 2467 deletions(-) delete mode 100644 activitysim/core/calibration.py create mode 100644 activitysim/core/calibration/README.md create mode 100644 activitysim/core/calibration/__init__.py create mode 100644 activitysim/core/calibration/coefficients.py create mode 100644 activitysim/core/calibration/component.py create mode 100644 activitysim/core/calibration/execution.py create mode 100644 activitysim/core/calibration/expressions.py create mode 100644 activitysim/core/calibration/multiprocess.py create mode 100644 activitysim/core/calibration/orchestrator.py create mode 100644 activitysim/core/calibration/recovery.py create mode 100644 activitysim/core/calibration/reporting.py create mode 100644 activitysim/core/calibration/settings.py diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py deleted file mode 100644 index cdeaa32e17..0000000000 --- a/activitysim/core/calibration.py +++ /dev/null @@ -1,2466 +0,0 @@ -# ActivitySim -# See full license in LICENSE.txt. -from __future__ import annotations - -import importlib -import importlib.util -import json -import logging -import math -import multiprocessing -import os -import re -import shutil -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Optional - -import numpy as np -import pandas as pd -import matplotlib -import matplotlib.pyplot as plt -from pydantic import model_validator - -from activitysim.core import simulate, workflow -from activitysim.core.configuration import PydanticReadable -from activitysim.core.configuration.base import PydanticBase -from activitysim.core.configuration.top import MultiprocessStep - -logger = logging.getLogger("calibration") - -plt.style.use("seaborn-v0_8-darkgrid") -matplotlib.use("Agg") # Forces non-interactive background rendering - -CALIBRATION_SETTINGS_FILE_NAME = "calibration.yaml" -CALIBRATION_OUTPUT_DIR = "calibration" -CALIBRATION_PROGRESS_FILE = "calibration/calibration_progress.json" -CALIBRATION_ITERATION_FILE = "calibration/calibration_iteration_records.csv" -CALIBRATION_SUMMARY_FILE = "calibration/calibration_iteration_summary.csv" -CALIBRATION_FINAL_COEFFICIENTS_FILE = "calibration/final_calibrated_coefficients.csv" -CALIBRATION_RECOVERY_DIR = "calibration/recovery" - -DEFAULT_INCREMENT = 2.0 -MAX_COEFFS_IN_GRAPH = 15 - -CALIBRATION_REQUIRED_COLUMNS = [ - "description", - "coefficient", - "model_value", - "target_value", - "hold_fast", - "min", - "max", - "damping", - "method", - "tolerance", -] - -MP_INJECTABLES = [ - "data_dir", - "configs_dir", - "data_model_dir", - "output_dir", - "cache_dir", - "settings_file_name", - "imported_extensions", - "run_timestamp", - "run_id", - "pipeline_file_name", -] - - -class CalibrationRunSettings(PydanticBase): - """Run-control settings for calibration.""" - - resume_after: Optional[str] = None - calibrate_models: list[str] - global_iterations: int = 1 - complete_steps: bool = False - invalidate_tables: list[str] | None = None - - @model_validator(mode="after") - def validate_run_settings(self): - if not self.calibrate_models: - raise ValueError( - "calibration.run.calibrate_models must contain at least one model name" - ) - return self - - """Tables to drop from state after each calibration restore so their - ``@workflow.table`` factories regenerate from current data. - - Default (None): invalidates ``["vehicles"]``. Set to ``[]`` to disable. - - A table should be listed here when ALL of the following are true: - - 1. It is created by a ``@workflow.table`` factory from another table's - values (not just from input data files). - 2. That source table is modified by a calibrated model or by a model - whose outputs change when calibrated coefficients change. - 3. The factory uses source-table values to determine **row identity** - (index values) or **row count**, not just column values. - - The canonical example is ``vehicles``: its factory repeats household - rows by ``households["auto_ownership"]`` and derives ``vehicle_id`` - from ``household_id``. When ``auto_ownership_simulate`` is calibrated, - different coefficients produce different ownership counts, so the - stale vehicles table loaded from a prior checkpoint would have the - wrong number of rows and wrong vehicle IDs. Dropping it forces the - factory to regenerate vehicles consistent with the current households. - - Tables that only read *column values* from upstream tables (without - affecting row identity) generally do NOT need invalidation — their - content will be correct as long as the upstream table is correct at - the restored checkpoint. - """ - - -class CalibrationReportsSettings(PydanticBase): - """Reporting settings for a calibrated component.""" - - generic: bool = True - bespoke: str | None = None - - -class CalibrationComponentSettings(PydanticBase): - """Settings for one calibratable model component.""" - - calibration_spec: str - helper_module: str | None = None - submodel_max_iterations: int = 1 - reports: CalibrationReportsSettings = CalibrationReportsSettings() - survey_file: Optional[str] = None - - -class CalibrationConfig(PydanticReadable): - """Top-level calibration configuration.""" - - enable: bool = False - run: CalibrationRunSettings - model_settings: dict[str, CalibrationComponentSettings] = {} - - @model_validator(mode="after") - def validate_model_settings(self): - """Validate that configured components are aligned with run settings.""" - for component in self.run.calibrate_models: - if component not in self.model_settings: - raise ValueError( - f"calibration model '{component}' is not in model_settings" - ) - - if self.run.global_iterations < 1: - raise ValueError("max_iterations must be >= 1") - - return self - - -@dataclass -class CalibrationComponentResult: - """Result details from calibrating one component.""" - - component: str - converged: bool - component_iterations: int - - -@dataclass -class CalibrationRunResult: - """Result details from a complete global calibration loop.""" - - converged: bool - completed_global_iterations: int - - -def read_calibration_settings(state: workflow.State) -> CalibrationConfig | None: - """Read and validate calibration settings if calibration.yaml exists.""" - return CalibrationConfig.read_settings_file( - state.filesystem, - CALIBRATION_SETTINGS_FILE_NAME, - mandatory=False, - ) - - -def calibration_enabled(state: workflow.State) -> bool: - """Return True when calibration.yaml exists and is enabled.""" - settings = read_calibration_settings(state) - return bool(settings and settings.enable) - - -def run_calibration_loop( - state: workflow.State, - models: list[str], -) -> CalibrationRunResult: - """ - Run the global calibration workflow. - - This function intentionally minimizes changes to the existing run mechanics: - it always reuses ActivitySim's normal model execution paths and only adds - calibration orchestration around them. - """ - calibration_settings = read_calibration_settings(state) - if not calibration_settings or not calibration_settings.enable: - raise RuntimeError("calibration loop called while calibration is disabled") - - if state.settings.duplicate_step_execution != "allow": - state.settings.duplicate_step_execution = "allow" - logger.warning( - "Overriding duplicate_step_execution setting: must be enabled for calibration" - ) - - if not calibration_settings.run.calibrate_models: - raise ValueError( - "calibration.run.calibrate_models must contain at least one model name" - ) - - missing_calibration_models = [ - component - for component in calibration_settings.run.calibrate_models - if component not in models - ] - if missing_calibration_models: - raise ValueError( - "settings.yaml models list does not include configured calibration " - f"model(s): {missing_calibration_models}" - ) - - resume_after = state.settings.resume_after - if resume_after is not None and resume_after not in models: - raise ValueError( - f"settings.yaml resume_after={resume_after!r} is not present in the " - "settings.yaml models list. Calibration requires resume_after to be " - "a model-level checkpoint name." - ) - - # sort calibration models into main model order - calibration_settings.run.calibrate_models = sorted( - calibration_settings.run.calibrate_models, key=lambda x: models.index(x) - ) - first_calib_model_idx = models.index(calibration_settings.run.calibrate_models[0]) - last_calib_model_idx = models.index(calibration_settings.run.calibrate_models[-1]) - first_model_idx = models.index(resume_after) + 1 if resume_after else None - first_calibration_restart_step = _prior_step_name( - models, calibration_settings.run.calibrate_models[0] - ) - - if resume_after is not None: - skipped_calibration_models = [ - component - for component in calibration_settings.run.calibrate_models - if models.index(component) <= models.index(resume_after) - ] - if skipped_calibration_models: - logger.warning( - "Calibration is honoring settings.yaml resume_after=%r using strict " - "ActivitySim semantics. The following calibrated model(s) occur at " - "or before resume_after and will be skipped during the first global " - "iteration: %s", - resume_after, - skipped_calibration_models, - ) - - _ensure_calibration_output_dir(state) - - progress = _read_progress(state) - if progress and progress.get("complete"): - logger.info( - "calibration progress is already complete; remove %s to start a " - "fresh calibration run", - CALIBRATION_PROGRESS_FILE, - ) - return CalibrationRunResult( - converged=bool(progress.get("converged", False)), - completed_global_iterations=int( - progress.get( - "last_completed_global_iteration", - calibration_settings.run.global_iterations, - ) - ), - ) - - interrupted_iteration = ( - progress.get("in_progress_iteration") if progress else None - ) - if interrupted_iteration is not None: - interrupted_iteration = int(interrupted_iteration) - logger.warning( - "recovering interrupted calibration global iteration %s", - interrupted_iteration, - ) - _restore_coefficient_backups(state, calibration_settings) - progress = { - "in_progress_iteration": None, - "next_global_iteration": interrupted_iteration, - "last_completed_global_iteration": interrupted_iteration - 1, - } - _write_progress(state, progress) - - # Progress files from earlier versions contain next_global_iteration, so - # they remain compatible with the corrected total-count semantics. - start_global_iter = int(progress.get("next_global_iteration", 1)) if progress else 1 - completed_global_iterations = start_global_iter - 1 - - if start_global_iter > calibration_settings.run.global_iterations: - logger.info( - "calibration progress already reached configured global_iterations=%s", - calibration_settings.run.global_iterations, - ) - converged = bool(progress.get("converged", False)) if progress else False - _write_final_coefficients_snapshot(state, calibration_settings) - _write_completed_progress( - state, - completed_global_iterations, - converged, - ) - return CalibrationRunResult( - converged=converged, - completed_global_iterations=completed_global_iterations, - ) - - if state.settings.resume_after is None: - # compute_accessibility requires its accessibility table to be empty; - # unlike most model steps, it will not overwrite a prior result. - # Remove a cached result before restore clears table-status metadata, - # so the table factory recreates its empty placeholder for the replay. - state.drop_table("accessibility") - state.checkpoint.restore() - - original_pipeline_name = state.filesystem.pipeline_file_name - - # Initialize shared resources for multiprocess mode (skims, shadow pricing). - # These are allocated once and reused across all calibration iterations. - shared_data_buffers = None - if state.settings.multiprocess: - shared_data_buffers = _initialize_mp_shared_resources(state) - - try: - # skip precursors if, on first iter, resume_after exists and is >= first_calib_model_idx - if ( - state.settings.resume_after is None - or first_model_idx < first_calib_model_idx - ): - # Run ActivitySim normally from resume_after through production model steps. - _run_precursor_components( - state, - models=models[:first_calib_model_idx] - if first_model_idx is None - else models[first_model_idx:first_calib_model_idx], - resume_after=state.settings.resume_after, - global_iter=start_global_iter, - shared_data_buffers=shared_data_buffers, - ) - else: - # Precursors skipped — but the pipeline must still be initialized - # at the resume_after point so that _calibrate_component (and its - # apportion subprocess) starts from the correct state without - # downstream model data. - extra_models = _prep_model_data( - state, resume_after=state.settings.resume_after - ) - if extra_models: - # No model-level checkpoint exists for resume_after; we must - # run models from the prior step through resume_after to - # recreate the correct intermediate state. - _run_in_configured_mode( - state, - models=extra_models, - resume_after=None, - shared_data_buffers=shared_data_buffers, - ) - elif not any( - cp.get("checkpoint_name") == state.settings.resume_after - for cp in state.checkpoint.checkpoints - ): - # _prep_model_data took its fallback path — the pipeline either - # doesn't exist or doesn't contain resume_after's checkpoint. - # The restored state is incomplete (precursor models never ran). - logger.warning( - "calibration: resume_after=%r not found in restored pipeline; " - "running precursor models", - state.settings.resume_after, - ) - _run_precursor_components( - state, - models=models[:first_calib_model_idx], - resume_after=None, - global_iter=start_global_iter, - shared_data_buffers=shared_data_buffers, - ) - else: - state.checkpoint.add(state.settings.resume_after) - state.checkpoint.close_store() - - for global_iter in range( - start_global_iter, - calibration_settings.run.global_iterations + 1, - ): - _begin_global_iteration_transaction( - state, - calibration_settings, - global_iter, - ) - - # Every global iteration after the first begins from the immutable - # checkpoint directly before the first calibrated model. This makes - # global reruns independent of the state left by the final calibrated - # model in the preceding iteration. - if global_iter > start_global_iter: - logger.info( - "Restarting global calibration iteration %s from checkpoint %r", - global_iter, - first_calibration_restart_step, - ) - extra_models = _prep_model_data( - state, resume_after=first_calibration_restart_step - ) - if extra_models: - _run_in_configured_mode( - state, - models=extra_models, - resume_after=None, - shared_data_buffers=shared_data_buffers, - ) - _invalidate_derived_tables(state) - if first_calibration_restart_step is not None: - state.checkpoint.add(first_calibration_restart_step) - state.checkpoint.close_store() - - logger.info( - "calibration global iteration %s/%s", - global_iter, - calibration_settings.run.global_iterations, - ) - - # suppress early termination on first iteration if resume_after is after all calibrated models - all_converged = ( - first_model_idx is not None and first_model_idx <= last_calib_model_idx - ) or global_iter > start_global_iter - - last_calibrated_component = None - for component in calibration_settings.run.calibrate_models: - # on the first global iter, skip model if it's before or == resume_after - if ( - global_iter == start_global_iter - and first_model_idx is not None - and first_model_idx > models.index(component) - ): - continue - component_settings = calibration_settings.model_settings[component] - - prior_step = _prior_step_name(models, component) - - if last_calibrated_component is not None: - - # run all models b/w the last calibrated model and the current one - _run_intermediate_components( - state, - models=models[ - models.index(last_calibrated_component) - + 1 : models.index(component) - ], - resume_after=last_calibrated_component, - shared_data_buffers=shared_data_buffers, - ) - - component_result = _calibrate_component( - state=state, - component_name=component, - component_settings=component_settings, - prior_step=prior_step, - global_iter=global_iter, - shared_data_buffers=shared_data_buffers, - ) - _write_component_plots(state, component) - - all_converged = all_converged and component_result.converged - - last_calibrated_component = component - - if ( - calibration_settings.run.complete_steps - or global_iter == calibration_settings.run.global_iterations - or ( - global_iter == start_global_iter - and state.settings.resume_after is not None - and first_model_idx > last_calib_model_idx - ) - ): - subsequent_components = ( - models[first_model_idx:] - if global_iter == start_global_iter - and first_model_idx > last_calib_model_idx - else models[models.index(last_calibrated_component) + 1 :] - ) - # finish the full model chain - _run_subsequent_components( - state, - models=subsequent_components, - resume_after=state.settings.resume_after - if global_iter == start_global_iter - and first_model_idx > last_calib_model_idx - else last_calibrated_component, - shared_data_buffers=shared_data_buffers, - ) - - completed_global_iterations = global_iter - iteration_is_complete = ( - all_converged - or global_iter == calibration_settings.run.global_iterations - ) - if not iteration_is_complete: - _write_progress( - state, - { - "in_progress_iteration": None, - "next_global_iteration": global_iter + 1, - "last_completed_global_iteration": global_iter, - "converged": all_converged, - }, - ) - - if all_converged: - logger.info( - "calibration converged after global iteration %s/%s", - global_iter, - calibration_settings.run.global_iterations, - ) - break - - _write_final_coefficients_snapshot(state, calibration_settings) - _write_completed_progress( - state, - completed_global_iterations, - all_converged, - ) - - return CalibrationRunResult( - converged=all_converged, - completed_global_iterations=completed_global_iterations, - ) - finally: - state.filesystem.pipeline_file_name = original_pipeline_name - - -def _run_precursor_components( - state: workflow.State, - models: list[str], - resume_after: str, - global_iter: int, - shared_data_buffers: dict | None = None, -) -> None: - """Run the normal ActivitySim model flow for one global calibration iteration.""" - - # if global_iter > 1 and resume_after is not None: - # # Seed a fresh pipeline from the configured resume checkpoint to avoid - # # duplicate checkpoint-name collisions across global calibration loops. - # prior_pipeline = state.checkpoint.store.filename - # state.checkpoint.close_store() - # state.filesystem.pipeline_file_name = f"pipeline_calibration_iter_{global_iter}" - # state.checkpoint.restore_from(prior_pipeline, checkpoint_name=resume_after) - # else: - - _run_in_configured_mode( - state, - models=models, - resume_after=resume_after, - shared_data_buffers=shared_data_buffers, - ) - - -def _run_intermediate_components( - state: workflow.State, - models: list[str], - resume_after: str, - shared_data_buffers: dict | None = None, -) -> None: - if len(models) == 0: - return - _run_in_configured_mode( - state, - models=models, - resume_after=resume_after, - shared_data_buffers=shared_data_buffers, - ) - - -def _run_subsequent_components( - state: workflow.State, - models: list[str], - resume_after: str, - shared_data_buffers: dict | None = None, -) -> None: - _run_in_configured_mode( - state, - models=models, - resume_after=resume_after, - shared_data_buffers=shared_data_buffers, - ) - - -def _calibrate_component( - state: workflow.State, - component_name: str, - component_settings: CalibrationComponentSettings, - prior_step: str, - global_iter: int, - shared_data_buffers: dict | None = None, -) -> CalibrationComponentResult: - """Run iterative coefficient calibration for one component.""" - model_settings_file = _infer_model_settings_file(component_name) - model_settings = state.filesystem.read_model_settings( - model_settings_file, mandatory=True - ) - - coefficients_df = state.filesystem.read_model_coefficients( - model_settings=model_settings - ) - helper_symbols, bespoke_callable, helper_module = _load_helper_symbols( - state, - component_settings, - ) - - calibration_spec_df = _read_calibration_spec( - state, component_settings.calibration_spec - ) - - utility_coeff_names = _extract_utility_coefficient_names(state, model_settings) - _validate_calibration_coefficients_against_utility_spec( - component_name, - calibration_spec_df, - utility_coeff_names, - ) - - coefficients_df = _ensure_coefficients_exist( - component_name, - calibration_spec_df, - coefficients_df, - ) - - _warn_if_initial_values_outside_bounds( - component_name, calibration_spec_df, coefficients_df - ) - - component_converged = False - component_iterations = 0 - - # Determine the checkpoint name to restore from for component re-runs. - # In MP mode, the checkpoint that represents prior_step's completed state - # is the last checkpoint in the pipeline before we run the component. - # We capture it once and reuse across component iterations. - mp_restore_checkpoint = None - if state.settings.multiprocess and shared_data_buffers is not None: - # The pipeline should already be open from _restore_parent_state_from_pipeline - # called after precursor/intermediate models ran. The last checkpoint - # in the pipeline represents the state at prior_step. - if state.checkpoint.checkpoints: - mp_restore_checkpoint = state.checkpoint.last_checkpoint.get( - "checkpoint_name", "_" - ) - else: - mp_restore_checkpoint = "_" - - for component_iter in range(1, component_settings.submodel_max_iterations + 1): - component_iterations = component_iter - run_model_name = f"{component_name}.c_i{component_iter};" f"g_i{global_iter}" - # Re-run only this component from its prior checkpoint so model values - # reflect the current candidate coefficients for this component. - if state.settings.multiprocess and shared_data_buffers is not None: - # Use direct MP orchestration with explicit checkpoint control. - # This ensures we always apportion from prior_step's state, - # even after multiple component iterations. - _run_mp_single_component( - state, - component_name=component_name, - # Always restore from the same immutable pre-component - # checkpoint. LAST_CHECKPOINT may point to the prior iteration's - # coalesced component output and is therefore not a safe baseline. - restore_checkpoint=mp_restore_checkpoint, - shared_data_buffers=shared_data_buffers, - ) - else: - # Restore to prior_step ourselves then run the model directly. - # state.run(resume_after=prior_step) would trigger - # checkpoint.restore → init_state which creates a fresh RNG. - # If prior_step is before the calibrated model created its table - # (e.g. vehicles), the table won't be in that checkpoint and the - # RNG channel won't be registered — causing a crash when the - # model tries to use it. By restoring here and calling by_name, - # we keep the RNG channels from _prep_model_data intact. - extra_models = _prep_model_data(state, resume_after=prior_step) - if extra_models: - # prior_step checkpoint not found directly; run intermediate - # models (e.g. annotators) to recreate the correct state. - for m in extra_models: - state.run.by_name(m) - _invalidate_derived_tables(state) - state.checkpoint.add(prior_step) - state.run.by_name(run_model_name) - - eval_context = _build_expression_context( - state, helper_symbols, component_name, component_settings - ) - - ( - row_records, - summary_record, - new_coefficients_df, - component_converged, - ) = _evaluate_and_update( - component_name=component_name, - calibration_spec_df=calibration_spec_df, - coefficients_df=coefficients_df, - eval_context=eval_context, - global_iter=global_iter, - component_iter=component_iter, - ) - - coefficients_df = new_coefficients_df - - _persist_coefficients_to_config(state, model_settings, coefficients_df) - _append_iteration_records(state, component_name, row_records) - _append_summary_records(state, [summary_record]) - - if component_settings.reports.generic: - _write_generic_report(state, component_name, row_records) - - if bespoke_callable is not None: - bespoke_callable(eval_context) - - if component_converged: - break - - state.checkpoint.add(component_name) - - return CalibrationComponentResult( - component=component_name, - converged=component_converged, - component_iterations=component_iterations, - ) - - -def _read_calibration_spec(state: workflow.State, file_name: str) -> pd.DataFrame: - """Read calibration spec CSV and validate required columns.""" - path = state.filesystem.get_config_file_path(file_name) - df = pd.read_csv(path, comment="#") - - missing = [c for c in CALIBRATION_REQUIRED_COLUMNS if c not in df.columns] - if missing: - raise ValueError( - f"calibration_spec {file_name} is missing required columns: {missing}" - ) - - df = df[CALIBRATION_REQUIRED_COLUMNS].copy() - df["description"] = df["description"].astype(str) - df["coefficient"] = df["coefficient"].astype(str) - - # Normalize booleans and defaults to keep row math deterministic. - df["hold_fast"] = ( - df["hold_fast"] - .fillna(False) - .astype(str) - .str.strip() - .str.lower() - .isin(["1", "true", "t", "yes", "y"]) - ) - - for c in ["min", "max", "damping", "tolerance"]: - df[c] = pd.to_numeric(df[c], errors="coerce") - - df["method"] = df["method"].astype(str).str.strip().str.lower() - - bad_methods = df.loc[~df["method"].isin(["log_ratio", "odds_ratio"]), "method"] - if not bad_methods.empty: - raise ValueError( - f"unsupported calibration method(s): {sorted(set(bad_methods))}" - ) - - if df["damping"].isna().any(): - raise ValueError("calibration_spec damping must be numeric") - if df["tolerance"].isna().any(): - raise ValueError("calibration_spec tolerance must be numeric") - - return df - - -def _extract_utility_coefficient_names( - state: workflow.State, - model_settings: dict[str, Any] | Any, -) -> set[str]: - """ - Extract coefficient names used by the configured utility specifications. - - Templated logit models map utility-spec row labels to actual coefficient - names by segment, so their template cell values are the source of truth. - Other models use coefficient tokens in utility-spec columns. - """ - if _setting_value(model_settings, "COEFFICIENT_TEMPLATE"): - template = simulate.read_model_coefficient_template( - state.filesystem, model_settings - ) - return {str(name) for name in template.to_numpy().ravel()} - - names: set[str] = set() - - model_settings_dict = _settings_to_dict(model_settings) - spec_keys = [ - k for k in model_settings_dict.keys() if str(k).upper().endswith("SPEC") - ] - for key in spec_keys: - spec_file = model_settings_dict.get(key) - if not spec_file: - continue - - spec_path = state.filesystem.get_config_file_path(spec_file) - try: - raw = pd.read_csv(spec_path, comment="#") - except Exception: - # Do not fail hard on optional or model-specific supplemental specs. - continue - - utility_columns = [ - c - for c in raw.columns - if c - not in [ - "Description", - "Expression", - "Label", - "description", - "expression", - "label", - ] - ] - - for col in utility_columns: - for value in raw[col].dropna().tolist(): - if isinstance(value, (int, float, np.number)): - continue - text = str(value) - for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", text): - names.add(token) - - return names - - -def _validate_calibration_coefficients_against_utility_spec( - component_name: str, - calibration_spec_df: pd.DataFrame, - utility_coeff_names: set[str], -) -> None: - """Ensure calibration coefficients are referenced in utility specifications.""" - missing = sorted( - set(calibration_spec_df["coefficient"].tolist()) - set(utility_coeff_names) - ) - if missing: - raise ValueError( - f"calibration coefficients not present in model utility specification for {component_name}: {missing}" - ) - - -def _ensure_coefficients_exist( - component_name: str, - calibration_spec_df: pd.DataFrame, - coefficients_df: pd.DataFrame, -) -> pd.DataFrame: - """Add calibration coefficients missing from coefficient file with value 0.0.""" - coeffs = coefficients_df.copy() - - for coefficient_name in calibration_spec_df["coefficient"].tolist(): - if coefficient_name not in coeffs.index: - logger.warning( - "component %s coefficient %s missing from coefficient file, adding with value 0.0", - component_name, - coefficient_name, - ) - coeffs.loc[coefficient_name, "value"] = 0.0 - - coeffs["value"] = pd.to_numeric(coeffs["value"], errors="coerce") - if coeffs["value"].isna().any(): - bad = coeffs[coeffs["value"].isna()].index.tolist() - raise ValueError(f"non-numeric coefficient values found: {bad}") - - return coeffs - - -def _warn_if_initial_values_outside_bounds( - component_name: str, - calibration_spec_df: pd.DataFrame, - coefficients_df: pd.DataFrame, -) -> None: - """Warn when initial coefficient values violate provided bounds.""" - for _, row in calibration_spec_df.iterrows(): - coefficient_name = row["coefficient"] - current_value = float(coefficients_df.loc[coefficient_name, "value"]) - lower = row["min"] - upper = row["max"] - - if not pd.isna(lower) and current_value < float(lower): - logger.warning( - "component %s coefficient %s starts below min bound (%s < %s)", - component_name, - coefficient_name, - current_value, - lower, - ) - if not pd.isna(upper) and current_value > float(upper): - logger.warning( - "component %s coefficient %s starts above max bound (%s > %s)", - component_name, - coefficient_name, - current_value, - upper, - ) - - -def _build_expression_context( - state: workflow.State, - helper_symbols: dict[str, Any], - component_name: str, - component_settings: CalibrationComponentSettings, -) -> dict[str, Any]: - """Create the evaluation context for model_value and target_value expressions.""" - context: dict[str, Any] = { - "state": state, - "np": np, - "pd": pd, - "component_output_dir": _component_output_dir(state, component_name), - "component_settings": component_settings, - } - - # Load active tables into context for direct expression access. - for table_name in list(state.existing_table_names): - try: - context[table_name] = state.get_dataframe(table_name, as_copy=False) - except Exception: - # Some entries may not be available as dataframes in all contexts. - continue - try: - network_los = state.get_injectable("network_los") - context["network_los"] = network_los - context["skim_dict"] = network_los.get_default_skim_dict() - except Exception: - # Network LOS may not be available in all contexts. - pass - - context.update(helper_symbols) - # Explicit function-call context used by calibration expressions. - context["context"] = context - return context - - -def _evaluate_and_update( - component_name: str, - calibration_spec_df: pd.DataFrame, - coefficients_df: pd.DataFrame, - eval_context: dict[str, Any], - global_iter: int, - component_iter: int, -) -> tuple[list[dict[str, Any]], dict[str, Any], pd.DataFrame, bool]: - """Evaluate spec rows, update coefficients, and return detailed records.""" - updated = coefficients_df.copy() - records: list[dict[str, Any]] = [] - - max_difference = -math.inf - max_difference_coefficient = "" - max_change = -math.inf - max_change_coefficient = "" - - num_converged = 0 - - for _, row in calibration_spec_df.iterrows(): - coefficient_name = row["coefficient"] - description = row["description"] - method = row["method"] - hold_fast = bool(row["hold_fast"]) - - default_increment = ( - row["default_increment"] - if "default_increment" in row.index - else DEFAULT_INCREMENT - ) - - prev_value = float(updated.loc[coefficient_name, "value"]) - - model_value = _eval_numeric_value( - row["model_value"], - eval_context, - component_name, - description, - "model_value", - ) - target_value = _eval_numeric_value( - row["target_value"], - eval_context, - component_name, - description, - "target_value", - ) - - difference = target_value - model_value - pct_difference = _safe_percent_difference(difference, target_value) - - tolerance = float(row["tolerance"]) - converged = abs(difference) <= tolerance - - damping = float(row["damping"]) - raw_delta = _compute_delta( - method=method, - model_value=model_value, - target_value=target_value, - damping=damping, - component_name=component_name, - description=description, - default_increment=default_increment, - ) - - candidate_value = prev_value if hold_fast else prev_value + raw_delta - - at_min = False - at_max = False - - lower = row["min"] - upper = row["max"] - - if not pd.isna(lower) and candidate_value <= float(lower): - candidate_value = float(lower) - at_min = True - if not pd.isna(upper) and candidate_value >= float(upper): - candidate_value = float(upper) - at_max = True - - if not np.isfinite(candidate_value): - raise RuntimeError( - f"non-finite next coefficient for {component_name} / {description} / {coefficient_name}" - ) - - updated.loc[coefficient_name, "value"] = candidate_value - - abs_diff = abs(difference) - abs_change = abs(candidate_value - prev_value) - - if abs_diff > max_difference: - max_difference = abs_diff - max_difference_coefficient = coefficient_name - - if abs_change > max_change: - max_change = abs_change - max_change_coefficient = coefficient_name - - if converged: - num_converged += 1 - - records.append( - { - "global_iter": global_iter, - "component_iter": component_iter, - "description": description, - "component": component_name, - "coefficient": coefficient_name, - "target_value": target_value, - "model_value": model_value, - "difference": difference, - "pct_difference": pct_difference, - "hold_fast": hold_fast, - "prev_coefficient": prev_value, - "coef_delta": abs_change, - "next_coefficient": candidate_value, - "converged": converged, - "at_min": at_min, - "at_max": at_max, - } - ) - - total_rows = len(calibration_spec_df) - num_unconverged = total_rows - num_converged - component_converged = num_unconverged == 0 - - summary_record = { - "global_iter": global_iter, - "component_iter": component_iter, - "component": component_name, - "max_difference": max_difference if max_difference != -math.inf else 0.0, - "max_difference_coefficient": max_difference_coefficient, - "max_change": max_change if max_change != -math.inf else 0.0, - "max_change_coefficient": max_change_coefficient, - "num_converged_iter": num_converged, - "tot_converged": num_converged, - "num_unconverged": num_unconverged, - } - - return records, summary_record, updated, component_converged - - -def _eval_numeric_value( - raw_value: Any, - context: dict[str, Any], - component_name: str, - description: str, - field_name: str, -) -> float: - """Evaluate numeric or expression value and enforce finite numeric result.""" - if isinstance(raw_value, (int, float, np.number)) and not pd.isna(raw_value): - value = float(raw_value) - else: - try: - value = eval(str(raw_value), {}, context) - except Exception as err: - raise RuntimeError( - f"error evaluating {field_name} for {component_name} / {description}: {raw_value}" - ) from err - - try: - value = float(value) - except Exception as err: - raise RuntimeError( - f"{field_name} did not evaluate to a numeric value for {component_name} / {description}: {value}" - ) from err - - if not np.isfinite(value): - raise RuntimeError( - f"{field_name} evaluated to non-finite value for {component_name} / {description}: {value}" - ) - - return value - - -def _compute_delta( - method: str, - model_value: float, - target_value: float, - damping: float, - component_name: str, - description: str, - default_increment: float, -) -> float: - """Compute damped coefficient delta using selected method.""" - if damping < 0: - raise RuntimeError( - f"negative damping not allowed for {component_name} / {description}: {damping}" - ) - - if method == "log_ratio": - if model_value <= 0 or target_value <= 0: - logger.warning( - f"log_ratio requires positive model and target values for {component_name} / {description}. Falling back to default increment {default_increment}" - ) - if model_value <= 0 and target_value > 0: - return default_increment - elif model_value > 0 and target_value <= 0: - return -default_increment - else: - return 0 - delta = math.log(target_value / model_value) * damping - - elif method == "odds_ratio": - # Formula requested by the calibration outline. - numerator = (target_value * model_value) - target_value - denominator = (target_value * model_value) - model_value - - if numerator <= 0 or denominator <= 0: - logger.warning( - f"odds_ratio produced invalid numerator/denominator for {component_name} / {description}. Falling back to default increment {default_increment}" - ) - if model_value <= 0 and target_value > 0: - return default_increment - elif model_value > 0 and target_value <= 0: - return -default_increment - else: - return 0 - - ratio = numerator / denominator - if ratio <= 0 or not np.isfinite(ratio): - raise RuntimeError( - f"odds_ratio produced invalid ratio for {component_name} / {description}" - ) - - delta = math.log(ratio) * damping - - else: - raise RuntimeError(f"unsupported calibration method: {method}") - - if not np.isfinite(delta): - raise RuntimeError( - f"coefficient delta is non-finite for {component_name} / {description}" - ) - - return delta - - -def _safe_percent_difference(difference: float, target_value: float) -> float: - """Return a stable percentage difference with zero-target handling.""" - if target_value == 0: - return math.inf if difference != 0 else 0.0 - return (difference / target_value) * 100.0 - - -def _persist_coefficients_to_config( - state: workflow.State, - model_settings: dict[str, Any] | Any, - coefficients_df: pd.DataFrame, -) -> None: - """Write updated coefficients back to the component coefficient file in configs.""" - coeff_file = _setting_value(model_settings, "COEFFICIENTS") - if not coeff_file: - raise RuntimeError("component model settings missing COEFFICIENTS") - - output = coefficients_df.copy() - output.index.name = "coefficient_name" - - coeff_path = state.filesystem.get_config_file_path(coeff_file) - output.to_csv(coeff_path) - - -def _append_iteration_records( - state: workflow.State, component_name: str, records: list[dict[str, Any]] -) -> None: - """Append per-coefficient calibration iteration records.""" - if not records: - return - df = pd.DataFrame(records) - - # Save a global iteration history file - global_path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) - _append_csv( - df, - global_path, - unique_on=["global_iter", "component_iter", "component", "coefficient"], - ) - - # Also write component-local iteration history - component_path = ( - _component_output_dir(state, component_name) - / Path(CALIBRATION_ITERATION_FILE).name - ) - _append_csv( - df, - component_path, - unique_on=["global_iter", "component_iter", "component", "coefficient"], - ) - - -def _append_summary_records( - state: workflow.State, records: list[dict[str, Any]] -) -> None: - """Append per-iteration summary records.""" - if not records: - return - path = state.get_output_file_path(CALIBRATION_SUMMARY_FILE) - df = pd.DataFrame(records) - _append_csv( - df, - path, - unique_on=["global_iter", "component_iter", "component"], - ) - - -def _append_csv( - df: pd.DataFrame, path: Path, unique_on: list[str] | None = None -) -> None: - """Append a dataframe to a CSV file, replacing rows with matching keys.""" - os.makedirs(path.parent, exist_ok=True) - if unique_on and path.exists(): - existing = pd.read_csv(path) - df = pd.concat([existing, df], ignore_index=True).drop_duplicates( - subset=unique_on, keep="last" - ) - df.to_csv(path, index=False) - return - - write_header = not path.exists() - df.to_csv(path, mode="a", index=False, header=write_header) - - -def _component_output_dir(state: workflow.State, component_name: str) -> Path: - """Return output/calibration/ and ensure it exists.""" - component_dir = state.get_output_file_path(f"calibration/{component_name}") - os.makedirs(component_dir, exist_ok=True) - return component_dir - - -def _write_component_plots(state: workflow.State, component_name: str) -> None: - """Write/update all standard plots for one calibrated component.""" - recs = _read_component_iteration_records(state, component_name) - if recs is None or recs.empty: - return - - # Segment coefficients into manageable sets for plotting - coefs = sorted(recs.index.get_level_values("coefficient").unique()) - n_sets = math.ceil(len(coefs) / MAX_COEFFS_IN_GRAPH) - for coef_set in range(n_sets): - set_coefs = coefs[ - coef_set - * MAX_COEFFS_IN_GRAPH : min( - len(coefs), (coef_set + 1) * MAX_COEFFS_IN_GRAPH - ) - ] - _plot_coefficient_progress(state, component_name, recs, set_coefs, coef_set) - last_records = _component_last_records(recs, set_coefs) - _plot_component_values(state, component_name, last_records, coef_set) - _plot_component_pct_change(state, component_name, last_records, coef_set) - - -def _read_component_iteration_records( - state: workflow.State, component_name: str -) -> pd.DataFrame | None: - """Read all iteration records for a single component.""" - path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) - if not path.exists(): - return None - - iteration_records = ( - pd.read_csv(path) - .set_index(["global_iter", "component_iter", "coefficient"]) - .sort_index() - ) - return iteration_records.loc[iteration_records.component == component_name] - - -def _plot_coefficient_progress( - state: workflow.State, - component_name: str, - recs: pd.DataFrame, - set_coefs: list[str], - coef_set: int, -) -> None: - """Plot coefficient value progression for one coefficient subset.""" - component_dir = _component_output_dir(state, component_name) - ax = ( - recs[recs.index.get_level_values("coefficient").isin(set_coefs)] - .next_coefficient.unstack("coefficient") - .plot(figsize=(10, 5)) - ) - ax.xaxis.set_label_text("Component iteration") - ax.yaxis.set_label_text("Coefficient value") - ax.legend(title="Coefficient label", loc="center left", bbox_to_anchor=(1.02, 0.5)) - plt.tight_layout() - ax.figure.savefig( - component_dir / f"coefficient_progress_set_{coef_set}.png", - bbox_inches="tight", - ) - plt.close(ax.figure) - - -def _component_last_records(recs: pd.DataFrame, set_coefs: list[str]) -> pd.DataFrame: - """Select target/model values for the latest iteration and coefficient subset.""" - filtered = recs[recs.index.get_level_values("coefficient").isin(set_coefs)] - last_global = filtered.index.get_level_values("global_iter")[-1] - last_comp = filtered.loc[last_global].index.get_level_values("component_iter")[-1] - return filtered.xs( - (last_global, last_comp), level=("global_iter", "component_iter") - )[["target_value", "model_value"]] - - -def _plot_component_values( - state: workflow.State, - component_name: str, - last_records: pd.DataFrame, - coef_set: int, -) -> None: - """Plot final target/model component values for one coefficient subset.""" - component_dir = _component_output_dir(state, component_name) - ax = last_records.plot.bar(figsize=(10, 5)) - ax.xaxis.set_tick_params(rotation=45) - ax.xaxis.set_label_text("Component value") - plt.tight_layout() - ax.figure.savefig(component_dir / f"final_components_set_{coef_set}.png") - plt.close(ax.figure) - - -def _plot_component_pct_change( - state: workflow.State, - component_name: str, - last_records: pd.DataFrame, - coef_set: int, -) -> None: - """Plot final percent difference for one coefficient subset.""" - component_dir = _component_output_dir(state, component_name) - fig, ax = plt.subplots(figsize=(10, 5)) - pct_diff = last_records.diff(axis=1).model_value / last_records.target_value - ax = pct_diff.plot.bar(ax=ax) - ax.xaxis.set_tick_params(rotation=45) - ax.xaxis.set_label_text("Coefficient") - ax.yaxis.set_label_text("% Difference between Model and Target") - plt.tight_layout() - ax.figure.savefig(component_dir / f"final_pct_change_set_{coef_set}.png") - plt.close(ax.figure) - - -def _write_generic_report( - state: workflow.State, - component_name: str, - row_records: list[dict[str, Any]], -) -> None: - """Write a simple dashboard-friendly generic report for the current component iteration.""" - if not row_records: - return - - df = pd.DataFrame(row_records) - report = ( - df[ - [ - "global_iter", - "component_iter", - "component", - "description", - "difference", - "pct_difference", - "converged", - ] - ] - .copy() - .sort_values(["global_iter", "component_iter", "description"]) - ) - - path = _component_output_dir(state, component_name) / "generic_report.csv" - _append_csv( - report, - path, - unique_on=["global_iter", "component_iter", "component", "description"], - ) - - -def _load_helper_symbols( - state: workflow.State, - component_settings: CalibrationComponentSettings, -) -> tuple[dict[str, Any], Any | None, Any | None]: - """Load helper module and return evaluation symbols and bespoke function.""" - if not component_settings.helper_module: - return {}, None, None - - module = _load_helper_module(state, component_settings.helper_module) - symbols = { - name: obj for name, obj in vars(module).items() if not name.startswith("__") - } - - bespoke = None - if component_settings.reports and component_settings.reports.bespoke: - fn_name = component_settings.reports.bespoke - if not hasattr(module, fn_name): - raise RuntimeError( - f"helper module does not define bespoke function {fn_name}" - ) - bespoke = getattr(module, fn_name) - - return symbols, bespoke, module - - -def _load_helper_module(state: workflow.State, helper_module: str): - """Load helper module by file path or import path.""" - if helper_module.endswith(".py"): - helper_path = state.filesystem.get_config_file_path(helper_module) - module_name = Path(helper_module).stem - - spec = importlib.util.spec_from_file_location(module_name, helper_path) - if spec is None or spec.loader is None: - raise RuntimeError(f"unable to load helper module from {helper_path}") - - module = importlib.util.module_from_spec(spec) - # Expose state as a global for compatibility with helper examples. - module.state = state - spec.loader.exec_module(module) - return module - - module = importlib.import_module(helper_module) - setattr(module, "state", state) - return module - - -def _infer_model_settings_file(component_name: str) -> str: - """Infer model settings yaml filename from component step name.""" - # This follows the dominant naming convention in the existing codebase. - if component_name.endswith("_simulate"): - base = component_name[: -len("_simulate")] - else: - base = component_name - return f"{base}.yaml" - - -def _settings_to_dict(model_settings: dict[str, Any] | Any) -> dict[str, Any]: - """Convert pydantic or dict model settings to a plain dictionary.""" - if isinstance(model_settings, dict): - return model_settings - if hasattr(model_settings, "model_dump"): - return model_settings.model_dump() - return dict(model_settings) - - -def _setting_value(model_settings: dict[str, Any] | Any, key: str, default=None): - """Read a setting value from dict-like or attribute-based settings.""" - if isinstance(model_settings, dict): - return model_settings.get(key, default) - return getattr(model_settings, key, default) - - -def _prior_step_name(models: list[str], component_name: str) -> str | None: - """Return the step name immediately preceding component_name in models.""" - if component_name not in models: - return None - idx = models.index(component_name) - if idx == 0: - return None - return models[idx - 1] - - -def _write_final_coefficients_snapshot( - state: workflow.State, - calibration_settings: CalibrationConfig, -) -> None: - """Write a combined final coefficients file snapshot for calibrated components.""" - frames = [] - for component_name in calibration_settings.run.calibrate_models: - model_settings_file = _infer_model_settings_file(component_name) - model_settings = state.filesystem.read_model_settings( - model_settings_file, mandatory=True - ) - coeff_df = state.filesystem.read_model_coefficients( - model_settings=model_settings - ).copy() - coeff_df = coeff_df.reset_index().rename(columns={"index": "coefficient_name"}) - coeff_df.insert(0, "component", component_name) - frames.append(coeff_df) - - if not frames: - return - - final_df = pd.concat(frames, ignore_index=True) - path = state.get_output_file_path(CALIBRATION_FINAL_COEFFICIENTS_FILE) - os.makedirs(path.parent, exist_ok=True) - final_df.to_csv(path, index=False) - - -def _ensure_calibration_output_dir(state: workflow.State) -> None: - """Ensure output/calibration exists.""" - path = state.get_output_file_path(CALIBRATION_OUTPUT_DIR) - os.makedirs(path, exist_ok=True) - - -def _calibration_coefficient_paths( - state: workflow.State, - calibration_settings: CalibrationConfig, -) -> list[Path]: - """Return the unique coefficient files modified by this calibration run.""" - paths: list[Path] = [] - seen: set[str] = set() - - for component_name in calibration_settings.run.calibrate_models: - model_settings_file = _infer_model_settings_file(component_name) - model_settings = state.filesystem.read_model_settings( - model_settings_file, mandatory=True - ) - coefficient_file = _setting_value(model_settings, "COEFFICIENTS") - if not coefficient_file: - raise RuntimeError( - f"component {component_name} model settings missing COEFFICIENTS" - ) - - path = Path(state.filesystem.get_config_file_path(coefficient_file)).resolve() - key = os.path.normcase(str(path)) - if key not in seen: - paths.append(path) - seen.add(key) - - return paths - - -def _begin_global_iteration_transaction( - state: workflow.State, - calibration_settings: CalibrationConfig, - global_iteration: int, -) -> None: - """Snapshot coefficients and durably mark a global iteration in progress.""" - recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) - os.makedirs(recovery_dir, exist_ok=True) - - for file_number, coefficient_path in enumerate( - _calibration_coefficient_paths(state, calibration_settings) - ): - if not coefficient_path.exists(): - raise FileNotFoundError( - f"calibration coefficient file not found: {coefficient_path}" - ) - - backup_name = f"{file_number:03d}_{coefficient_path.name}" - shutil.copyfile(coefficient_path, recovery_dir / backup_name) - - # Write the marker only after all backups exist. If backup creation is - # interrupted, the previous between-iteration progress remains valid. - _write_progress( - state, - { - "in_progress_iteration": global_iteration, - "next_global_iteration": global_iteration, - "last_completed_global_iteration": global_iteration - 1, - }, - ) - - -def _restore_coefficient_backups( - state: workflow.State, - calibration_settings: CalibrationConfig, -) -> None: - """Restore the coefficient backups for an interrupted global iteration.""" - recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) - for file_number, coefficient_path in enumerate( - _calibration_coefficient_paths(state, calibration_settings) - ): - backup_path = recovery_dir / f"{file_number:03d}_{coefficient_path.name}" - if not backup_path.exists(): - raise RuntimeError( - f"cannot recover interrupted calibration iteration: missing {backup_path}" - ) - shutil.copyfile(backup_path, coefficient_path) - - -def _read_progress(state: workflow.State) -> dict[str, Any] | None: - """Read persisted calibration progress metadata if it exists.""" - path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) - if not path.exists(): - return None - with open(path, "r", encoding="utf-8") as f: - return json.load(f) - - -def _write_progress(state: workflow.State, payload: dict[str, Any]) -> None: - """Atomically write calibration progress metadata.""" - path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) - os.makedirs(path.parent, exist_ok=True) - temporary_path = path.with_name(f".{path.name}.tmp") - with open(temporary_path, "w", encoding="utf-8") as f: - json.dump(payload, f, indent=2) - os.replace(temporary_path, path) - - -def _write_completed_progress( - state: workflow.State, - completed_global_iterations: int, - converged: bool, -) -> None: - """Mark calibration complete after all final output has been written.""" - _write_progress( - state, - { - "complete": True, - "in_progress_iteration": None, - "next_global_iteration": completed_global_iterations + 1, - "last_completed_global_iteration": completed_global_iterations, - "converged": converged, - }, - ) - - -def _run_mp_single_component( - state: workflow.State, - component_name: str, - restore_checkpoint: str, - shared_data_buffers: dict, -) -> None: - """Run a single component in multiprocess mode with explicit checkpoint control. - - This directly orchestrates the apportion → simulate → coalesce flow - without going through run_multiprocess/get_run_list, giving us precise - control over which checkpoint to apportion from. This is essential for - calibration component re-runs where we must always restart from prior_step's - state regardless of what other checkpoints exist in the pipeline. - - Parameters - ---------- - state : workflow.State - component_name : str - The model component to run. - restore_checkpoint : str - The checkpoint name to restore from before apportioning. - This should be the checkpoint representing prior_step's state. - shared_data_buffers : dict - Pre-allocated shared memory buffers for skims/shadow pricing. - """ - from activitysim.core import mp_tasks - - # Determine slice info from original settings - original_steps = state.settings.multiprocess_steps - all_models = state.settings.models - - slice_info = None - num_processes = state.settings.num_processes or 2 - chunk_size = state.settings.chunk_size or 0 - - # Find which original step this component belongs to - step_boundaries = [] - for i, step in enumerate(original_steps): - step_boundaries.append(all_models.index(step.begin)) - step_boundaries.append(len(all_models)) - - component_idx = all_models.index(component_name) - for i, step in enumerate(original_steps): - if step_boundaries[i] <= component_idx < step_boundaries[i + 1]: - if step.slice: - slice_info = step.slice.model_dump() - if step.num_processes: - num_processes = step.num_processes - if step.chunk_size: - chunk_size = step.chunk_size - break - - # Build step_info dict matching what mp_tasks functions expect - step_info = { - "name": component_name, - "models": [component_name], - "num_processes": num_processes, - "chunk_size": chunk_size, - "step_num": 0, - "slice": slice_info, - "last_checkpoint_in_previous_multiprocess_step": restore_checkpoint, - } - - injectables = _build_calibration_injectables(state) - - if num_processes == 1: - sub_proc_names = [component_name] - else: - sub_proc_names = [f"{component_name}_{i}" for i in range(num_processes)] - - fail_fast = state.settings.fail_fast - - # Apportion pipeline (split tables across sub-processes) - if num_processes > 1 and slice_info is not None: - mp_tasks.run_sub_task( - state, - multiprocessing.Process( - target=mp_tasks.mp_apportion_pipeline, - name=f"{component_name}_apportion", - args=(injectables, sub_proc_names, step_info), - ), - ) - - # For multi-process runs, subprocesses must restore from the apportioned - # pipeline (which has one checkpoint). Use LAST_CHECKPOINT so they don't - # overwrite the apportioned data with a fresh pipeline. - # For single-process runs (no apportion), use restore_checkpoint to resume - # from the correct point in the main pipeline. - if num_processes > 1: - sim_resume_after = "_" # LAST_CHECKPOINT in apportioned sub-pipeline - else: - sim_resume_after = restore_checkpoint - - # Run simulations in sub-processes - completed = mp_tasks.run_sub_simulations( - state, - injectables, - shared_data_buffers, - step_info, - sub_proc_names, - sim_resume_after, - [], # previously_completed - fail_fast, - ) - - if len(completed) != num_processes: - from activitysim.core.exceptions import SubprocessError - - raise SubprocessError( - f"{num_processes - len(completed)} processes failed in " - f"calibration step {component_name}" - ) - - # Coalesce sub-process pipelines back into main pipeline - if num_processes > 1 and slice_info is not None: - mp_tasks.run_sub_task( - state, - multiprocessing.Process( - target=mp_tasks.mp_coalesce_pipelines, - name=f"{component_name}_coalesce", - args=(injectables, sub_proc_names, slice_info), - ), - ) - - # Restore coalesced results into parent state - _restore_parent_state_from_pipeline(state) - - -def _run_in_configured_mode( - state: workflow.State, - models: list[str], - resume_after: str | None, - shared_data_buffers: dict | None = None, -) -> None: - """Run models using the same single/multiprocess mode as the parent run.""" - if not models: - return - - extra_models = _prep_model_data(state, resume_after=resume_after) - if extra_models: - # Models from the step beginning through resume_after must run first - # to recreate the correct intermediate state (since no model-level - # checkpoint existed for resume_after). - models = extra_models + models - - if state.settings.multiprocess: - # Write the restored state as a checkpoint so LAST_CHECKPOINT on disk - # reflects the correct (clean) state for the apportion subprocess. - # When extra_models were prepended, the state is from a PRIOR step - # (not the actual resume_after point) — use a non-conflicting name - # so it won't be mistakenly loaded as the resume_after state on a - # subsequent restart. - if extra_models: - state.checkpoint.add(f"_calibration_staging") - else: - state.checkpoint.add(resume_after or models[0]) - state.checkpoint.close_store() - - # When subprocess pipelines from a prior run already have the - # resume_after checkpoint (Path 2 in _prep_model_data), subprocesses - # can skip models before resume_after by reusing those pipelines - # instead of freshly apportioning. Signal this by passing - # can_reuse_subprocs=True. - can_reuse = not extra_models and resume_after is not None - - _run_multiprocess_with_overrides( - state, - models=models, - resume_after=resume_after, - shared_data_buffers=shared_data_buffers, - can_reuse_subprocs=can_reuse, - ) - # After multiprocess completes, the coalesced pipeline exists on disk. - # Restore it into the parent process state so tables are accessible - # for calibration expression evaluation. - _restore_parent_state_from_pipeline(state) - # Add a checkpoint named after the last model so that model-name - # references (e.g. _prior_step_name, resume_after on global_iter > 1) - # resolve correctly. Without this, only the step-level coalesce name - # exists in the pipeline. - state.checkpoint.add(models[-1]) - return - - # State is already at the correct point from _prep_model_data above. - # Do NOT call _prep_model_data again — the second call would build its - # table_checkpoint_map from the now-truncated in-memory checkpoint history, - # losing references to tables created after resume_after (e.g. vehicles). - state.checkpoint.add(resume_after or models[0]) - for model in models: - state.run.by_name(model) - # Ensure final model's state is persisted even if should_save_checkpoint - # returned False for it — _calibrate_component needs to restore to it. - if models: - state.checkpoint.add(models[-1]) - - -def _prep_model_data(state, resume_after=None): - """Restore the pipeline to the correct state before running models. - - Resolution priority: - 1. Direct model-level checkpoint in the main pipeline (fastest, exact). - 2. Subprocess pipelines from a prior multiprocess run — performs a - "coalesce at specific checkpoint" to recover the exact intermediate - state without re-running anything. - 3. Previous step checkpoint + re-run models from step begin through - resume_after (slowest but always works). - - Returns - ------- - list[str] - Models that must be prepended to the caller's models list to reach - the correct state at ``resume_after``. Empty when the exact - checkpoint was found and restored directly (paths 1 or 2). - """ - if resume_after: - try: - if state.checkpoint.store_is_open(): - checkpoint_names = [ - cp.get("checkpoint_name", "") for cp in state.checkpoint.checkpoints - ] - else: - from activitysim.core.workflow.checkpoint import HdfStore, ParquetStore - - pipeline_path = Path(state.checkpoint.default_pipeline_file_path()) - if state.settings.checkpoint_format == "hdf": - store = HdfStore(pipeline_path, mode="r") - else: - store = ParquetStore(pipeline_path, mode="r") - try: - checkpoint_names = store.list_checkpoint_names() - finally: - store.close() - - # Path 1: direct model-level checkpoint in main pipeline - if resume_after in checkpoint_names: - _restore_parent_state_from_pipeline(state, checkpoint_name=resume_after) - return [] - - # Path 2: subprocess pipelines (model-level checkpoints preserved) - if _restore_from_subprocess_pipelines(state, resume_after): - return [] - - # Path 3: restore from previous step and re-run - all_models = state.settings.models - mp_steps = state.settings.multiprocess_steps - if mp_steps and resume_after in all_models: - resume_idx = all_models.index(resume_after) - step_boundaries = [all_models.index(s.begin) for s in mp_steps] - step_boundaries.append(len(all_models)) - for i, step in enumerate(mp_steps): - if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: - if i > 0 and mp_steps[i - 1].name in checkpoint_names: - _restore_parent_state_from_pipeline( - state, checkpoint_name=mp_steps[i - 1].name - ) - step_begin_idx = step_boundaries[i] - extra_models = all_models[step_begin_idx : resume_idx + 1] - return extra_models - elif i == 0: - _restore_parent_state_from_pipeline( - state, checkpoint_name="_" - ) - extra_models = all_models[: resume_idx + 1] - return extra_models - break - except Exception: - logger.warning( - "calibration: could not restore from checkpoint %r, " - "falling back to LAST_CHECKPOINT", - resume_after, - ) - - # Fallback: load LAST_CHECKPOINT (appropriate after a coalesce that - # only ran the desired models) - _restore_parent_state_from_pipeline(state) - return [] - - -def _restore_from_subprocess_pipelines( - state: workflow.State, resume_after: str -) -> bool: - """Restore state from subprocess pipelines at a specific model checkpoint. - - Subprocess pipelines retain model-level checkpoints that don't exist in - the main pipeline. This performs a "coalesce at checkpoint" — reading - mirrored tables from one subprocess and concatenating sliced tables from - all subprocesses at the specified checkpoint name. - - Parameters - ---------- - state : workflow.State - resume_after : str - Model-level checkpoint name to restore from. - - Returns - ------- - bool - True if the restore succeeded; False if subprocess pipelines don't - exist or don't contain the requested checkpoint. - """ - from activitysim.core.workflow.checkpoint import ( - CHECKPOINT_NAME, - CHECKPOINT_TABLE_NAME, - HdfStore, - NON_TABLE_COLUMNS, - ParquetStore, - ) - - all_models = state.settings.models - mp_steps = state.settings.multiprocess_steps - if not mp_steps or resume_after not in all_models: - return False - - # Find the multiprocess step containing resume_after - resume_idx = all_models.index(resume_after) - step_boundaries = [all_models.index(s.begin) for s in mp_steps] - step_boundaries.append(len(all_models)) - - enclosing_step = None - num_processes = state.settings.num_processes or 2 - slice_info = None - for i, step in enumerate(mp_steps): - if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: - enclosing_step = step - if step.num_processes: - num_processes = step.num_processes - if step.slice: - slice_info = ( - step.slice.model_dump() - if hasattr(step.slice, "model_dump") - else step.slice - ) - break - - if enclosing_step is None or num_processes <= 1: - return False - - # Build subprocess pipeline file paths - step_name = enclosing_step.name - pipeline_file_name = state.filesystem.pipeline_file_name - sub_proc_names = [f"{step_name}_{i}" for i in range(num_processes)] - - def _subprocess_path(proc_name): - base = state.get_output_file_path(pipeline_file_name, prefix=proc_name) - if state.settings.checkpoint_format == "hdf": - return base - pq = Path(str(base)).with_suffix(ParquetStore.extension) - return pq if pq.exists() else base - - first_path = _subprocess_path(sub_proc_names[0]) - if not first_path.exists(): - return False - - # Open first subprocess pipeline and verify checkpoint exists - if state.settings.checkpoint_format == "hdf": - first_store = HdfStore(first_path, mode="r") - else: - first_store = ParquetStore(first_path, mode="r") - - try: - cp_names = first_store.list_checkpoint_names() - if resume_after not in cp_names: - return False - - # Read checkpoint row to get table→checkpoint mapping - cp_df = first_store.get_dataframe(CHECKPOINT_TABLE_NAME) - cp_row = cp_df[cp_df[CHECKPOINT_NAME] == resume_after].iloc[-1] - - table_map = {} - for col in cp_row.index: - if col not in NON_TABLE_COLUMNS and cp_row[col]: - table_map[col] = cp_row[col] - - # Read all tables from first subprocess at this checkpoint - tables = {} - for table_name, cp_for_table in table_map.items(): - try: - tables[table_name] = first_store.get_dataframe(table_name, cp_for_table) - except (FileNotFoundError, KeyError): - logger.warning( - f"calibration: subprocess pipeline missing table " - f"{table_name} at {cp_for_table}" - ) - finally: - first_store.close() - - if not tables: - return False - - # Determine sliced tables that need concatenation across processes - sliced_table_names = set(slice_info.get("tables", [])) if slice_info else set() - - # Read sliced tables from remaining subprocesses and concatenate - if num_processes > 1 and sliced_table_names: - omnibus = {t: [tables[t]] for t in sliced_table_names if t in tables} - - for proc_name in sub_proc_names[1:]: - proc_path = _subprocess_path(proc_name) - if not proc_path.exists(): - logger.warning( - f"calibration: subprocess pipeline not found: {proc_path}" - ) - return False - - if state.settings.checkpoint_format == "hdf": - proc_store = HdfStore(proc_path, mode="r") - else: - proc_store = ParquetStore(proc_path, mode="r") - - try: - proc_cp_df = proc_store.get_dataframe(CHECKPOINT_TABLE_NAME) - proc_row = proc_cp_df[proc_cp_df[CHECKPOINT_NAME] == resume_after].iloc[ - -1 - ] - - for table_name in list(omnibus.keys()): - cp_for_table = proc_row.get(table_name, "") - if cp_for_table: - omnibus[table_name].append( - proc_store.get_dataframe(table_name, cp_for_table) - ) - finally: - proc_store.close() - - # Replace sliced tables with concatenated versions - for table_name, dfs in omnibus.items(): - tables[table_name] = pd.concat(dfs, sort=False) - - # Load into parent state - prior_rng_channels = list(state.get_injectable("rng_channels", [])) - prior_index_to_channel = ( - dict(state.rng().index_to_channel) - if hasattr(state.rng(), "index_to_channel") - else {} - ) - - state.init_state() - if state.checkpoint.store_is_open(): - state.checkpoint.close_store() - state.checkpoint.open_store(overwrite=False) - - for table_name, df in tables.items(): - state.add_table(table_name, df) - - _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) - - # Mark all tables dirty for subsequent checkpoint.add - for table_name in list(state.existing_table_names): - state.existing_table_status[table_name] = True - - logger.info( - "calibration: restored %d tables from subprocess pipelines at " - "checkpoint '%s'", - len(tables), - resume_after, - ) - return True - - -def _run_multiprocess_with_overrides( - state: workflow.State, - models: list[str], - resume_after: str | None, - shared_data_buffers: dict | None = None, - can_reuse_subprocs: bool = False, -) -> None: - """Run multiprocess with temporary settings overrides for calibration passes. - - Parameters - ---------- - can_reuse_subprocs : bool, default False - When True, subprocess pipelines from a prior run are assumed to exist - and contain the ``resume_after`` checkpoint. Breadcrumbs are written - so that ``get_run_list`` populates ``step_info["resume_after"]``, - apportion is skipped (reusing existing subprocess pipelines), and - subprocesses resume from their model-level checkpoint — skipping - already-completed models. - """ - from collections import OrderedDict - - from activitysim.core import mp_tasks - - original_models = state.settings.models - original_mp_steps = state.settings.multiprocess_steps - original_resume_after = state.settings.resume_after - - # Build valid multiprocess_steps for the requested model subset. - calibration_mp_steps = _build_calibration_mp_steps( - models=models, - original_steps=original_mp_steps, - all_models=original_models, - ) - - state.settings.models = models - state.settings.multiprocess_steps = calibration_mp_steps - - if can_reuse_subprocs and resume_after: - # Include resume_after in the models list so get_breadcrumbs can - # locate the step containing it. Subprocesses will skip this model - # (it's already checkpointed in their pipeline) and run the rest. - models = [resume_after] + models - - # Rebuild steps with resume_after included. - calibration_mp_steps = _build_calibration_mp_steps( - models=models, - original_steps=original_mp_steps, - all_models=original_models, - ) - state.settings.models = models - state.settings.multiprocess_steps = calibration_mp_steps - state.settings.resume_after = resume_after - - # Write minimal breadcrumbs indicating the step containing - # resume_after has completed apportion (so it's skipped) but - # simulate/coalesce need re-running. - breadcrumbs = OrderedDict() - for step in calibration_mp_steps: - step_dict = {"name": step.name, "apportion": True} - breadcrumbs[step.name] = step_dict - # Find the step containing resume_after - all_models = state.settings.models - if resume_after in all_models: - step_begin = all_models.index(step.begin) - step_models_in_step = [ - m for m in all_models[step_begin:] if m in models - ] - if resume_after in step_models_in_step: - # This step contains resume_after — stop here. - # get_breadcrumbs will mark simulate/coalesce for re-run. - break - - mp_tasks.write_breadcrumbs(state, breadcrumbs) - else: - # No reuse: calibration manages pipeline state externally via - # _restore_parent_state_from_pipeline and checkpoint.add, so the MP - # system's breadcrumb-based resume logic must not be triggered. - state.settings.resume_after = None - - try: - injectables = _build_calibration_injectables(state) - mp_tasks.run_multiprocess( - state, - injectables, - shared_data_buffers=shared_data_buffers, - skip_final_checkpoint=True, - force_resume=resume_after is not None and not can_reuse_subprocs, - ) - finally: - state.settings.models = original_models - state.settings.resume_after = original_resume_after - state.settings.multiprocess_steps = original_mp_steps - - -def _reregister_rng_channels( - state: workflow.State, - prior_channels: list[str], - prior_index_to_channel: dict[str, str] = None, -) -> None: - """Re-register RNG channels that were lost during init_state().""" - current_channels = set(state.get_injectable("rng_channels", [])) - for channel_name in prior_channels: - if channel_name not in current_channels and state.is_table(channel_name): - try: - state.rng().add_channel(channel_name, state.get_dataframe(channel_name)) - current_channels.add(channel_name) - except Exception: - pass - # For channels whose tables don't exist at the restored checkpoint, - # register an empty channel. Do NOT pre-load from a later checkpoint - # in the store — that data may include modifications from downstream - # models and would pollute the pre-model state. The empty channel - # allows the model's normal table factory to create the table fresh - # and extend the channel without hitting the disjoint-index assertion. - if prior_index_to_channel: - for index_name, channel_name in prior_index_to_channel.items(): - if index_name not in state.rng().index_to_channel: - if channel_name not in state.rng().channels: - empty_df = pd.DataFrame( - index=pd.Index([], dtype="int64", name=index_name) - ) - state.rng().add_channel(channel_name, empty_df) - else: - state.rng().index_to_channel[index_name] = channel_name - current_channels.add(channel_name) - state.add_injectable("rng_channels", list(current_channels)) - - -def _invalidate_derived_tables(state: workflow.State) -> None: - """Drop factory-produced tables that may be stale after a calibration restore. - - When a calibrated model (e.g. auto_ownership) changes a table that a - @workflow.table factory depends on (e.g. vehicles depends on households), - the checkpoint may contain a stale version of that factory table. Dropping - it forces the factory to regenerate from current data on next access. - - Auto-detection rule: invalidate any table that is (a) registered as a - @workflow.table factory, (b) has DataFrame parameters (= table dependencies), - and (c) is in RANDOM_CHANNELS. This currently matches only 'vehicles' but - will automatically cover future factory tables with the same pattern. - """ - settings = read_calibration_settings(state) - if not settings: - return - - tables_to_invalidate = settings.run.invalidate_tables - if tables_to_invalidate is None: - # Vehicles needs regeneration only when calibration changes - # households.auto_ownership. Downstream calibration components must - # retain vehicle_type_choice's vehicle attributes. - tables_to_invalidate = ( - ["vehicles"] - if "auto_ownership_simulate" in settings.run.calibrate_models - else [] - ) - - logger.debug( - "calibration: tables detected for invalidation: %s", tables_to_invalidate - ) - tables_before = set(state.existing_table_names) - - for table_name in tables_to_invalidate: - if state.is_table(table_name): - state.drop_table(table_name) - state.rng().drop_channel(table_name) - state.get_dataframe(table_name, as_copy=False) - logger.debug("calibration: invalidated derived table '%s'", table_name) - - tables_after = set(state.existing_table_names) - lost = tables_before - tables_after - set(tables_to_invalidate) - if lost: - logger.error( - "calibration: tables unexpectedly removed during invalidation: %s", lost - ) - - -def _restore_parent_state_from_pipeline( - state: workflow.State, checkpoint_name: str = "_" -) -> None: - """Restore pipeline tables into the parent process state. - - After a multiprocess run, the parent's in-memory state is stale. - This loads a specific checkpoint from the pipeline store so that - calibration expressions can evaluate against model outputs. - - Parameters - ---------- - checkpoint_name : str, default "_" - The checkpoint to restore from. Use a model-level checkpoint name - (e.g. the prior step name) to get the exact state at that point, - avoiding pollution from downstream models that may have added rows - to shared tables like ``tours``. The default ``"_"`` loads the - last checkpoint, which is appropriate immediately after a coalesce - that only ran the desired models. - - All tables are explicitly re-checkpointed so that subsequent apportion - subprocesses can load them from a direct file path without relying on - checkpoint backtracking through potentially ambiguous checkpoint history. - """ - # Capture RNG state before restore — models may have dynamically - # added channels (e.g. "vehicles") that aren't in the default - # rng_channels injectable and would be lost by init_state(). - prior_rng_channels = list(state.get_injectable("rng_channels", [])) - prior_index_to_channel = ( - dict(state.rng().index_to_channel) - if hasattr(state.rng(), "index_to_channel") - else {} - ) - - if state.checkpoint.store_is_open(): - state.checkpoint.close_store() - state.checkpoint.restore(resume_after=checkpoint_name) - - _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) - - # After restore, all tables are clean (status=False). Mark them dirty so - # the next checkpoint.add() writes them to disk at a known checkpoint name. - # This ensures apportion subprocesses find table files at a single, - # unambiguous checkpoint rather than needing to backtrack through history. - for table_name in list(state.existing_table_names): - state.existing_table_status[table_name] = True - - -def _initialize_mp_shared_resources(state: workflow.State) -> dict: - """Allocate shared data buffers (skims, shadow pricing) once for reuse. - - This mirrors the allocation logic in mp_tasks.run_multiprocess but - is called once at calibration start rather than on every sub-run. - """ - from activitysim.core import mp_tasks, tracing - - shared_data_buffers = {} - sharrow_enabled = state.settings.sharrow - - t0 = tracing.print_elapsed_time() - if not sharrow_enabled: - shared_data_buffers.update(mp_tasks.allocate_shared_skim_buffers(state)) - t0 = tracing.print_elapsed_time("calibration: allocate shared skim buffer", t0) - - shared_data_buffers.update(mp_tasks.allocate_shared_shadow_pricing_buffers(state)) - t0 = tracing.print_elapsed_time( - "calibration: allocate shared shadow_pricing buffer", t0 - ) - - shared_data_buffers.update( - mp_tasks.allocate_shared_shadow_pricing_buffers_choice(state) - ) - t0 = tracing.print_elapsed_time( - "calibration: allocate shared shadow_pricing choice buffer", t0 - ) - - # Load skim data into the shared buffers. - if sharrow_enabled: - shared_data_buffers["skim_dataset"] = "sh.Dataset:skim_dataset" - from activitysim.core import flow, skim_dataset # noqa: F401 - - state.get_injectable("skim_dataset") - else: - if len(shared_data_buffers) > 0: - injectables = _build_calibration_injectables(state) - mp_tasks.run_sub_task( - state, - multiprocessing.Process( - target=mp_tasks.mp_setup_skims, - name="mp_setup_skims_calibration", - args=(injectables,), - kwargs=shared_data_buffers, - ), - ) - - # Make skims available in the parent process for expression evaluation. - state.add_injectable("data_buffers", shared_data_buffers) - try: - state.get_injectable("network_los") - except Exception: - logger.warning( - "calibration: could not resolve network_los in parent process; " - "skim-dependent expressions may fail" - ) - - return shared_data_buffers - - -def _build_calibration_mp_steps( - models: list[str], - original_steps: list[MultiprocessStep], - all_models: list[str], -) -> list[MultiprocessStep]: - """Build valid MultiprocessStep objects for a calibration model subset. - - The key challenge is that get_run_list() in mp_tasks requires: - - The first step's begin == models[0] - - Steps are ordered and non-overlapping - - Each step's begin is in the models list - - We intersect the original multiprocess_steps with the requested model - subset and construct new steps that satisfy these constraints. - """ - if not models: - return [] - - # Determine which original step each model in the full list belongs to. - # Build a mapping: model_name -> original step index - model_to_step: dict[str, int] = {} - step_boundaries = [] - for i, step in enumerate(original_steps): - begin_idx = all_models.index(step.begin) - step_boundaries.append(begin_idx) - step_boundaries.append(len(all_models)) - - for i, step in enumerate(original_steps): - for model_idx in range(step_boundaries[i], step_boundaries[i + 1]): - model_to_step[all_models[model_idx]] = i - - # Group the requested models by their original step - from collections import OrderedDict - - step_model_groups: OrderedDict[int, list[str]] = OrderedDict() - for model in models: - step_idx = model_to_step.get(model) - if step_idx is None: - continue - step_model_groups.setdefault(step_idx, []).append(model) - - # Build new MultiprocessStep for each group. - # Some original steps (e.g. mp_initialize) omit num_processes, slice, - # and chunk_size — these default to None on MultiprocessStep and - # get_run_list() applies global defaults when they are absent. - # Step names include the first model to ensure uniqueness across multiple - # intermediate runs that draw from the same original step. - new_steps = [] - for step_idx, step_models in step_model_groups.items(): - orig_step = original_steps[step_idx] - kwargs: dict[str, Any] = { - "name": orig_step.name, - "begin": step_models[0], - } - if orig_step.num_processes is not None: - kwargs["num_processes"] = orig_step.num_processes - if orig_step.slice is not None: - kwargs["slice"] = orig_step.slice - if orig_step.chunk_size is not None: - kwargs["chunk_size"] = orig_step.chunk_size - new_steps.append(MultiprocessStep(**kwargs)) - - return new_steps - - -def _build_calibration_injectables(state: workflow.State) -> dict: - """Build the injectables dict for multiprocess sub-processes.""" - injectables = {} - for key in MP_INJECTABLES: - try: - injectables[key] = state.get_injectable(key) - except KeyError: - pass - injectables["settings"] = state.settings - return injectables diff --git a/activitysim/core/calibration/README.md b/activitysim/core/calibration/README.md new file mode 100644 index 0000000000..66c92b3802 --- /dev/null +++ b/activitysim/core/calibration/README.md @@ -0,0 +1,27 @@ +# ActivitySim calibration package + +This package implements ActivitySim's automated coefficient calibration workflow. +The modules are divided by responsibility so orchestration, model execution, and +calibration math can evolve independently. + +- `__init__.py` defines the public API and compatibility access to former private + attributes from `activitysim.core.calibration`. +- `settings.py` defines calibration configuration models, result types, and + configuration loading. +- `orchestrator.py` coordinates global iterations and model-component sequencing. +- `component.py` runs component iterations, validates calibration specifications, + and calculates coefficient updates. +- `expressions.py` builds expression contexts, evaluates target/model expressions, + and loads optional helper modules. +- `coefficients.py` locates, reads, and writes model coefficient files. +- `reporting.py` writes iteration histories, summaries, plots, and final snapshots. +- `recovery.py` maintains calibration progress and start-of-iteration coefficient + backups. +- `execution.py` restores pipeline state and dispatches model execution. +- `multiprocess.py` contains subprocess orchestration, shared-resource setup, and + multiprocess pipeline restoration. + +Dependencies should flow from `orchestrator.py` into the focused modules. Lower +level modules should not import the orchestrator. Multiprocess code imports back +into `execution.py` only inside the single-component runner to avoid an import-time +cycle. diff --git a/activitysim/core/calibration/__init__.py b/activitysim/core/calibration/__init__.py new file mode 100644 index 0000000000..442046abd7 --- /dev/null +++ b/activitysim/core/calibration/__init__.py @@ -0,0 +1,70 @@ +# ActivitySim +# See full license in LICENSE.txt. +"""Automated calibration support for ActivitySim model runs. + +The public API is intentionally small. Private attributes from the former +single-file module remain available through ``__getattr__`` for compatibility +while callers migrate to the package modules. +""" + +from . import ( + coefficients, + component, + execution, + expressions, + multiprocess, + recovery, + reporting, + settings, +) +from .orchestrator import run_calibration_loop +from .settings import ( + CalibrationComponentResult, + CalibrationComponentSettings, + CalibrationConfig, + CalibrationReportsSettings, + CalibrationRunResult, + CalibrationRunSettings, + calibration_enabled, + read_calibration_settings, +) + +__all__ = [ + "CalibrationComponentResult", + "CalibrationComponentSettings", + "CalibrationConfig", + "CalibrationReportsSettings", + "CalibrationRunResult", + "CalibrationRunSettings", + "calibration_enabled", + "read_calibration_settings", + "run_calibration_loop", +] + +_COMPATIBILITY_MODULES = ( + settings, + component, + expressions, + coefficients, + reporting, + recovery, + execution, + multiprocess, +) + + +def __getattr__(name): + for module in _COMPATIBILITY_MODULES: + if hasattr(module, name): + return getattr(module, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__(): + compatibility_names = { + name + for module in _COMPATIBILITY_MODULES + for name in vars(module) + if not name.startswith("__") + } + return sorted(set(globals()) | compatibility_names) diff --git a/activitysim/core/calibration/coefficients.py b/activitysim/core/calibration/coefficients.py new file mode 100644 index 0000000000..9aa15bcd4c --- /dev/null +++ b/activitysim/core/calibration/coefficients.py @@ -0,0 +1,85 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +import pandas as pd + +from activitysim.core import workflow + +from .settings import CalibrationConfig + + +def _persist_coefficients_to_config( + state: workflow.State, + model_settings: dict[str, Any] | Any, + coefficients_df: pd.DataFrame, +) -> None: + """Write updated coefficients back to the component coefficient file in configs.""" + coeff_file = _setting_value(model_settings, "COEFFICIENTS") + if not coeff_file: + raise RuntimeError("component model settings missing COEFFICIENTS") + + output = coefficients_df.copy() + output.index.name = "coefficient_name" + + coeff_path = state.filesystem.get_config_file_path(coeff_file) + output.to_csv(coeff_path) + + +def _infer_model_settings_file(component_name: str) -> str: + """Infer model settings yaml filename from component step name.""" + # This follows the dominant naming convention in the existing codebase. + if component_name.endswith("_simulate"): + base = component_name[: -len("_simulate")] + else: + base = component_name + return f"{base}.yaml" + + +def _settings_to_dict(model_settings: dict[str, Any] | Any) -> dict[str, Any]: + """Convert pydantic or dict model settings to a plain dictionary.""" + if isinstance(model_settings, dict): + return model_settings + if hasattr(model_settings, "model_dump"): + return model_settings.model_dump() + return dict(model_settings) + + +def _setting_value(model_settings: dict[str, Any] | Any, key: str, default=None): + """Read a setting value from dict-like or attribute-based settings.""" + if isinstance(model_settings, dict): + return model_settings.get(key, default) + return getattr(model_settings, key, default) + + +def _calibration_coefficient_paths( + state: workflow.State, + calibration_settings: CalibrationConfig, +) -> list[Path]: + """Return the unique coefficient files modified by this calibration run.""" + paths: list[Path] = [] + seen: set[str] = set() + + for component_name in calibration_settings.run.calibrate_models: + model_settings_file = _infer_model_settings_file(component_name) + model_settings = state.filesystem.read_model_settings( + model_settings_file, mandatory=True + ) + coefficient_file = _setting_value(model_settings, "COEFFICIENTS") + if not coefficient_file: + raise RuntimeError( + f"component {component_name} model settings missing COEFFICIENTS" + ) + + path = Path(state.filesystem.get_config_file_path(coefficient_file)).resolve() + key = os.path.normcase(str(path)) + if key not in seen: + paths.append(path) + seen.add(key) + + return paths + diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py new file mode 100644 index 0000000000..7a3285ca3a --- /dev/null +++ b/activitysim/core/calibration/component.py @@ -0,0 +1,514 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import logging +import math +import re +from typing import Any + +import numpy as np +import pandas as pd + +from activitysim.core import simulate, workflow + +from .coefficients import ( + _infer_model_settings_file, + _persist_coefficients_to_config, + _setting_value, + _settings_to_dict, +) +from .execution import _invalidate_derived_tables, _prep_model_data +from .expressions import ( + _build_expression_context, + _compute_delta, + _eval_numeric_value, + _load_helper_symbols, +) +from .multiprocess import _run_mp_single_component +from .reporting import ( + _append_iteration_records, + _append_summary_records, + _write_generic_report, +) +from .settings import CalibrationComponentResult, CalibrationComponentSettings + +logger = logging.getLogger("calibration") + +DEFAULT_INCREMENT = 2.0 +CALIBRATION_REQUIRED_COLUMNS = [ + "description", + "coefficient", + "model_value", + "target_value", + "hold_fast", + "min", + "max", + "damping", + "method", + "tolerance", +] + + +def _calibrate_component( + state: workflow.State, + component_name: str, + component_settings: CalibrationComponentSettings, + prior_step: str, + global_iter: int, + shared_data_buffers: dict | None = None, +) -> CalibrationComponentResult: + """Run iterative coefficient calibration for one component.""" + model_settings_file = _infer_model_settings_file(component_name) + model_settings = state.filesystem.read_model_settings( + model_settings_file, mandatory=True + ) + + coefficients_df = state.filesystem.read_model_coefficients( + model_settings=model_settings + ) + helper_symbols, bespoke_callable, helper_module = _load_helper_symbols( + state, + component_settings, + ) + + calibration_spec_df = _read_calibration_spec( + state, component_settings.calibration_spec + ) + + utility_coeff_names = _extract_utility_coefficient_names(state, model_settings) + _validate_calibration_coefficients_against_utility_spec( + component_name, + calibration_spec_df, + utility_coeff_names, + ) + + coefficients_df = _ensure_coefficients_exist( + component_name, + calibration_spec_df, + coefficients_df, + ) + + _warn_if_initial_values_outside_bounds( + component_name, calibration_spec_df, coefficients_df + ) + + component_converged = False + component_iterations = 0 + + # Determine the checkpoint name to restore from for component re-runs. + # In MP mode, the checkpoint that represents prior_step's completed state + # is the last checkpoint in the pipeline before we run the component. + # We capture it once and reuse across component iterations. + mp_restore_checkpoint = None + if state.settings.multiprocess and shared_data_buffers is not None: + # The pipeline should already be open from _restore_parent_state_from_pipeline + # called after precursor/intermediate models ran. The last checkpoint + # in the pipeline represents the state at prior_step. + if state.checkpoint.checkpoints: + mp_restore_checkpoint = state.checkpoint.last_checkpoint.get( + "checkpoint_name", "_" + ) + else: + mp_restore_checkpoint = "_" + + for component_iter in range(1, component_settings.submodel_max_iterations + 1): + component_iterations = component_iter + run_model_name = f"{component_name}.c_i{component_iter};" f"g_i{global_iter}" + # Re-run only this component from its prior checkpoint so model values + # reflect the current candidate coefficients for this component. + if state.settings.multiprocess and shared_data_buffers is not None: + # Use direct MP orchestration with explicit checkpoint control. + # This ensures we always apportion from prior_step's state, + # even after multiple component iterations. + _run_mp_single_component( + state, + component_name=component_name, + # Always restore from the same immutable pre-component + # checkpoint. LAST_CHECKPOINT may point to the prior iteration's + # coalesced component output and is therefore not a safe baseline. + restore_checkpoint=mp_restore_checkpoint, + shared_data_buffers=shared_data_buffers, + ) + else: + # Restore to prior_step ourselves then run the model directly. + # state.run(resume_after=prior_step) would trigger + # checkpoint.restore → init_state which creates a fresh RNG. + # If prior_step is before the calibrated model created its table + # (e.g. vehicles), the table won't be in that checkpoint and the + # RNG channel won't be registered — causing a crash when the + # model tries to use it. By restoring here and calling by_name, + # we keep the RNG channels from _prep_model_data intact. + extra_models = _prep_model_data(state, resume_after=prior_step) + if extra_models: + # prior_step checkpoint not found directly; run intermediate + # models (e.g. annotators) to recreate the correct state. + for m in extra_models: + state.run.by_name(m) + _invalidate_derived_tables(state) + state.checkpoint.add(prior_step) + state.run.by_name(run_model_name) + + eval_context = _build_expression_context( + state, helper_symbols, component_name, component_settings + ) + + ( + row_records, + summary_record, + new_coefficients_df, + component_converged, + ) = _evaluate_and_update( + component_name=component_name, + calibration_spec_df=calibration_spec_df, + coefficients_df=coefficients_df, + eval_context=eval_context, + global_iter=global_iter, + component_iter=component_iter, + ) + + coefficients_df = new_coefficients_df + + _persist_coefficients_to_config(state, model_settings, coefficients_df) + _append_iteration_records(state, component_name, row_records) + _append_summary_records(state, [summary_record]) + + if component_settings.reports.generic: + _write_generic_report(state, component_name, row_records) + + if bespoke_callable is not None: + bespoke_callable(eval_context) + + if component_converged: + break + + state.checkpoint.add(component_name) + + return CalibrationComponentResult( + component=component_name, + converged=component_converged, + component_iterations=component_iterations, + ) + + +def _read_calibration_spec(state: workflow.State, file_name: str) -> pd.DataFrame: + """Read calibration spec CSV and validate required columns.""" + path = state.filesystem.get_config_file_path(file_name) + df = pd.read_csv(path, comment="#") + + missing = [c for c in CALIBRATION_REQUIRED_COLUMNS if c not in df.columns] + if missing: + raise ValueError( + f"calibration_spec {file_name} is missing required columns: {missing}" + ) + + df = df[CALIBRATION_REQUIRED_COLUMNS].copy() + df["description"] = df["description"].astype(str) + df["coefficient"] = df["coefficient"].astype(str) + + # Normalize booleans and defaults to keep row math deterministic. + df["hold_fast"] = ( + df["hold_fast"] + .fillna(False) + .astype(str) + .str.strip() + .str.lower() + .isin(["1", "true", "t", "yes", "y"]) + ) + + for c in ["min", "max", "damping", "tolerance"]: + df[c] = pd.to_numeric(df[c], errors="coerce") + + df["method"] = df["method"].astype(str).str.strip().str.lower() + + bad_methods = df.loc[~df["method"].isin(["log_ratio", "odds_ratio"]), "method"] + if not bad_methods.empty: + raise ValueError( + f"unsupported calibration method(s): {sorted(set(bad_methods))}" + ) + + if df["damping"].isna().any(): + raise ValueError("calibration_spec damping must be numeric") + if df["tolerance"].isna().any(): + raise ValueError("calibration_spec tolerance must be numeric") + + return df + + +def _extract_utility_coefficient_names( + state: workflow.State, + model_settings: dict[str, Any] | Any, +) -> set[str]: + """ + Extract coefficient names used by the configured utility specifications. + + Templated logit models map utility-spec row labels to actual coefficient + names by segment, so their template cell values are the source of truth. + Other models use coefficient tokens in utility-spec columns. + """ + if _setting_value(model_settings, "COEFFICIENT_TEMPLATE"): + template = simulate.read_model_coefficient_template( + state.filesystem, model_settings + ) + return {str(name) for name in template.to_numpy().ravel()} + + names: set[str] = set() + + model_settings_dict = _settings_to_dict(model_settings) + spec_keys = [ + k for k in model_settings_dict.keys() if str(k).upper().endswith("SPEC") + ] + for key in spec_keys: + spec_file = model_settings_dict.get(key) + if not spec_file: + continue + + spec_path = state.filesystem.get_config_file_path(spec_file) + try: + raw = pd.read_csv(spec_path, comment="#") + except Exception: + # Do not fail hard on optional or model-specific supplemental specs. + continue + + utility_columns = [ + c + for c in raw.columns + if c + not in [ + "Description", + "Expression", + "Label", + "description", + "expression", + "label", + ] + ] + + for col in utility_columns: + for value in raw[col].dropna().tolist(): + if isinstance(value, (int, float, np.number)): + continue + text = str(value) + for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", text): + names.add(token) + + return names + + +def _validate_calibration_coefficients_against_utility_spec( + component_name: str, + calibration_spec_df: pd.DataFrame, + utility_coeff_names: set[str], +) -> None: + """Ensure calibration coefficients are referenced in utility specifications.""" + missing = sorted( + set(calibration_spec_df["coefficient"].tolist()) - set(utility_coeff_names) + ) + if missing: + raise ValueError( + f"calibration coefficients not present in model utility specification for {component_name}: {missing}" + ) + + +def _ensure_coefficients_exist( + component_name: str, + calibration_spec_df: pd.DataFrame, + coefficients_df: pd.DataFrame, +) -> pd.DataFrame: + """Add calibration coefficients missing from coefficient file with value 0.0.""" + coeffs = coefficients_df.copy() + + for coefficient_name in calibration_spec_df["coefficient"].tolist(): + if coefficient_name not in coeffs.index: + logger.warning( + "component %s coefficient %s missing from coefficient file, adding with value 0.0", + component_name, + coefficient_name, + ) + coeffs.loc[coefficient_name, "value"] = 0.0 + + coeffs["value"] = pd.to_numeric(coeffs["value"], errors="coerce") + if coeffs["value"].isna().any(): + bad = coeffs[coeffs["value"].isna()].index.tolist() + raise ValueError(f"non-numeric coefficient values found: {bad}") + + return coeffs + + +def _warn_if_initial_values_outside_bounds( + component_name: str, + calibration_spec_df: pd.DataFrame, + coefficients_df: pd.DataFrame, +) -> None: + """Warn when initial coefficient values violate provided bounds.""" + for _, row in calibration_spec_df.iterrows(): + coefficient_name = row["coefficient"] + current_value = float(coefficients_df.loc[coefficient_name, "value"]) + lower = row["min"] + upper = row["max"] + + if not pd.isna(lower) and current_value < float(lower): + logger.warning( + "component %s coefficient %s starts below min bound (%s < %s)", + component_name, + coefficient_name, + current_value, + lower, + ) + if not pd.isna(upper) and current_value > float(upper): + logger.warning( + "component %s coefficient %s starts above max bound (%s > %s)", + component_name, + coefficient_name, + current_value, + upper, + ) + + +def _evaluate_and_update( + component_name: str, + calibration_spec_df: pd.DataFrame, + coefficients_df: pd.DataFrame, + eval_context: dict[str, Any], + global_iter: int, + component_iter: int, +) -> tuple[list[dict[str, Any]], dict[str, Any], pd.DataFrame, bool]: + """Evaluate spec rows, update coefficients, and return detailed records.""" + updated = coefficients_df.copy() + records: list[dict[str, Any]] = [] + + max_difference = -math.inf + max_difference_coefficient = "" + max_change = -math.inf + max_change_coefficient = "" + + num_converged = 0 + + for _, row in calibration_spec_df.iterrows(): + coefficient_name = row["coefficient"] + description = row["description"] + method = row["method"] + hold_fast = bool(row["hold_fast"]) + + default_increment = ( + row["default_increment"] + if "default_increment" in row.index + else DEFAULT_INCREMENT + ) + + prev_value = float(updated.loc[coefficient_name, "value"]) + + model_value = _eval_numeric_value( + row["model_value"], + eval_context, + component_name, + description, + "model_value", + ) + target_value = _eval_numeric_value( + row["target_value"], + eval_context, + component_name, + description, + "target_value", + ) + + difference = target_value - model_value + pct_difference = _safe_percent_difference(difference, target_value) + + tolerance = float(row["tolerance"]) + converged = abs(difference) <= tolerance + + damping = float(row["damping"]) + raw_delta = _compute_delta( + method=method, + model_value=model_value, + target_value=target_value, + damping=damping, + component_name=component_name, + description=description, + default_increment=default_increment, + ) + + candidate_value = prev_value if hold_fast else prev_value + raw_delta + + at_min = False + at_max = False + + lower = row["min"] + upper = row["max"] + + if not pd.isna(lower) and candidate_value <= float(lower): + candidate_value = float(lower) + at_min = True + if not pd.isna(upper) and candidate_value >= float(upper): + candidate_value = float(upper) + at_max = True + + if not np.isfinite(candidate_value): + raise RuntimeError( + f"non-finite next coefficient for {component_name} / {description} / {coefficient_name}" + ) + + updated.loc[coefficient_name, "value"] = candidate_value + + abs_diff = abs(difference) + abs_change = abs(candidate_value - prev_value) + + if abs_diff > max_difference: + max_difference = abs_diff + max_difference_coefficient = coefficient_name + + if abs_change > max_change: + max_change = abs_change + max_change_coefficient = coefficient_name + + if converged: + num_converged += 1 + + records.append( + { + "global_iter": global_iter, + "component_iter": component_iter, + "description": description, + "component": component_name, + "coefficient": coefficient_name, + "target_value": target_value, + "model_value": model_value, + "difference": difference, + "pct_difference": pct_difference, + "hold_fast": hold_fast, + "prev_coefficient": prev_value, + "coef_delta": abs_change, + "next_coefficient": candidate_value, + "converged": converged, + "at_min": at_min, + "at_max": at_max, + } + ) + + total_rows = len(calibration_spec_df) + num_unconverged = total_rows - num_converged + component_converged = num_unconverged == 0 + + summary_record = { + "global_iter": global_iter, + "component_iter": component_iter, + "component": component_name, + "max_difference": max_difference if max_difference != -math.inf else 0.0, + "max_difference_coefficient": max_difference_coefficient, + "max_change": max_change if max_change != -math.inf else 0.0, + "max_change_coefficient": max_change_coefficient, + "num_converged_iter": num_converged, + "tot_converged": num_converged, + "num_unconverged": num_unconverged, + } + + return records, summary_record, updated, component_converged + + +def _safe_percent_difference(difference: float, target_value: float) -> float: + """Return a stable percentage difference with zero-target handling.""" + if target_value == 0: + return math.inf if difference != 0 else 0.0 + return (difference / target_value) * 100.0 diff --git a/activitysim/core/calibration/execution.py b/activitysim/core/calibration/execution.py new file mode 100644 index 0000000000..0c7f8e98ab --- /dev/null +++ b/activitysim/core/calibration/execution.py @@ -0,0 +1,262 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import logging +from pathlib import Path + +from activitysim.core import workflow + +from .multiprocess import ( + _reregister_rng_channels, + _restore_from_subprocess_pipelines, + _run_multiprocess_with_overrides, +) +from .settings import read_calibration_settings + +logger = logging.getLogger("calibration") + + +def _run_in_configured_mode( + state: workflow.State, + models: list[str], + resume_after: str | None, + shared_data_buffers: dict | None = None, +) -> None: + """Run models using the same single/multiprocess mode as the parent run.""" + if not models: + return + + extra_models = _prep_model_data(state, resume_after=resume_after) + if extra_models: + # Models from the step beginning through resume_after must run first + # to recreate the correct intermediate state (since no model-level + # checkpoint existed for resume_after). + models = extra_models + models + + if state.settings.multiprocess: + # Write the restored state as a checkpoint so LAST_CHECKPOINT on disk + # reflects the correct (clean) state for the apportion subprocess. + # When extra_models were prepended, the state is from a PRIOR step + # (not the actual resume_after point) — use a non-conflicting name + # so it won't be mistakenly loaded as the resume_after state on a + # subsequent restart. + if extra_models: + state.checkpoint.add("_calibration_staging") + else: + state.checkpoint.add(resume_after or models[0]) + state.checkpoint.close_store() + + # When subprocess pipelines from a prior run already have the + # resume_after checkpoint (Path 2 in _prep_model_data), subprocesses + # can skip models before resume_after by reusing those pipelines + # instead of freshly apportioning. Signal this by passing + # can_reuse_subprocs=True. + can_reuse = not extra_models and resume_after is not None + + _run_multiprocess_with_overrides( + state, + models=models, + resume_after=resume_after, + shared_data_buffers=shared_data_buffers, + can_reuse_subprocs=can_reuse, + ) + # After multiprocess completes, the coalesced pipeline exists on disk. + # Restore it into the parent process state so tables are accessible + # for calibration expression evaluation. + _restore_parent_state_from_pipeline(state) + # Add a checkpoint named after the last model so that model-name + # references (e.g. _prior_step_name, resume_after on global_iter > 1) + # resolve correctly. Without this, only the step-level coalesce name + # exists in the pipeline. + state.checkpoint.add(models[-1]) + return + + # State is already at the correct point from _prep_model_data above. + # Do NOT call _prep_model_data again — the second call would build its + # table_checkpoint_map from the now-truncated in-memory checkpoint history, + # losing references to tables created after resume_after (e.g. vehicles). + state.checkpoint.add(resume_after or models[0]) + for model in models: + state.run.by_name(model) + # Ensure final model's state is persisted even if should_save_checkpoint + # returned False for it — _calibrate_component needs to restore to it. + if models: + state.checkpoint.add(models[-1]) + + +def _prep_model_data(state, resume_after=None): + """Restore the pipeline to the correct state before running models. + + Resolution priority: + 1. Direct model-level checkpoint in the main pipeline (fastest, exact). + 2. Subprocess pipelines from a prior multiprocess run — performs a + "coalesce at specific checkpoint" to recover the exact intermediate + state without re-running anything. + 3. Previous step checkpoint + re-run models from step begin through + resume_after (slowest but always works). + + Returns + ------- + list[str] + Models that must be prepended to the caller's models list to reach + the correct state at ``resume_after``. Empty when the exact + checkpoint was found and restored directly (paths 1 or 2). + """ + if resume_after: + try: + if state.checkpoint.store_is_open(): + checkpoint_names = [ + cp.get("checkpoint_name", "") for cp in state.checkpoint.checkpoints + ] + else: + from activitysim.core.workflow.checkpoint import HdfStore, ParquetStore + + pipeline_path = Path(state.checkpoint.default_pipeline_file_path()) + if state.settings.checkpoint_format == "hdf": + store = HdfStore(pipeline_path, mode="r") + else: + store = ParquetStore(pipeline_path, mode="r") + try: + checkpoint_names = store.list_checkpoint_names() + finally: + store.close() + + # Path 1: direct model-level checkpoint in main pipeline + if resume_after in checkpoint_names: + _restore_parent_state_from_pipeline(state, checkpoint_name=resume_after) + return [] + + # Path 2: subprocess pipelines (model-level checkpoints preserved) + if _restore_from_subprocess_pipelines(state, resume_after): + return [] + + # Path 3: restore from previous step and re-run + all_models = state.settings.models + mp_steps = state.settings.multiprocess_steps + if mp_steps and resume_after in all_models: + resume_idx = all_models.index(resume_after) + step_boundaries = [all_models.index(s.begin) for s in mp_steps] + step_boundaries.append(len(all_models)) + for i, step in enumerate(mp_steps): + if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: + if i > 0 and mp_steps[i - 1].name in checkpoint_names: + _restore_parent_state_from_pipeline( + state, checkpoint_name=mp_steps[i - 1].name + ) + step_begin_idx = step_boundaries[i] + extra_models = all_models[step_begin_idx : resume_idx + 1] + return extra_models + elif i == 0: + _restore_parent_state_from_pipeline( + state, checkpoint_name="_" + ) + extra_models = all_models[: resume_idx + 1] + return extra_models + break + except Exception: + logger.warning( + "calibration: could not restore from checkpoint %r, " + "falling back to LAST_CHECKPOINT", + resume_after, + ) + + # Fallback: load LAST_CHECKPOINT (appropriate after a coalesce that + # only ran the desired models) + _restore_parent_state_from_pipeline(state) + return [] + + +def _invalidate_derived_tables(state: workflow.State) -> None: + """Drop factory-produced tables that may be stale after a calibration restore. + + When a calibrated model (e.g. auto_ownership) changes a table that a + @workflow.table factory depends on (e.g. vehicles depends on households), + the checkpoint may contain a stale version of that factory table. Dropping + it forces the factory to regenerate from current data on next access. + + Auto-detection rule: invalidate any table that is (a) registered as a + @workflow.table factory, (b) has DataFrame parameters (= table dependencies), + and (c) is in RANDOM_CHANNELS. This currently matches only 'vehicles' but + will automatically cover future factory tables with the same pattern. + """ + settings = read_calibration_settings(state) + if not settings: + return + + tables_to_invalidate = settings.run.invalidate_tables + if tables_to_invalidate is None: + # Vehicles needs regeneration only when calibration changes + # households.auto_ownership. Downstream calibration components must + # retain vehicle_type_choice's vehicle attributes. + tables_to_invalidate = ( + ["vehicles"] + if "auto_ownership_simulate" in settings.run.calibrate_models + else [] + ) + + logger.debug( + "calibration: tables detected for invalidation: %s", tables_to_invalidate + ) + tables_before = set(state.existing_table_names) + + for table_name in tables_to_invalidate: + if state.is_table(table_name): + state.drop_table(table_name) + state.rng().drop_channel(table_name) + state.get_dataframe(table_name, as_copy=False) + logger.debug("calibration: invalidated derived table '%s'", table_name) + + tables_after = set(state.existing_table_names) + lost = tables_before - tables_after - set(tables_to_invalidate) + if lost: + logger.error( + "calibration: tables unexpectedly removed during invalidation: %s", lost + ) + + +def _restore_parent_state_from_pipeline( + state: workflow.State, checkpoint_name: str = "_" +) -> None: + """Restore pipeline tables into the parent process state. + + After a multiprocess run, the parent's in-memory state is stale. + This loads a specific checkpoint from the pipeline store so that + calibration expressions can evaluate against model outputs. + + Parameters + ---------- + checkpoint_name : str, default "_" + The checkpoint to restore from. Use a model-level checkpoint name + (e.g. the prior step name) to get the exact state at that point, + avoiding pollution from downstream models that may have added rows + to shared tables like ``tours``. The default ``"_"`` loads the + last checkpoint, which is appropriate immediately after a coalesce + that only ran the desired models. + + All tables are explicitly re-checkpointed so that subsequent apportion + subprocesses can load them from a direct file path without relying on + checkpoint backtracking through potentially ambiguous checkpoint history. + """ + # Capture RNG state before restore — models may have dynamically + # added channels (e.g. "vehicles") that aren't in the default + # rng_channels injectable and would be lost by init_state(). + prior_rng_channels = list(state.get_injectable("rng_channels", [])) + prior_index_to_channel = ( + dict(state.rng().index_to_channel) + if hasattr(state.rng(), "index_to_channel") + else {} + ) + + if state.checkpoint.store_is_open(): + state.checkpoint.close_store() + state.checkpoint.restore(resume_after=checkpoint_name) + + _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) + + # After restore, all tables are clean (status=False). Mark them dirty so + # the next checkpoint.add() writes them to disk at a known checkpoint name. + # This ensures apportion subprocesses find table files at a single, + # unambiguous checkpoint rather than needing to backtrack through history. + for table_name in list(state.existing_table_names): + state.existing_table_status[table_name] = True diff --git a/activitysim/core/calibration/expressions.py b/activitysim/core/calibration/expressions.py new file mode 100644 index 0000000000..9338b7b39c --- /dev/null +++ b/activitysim/core/calibration/expressions.py @@ -0,0 +1,199 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import importlib +import importlib.util +import logging +import math +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +from activitysim.core import workflow + +from .reporting import _component_output_dir +from .settings import CalibrationComponentSettings + +logger = logging.getLogger("calibration") + + +def _build_expression_context( + state: workflow.State, + helper_symbols: dict[str, Any], + component_name: str, + component_settings: CalibrationComponentSettings, +) -> dict[str, Any]: + """Create the evaluation context for model_value and target_value expressions.""" + context: dict[str, Any] = { + "state": state, + "np": np, + "pd": pd, + "component_output_dir": _component_output_dir(state, component_name), + "component_settings": component_settings, + } + + # Load active tables into context for direct expression access. + for table_name in list(state.existing_table_names): + try: + context[table_name] = state.get_dataframe(table_name, as_copy=False) + except Exception: + # Some entries may not be available as dataframes in all contexts. + continue + try: + network_los = state.get_injectable("network_los") + context["network_los"] = network_los + context["skim_dict"] = network_los.get_default_skim_dict() + except Exception: + # Network LOS may not be available in all contexts. + pass + + context.update(helper_symbols) + # Explicit function-call context used by calibration expressions. + context["context"] = context + return context + + +def _eval_numeric_value( + raw_value: Any, + context: dict[str, Any], + component_name: str, + description: str, + field_name: str, +) -> float: + """Evaluate numeric or expression value and enforce finite numeric result.""" + if isinstance(raw_value, (int, float, np.number)) and not pd.isna(raw_value): + value = float(raw_value) + else: + try: + value = eval(str(raw_value), {}, context) + except Exception as err: + raise RuntimeError( + f"error evaluating {field_name} for {component_name} / {description}: {raw_value}" + ) from err + + try: + value = float(value) + except Exception as err: + raise RuntimeError( + f"{field_name} did not evaluate to a numeric value for {component_name} / {description}: {value}" + ) from err + + if not np.isfinite(value): + raise RuntimeError( + f"{field_name} evaluated to non-finite value for {component_name} / {description}: {value}" + ) + + return value + + +def _compute_delta( + method: str, + model_value: float, + target_value: float, + damping: float, + component_name: str, + description: str, + default_increment: float, +) -> float: + """Compute damped coefficient delta using selected method.""" + if damping < 0: + raise RuntimeError( + f"negative damping not allowed for {component_name} / {description}: {damping}" + ) + + if method == "log_ratio": + if model_value <= 0 or target_value <= 0: + logger.warning( + f"log_ratio requires positive model and target values for {component_name} / {description}. Falling back to default increment {default_increment}" + ) + if model_value <= 0 and target_value > 0: + return default_increment + elif model_value > 0 and target_value <= 0: + return -default_increment + else: + return 0 + delta = math.log(target_value / model_value) * damping + + elif method == "odds_ratio": + # Formula requested by the calibration outline. + numerator = (target_value * model_value) - target_value + denominator = (target_value * model_value) - model_value + + if numerator <= 0 or denominator <= 0: + logger.warning( + f"odds_ratio produced invalid numerator/denominator for {component_name} / {description}. Falling back to default increment {default_increment}" + ) + if model_value <= 0 and target_value > 0: + return default_increment + elif model_value > 0 and target_value <= 0: + return -default_increment + else: + return 0 + + ratio = numerator / denominator + if ratio <= 0 or not np.isfinite(ratio): + raise RuntimeError( + f"odds_ratio produced invalid ratio for {component_name} / {description}" + ) + + delta = math.log(ratio) * damping + + else: + raise RuntimeError(f"unsupported calibration method: {method}") + + if not np.isfinite(delta): + raise RuntimeError( + f"coefficient delta is non-finite for {component_name} / {description}" + ) + + return delta + + +def _load_helper_symbols( + state: workflow.State, + component_settings: CalibrationComponentSettings, +) -> tuple[dict[str, Any], Any | None, Any | None]: + """Load helper module and return evaluation symbols and bespoke function.""" + if not component_settings.helper_module: + return {}, None, None + + module = _load_helper_module(state, component_settings.helper_module) + symbols = { + name: obj for name, obj in vars(module).items() if not name.startswith("__") + } + + bespoke = None + if component_settings.reports and component_settings.reports.bespoke: + fn_name = component_settings.reports.bespoke + if not hasattr(module, fn_name): + raise RuntimeError( + f"helper module does not define bespoke function {fn_name}" + ) + bespoke = getattr(module, fn_name) + + return symbols, bespoke, module + + +def _load_helper_module(state: workflow.State, helper_module: str): + """Load helper module by file path or import path.""" + if helper_module.endswith(".py"): + helper_path = state.filesystem.get_config_file_path(helper_module) + module_name = Path(helper_module).stem + + spec = importlib.util.spec_from_file_location(module_name, helper_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"unable to load helper module from {helper_path}") + + module = importlib.util.module_from_spec(spec) + # Expose state as a global for compatibility with helper examples. + module.state = state + spec.loader.exec_module(module) + return module + + module = importlib.import_module(helper_module) + setattr(module, "state", state) + return module + diff --git a/activitysim/core/calibration/multiprocess.py b/activitysim/core/calibration/multiprocess.py new file mode 100644 index 0000000000..b828005007 --- /dev/null +++ b/activitysim/core/calibration/multiprocess.py @@ -0,0 +1,604 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import logging +import multiprocessing +from pathlib import Path +from typing import Any + +import pandas as pd + +from activitysim.core import workflow +from activitysim.core.configuration.top import MultiprocessStep + +logger = logging.getLogger("calibration") + +MP_INJECTABLES = [ + "data_dir", + "configs_dir", + "data_model_dir", + "output_dir", + "cache_dir", + "settings_file_name", + "imported_extensions", + "run_timestamp", + "run_id", + "pipeline_file_name", +] + + +def _run_mp_single_component( + state: workflow.State, + component_name: str, + restore_checkpoint: str, + shared_data_buffers: dict, +) -> None: + """Run a single component in multiprocess mode with explicit checkpoint control. + + This directly orchestrates the apportion → simulate → coalesce flow + without going through run_multiprocess/get_run_list, giving us precise + control over which checkpoint to apportion from. This is essential for + calibration component re-runs where we must always restart from prior_step's + state regardless of what other checkpoints exist in the pipeline. + + Parameters + ---------- + state : workflow.State + component_name : str + The model component to run. + restore_checkpoint : str + The checkpoint name to restore from before apportioning. + This should be the checkpoint representing prior_step's state. + shared_data_buffers : dict + Pre-allocated shared memory buffers for skims/shadow pricing. + """ + from activitysim.core import mp_tasks + + from .execution import _restore_parent_state_from_pipeline + + # Determine slice info from original settings + original_steps = state.settings.multiprocess_steps + all_models = state.settings.models + + slice_info = None + num_processes = state.settings.num_processes or 2 + chunk_size = state.settings.chunk_size or 0 + + # Find which original step this component belongs to + step_boundaries = [] + for i, step in enumerate(original_steps): + step_boundaries.append(all_models.index(step.begin)) + step_boundaries.append(len(all_models)) + + component_idx = all_models.index(component_name) + for i, step in enumerate(original_steps): + if step_boundaries[i] <= component_idx < step_boundaries[i + 1]: + if step.slice: + slice_info = step.slice.model_dump() + if step.num_processes: + num_processes = step.num_processes + if step.chunk_size: + chunk_size = step.chunk_size + break + + # Build step_info dict matching what mp_tasks functions expect + step_info = { + "name": component_name, + "models": [component_name], + "num_processes": num_processes, + "chunk_size": chunk_size, + "step_num": 0, + "slice": slice_info, + "last_checkpoint_in_previous_multiprocess_step": restore_checkpoint, + } + + injectables = _build_calibration_injectables(state) + + if num_processes == 1: + sub_proc_names = [component_name] + else: + sub_proc_names = [f"{component_name}_{i}" for i in range(num_processes)] + + fail_fast = state.settings.fail_fast + + # Apportion pipeline (split tables across sub-processes) + if num_processes > 1 and slice_info is not None: + mp_tasks.run_sub_task( + state, + multiprocessing.Process( + target=mp_tasks.mp_apportion_pipeline, + name=f"{component_name}_apportion", + args=(injectables, sub_proc_names, step_info), + ), + ) + + # For multi-process runs, subprocesses must restore from the apportioned + # pipeline (which has one checkpoint). Use LAST_CHECKPOINT so they don't + # overwrite the apportioned data with a fresh pipeline. + # For single-process runs (no apportion), use restore_checkpoint to resume + # from the correct point in the main pipeline. + if num_processes > 1: + sim_resume_after = "_" # LAST_CHECKPOINT in apportioned sub-pipeline + else: + sim_resume_after = restore_checkpoint + + # Run simulations in sub-processes + completed = mp_tasks.run_sub_simulations( + state, + injectables, + shared_data_buffers, + step_info, + sub_proc_names, + sim_resume_after, + [], # previously_completed + fail_fast, + ) + + if len(completed) != num_processes: + from activitysim.core.exceptions import SubprocessError + + raise SubprocessError( + f"{num_processes - len(completed)} processes failed in " + f"calibration step {component_name}" + ) + + # Coalesce sub-process pipelines back into main pipeline + if num_processes > 1 and slice_info is not None: + mp_tasks.run_sub_task( + state, + multiprocessing.Process( + target=mp_tasks.mp_coalesce_pipelines, + name=f"{component_name}_coalesce", + args=(injectables, sub_proc_names, slice_info), + ), + ) + + # Restore coalesced results into parent state + _restore_parent_state_from_pipeline(state) + + +def _restore_from_subprocess_pipelines( + state: workflow.State, resume_after: str +) -> bool: + """Restore state from subprocess pipelines at a specific model checkpoint. + + Subprocess pipelines retain model-level checkpoints that don't exist in + the main pipeline. This performs a "coalesce at checkpoint" — reading + mirrored tables from one subprocess and concatenating sliced tables from + all subprocesses at the specified checkpoint name. + + Parameters + ---------- + state : workflow.State + resume_after : str + Model-level checkpoint name to restore from. + + Returns + ------- + bool + True if the restore succeeded; False if subprocess pipelines don't + exist or don't contain the requested checkpoint. + """ + from activitysim.core.workflow.checkpoint import ( + CHECKPOINT_NAME, + CHECKPOINT_TABLE_NAME, + HdfStore, + NON_TABLE_COLUMNS, + ParquetStore, + ) + + all_models = state.settings.models + mp_steps = state.settings.multiprocess_steps + if not mp_steps or resume_after not in all_models: + return False + + # Find the multiprocess step containing resume_after + resume_idx = all_models.index(resume_after) + step_boundaries = [all_models.index(s.begin) for s in mp_steps] + step_boundaries.append(len(all_models)) + + enclosing_step = None + num_processes = state.settings.num_processes or 2 + slice_info = None + for i, step in enumerate(mp_steps): + if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: + enclosing_step = step + if step.num_processes: + num_processes = step.num_processes + if step.slice: + slice_info = ( + step.slice.model_dump() + if hasattr(step.slice, "model_dump") + else step.slice + ) + break + + if enclosing_step is None or num_processes <= 1: + return False + + # Build subprocess pipeline file paths + step_name = enclosing_step.name + pipeline_file_name = state.filesystem.pipeline_file_name + sub_proc_names = [f"{step_name}_{i}" for i in range(num_processes)] + + def _subprocess_path(proc_name): + base = state.get_output_file_path(pipeline_file_name, prefix=proc_name) + if state.settings.checkpoint_format == "hdf": + return base + pq = Path(str(base)).with_suffix(ParquetStore.extension) + return pq if pq.exists() else base + + first_path = _subprocess_path(sub_proc_names[0]) + if not first_path.exists(): + return False + + # Open first subprocess pipeline and verify checkpoint exists + if state.settings.checkpoint_format == "hdf": + first_store = HdfStore(first_path, mode="r") + else: + first_store = ParquetStore(first_path, mode="r") + + try: + cp_names = first_store.list_checkpoint_names() + if resume_after not in cp_names: + return False + + # Read checkpoint row to get table→checkpoint mapping + cp_df = first_store.get_dataframe(CHECKPOINT_TABLE_NAME) + cp_row = cp_df[cp_df[CHECKPOINT_NAME] == resume_after].iloc[-1] + + table_map = {} + for col in cp_row.index: + if col not in NON_TABLE_COLUMNS and cp_row[col]: + table_map[col] = cp_row[col] + + # Read all tables from first subprocess at this checkpoint + tables = {} + for table_name, cp_for_table in table_map.items(): + try: + tables[table_name] = first_store.get_dataframe(table_name, cp_for_table) + except (FileNotFoundError, KeyError): + logger.warning( + f"calibration: subprocess pipeline missing table " + f"{table_name} at {cp_for_table}" + ) + finally: + first_store.close() + + if not tables: + return False + + # Determine sliced tables that need concatenation across processes + sliced_table_names = set(slice_info.get("tables", [])) if slice_info else set() + + # Read sliced tables from remaining subprocesses and concatenate + if num_processes > 1 and sliced_table_names: + omnibus = {t: [tables[t]] for t in sliced_table_names if t in tables} + + for proc_name in sub_proc_names[1:]: + proc_path = _subprocess_path(proc_name) + if not proc_path.exists(): + logger.warning( + f"calibration: subprocess pipeline not found: {proc_path}" + ) + return False + + if state.settings.checkpoint_format == "hdf": + proc_store = HdfStore(proc_path, mode="r") + else: + proc_store = ParquetStore(proc_path, mode="r") + + try: + proc_cp_df = proc_store.get_dataframe(CHECKPOINT_TABLE_NAME) + proc_row = proc_cp_df[proc_cp_df[CHECKPOINT_NAME] == resume_after].iloc[ + -1 + ] + + for table_name in list(omnibus.keys()): + cp_for_table = proc_row.get(table_name, "") + if cp_for_table: + omnibus[table_name].append( + proc_store.get_dataframe(table_name, cp_for_table) + ) + finally: + proc_store.close() + + # Replace sliced tables with concatenated versions + for table_name, dfs in omnibus.items(): + tables[table_name] = pd.concat(dfs, sort=False) + + # Load into parent state + prior_rng_channels = list(state.get_injectable("rng_channels", [])) + prior_index_to_channel = ( + dict(state.rng().index_to_channel) + if hasattr(state.rng(), "index_to_channel") + else {} + ) + + state.init_state() + if state.checkpoint.store_is_open(): + state.checkpoint.close_store() + state.checkpoint.open_store(overwrite=False) + + for table_name, df in tables.items(): + state.add_table(table_name, df) + + _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) + + # Mark all tables dirty for subsequent checkpoint.add + for table_name in list(state.existing_table_names): + state.existing_table_status[table_name] = True + + logger.info( + "calibration: restored %d tables from subprocess pipelines at " + "checkpoint '%s'", + len(tables), + resume_after, + ) + return True + + +def _run_multiprocess_with_overrides( + state: workflow.State, + models: list[str], + resume_after: str | None, + shared_data_buffers: dict | None = None, + can_reuse_subprocs: bool = False, +) -> None: + """Run multiprocess with temporary settings overrides for calibration passes. + + Parameters + ---------- + can_reuse_subprocs : bool, default False + When True, subprocess pipelines from a prior run are assumed to exist + and contain the ``resume_after`` checkpoint. Breadcrumbs are written + so that ``get_run_list`` populates ``step_info["resume_after"]``, + apportion is skipped (reusing existing subprocess pipelines), and + subprocesses resume from their model-level checkpoint — skipping + already-completed models. + """ + from collections import OrderedDict + + from activitysim.core import mp_tasks + + original_models = state.settings.models + original_mp_steps = state.settings.multiprocess_steps + original_resume_after = state.settings.resume_after + + # Build valid multiprocess_steps for the requested model subset. + calibration_mp_steps = _build_calibration_mp_steps( + models=models, + original_steps=original_mp_steps, + all_models=original_models, + ) + + state.settings.models = models + state.settings.multiprocess_steps = calibration_mp_steps + + if can_reuse_subprocs and resume_after: + # Include resume_after in the models list so get_breadcrumbs can + # locate the step containing it. Subprocesses will skip this model + # (it's already checkpointed in their pipeline) and run the rest. + models = [resume_after] + models + + # Rebuild steps with resume_after included. + calibration_mp_steps = _build_calibration_mp_steps( + models=models, + original_steps=original_mp_steps, + all_models=original_models, + ) + state.settings.models = models + state.settings.multiprocess_steps = calibration_mp_steps + state.settings.resume_after = resume_after + + # Write minimal breadcrumbs indicating the step containing + # resume_after has completed apportion (so it's skipped) but + # simulate/coalesce need re-running. + breadcrumbs = OrderedDict() + for step in calibration_mp_steps: + step_dict = {"name": step.name, "apportion": True} + breadcrumbs[step.name] = step_dict + # Find the step containing resume_after + all_models = state.settings.models + if resume_after in all_models: + step_begin = all_models.index(step.begin) + step_models_in_step = [ + m for m in all_models[step_begin:] if m in models + ] + if resume_after in step_models_in_step: + # This step contains resume_after — stop here. + # get_breadcrumbs will mark simulate/coalesce for re-run. + break + + mp_tasks.write_breadcrumbs(state, breadcrumbs) + else: + # No reuse: calibration manages pipeline state externally via + # _restore_parent_state_from_pipeline and checkpoint.add, so the MP + # system's breadcrumb-based resume logic must not be triggered. + state.settings.resume_after = None + + try: + injectables = _build_calibration_injectables(state) + mp_tasks.run_multiprocess( + state, + injectables, + shared_data_buffers=shared_data_buffers, + skip_final_checkpoint=True, + force_resume=resume_after is not None and not can_reuse_subprocs, + ) + finally: + state.settings.models = original_models + state.settings.resume_after = original_resume_after + state.settings.multiprocess_steps = original_mp_steps + + +def _reregister_rng_channels( + state: workflow.State, + prior_channels: list[str], + prior_index_to_channel: dict[str, str] = None, +) -> None: + """Re-register RNG channels that were lost during init_state().""" + current_channels = set(state.get_injectable("rng_channels", [])) + for channel_name in prior_channels: + if channel_name not in current_channels and state.is_table(channel_name): + try: + state.rng().add_channel(channel_name, state.get_dataframe(channel_name)) + current_channels.add(channel_name) + except Exception: + pass + # For channels whose tables don't exist at the restored checkpoint, + # register an empty channel. Do NOT pre-load from a later checkpoint + # in the store — that data may include modifications from downstream + # models and would pollute the pre-model state. The empty channel + # allows the model's normal table factory to create the table fresh + # and extend the channel without hitting the disjoint-index assertion. + if prior_index_to_channel: + for index_name, channel_name in prior_index_to_channel.items(): + if index_name not in state.rng().index_to_channel: + if channel_name not in state.rng().channels: + empty_df = pd.DataFrame( + index=pd.Index([], dtype="int64", name=index_name) + ) + state.rng().add_channel(channel_name, empty_df) + else: + state.rng().index_to_channel[index_name] = channel_name + current_channels.add(channel_name) + state.add_injectable("rng_channels", list(current_channels)) + + +def _initialize_mp_shared_resources(state: workflow.State) -> dict: + """Allocate shared data buffers (skims, shadow pricing) once for reuse. + + This mirrors the allocation logic in mp_tasks.run_multiprocess but + is called once at calibration start rather than on every sub-run. + """ + from activitysim.core import mp_tasks, tracing + + shared_data_buffers = {} + sharrow_enabled = state.settings.sharrow + + t0 = tracing.print_elapsed_time() + if not sharrow_enabled: + shared_data_buffers.update(mp_tasks.allocate_shared_skim_buffers(state)) + t0 = tracing.print_elapsed_time("calibration: allocate shared skim buffer", t0) + + shared_data_buffers.update(mp_tasks.allocate_shared_shadow_pricing_buffers(state)) + t0 = tracing.print_elapsed_time( + "calibration: allocate shared shadow_pricing buffer", t0 + ) + + shared_data_buffers.update( + mp_tasks.allocate_shared_shadow_pricing_buffers_choice(state) + ) + t0 = tracing.print_elapsed_time( + "calibration: allocate shared shadow_pricing choice buffer", t0 + ) + + # Load skim data into the shared buffers. + if sharrow_enabled: + shared_data_buffers["skim_dataset"] = "sh.Dataset:skim_dataset" + from activitysim.core import flow, skim_dataset # noqa: F401 + + state.get_injectable("skim_dataset") + else: + if len(shared_data_buffers) > 0: + injectables = _build_calibration_injectables(state) + mp_tasks.run_sub_task( + state, + multiprocessing.Process( + target=mp_tasks.mp_setup_skims, + name="mp_setup_skims_calibration", + args=(injectables,), + kwargs=shared_data_buffers, + ), + ) + + # Make skims available in the parent process for expression evaluation. + state.add_injectable("data_buffers", shared_data_buffers) + try: + state.get_injectable("network_los") + except Exception: + logger.warning( + "calibration: could not resolve network_los in parent process; " + "skim-dependent expressions may fail" + ) + + return shared_data_buffers + + +def _build_calibration_mp_steps( + models: list[str], + original_steps: list[MultiprocessStep], + all_models: list[str], +) -> list[MultiprocessStep]: + """Build valid MultiprocessStep objects for a calibration model subset. + + The key challenge is that get_run_list() in mp_tasks requires: + - The first step's begin == models[0] + - Steps are ordered and non-overlapping + - Each step's begin is in the models list + + We intersect the original multiprocess_steps with the requested model + subset and construct new steps that satisfy these constraints. + """ + if not models: + return [] + + # Determine which original step each model in the full list belongs to. + # Build a mapping: model_name -> original step index + model_to_step: dict[str, int] = {} + step_boundaries = [] + for i, step in enumerate(original_steps): + begin_idx = all_models.index(step.begin) + step_boundaries.append(begin_idx) + step_boundaries.append(len(all_models)) + + for i, step in enumerate(original_steps): + for model_idx in range(step_boundaries[i], step_boundaries[i + 1]): + model_to_step[all_models[model_idx]] = i + + # Group the requested models by their original step + from collections import OrderedDict + + step_model_groups: OrderedDict[int, list[str]] = OrderedDict() + for model in models: + step_idx = model_to_step.get(model) + if step_idx is None: + continue + step_model_groups.setdefault(step_idx, []).append(model) + + # Build new MultiprocessStep for each group. + # Some original steps (e.g. mp_initialize) omit num_processes, slice, + # and chunk_size — these default to None on MultiprocessStep and + # get_run_list() applies global defaults when they are absent. + # Step names include the first model to ensure uniqueness across multiple + # intermediate runs that draw from the same original step. + new_steps = [] + for step_idx, step_models in step_model_groups.items(): + orig_step = original_steps[step_idx] + kwargs: dict[str, Any] = { + "name": orig_step.name, + "begin": step_models[0], + } + if orig_step.num_processes is not None: + kwargs["num_processes"] = orig_step.num_processes + if orig_step.slice is not None: + kwargs["slice"] = orig_step.slice + if orig_step.chunk_size is not None: + kwargs["chunk_size"] = orig_step.chunk_size + new_steps.append(MultiprocessStep(**kwargs)) + + return new_steps + + +def _build_calibration_injectables(state: workflow.State) -> dict: + """Build the injectables dict for multiprocess sub-processes.""" + injectables = {} + for key in MP_INJECTABLES: + try: + injectables[key] = state.get_injectable(key) + except KeyError: + pass + injectables["settings"] = state.settings + return injectables diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py new file mode 100644 index 0000000000..a4973f9625 --- /dev/null +++ b/activitysim/core/calibration/orchestrator.py @@ -0,0 +1,455 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import logging + +from activitysim.core import workflow + +from .component import _calibrate_component +from .execution import ( + _invalidate_derived_tables, + _prep_model_data, + _run_in_configured_mode, +) +from .multiprocess import _initialize_mp_shared_resources +from .recovery import ( + CALIBRATION_PROGRESS_FILE, + _begin_global_iteration_transaction, + _read_progress, + _restore_coefficient_backups, + _write_completed_progress, + _write_progress, +) +from .reporting import ( + _ensure_calibration_output_dir, + _write_component_plots, + _write_final_coefficients_snapshot, +) +from .settings import ( + CalibrationRunResult, + read_calibration_settings, +) + +logger = logging.getLogger("calibration") + + +def run_calibration_loop( + state: workflow.State, + models: list[str], +) -> CalibrationRunResult: + """ + Run the global calibration workflow. + + This function intentionally minimizes changes to the existing run mechanics: + it always reuses ActivitySim's normal model execution paths and only adds + calibration orchestration around them. + """ + calibration_settings = read_calibration_settings(state) + if not calibration_settings or not calibration_settings.enable: + raise RuntimeError("calibration loop called while calibration is disabled") + + if state.settings.duplicate_step_execution != "allow": + state.settings.duplicate_step_execution = "allow" + logger.warning( + "Overriding duplicate_step_execution setting: must be enabled for calibration" + ) + + if not calibration_settings.run.calibrate_models: + raise ValueError( + "calibration.run.calibrate_models must contain at least one model name" + ) + + missing_calibration_models = [ + component + for component in calibration_settings.run.calibrate_models + if component not in models + ] + if missing_calibration_models: + raise ValueError( + "settings.yaml models list does not include configured calibration " + f"model(s): {missing_calibration_models}" + ) + + resume_after = state.settings.resume_after + if resume_after is not None and resume_after not in models: + raise ValueError( + f"settings.yaml resume_after={resume_after!r} is not present in the " + "settings.yaml models list. Calibration requires resume_after to be " + "a model-level checkpoint name." + ) + + # sort calibration models into main model order + calibration_settings.run.calibrate_models = sorted( + calibration_settings.run.calibrate_models, key=lambda x: models.index(x) + ) + first_calib_model_idx = models.index(calibration_settings.run.calibrate_models[0]) + last_calib_model_idx = models.index(calibration_settings.run.calibrate_models[-1]) + first_model_idx = models.index(resume_after) + 1 if resume_after else None + first_calibration_restart_step = _prior_step_name( + models, calibration_settings.run.calibrate_models[0] + ) + + if resume_after is not None: + skipped_calibration_models = [ + component + for component in calibration_settings.run.calibrate_models + if models.index(component) <= models.index(resume_after) + ] + if skipped_calibration_models: + logger.warning( + "Calibration is honoring settings.yaml resume_after=%r using strict " + "ActivitySim semantics. The following calibrated model(s) occur at " + "or before resume_after and will be skipped during the first global " + "iteration: %s", + resume_after, + skipped_calibration_models, + ) + + _ensure_calibration_output_dir(state) + + progress = _read_progress(state) + if progress and progress.get("complete"): + logger.info( + "calibration progress is already complete; remove %s to start a " + "fresh calibration run", + CALIBRATION_PROGRESS_FILE, + ) + return CalibrationRunResult( + converged=bool(progress.get("converged", False)), + completed_global_iterations=int( + progress.get( + "last_completed_global_iteration", + calibration_settings.run.global_iterations, + ) + ), + ) + + interrupted_iteration = ( + progress.get("in_progress_iteration") if progress else None + ) + if interrupted_iteration is not None: + interrupted_iteration = int(interrupted_iteration) + logger.warning( + "recovering interrupted calibration global iteration %s", + interrupted_iteration, + ) + _restore_coefficient_backups(state, calibration_settings) + progress = { + "in_progress_iteration": None, + "next_global_iteration": interrupted_iteration, + "last_completed_global_iteration": interrupted_iteration - 1, + } + _write_progress(state, progress) + + # Progress files from earlier versions contain next_global_iteration, so + # they remain compatible with the corrected total-count semantics. + start_global_iter = int(progress.get("next_global_iteration", 1)) if progress else 1 + completed_global_iterations = start_global_iter - 1 + + if start_global_iter > calibration_settings.run.global_iterations: + logger.info( + "calibration progress already reached configured global_iterations=%s", + calibration_settings.run.global_iterations, + ) + converged = bool(progress.get("converged", False)) if progress else False + _write_final_coefficients_snapshot(state, calibration_settings) + _write_completed_progress( + state, + completed_global_iterations, + converged, + ) + return CalibrationRunResult( + converged=converged, + completed_global_iterations=completed_global_iterations, + ) + + if state.settings.resume_after is None: + # compute_accessibility requires its accessibility table to be empty; + # unlike most model steps, it will not overwrite a prior result. + # Remove a cached result before restore clears table-status metadata, + # so the table factory recreates its empty placeholder for the replay. + state.drop_table("accessibility") + state.checkpoint.restore() + + original_pipeline_name = state.filesystem.pipeline_file_name + + # Initialize shared resources for multiprocess mode (skims, shadow pricing). + # These are allocated once and reused across all calibration iterations. + shared_data_buffers = None + if state.settings.multiprocess: + shared_data_buffers = _initialize_mp_shared_resources(state) + + try: + # skip precursors if, on first iter, resume_after exists and is >= first_calib_model_idx + if ( + state.settings.resume_after is None + or first_model_idx < first_calib_model_idx + ): + # Run ActivitySim normally from resume_after through production model steps. + _run_precursor_components( + state, + models=models[:first_calib_model_idx] + if first_model_idx is None + else models[first_model_idx:first_calib_model_idx], + resume_after=state.settings.resume_after, + global_iter=start_global_iter, + shared_data_buffers=shared_data_buffers, + ) + else: + # Precursors skipped — but the pipeline must still be initialized + # at the resume_after point so that _calibrate_component (and its + # apportion subprocess) starts from the correct state without + # downstream model data. + extra_models = _prep_model_data( + state, resume_after=state.settings.resume_after + ) + if extra_models: + # No model-level checkpoint exists for resume_after; we must + # run models from the prior step through resume_after to + # recreate the correct intermediate state. + _run_in_configured_mode( + state, + models=extra_models, + resume_after=None, + shared_data_buffers=shared_data_buffers, + ) + elif not any( + cp.get("checkpoint_name") == state.settings.resume_after + for cp in state.checkpoint.checkpoints + ): + # _prep_model_data took its fallback path — the pipeline either + # doesn't exist or doesn't contain resume_after's checkpoint. + # The restored state is incomplete (precursor models never ran). + logger.warning( + "calibration: resume_after=%r not found in restored pipeline; " + "running precursor models", + state.settings.resume_after, + ) + _run_precursor_components( + state, + models=models[:first_calib_model_idx], + resume_after=None, + global_iter=start_global_iter, + shared_data_buffers=shared_data_buffers, + ) + else: + state.checkpoint.add(state.settings.resume_after) + state.checkpoint.close_store() + + for global_iter in range( + start_global_iter, + calibration_settings.run.global_iterations + 1, + ): + _begin_global_iteration_transaction( + state, + calibration_settings, + global_iter, + ) + + # Every global iteration after the first begins from the immutable + # checkpoint directly before the first calibrated model. This makes + # global reruns independent of the state left by the final calibrated + # model in the preceding iteration. + if global_iter > start_global_iter: + logger.info( + "Restarting global calibration iteration %s from checkpoint %r", + global_iter, + first_calibration_restart_step, + ) + extra_models = _prep_model_data( + state, resume_after=first_calibration_restart_step + ) + if extra_models: + _run_in_configured_mode( + state, + models=extra_models, + resume_after=None, + shared_data_buffers=shared_data_buffers, + ) + _invalidate_derived_tables(state) + if first_calibration_restart_step is not None: + state.checkpoint.add(first_calibration_restart_step) + state.checkpoint.close_store() + + logger.info( + "calibration global iteration %s/%s", + global_iter, + calibration_settings.run.global_iterations, + ) + + # suppress early termination on first iteration if resume_after is after all calibrated models + all_converged = ( + first_model_idx is not None and first_model_idx <= last_calib_model_idx + ) or global_iter > start_global_iter + + last_calibrated_component = None + for component in calibration_settings.run.calibrate_models: + # on the first global iter, skip model if it's before or == resume_after + if ( + global_iter == start_global_iter + and first_model_idx is not None + and first_model_idx > models.index(component) + ): + continue + component_settings = calibration_settings.model_settings[component] + + prior_step = _prior_step_name(models, component) + + if last_calibrated_component is not None: + + # run all models b/w the last calibrated model and the current one + _run_intermediate_components( + state, + models=models[ + models.index(last_calibrated_component) + + 1 : models.index(component) + ], + resume_after=last_calibrated_component, + shared_data_buffers=shared_data_buffers, + ) + + component_result = _calibrate_component( + state=state, + component_name=component, + component_settings=component_settings, + prior_step=prior_step, + global_iter=global_iter, + shared_data_buffers=shared_data_buffers, + ) + _write_component_plots(state, component) + + all_converged = all_converged and component_result.converged + + last_calibrated_component = component + + if ( + calibration_settings.run.complete_steps + or global_iter == calibration_settings.run.global_iterations + or ( + global_iter == start_global_iter + and state.settings.resume_after is not None + and first_model_idx > last_calib_model_idx + ) + ): + subsequent_components = ( + models[first_model_idx:] + if global_iter == start_global_iter + and first_model_idx > last_calib_model_idx + else models[models.index(last_calibrated_component) + 1 :] + ) + # finish the full model chain + _run_subsequent_components( + state, + models=subsequent_components, + resume_after=state.settings.resume_after + if global_iter == start_global_iter + and first_model_idx > last_calib_model_idx + else last_calibrated_component, + shared_data_buffers=shared_data_buffers, + ) + + completed_global_iterations = global_iter + iteration_is_complete = ( + all_converged + or global_iter == calibration_settings.run.global_iterations + ) + if not iteration_is_complete: + _write_progress( + state, + { + "in_progress_iteration": None, + "next_global_iteration": global_iter + 1, + "last_completed_global_iteration": global_iter, + "converged": all_converged, + }, + ) + + if all_converged: + logger.info( + "calibration converged after global iteration %s/%s", + global_iter, + calibration_settings.run.global_iterations, + ) + break + + _write_final_coefficients_snapshot(state, calibration_settings) + _write_completed_progress( + state, + completed_global_iterations, + all_converged, + ) + + return CalibrationRunResult( + converged=all_converged, + completed_global_iterations=completed_global_iterations, + ) + finally: + state.filesystem.pipeline_file_name = original_pipeline_name + + +def _run_precursor_components( + state: workflow.State, + models: list[str], + resume_after: str, + global_iter: int, + shared_data_buffers: dict | None = None, +) -> None: + """Run the normal ActivitySim model flow for one global calibration iteration.""" + + # if global_iter > 1 and resume_after is not None: + # # Seed a fresh pipeline from the configured resume checkpoint to avoid + # # duplicate checkpoint-name collisions across global calibration loops. + # prior_pipeline = state.checkpoint.store.filename + # state.checkpoint.close_store() + # state.filesystem.pipeline_file_name = f"pipeline_calibration_iter_{global_iter}" + # state.checkpoint.restore_from(prior_pipeline, checkpoint_name=resume_after) + # else: + + _run_in_configured_mode( + state, + models=models, + resume_after=resume_after, + shared_data_buffers=shared_data_buffers, + ) + + +def _run_intermediate_components( + state: workflow.State, + models: list[str], + resume_after: str, + shared_data_buffers: dict | None = None, +) -> None: + if len(models) == 0: + return + _run_in_configured_mode( + state, + models=models, + resume_after=resume_after, + shared_data_buffers=shared_data_buffers, + ) + + +def _run_subsequent_components( + state: workflow.State, + models: list[str], + resume_after: str, + shared_data_buffers: dict | None = None, +) -> None: + _run_in_configured_mode( + state, + models=models, + resume_after=resume_after, + shared_data_buffers=shared_data_buffers, + ) + + +def _prior_step_name(models: list[str], component_name: str) -> str | None: + """Return the step name immediately preceding component_name in models.""" + if component_name not in models: + return None + idx = models.index(component_name) + if idx == 0: + return None + return models[idx - 1] + diff --git a/activitysim/core/calibration/recovery.py b/activitysim/core/calibration/recovery.py new file mode 100644 index 0000000000..b46c779ba8 --- /dev/null +++ b/activitysim/core/calibration/recovery.py @@ -0,0 +1,103 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import json +import os +import shutil +from typing import Any + +from activitysim.core import workflow + +from .coefficients import _calibration_coefficient_paths +from .settings import CalibrationConfig + +CALIBRATION_PROGRESS_FILE = "calibration/calibration_progress.json" +CALIBRATION_RECOVERY_DIR = "calibration/recovery" + + +def _begin_global_iteration_transaction( + state: workflow.State, + calibration_settings: CalibrationConfig, + global_iteration: int, +) -> None: + """Snapshot coefficients and durably mark a global iteration in progress.""" + recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) + os.makedirs(recovery_dir, exist_ok=True) + + for file_number, coefficient_path in enumerate( + _calibration_coefficient_paths(state, calibration_settings) + ): + if not coefficient_path.exists(): + raise FileNotFoundError( + f"calibration coefficient file not found: {coefficient_path}" + ) + + backup_name = f"{file_number:03d}_{coefficient_path.name}" + shutil.copyfile(coefficient_path, recovery_dir / backup_name) + + # Write the marker only after all backups exist. If backup creation is + # interrupted, the previous between-iteration progress remains valid. + _write_progress( + state, + { + "in_progress_iteration": global_iteration, + "next_global_iteration": global_iteration, + "last_completed_global_iteration": global_iteration - 1, + }, + ) + + +def _restore_coefficient_backups( + state: workflow.State, + calibration_settings: CalibrationConfig, +) -> None: + """Restore the coefficient backups for an interrupted global iteration.""" + recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) + for file_number, coefficient_path in enumerate( + _calibration_coefficient_paths(state, calibration_settings) + ): + backup_path = recovery_dir / f"{file_number:03d}_{coefficient_path.name}" + if not backup_path.exists(): + raise RuntimeError( + f"cannot recover interrupted calibration iteration: missing {backup_path}" + ) + shutil.copyfile(backup_path, coefficient_path) + + +def _read_progress(state: workflow.State) -> dict[str, Any] | None: + """Read persisted calibration progress metadata if it exists.""" + path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) + if not path.exists(): + return None + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _write_progress(state: workflow.State, payload: dict[str, Any]) -> None: + """Atomically write calibration progress metadata.""" + path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) + os.makedirs(path.parent, exist_ok=True) + temporary_path = path.with_name(f".{path.name}.tmp") + with open(temporary_path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + os.replace(temporary_path, path) + + +def _write_completed_progress( + state: workflow.State, + completed_global_iterations: int, + converged: bool, +) -> None: + """Mark calibration complete after all final output has been written.""" + _write_progress( + state, + { + "complete": True, + "in_progress_iteration": None, + "next_global_iteration": completed_global_iterations + 1, + "last_completed_global_iteration": completed_global_iterations, + "converged": converged, + }, + ) + diff --git a/activitysim/core/calibration/reporting.py b/activitysim/core/calibration/reporting.py new file mode 100644 index 0000000000..ff09e8ab1f --- /dev/null +++ b/activitysim/core/calibration/reporting.py @@ -0,0 +1,268 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import math +import os +from pathlib import Path +from typing import Any + +import matplotlib +import matplotlib.pyplot as plt +import pandas as pd + +from activitysim.core import workflow + +from .coefficients import _infer_model_settings_file +from .settings import CalibrationConfig + +plt.style.use("seaborn-v0_8-darkgrid") +matplotlib.use("Agg") + +CALIBRATION_OUTPUT_DIR = "calibration" +CALIBRATION_ITERATION_FILE = "calibration/calibration_iteration_records.csv" +CALIBRATION_SUMMARY_FILE = "calibration/calibration_iteration_summary.csv" +CALIBRATION_FINAL_COEFFICIENTS_FILE = "calibration/final_calibrated_coefficients.csv" +MAX_COEFFS_IN_GRAPH = 15 + + +def _append_iteration_records( + state: workflow.State, component_name: str, records: list[dict[str, Any]] +) -> None: + """Append per-coefficient calibration iteration records.""" + if not records: + return + df = pd.DataFrame(records) + + # Save a global iteration history file + global_path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) + _append_csv( + df, + global_path, + unique_on=["global_iter", "component_iter", "component", "coefficient"], + ) + + # Also write component-local iteration history + component_path = ( + _component_output_dir(state, component_name) + / Path(CALIBRATION_ITERATION_FILE).name + ) + _append_csv( + df, + component_path, + unique_on=["global_iter", "component_iter", "component", "coefficient"], + ) + + +def _append_summary_records( + state: workflow.State, records: list[dict[str, Any]] +) -> None: + """Append per-iteration summary records.""" + if not records: + return + path = state.get_output_file_path(CALIBRATION_SUMMARY_FILE) + df = pd.DataFrame(records) + _append_csv( + df, + path, + unique_on=["global_iter", "component_iter", "component"], + ) + + +def _append_csv( + df: pd.DataFrame, path: Path, unique_on: list[str] | None = None +) -> None: + """Append a dataframe to a CSV file, replacing rows with matching keys.""" + os.makedirs(path.parent, exist_ok=True) + if unique_on and path.exists(): + existing = pd.read_csv(path) + df = pd.concat([existing, df], ignore_index=True).drop_duplicates( + subset=unique_on, keep="last" + ) + df.to_csv(path, index=False) + return + + write_header = not path.exists() + df.to_csv(path, mode="a", index=False, header=write_header) + + +def _component_output_dir(state: workflow.State, component_name: str) -> Path: + """Return output/calibration/ and ensure it exists.""" + component_dir = state.get_output_file_path(f"calibration/{component_name}") + os.makedirs(component_dir, exist_ok=True) + return component_dir + + +def _write_component_plots(state: workflow.State, component_name: str) -> None: + """Write/update all standard plots for one calibrated component.""" + recs = _read_component_iteration_records(state, component_name) + if recs is None or recs.empty: + return + + # Segment coefficients into manageable sets for plotting + coefs = sorted(recs.index.get_level_values("coefficient").unique()) + n_sets = math.ceil(len(coefs) / MAX_COEFFS_IN_GRAPH) + for coef_set in range(n_sets): + set_coefs = coefs[ + coef_set + * MAX_COEFFS_IN_GRAPH : min( + len(coefs), (coef_set + 1) * MAX_COEFFS_IN_GRAPH + ) + ] + _plot_coefficient_progress(state, component_name, recs, set_coefs, coef_set) + last_records = _component_last_records(recs, set_coefs) + _plot_component_values(state, component_name, last_records, coef_set) + _plot_component_pct_change(state, component_name, last_records, coef_set) + + +def _read_component_iteration_records( + state: workflow.State, component_name: str +) -> pd.DataFrame | None: + """Read all iteration records for a single component.""" + path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) + if not path.exists(): + return None + + iteration_records = ( + pd.read_csv(path) + .set_index(["global_iter", "component_iter", "coefficient"]) + .sort_index() + ) + return iteration_records.loc[iteration_records.component == component_name] + + +def _plot_coefficient_progress( + state: workflow.State, + component_name: str, + recs: pd.DataFrame, + set_coefs: list[str], + coef_set: int, +) -> None: + """Plot coefficient value progression for one coefficient subset.""" + component_dir = _component_output_dir(state, component_name) + ax = ( + recs[recs.index.get_level_values("coefficient").isin(set_coefs)] + .next_coefficient.unstack("coefficient") + .plot(figsize=(10, 5)) + ) + ax.xaxis.set_label_text("Component iteration") + ax.yaxis.set_label_text("Coefficient value") + ax.legend(title="Coefficient label", loc="center left", bbox_to_anchor=(1.02, 0.5)) + plt.tight_layout() + ax.figure.savefig( + component_dir / f"coefficient_progress_set_{coef_set}.png", + bbox_inches="tight", + ) + plt.close(ax.figure) + + +def _component_last_records(recs: pd.DataFrame, set_coefs: list[str]) -> pd.DataFrame: + """Select target/model values for the latest iteration and coefficient subset.""" + filtered = recs[recs.index.get_level_values("coefficient").isin(set_coefs)] + last_global = filtered.index.get_level_values("global_iter")[-1] + last_comp = filtered.loc[last_global].index.get_level_values("component_iter")[-1] + return filtered.xs( + (last_global, last_comp), level=("global_iter", "component_iter") + )[["target_value", "model_value"]] + + +def _plot_component_values( + state: workflow.State, + component_name: str, + last_records: pd.DataFrame, + coef_set: int, +) -> None: + """Plot final target/model component values for one coefficient subset.""" + component_dir = _component_output_dir(state, component_name) + ax = last_records.plot.bar(figsize=(10, 5)) + ax.xaxis.set_tick_params(rotation=45) + ax.xaxis.set_label_text("Component value") + plt.tight_layout() + ax.figure.savefig(component_dir / f"final_components_set_{coef_set}.png") + plt.close(ax.figure) + + +def _plot_component_pct_change( + state: workflow.State, + component_name: str, + last_records: pd.DataFrame, + coef_set: int, +) -> None: + """Plot final percent difference for one coefficient subset.""" + component_dir = _component_output_dir(state, component_name) + fig, ax = plt.subplots(figsize=(10, 5)) + pct_diff = last_records.diff(axis=1).model_value / last_records.target_value + ax = pct_diff.plot.bar(ax=ax) + ax.xaxis.set_tick_params(rotation=45) + ax.xaxis.set_label_text("Coefficient") + ax.yaxis.set_label_text("% Difference between Model and Target") + plt.tight_layout() + ax.figure.savefig(component_dir / f"final_pct_change_set_{coef_set}.png") + plt.close(ax.figure) + + +def _write_generic_report( + state: workflow.State, + component_name: str, + row_records: list[dict[str, Any]], +) -> None: + """Write a simple dashboard-friendly generic report for the current component iteration.""" + if not row_records: + return + + df = pd.DataFrame(row_records) + report = ( + df[ + [ + "global_iter", + "component_iter", + "component", + "description", + "difference", + "pct_difference", + "converged", + ] + ] + .copy() + .sort_values(["global_iter", "component_iter", "description"]) + ) + + path = _component_output_dir(state, component_name) / "generic_report.csv" + _append_csv( + report, + path, + unique_on=["global_iter", "component_iter", "component", "description"], + ) + + +def _write_final_coefficients_snapshot( + state: workflow.State, + calibration_settings: CalibrationConfig, +) -> None: + """Write a combined final coefficients file snapshot for calibrated components.""" + frames = [] + for component_name in calibration_settings.run.calibrate_models: + model_settings_file = _infer_model_settings_file(component_name) + model_settings = state.filesystem.read_model_settings( + model_settings_file, mandatory=True + ) + coeff_df = state.filesystem.read_model_coefficients( + model_settings=model_settings + ).copy() + coeff_df = coeff_df.reset_index().rename(columns={"index": "coefficient_name"}) + coeff_df.insert(0, "component", component_name) + frames.append(coeff_df) + + if not frames: + return + + final_df = pd.concat(frames, ignore_index=True) + path = state.get_output_file_path(CALIBRATION_FINAL_COEFFICIENTS_FILE) + os.makedirs(path.parent, exist_ok=True) + final_df.to_csv(path, index=False) + + +def _ensure_calibration_output_dir(state: workflow.State) -> None: + """Ensure output/calibration exists.""" + path = state.get_output_file_path(CALIBRATION_OUTPUT_DIR) + os.makedirs(path, exist_ok=True) diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py new file mode 100644 index 0000000000..b88db13da3 --- /dev/null +++ b/activitysim/core/calibration/settings.py @@ -0,0 +1,132 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +from pydantic import model_validator + +from activitysim.core import workflow +from activitysim.core.configuration import PydanticReadable +from activitysim.core.configuration.base import PydanticBase + +CALIBRATION_SETTINGS_FILE_NAME = "calibration.yaml" + + +class CalibrationRunSettings(PydanticBase): + """Run-control settings for calibration.""" + + resume_after: Optional[str] = None + calibrate_models: list[str] + global_iterations: int = 1 + complete_steps: bool = False + invalidate_tables: list[str] | None = None + + @model_validator(mode="after") + def validate_run_settings(self): + if not self.calibrate_models: + raise ValueError( + "calibration.run.calibrate_models must contain at least one model name" + ) + return self + + """Tables to drop from state after each calibration restore so their + ``@workflow.table`` factories regenerate from current data. + + Default (None): invalidates ``["vehicles"]``. Set to ``[]`` to disable. + + A table should be listed here when ALL of the following are true: + + 1. It is created by a ``@workflow.table`` factory from another table's + values (not just from input data files). + 2. That source table is modified by a calibrated model or by a model + whose outputs change when calibrated coefficients change. + 3. The factory uses source-table values to determine **row identity** + (index values) or **row count**, not just column values. + + The canonical example is ``vehicles``: its factory repeats household + rows by ``households["auto_ownership"]`` and derives ``vehicle_id`` + from ``household_id``. When ``auto_ownership_simulate`` is calibrated, + different coefficients produce different ownership counts, so the + stale vehicles table loaded from a prior checkpoint would have the + wrong number of rows and wrong vehicle IDs. Dropping it forces the + factory to regenerate vehicles consistent with the current households. + + Tables that only read *column values* from upstream tables (without + affecting row identity) generally do NOT need invalidation — their + content will be correct as long as the upstream table is correct at + the restored checkpoint. + """ + + +class CalibrationReportsSettings(PydanticBase): + """Reporting settings for a calibrated component.""" + + generic: bool = True + bespoke: str | None = None + + +class CalibrationComponentSettings(PydanticBase): + """Settings for one calibratable model component.""" + + calibration_spec: str + helper_module: str | None = None + submodel_max_iterations: int = 1 + reports: CalibrationReportsSettings = CalibrationReportsSettings() + survey_file: Optional[str] = None + + +class CalibrationConfig(PydanticReadable): + """Top-level calibration configuration.""" + + enable: bool = False + run: CalibrationRunSettings + model_settings: dict[str, CalibrationComponentSettings] = {} + + @model_validator(mode="after") + def validate_model_settings(self): + """Validate that configured components are aligned with run settings.""" + for component in self.run.calibrate_models: + if component not in self.model_settings: + raise ValueError( + f"calibration model '{component}' is not in model_settings" + ) + + if self.run.global_iterations < 1: + raise ValueError("max_iterations must be >= 1") + + return self + + +@dataclass +class CalibrationComponentResult: + """Result details from calibrating one component.""" + + component: str + converged: bool + component_iterations: int + + +@dataclass +class CalibrationRunResult: + """Result details from a complete global calibration loop.""" + + converged: bool + completed_global_iterations: int + + +def read_calibration_settings(state: workflow.State) -> CalibrationConfig | None: + """Read and validate calibration settings if calibration.yaml exists.""" + return CalibrationConfig.read_settings_file( + state.filesystem, + CALIBRATION_SETTINGS_FILE_NAME, + mandatory=False, + ) + + +def calibration_enabled(state: workflow.State) -> bool: + """Return True when calibration.yaml exists and is enabled.""" + settings = read_calibration_settings(state) + return bool(settings and settings.enable) + diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py index b9926eb0a1..217cb46580 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py @@ -3,7 +3,8 @@ Notes: The lru_cache decorator to ensure that survey data is loaded and computed only once per Python process. - Context is developed in _build_expression_context() in activitysim.core.calibration. +Context is developed in _build_expression_context() in +activitysim.core.calibration.expressions. """ import matplotlib.pyplot as plt From ce7844f9550a9a3db84911ed4d8e5ca74513d679 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:25:54 -0700 Subject: [PATCH 78/90] if everything is converged, will continue to run downstream models --- activitysim/core/calibration/orchestrator.py | 29 ++++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index a4973f9625..97953bb1a7 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -323,19 +323,24 @@ def run_calibration_loop( last_calibrated_component = component + iteration_is_complete = ( + all_converged + or global_iter == calibration_settings.run.global_iterations + ) + resumed_after_all_calibrated_models = ( + global_iter == start_global_iter + and state.settings.resume_after is not None + and first_model_idx > last_calib_model_idx + ) + if ( calibration_settings.run.complete_steps - or global_iter == calibration_settings.run.global_iterations - or ( - global_iter == start_global_iter - and state.settings.resume_after is not None - and first_model_idx > last_calib_model_idx - ) + or iteration_is_complete + or resumed_after_all_calibrated_models ): subsequent_components = ( models[first_model_idx:] - if global_iter == start_global_iter - and first_model_idx > last_calib_model_idx + if resumed_after_all_calibrated_models else models[models.index(last_calibrated_component) + 1 :] ) # finish the full model chain @@ -343,17 +348,12 @@ def run_calibration_loop( state, models=subsequent_components, resume_after=state.settings.resume_after - if global_iter == start_global_iter - and first_model_idx > last_calib_model_idx + if resumed_after_all_calibrated_models else last_calibrated_component, shared_data_buffers=shared_data_buffers, ) completed_global_iterations = global_iter - iteration_is_complete = ( - all_converged - or global_iter == calibration_settings.run.global_iterations - ) if not iteration_is_complete: _write_progress( state, @@ -452,4 +452,3 @@ def _prior_step_name(models: list[str], component_name: str) -> str | None: if idx == 0: return None return models[idx - 1] - From e9dcffbebb12e3fa02580ea26142e58bb9fc637b Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:26:38 -0700 Subject: [PATCH 79/90] blacken --- activitysim/core/calibration/coefficients.py | 1 - activitysim/core/calibration/expressions.py | 1 - activitysim/core/calibration/orchestrator.py | 4 +--- activitysim/core/calibration/recovery.py | 1 - activitysim/core/calibration/settings.py | 1 - 5 files changed, 1 insertion(+), 7 deletions(-) diff --git a/activitysim/core/calibration/coefficients.py b/activitysim/core/calibration/coefficients.py index 9aa15bcd4c..a41d36f7ad 100644 --- a/activitysim/core/calibration/coefficients.py +++ b/activitysim/core/calibration/coefficients.py @@ -82,4 +82,3 @@ def _calibration_coefficient_paths( seen.add(key) return paths - diff --git a/activitysim/core/calibration/expressions.py b/activitysim/core/calibration/expressions.py index 9338b7b39c..72263c9c96 100644 --- a/activitysim/core/calibration/expressions.py +++ b/activitysim/core/calibration/expressions.py @@ -196,4 +196,3 @@ def _load_helper_module(state: workflow.State, helper_module: str): module = importlib.import_module(helper_module) setattr(module, "state", state) return module - diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index 97953bb1a7..2da1292304 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -125,9 +125,7 @@ def run_calibration_loop( ), ) - interrupted_iteration = ( - progress.get("in_progress_iteration") if progress else None - ) + interrupted_iteration = progress.get("in_progress_iteration") if progress else None if interrupted_iteration is not None: interrupted_iteration = int(interrupted_iteration) logger.warning( diff --git a/activitysim/core/calibration/recovery.py b/activitysim/core/calibration/recovery.py index b46c779ba8..2128480d94 100644 --- a/activitysim/core/calibration/recovery.py +++ b/activitysim/core/calibration/recovery.py @@ -100,4 +100,3 @@ def _write_completed_progress( "converged": converged, }, ) - diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index b88db13da3..b99964796c 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -129,4 +129,3 @@ def calibration_enabled(state: workflow.State) -> bool: """Return True when calibration.yaml exists and is enabled.""" settings = read_calibration_settings(state) return bool(settings and settings.enable) - From adf17f94d29dac4990f5d8e8f56f2875649a633b Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:06:38 -0700 Subject: [PATCH 80/90] remove coefficient rollback functionality --- activitysim/core/calibration/README.md | 14 +++++- activitysim/core/calibration/coefficients.py | 46 +++++++------------- activitysim/core/calibration/component.py | 6 ++- activitysim/core/calibration/orchestrator.py | 38 ++++++++-------- activitysim/core/calibration/recovery.py | 45 ++----------------- activitysim/core/calibration/reporting.py | 7 ++- activitysim/core/calibration/settings.py | 1 + 7 files changed, 60 insertions(+), 97 deletions(-) diff --git a/activitysim/core/calibration/README.md b/activitysim/core/calibration/README.md index 66c92b3802..6e7028f8c2 100644 --- a/activitysim/core/calibration/README.md +++ b/activitysim/core/calibration/README.md @@ -15,12 +15,22 @@ calibration math can evolve independently. and loads optional helper modules. - `coefficients.py` locates, reads, and writes model coefficient files. - `reporting.py` writes iteration histories, summaries, plots, and final snapshots. -- `recovery.py` maintains calibration progress and start-of-iteration coefficient - backups. +- `recovery.py` durably tracks calibration progress. Coefficient files are the + authoritative current state and are never rolled back when a run resumes. - `execution.py` restores pipeline state and dispatches model execution. - `multiprocess.py` contains subprocess orchestration, shared-resource setup, and multiprocess pipeline restoration. +Component settings may specify `model_settings_file` when a workflow step does +not follow the conventional `.yaml` naming pattern. Components +that share model settings may point to the same coefficient file. + +When a run is interrupted, calibration continues the active global iteration. +Any coefficient updates written before the interruption, including subsequent +manual edits, are preserved. On the first resumed iteration, top-level +`settings.yaml` `resume_after` uses standard ActivitySim semantics: the named +model is treated as complete and execution begins with the following model. + Dependencies should flow from `orchestrator.py` into the focused modules. Lower level modules should not import the orchestrator. Multiprocess code imports back into `execution.py` only inside the single-component runner to avoid an import-time diff --git a/activitysim/core/calibration/coefficients.py b/activitysim/core/calibration/coefficients.py index a41d36f7ad..134298e040 100644 --- a/activitysim/core/calibration/coefficients.py +++ b/activitysim/core/calibration/coefficients.py @@ -10,7 +10,7 @@ from activitysim.core import workflow -from .settings import CalibrationConfig +from .settings import CalibrationComponentSettings def _persist_coefficients_to_config( @@ -26,8 +26,10 @@ def _persist_coefficients_to_config( output = coefficients_df.copy() output.index.name = "coefficient_name" - coeff_path = state.filesystem.get_config_file_path(coeff_file) - output.to_csv(coeff_path) + coeff_path = Path(state.filesystem.get_config_file_path(coeff_file)) + temporary_path = coeff_path.with_name(f".{coeff_path.name}.tmp") + output.to_csv(temporary_path) + os.replace(temporary_path, coeff_path) def _infer_model_settings_file(component_name: str) -> str: @@ -40,6 +42,16 @@ def _infer_model_settings_file(component_name: str) -> str: return f"{base}.yaml" +def _resolve_model_settings_file( + component_name: str, + component_settings: CalibrationComponentSettings, +) -> str: + """Return an explicit component settings file or infer the conventional name.""" + return component_settings.model_settings_file or _infer_model_settings_file( + component_name + ) + + def _settings_to_dict(model_settings: dict[str, Any] | Any) -> dict[str, Any]: """Convert pydantic or dict model settings to a plain dictionary.""" if isinstance(model_settings, dict): @@ -54,31 +66,3 @@ def _setting_value(model_settings: dict[str, Any] | Any, key: str, default=None) if isinstance(model_settings, dict): return model_settings.get(key, default) return getattr(model_settings, key, default) - - -def _calibration_coefficient_paths( - state: workflow.State, - calibration_settings: CalibrationConfig, -) -> list[Path]: - """Return the unique coefficient files modified by this calibration run.""" - paths: list[Path] = [] - seen: set[str] = set() - - for component_name in calibration_settings.run.calibrate_models: - model_settings_file = _infer_model_settings_file(component_name) - model_settings = state.filesystem.read_model_settings( - model_settings_file, mandatory=True - ) - coefficient_file = _setting_value(model_settings, "COEFFICIENTS") - if not coefficient_file: - raise RuntimeError( - f"component {component_name} model settings missing COEFFICIENTS" - ) - - path = Path(state.filesystem.get_config_file_path(coefficient_file)).resolve() - key = os.path.normcase(str(path)) - if key not in seen: - paths.append(path) - seen.add(key) - - return paths diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index 7a3285ca3a..82539ebf1f 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -13,8 +13,8 @@ from activitysim.core import simulate, workflow from .coefficients import ( - _infer_model_settings_file, _persist_coefficients_to_config, + _resolve_model_settings_file, _setting_value, _settings_to_dict, ) @@ -59,7 +59,9 @@ def _calibrate_component( shared_data_buffers: dict | None = None, ) -> CalibrationComponentResult: """Run iterative coefficient calibration for one component.""" - model_settings_file = _infer_model_settings_file(component_name) + model_settings_file = _resolve_model_settings_file( + component_name, component_settings + ) model_settings = state.filesystem.read_model_settings( model_settings_file, mandatory=True ) diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index 2da1292304..a93d5205d0 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -15,9 +15,8 @@ from .multiprocess import _initialize_mp_shared_resources from .recovery import ( CALIBRATION_PROGRESS_FILE, - _begin_global_iteration_transaction, + _mark_global_iteration_in_progress, _read_progress, - _restore_coefficient_backups, _write_completed_progress, _write_progress, ) @@ -110,6 +109,14 @@ def run_calibration_loop( progress = _read_progress(state) if progress and progress.get("complete"): + if resume_after is not None: + raise RuntimeError( + f"settings.yaml resume_after={resume_after!r} cannot be honored " + "because calibration progress is already complete. Remove " + f"{CALIBRATION_PROGRESS_FILE} or use a new output directory to " + "start a new calibration run; current coefficient values will " + "be preserved." + ) logger.info( "calibration progress is already complete; remove %s to start a " "fresh calibration run", @@ -127,22 +134,18 @@ def run_calibration_loop( interrupted_iteration = progress.get("in_progress_iteration") if progress else None if interrupted_iteration is not None: - interrupted_iteration = int(interrupted_iteration) + start_global_iter = int(interrupted_iteration) logger.warning( - "recovering interrupted calibration global iteration %s", - interrupted_iteration, + "continuing interrupted calibration global iteration %s using the " + "current coefficient files", + start_global_iter, + ) + else: + # Progress files from earlier versions contain next_global_iteration, so + # they remain compatible with the corrected total-count semantics. + start_global_iter = ( + int(progress.get("next_global_iteration", 1)) if progress else 1 ) - _restore_coefficient_backups(state, calibration_settings) - progress = { - "in_progress_iteration": None, - "next_global_iteration": interrupted_iteration, - "last_completed_global_iteration": interrupted_iteration - 1, - } - _write_progress(state, progress) - - # Progress files from earlier versions contain next_global_iteration, so - # they remain compatible with the corrected total-count semantics. - start_global_iter = int(progress.get("next_global_iteration", 1)) if progress else 1 completed_global_iterations = start_global_iter - 1 if start_global_iter > calibration_settings.run.global_iterations: @@ -239,9 +242,8 @@ def run_calibration_loop( start_global_iter, calibration_settings.run.global_iterations + 1, ): - _begin_global_iteration_transaction( + _mark_global_iteration_in_progress( state, - calibration_settings, global_iter, ) diff --git a/activitysim/core/calibration/recovery.py b/activitysim/core/calibration/recovery.py index 2128480d94..c7a4dc28dd 100644 --- a/activitysim/core/calibration/recovery.py +++ b/activitysim/core/calibration/recovery.py @@ -4,40 +4,18 @@ import json import os -import shutil from typing import Any from activitysim.core import workflow -from .coefficients import _calibration_coefficient_paths -from .settings import CalibrationConfig - CALIBRATION_PROGRESS_FILE = "calibration/calibration_progress.json" -CALIBRATION_RECOVERY_DIR = "calibration/recovery" -def _begin_global_iteration_transaction( +def _mark_global_iteration_in_progress( state: workflow.State, - calibration_settings: CalibrationConfig, global_iteration: int, ) -> None: - """Snapshot coefficients and durably mark a global iteration in progress.""" - recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) - os.makedirs(recovery_dir, exist_ok=True) - - for file_number, coefficient_path in enumerate( - _calibration_coefficient_paths(state, calibration_settings) - ): - if not coefficient_path.exists(): - raise FileNotFoundError( - f"calibration coefficient file not found: {coefficient_path}" - ) - - backup_name = f"{file_number:03d}_{coefficient_path.name}" - shutil.copyfile(coefficient_path, recovery_dir / backup_name) - - # Write the marker only after all backups exist. If backup creation is - # interrupted, the previous between-iteration progress remains valid. + """Durably mark a global iteration in progress.""" _write_progress( state, { @@ -48,29 +26,12 @@ def _begin_global_iteration_transaction( ) -def _restore_coefficient_backups( - state: workflow.State, - calibration_settings: CalibrationConfig, -) -> None: - """Restore the coefficient backups for an interrupted global iteration.""" - recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) - for file_number, coefficient_path in enumerate( - _calibration_coefficient_paths(state, calibration_settings) - ): - backup_path = recovery_dir / f"{file_number:03d}_{coefficient_path.name}" - if not backup_path.exists(): - raise RuntimeError( - f"cannot recover interrupted calibration iteration: missing {backup_path}" - ) - shutil.copyfile(backup_path, coefficient_path) - - def _read_progress(state: workflow.State) -> dict[str, Any] | None: """Read persisted calibration progress metadata if it exists.""" path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) if not path.exists(): return None - with open(path, "r", encoding="utf-8") as f: + with open(path, encoding="utf-8") as f: return json.load(f) diff --git a/activitysim/core/calibration/reporting.py b/activitysim/core/calibration/reporting.py index ff09e8ab1f..a4fbd60010 100644 --- a/activitysim/core/calibration/reporting.py +++ b/activitysim/core/calibration/reporting.py @@ -13,7 +13,7 @@ from activitysim.core import workflow -from .coefficients import _infer_model_settings_file +from .coefficients import _resolve_model_settings_file from .settings import CalibrationConfig plt.style.use("seaborn-v0_8-darkgrid") @@ -242,7 +242,10 @@ def _write_final_coefficients_snapshot( """Write a combined final coefficients file snapshot for calibrated components.""" frames = [] for component_name in calibration_settings.run.calibrate_models: - model_settings_file = _infer_model_settings_file(component_name) + component_settings = calibration_settings.model_settings[component_name] + model_settings_file = _resolve_model_settings_file( + component_name, component_settings + ) model_settings = state.filesystem.read_model_settings( model_settings_file, mandatory=True ) diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index b99964796c..7bbf5d6250 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -71,6 +71,7 @@ class CalibrationComponentSettings(PydanticBase): """Settings for one calibratable model component.""" calibration_spec: str + model_settings_file: str | None = None helper_module: str | None = None submodel_max_iterations: int = 1 reports: CalibrationReportsSettings = CalibrationReportsSettings() From 66e1037f5ef97de9c15c99d9889a9db7f9bd2724 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:44:46 -0700 Subject: [PATCH 81/90] better handling of internal memory handling between iterations --- activitysim/core/calibration/component.py | 11 +- activitysim/core/calibration/execution.py | 133 ++++++++++--------- activitysim/core/calibration/orchestrator.py | 2 - activitysim/core/calibration/settings.py | 31 +---- 4 files changed, 77 insertions(+), 100 deletions(-) diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index 82539ebf1f..caa6b1d8d1 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -18,7 +18,7 @@ _setting_value, _settings_to_dict, ) -from .execution import _invalidate_derived_tables, _prep_model_data +from .execution import _prep_model_data from .expressions import ( _build_expression_context, _compute_delta, @@ -136,18 +136,15 @@ def _calibrate_component( # Restore to prior_step ourselves then run the model directly. # state.run(resume_after=prior_step) would trigger # checkpoint.restore → init_state which creates a fresh RNG. - # If prior_step is before the calibrated model created its table - # (e.g. vehicles), the table won't be in that checkpoint and the - # RNG channel won't be registered — causing a crash when the - # model tries to use it. By restoring here and calling by_name, - # we keep the RNG channels from _prep_model_data intact. + # _prep_model_data also performs an exact calibration rewind, + # removing tables and RNG channels that do not exist at prior_step + # so normal model execution can recreate them at the right point. extra_models = _prep_model_data(state, resume_after=prior_step) if extra_models: # prior_step checkpoint not found directly; run intermediate # models (e.g. annotators) to recreate the correct state. for m in extra_models: state.run.by_name(m) - _invalidate_derived_tables(state) state.checkpoint.add(prior_step) state.run.by_name(run_model_name) diff --git a/activitysim/core/calibration/execution.py b/activitysim/core/calibration/execution.py index 0c7f8e98ab..8180ae22a9 100644 --- a/activitysim/core/calibration/execution.py +++ b/activitysim/core/calibration/execution.py @@ -5,15 +5,21 @@ import logging from pathlib import Path +import pandas as pd + from activitysim.core import workflow +from activitysim.core.workflow.checkpoint import ( + CHECKPOINT_NAME, + CHECKPOINT_TABLE_NAME, + LAST_CHECKPOINT, + NON_TABLE_COLUMNS, +) from .multiprocess import ( _reregister_rng_channels, _restore_from_subprocess_pipelines, _run_multiprocess_with_overrides, ) -from .settings import read_calibration_settings - logger = logging.getLogger("calibration") @@ -167,62 +173,15 @@ def _prep_model_data(state, resume_after=None): return [] -def _invalidate_derived_tables(state: workflow.State) -> None: - """Drop factory-produced tables that may be stale after a calibration restore. - - When a calibrated model (e.g. auto_ownership) changes a table that a - @workflow.table factory depends on (e.g. vehicles depends on households), - the checkpoint may contain a stale version of that factory table. Dropping - it forces the factory to regenerate from current data on next access. - - Auto-detection rule: invalidate any table that is (a) registered as a - @workflow.table factory, (b) has DataFrame parameters (= table dependencies), - and (c) is in RANDOM_CHANNELS. This currently matches only 'vehicles' but - will automatically cover future factory tables with the same pattern. - """ - settings = read_calibration_settings(state) - if not settings: - return - - tables_to_invalidate = settings.run.invalidate_tables - if tables_to_invalidate is None: - # Vehicles needs regeneration only when calibration changes - # households.auto_ownership. Downstream calibration components must - # retain vehicle_type_choice's vehicle attributes. - tables_to_invalidate = ( - ["vehicles"] - if "auto_ownership_simulate" in settings.run.calibrate_models - else [] - ) - - logger.debug( - "calibration: tables detected for invalidation: %s", tables_to_invalidate - ) - tables_before = set(state.existing_table_names) - - for table_name in tables_to_invalidate: - if state.is_table(table_name): - state.drop_table(table_name) - state.rng().drop_channel(table_name) - state.get_dataframe(table_name, as_copy=False) - logger.debug("calibration: invalidated derived table '%s'", table_name) - - tables_after = set(state.existing_table_names) - lost = tables_before - tables_after - set(tables_to_invalidate) - if lost: - logger.error( - "calibration: tables unexpectedly removed during invalidation: %s", lost - ) - - def _restore_parent_state_from_pipeline( state: workflow.State, checkpoint_name: str = "_" ) -> None: """Restore pipeline tables into the parent process state. - After a multiprocess run, the parent's in-memory state is stale. - This loads a specific checkpoint from the pipeline store so that - calibration expressions can evaluate against model outputs. + After a multiprocess run or calibration rewind, the parent's in-memory + state may contain tables created after the requested checkpoint. Remove + those tables before loading the checkpoint so the restored state exactly + represents that point in the model sequence. Parameters ---------- @@ -238,15 +197,65 @@ def _restore_parent_state_from_pipeline( subprocesses can load them from a direct file path without relying on checkpoint backtracking through potentially ambiguous checkpoint history. """ - # Capture RNG state before restore — models may have dynamically - # added channels (e.g. "vehicles") that aren't in the default - # rng_channels injectable and would be lost by init_state(). - prior_rng_channels = list(state.get_injectable("rng_channels", [])) - prior_index_to_channel = ( - dict(state.rng().index_to_channel) - if hasattr(state.rng(), "index_to_channel") - else {} - ) + # Read the target checkpoint manifest before restore truncates checkpoint + # history. Tables represented by columns in this manifest are pipeline- + # managed; a false/empty value means the table did not yet exist. + checkpoints = state.checkpoint.store.get_dataframe(CHECKPOINT_TABLE_NAME) + if checkpoint_name == LAST_CHECKPOINT: + target_checkpoint = checkpoints.iloc[-1] + else: + matching_checkpoints = checkpoints[ + checkpoints[CHECKPOINT_NAME] == checkpoint_name + ] + if matching_checkpoints.empty: + # Let checkpoint.restore raise its normal, more specific exception. + target_checkpoint = None + else: + target_checkpoint = matching_checkpoints.iloc[-1] + + stale_tables: set[str] = set() + if target_checkpoint is not None: + pipeline_tables = set(checkpoints.columns) - set(NON_TABLE_COLUMNS) + target_tables = { + table_name + for table_name in pipeline_tables + if pd.notna(target_checkpoint[table_name]) + and bool(target_checkpoint[table_name]) + } + stale_tables = pipeline_tables - target_tables + + # Capture RNG state before restore — models may have dynamically added + # channels that aren't in the default rng_channels injectable. Do not carry + # channels for stale tables across the rewind; their table factories will + # register fresh channels when normal downstream execution recreates them. + prior_rng_channels = [ + channel_name + for channel_name in state.get_injectable("rng_channels", []) + if channel_name not in stale_tables + ] + prior_index_to_channel = { + index_name: channel_name + for index_name, channel_name in getattr( + state.rng(), "index_to_channel", {} + ).items() + if channel_name not in stale_tables + } + + for table_name in stale_tables: + # Use State.drop rather than drop_table: a prior restore may have reset + # salient-table metadata while leaving the cached DataFrame in context. + if table_name in state: + state.drop(table_name) + logger.debug( + "calibration: exact restore removed post-checkpoint table '%s'", + table_name, + ) + if table_name in state.rng().channels: + state.rng().drop_channel(table_name) + + # checkpoint.load uses this injectable to decide which restored tables get + # RNG channels. Remove stale channel names before it performs that work. + state.add_injectable("rng_channels", prior_rng_channels) if state.checkpoint.store_is_open(): state.checkpoint.close_store() diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index a93d5205d0..facfa88eea 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -8,7 +8,6 @@ from .component import _calibrate_component from .execution import ( - _invalidate_derived_tables, _prep_model_data, _run_in_configured_mode, ) @@ -267,7 +266,6 @@ def run_calibration_loop( resume_after=None, shared_data_buffers=shared_data_buffers, ) - _invalidate_derived_tables(state) if first_calibration_restart_step is not None: state.checkpoint.add(first_calibration_restart_step) state.checkpoint.close_store() diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index 7bbf5d6250..23b1588b47 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -21,6 +21,8 @@ class CalibrationRunSettings(PydanticBase): calibrate_models: list[str] global_iterations: int = 1 complete_steps: bool = False + # Deprecated compatibility setting retained so existing configurations + # continue to parse. Exact calibration restores supersede this setting. invalidate_tables: list[str] | None = None @model_validator(mode="after") @@ -31,35 +33,6 @@ def validate_run_settings(self): ) return self - """Tables to drop from state after each calibration restore so their - ``@workflow.table`` factories regenerate from current data. - - Default (None): invalidates ``["vehicles"]``. Set to ``[]`` to disable. - - A table should be listed here when ALL of the following are true: - - 1. It is created by a ``@workflow.table`` factory from another table's - values (not just from input data files). - 2. That source table is modified by a calibrated model or by a model - whose outputs change when calibrated coefficients change. - 3. The factory uses source-table values to determine **row identity** - (index values) or **row count**, not just column values. - - The canonical example is ``vehicles``: its factory repeats household - rows by ``households["auto_ownership"]`` and derives ``vehicle_id`` - from ``household_id``. When ``auto_ownership_simulate`` is calibrated, - different coefficients produce different ownership counts, so the - stale vehicles table loaded from a prior checkpoint would have the - wrong number of rows and wrong vehicle IDs. Dropping it forces the - factory to regenerate vehicles consistent with the current households. - - Tables that only read *column values* from upstream tables (without - affecting row identity) generally do NOT need invalidation — their - content will be correct as long as the upstream table is correct at - the restored checkpoint. - """ - - class CalibrationReportsSettings(PydanticBase): """Reporting settings for a calibrated component.""" From 1f5586322bf0afaf86bbc7b18153308e1c77d599 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:29:41 -0700 Subject: [PATCH 82/90] run modes and resume after tests --- activitysim/core/calibration/execution.py | 16 +- activitysim/core/calibration/multiprocess.py | 24 ++- activitysim/core/calibration/settings.py | 1 + .../prototype_mtc/test/calibration/README.md | 26 +++ .../configs/auto_ownership_calibration.csv | 2 + .../test/calibration/configs/calibration.yaml | 29 +++ .../test/calibration/configs/settings.yaml | 42 ++++ .../configs/tour_mode_choice_calibration.csv | 2 + .../tour_mode_choice_calibration_failing.csv | 2 + .../workplace_location_calib_helper.py | 13 ++ .../workplace_location_calibration.csv | 2 + .../test/calibration/configs_mp/settings.yaml | 62 ++++++ .../test/calibration/test_run_modes.py | 202 ++++++++++++++++++ 13 files changed, 408 insertions(+), 15 deletions(-) create mode 100644 activitysim/examples/prototype_mtc/test/calibration/README.md create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs/auto_ownership_calibration.csv create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs/calibration.yaml create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs/settings.yaml create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration.csv create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration_failing.csv create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calib_helper.py create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calibration.csv create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs_mp/settings.yaml create mode 100644 activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py diff --git a/activitysim/core/calibration/execution.py b/activitysim/core/calibration/execution.py index 8180ae22a9..7afe6bab81 100644 --- a/activitysim/core/calibration/execution.py +++ b/activitysim/core/calibration/execution.py @@ -20,6 +20,7 @@ _restore_from_subprocess_pipelines, _run_multiprocess_with_overrides, ) + logger = logging.getLogger("calibration") @@ -53,19 +54,16 @@ def _run_in_configured_mode( state.checkpoint.add(resume_after or models[0]) state.checkpoint.close_store() - # When subprocess pipelines from a prior run already have the - # resume_after checkpoint (Path 2 in _prep_model_data), subprocesses - # can skip models before resume_after by reusing those pipelines - # instead of freshly apportioning. Signal this by passing - # can_reuse_subprocs=True. - can_reuse = not extra_models and resume_after is not None - _run_multiprocess_with_overrides( state, models=models, resume_after=resume_after, shared_data_buffers=shared_data_buffers, - can_reuse_subprocs=can_reuse, + # _prep_model_data may have restored either the main pipeline or + # coalesced subprocess pipelines. Freshly apportion the exact + # restored parent state in both cases; an empty extra_models list + # alone is not evidence that subprocess pipelines are reusable. + can_reuse_subprocs=False, ) # After multiprocess completes, the coalesced pipeline exists on disk. # Restore it into the parent process state so tables are accessible @@ -144,7 +142,7 @@ def _prep_model_data(state, resume_after=None): resume_idx = all_models.index(resume_after) step_boundaries = [all_models.index(s.begin) for s in mp_steps] step_boundaries.append(len(all_models)) - for i, step in enumerate(mp_steps): + for i, _step in enumerate(mp_steps): if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: if i > 0 and mp_steps[i - 1].name in checkpoint_names: _restore_parent_state_from_pipeline( diff --git a/activitysim/core/calibration/multiprocess.py b/activitysim/core/calibration/multiprocess.py index b828005007..9408787ffb 100644 --- a/activitysim/core/calibration/multiprocess.py +++ b/activitysim/core/calibration/multiprocess.py @@ -441,12 +441,13 @@ def _reregister_rng_channels( """Re-register RNG channels that were lost during init_state().""" current_channels = set(state.get_injectable("rng_channels", [])) for channel_name in prior_channels: - if channel_name not in current_channels and state.is_table(channel_name): + if channel_name not in state.rng().channels and state.is_table(channel_name): try: state.rng().add_channel(channel_name, state.get_dataframe(channel_name)) - current_channels.add(channel_name) except Exception: pass + if channel_name in state.rng().channels: + current_channels.add(channel_name) # For channels whose tables don't exist at the restored checkpoint, # register an empty channel. Do NOT pre-load from a later checkpoint # in the store — that data may include modifications from downstream @@ -457,12 +458,23 @@ def _reregister_rng_channels( for index_name, channel_name in prior_index_to_channel.items(): if index_name not in state.rng().index_to_channel: if channel_name not in state.rng().channels: - empty_df = pd.DataFrame( - index=pd.Index([], dtype="int64", name=index_name) - ) - state.rng().add_channel(channel_name, empty_df) + if state.is_table(channel_name): + # The channel may have been registered dynamically and + # therefore be absent from the rng_channels injectable. + # Populate it from the exact table restored from the + # target checkpoint rather than creating an empty + # channel for an existing domain. + state.rng().add_channel( + channel_name, state.get_dataframe(channel_name) + ) + else: + empty_df = pd.DataFrame( + index=pd.Index([], dtype="int64", name=index_name) + ) + state.rng().add_channel(channel_name, empty_df) else: state.rng().index_to_channel[index_name] = channel_name + if channel_name in state.rng().channels: current_channels.add(channel_name) state.add_injectable("rng_channels", list(current_channels)) diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index 23b1588b47..dbd8accaab 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -33,6 +33,7 @@ def validate_run_settings(self): ) return self + class CalibrationReportsSettings(PydanticBase): """Reporting settings for a calibrated component.""" diff --git a/activitysim/examples/prototype_mtc/test/calibration/README.md b/activitysim/examples/prototype_mtc/test/calibration/README.md new file mode 100644 index 0000000000..f8288ef562 --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/README.md @@ -0,0 +1,26 @@ +# Prototype MTC calibration run-mode test + +This fixture exercises calibration with three real prototype MTC models: + +- `workplace_location` evaluates its target with the `DIST` network skim; +- `auto_ownership_simulate` provides a second upstream coefficient update; and +- `tour_mode_choice_simulate` is the downstream restart target. + +The test runs uninterrupted single- and multiprocess references. It then runs +each mode with a deliberately invalid tour-mode calibration expression, verifies +that global iteration 1 remains in progress, replaces the expression, and +restarts with `resume_after: non_mandatory_tour_scheduling`. The resumed runs +must preserve the workplace and auto-ownership coefficient files and match the +uninterrupted final tables and calibrated coefficients across both modes. + +Each run copies mutable coefficient files into its temporary config directory; +the shared prototype MTC configs and data remain unchanged. The 25-household +sample minimizes simulation work, although process startup and pipeline +apportion/coalesce make the full matrix an integration test rather than a fast +unit test. + +Run from the repository root with: + +```powershell +.venv\Scripts\python.exe -m pytest activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py -q +``` diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs/auto_ownership_calibration.csv b/activitysim/examples/prototype_mtc/test/calibration/configs/auto_ownership_calibration.csv new file mode 100644 index 0000000000..ecddc7f5dd --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs/auto_ownership_calibration.csv @@ -0,0 +1,2 @@ +description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance +Zero-auto household share,coef_calib_auto_0,len(households[households.auto_ownership == 0]) / len(households),0.271828,FALSE,-10,10,1,log_ratio,0 diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs/calibration.yaml b/activitysim/examples/prototype_mtc/test/calibration/configs/calibration.yaml new file mode 100644 index 0000000000..2b4aa8d159 --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs/calibration.yaml @@ -0,0 +1,29 @@ +enable: true + +run: + calibrate_models: + - workplace_location + - auto_ownership_simulate + - tour_mode_choice_simulate + global_iterations: 1 + complete_steps: true + +model_settings: + workplace_location: + calibration_spec: workplace_location_calibration.csv + helper_module: workplace_location_calib_helper.py + submodel_max_iterations: 1 + reports: + generic: false + + auto_ownership_simulate: + calibration_spec: auto_ownership_calibration.csv + submodel_max_iterations: 1 + reports: + generic: false + + tour_mode_choice_simulate: + calibration_spec: tour_mode_choice_calibration.csv + submodel_max_iterations: 1 + reports: + generic: false diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs/settings.yaml b/activitysim/examples/prototype_mtc/test/calibration/configs/settings.yaml new file mode 100644 index 0000000000..897735df4a --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs/settings.yaml @@ -0,0 +1,42 @@ +inherit_settings: true + +strict: false +multiprocess: false +rng_base_seed: 0 +households_sample_size: 25 +chunk_size: 0 +use_shadow_pricing: false +want_dest_choice_sample_tables: false +cleanup_pipeline_after_run: false + +models: + - initialize_landuse + - initialize_households + - compute_accessibility + - school_location + - workplace_location + - auto_ownership_simulate + - free_parking + - cdap_simulate + - mandatory_tour_frequency + - mandatory_tour_scheduling + - joint_tour_frequency + - joint_tour_composition + - joint_tour_participation + - joint_tour_destination + - joint_tour_scheduling + - non_mandatory_tour_frequency + - non_mandatory_tour_destination + - non_mandatory_tour_scheduling + - tour_mode_choice_simulate + - write_tables + +output_tables: + h5_store: false + action: include + prefix: final_ + sort: true + tables: + - households + - persons + - tours diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration.csv b/activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration.csv new file mode 100644 index 0000000000..048b703c76 --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration.csv @@ -0,0 +1,2 @@ +description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance +Walk tour share for auto-sufficient households,coef_calib_WALK_auto_suff,len(tours[tours.tour_mode == 'WALK']) / len(tours),0.271828,FALSE,-10,10,1,log_ratio,0 diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration_failing.csv b/activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration_failing.csv new file mode 100644 index 0000000000..3a5922f642 --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration_failing.csv @@ -0,0 +1,2 @@ +description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance +Walk tour share for auto-sufficient households,coef_calib_WALK_auto_suff,missing_model_value,0.271828,FALSE,-10,10,1,log_ratio,0 diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calib_helper.py b/activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calib_helper.py new file mode 100644 index 0000000000..fd081a8ef2 --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calib_helper.py @@ -0,0 +1,13 @@ +from __future__ import annotations + + +def workplace_distance_share(context, minimum, maximum): + """Return the modeled worker share in a DIST skim interval.""" + persons = context["persons"] + workers = persons[persons["workplace_zone_id"] > 0] + distances = context["skim_dict"].lookup( + workers["home_zone_id"], + workers["workplace_zone_id"], + "DIST", + ) + return ((distances >= minimum) & (distances < maximum)).mean() diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calibration.csv b/activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calibration.csv new file mode 100644 index 0000000000..2eb39163d6 --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calibration.csv @@ -0,0 +1,2 @@ +description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance +Workplace distance under 2 miles,coef_calib_dist_0_2,"workplace_distance_share(context, 0, 2)",0.271828,FALSE,-10,10,1,log_ratio,0 diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs_mp/settings.yaml b/activitysim/examples/prototype_mtc/test/calibration/configs_mp/settings.yaml new file mode 100644 index 0000000000..33d33c537b --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs_mp/settings.yaml @@ -0,0 +1,62 @@ +inherit_settings: true + +strict: false +multiprocess: true +fail_fast: true +num_processes: 2 +rng_base_seed: 0 +households_sample_size: 25 +chunk_size: 0 +use_shadow_pricing: false +want_dest_choice_sample_tables: false +cleanup_pipeline_after_run: false + +models: + - initialize_landuse + - initialize_households + - compute_accessibility + - school_location + - workplace_location + - auto_ownership_simulate + - free_parking + - cdap_simulate + - mandatory_tour_frequency + - mandatory_tour_scheduling + - joint_tour_frequency + - joint_tour_composition + - joint_tour_participation + - joint_tour_destination + - joint_tour_scheduling + - non_mandatory_tour_frequency + - non_mandatory_tour_destination + - non_mandatory_tour_scheduling + - tour_mode_choice_simulate + - write_tables + +multiprocess_steps: + - name: mp_initialize + begin: initialize_landuse + - name: mp_accessibility + begin: compute_accessibility + slice: + tables: + - accessibility + exclude: true + - name: mp_households + begin: school_location + slice: + tables: + - households + - persons + - name: mp_summarize + begin: write_tables + +output_tables: + h5_store: false + action: include + prefix: final_ + sort: true + tables: + - households + - persons + - tours diff --git a/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py b/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py new file mode 100644 index 0000000000..607efdcd24 --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pandas as pd +import pandas.testing as pdt +import pytest +import yaml + + +HERE = Path(__file__).resolve().parent +EXAMPLE_ROOT = HERE.parents[1] +TEST_ROOT = EXAMPLE_ROOT / "test" +BASE_CONFIGS = EXAMPLE_ROOT / "configs" +BASE_MP_CONFIGS = EXAMPLE_ROOT / "configs_mp" +DATA = EXAMPLE_ROOT / "data" +SIMULATION = TEST_ROOT / "simulation.py" + +COEFFICIENT_FILES = ( + "workplace_location_coefficients.csv", + "auto_ownership_coefficients.csv", + "tour_mode_choice_coefficients.csv", +) + +RESULT_TABLES = ( + "final_households.csv", + "final_persons.csv", + "final_tours.csv", +) + + +def _prepare_run(tmp_path: Path, name: str, multiprocess: bool, failing: bool): + run_dir = tmp_path / name + configs_dir = run_dir / "configs" + output_dir = run_dir / "output" + shutil.copytree(HERE / "configs", configs_dir) + output_dir.mkdir(parents=True) + + settings_source = ( + HERE / "configs_mp" / "settings.yaml" + if multiprocess + else HERE / "configs" / "settings.yaml" + ) + shutil.copyfile(settings_source, configs_dir / "settings.yaml") + + for file_name in COEFFICIENT_FILES: + shutil.copyfile(BASE_CONFIGS / file_name, configs_dir / file_name) + + if failing: + shutil.copyfile( + configs_dir / "tour_mode_choice_calibration_failing.csv", + configs_dir / "tour_mode_choice_calibration.csv", + ) + + return configs_dir, output_dir + + +def _run(configs_dir: Path, output_dir: Path, multiprocess: bool): + args = [ + sys.executable, + str(SIMULATION), + "-c", + str(configs_dir), + ] + if multiprocess: + args.extend(["-c", str(BASE_MP_CONFIGS)]) + args.extend( + [ + "-c", + str(BASE_CONFIGS), + "-d", + str(DATA), + "-o", + str(output_dir), + ] + ) + return subprocess.run(args, check=False, capture_output=True, text=True) + + +def _resume(configs_dir: Path): + shutil.copyfile( + HERE / "configs" / "tour_mode_choice_calibration.csv", + configs_dir / "tour_mode_choice_calibration.csv", + ) + settings_path = configs_dir / "settings.yaml" + with open(settings_path, encoding="utf-8") as stream: + settings = yaml.safe_load(stream) + settings["resume_after"] = "non_mandatory_tour_scheduling" + with open(settings_path, "w", encoding="utf-8") as stream: + yaml.safe_dump(settings, stream, sort_keys=False) + + +def _assert_success(result: subprocess.CompletedProcess): + assert result.returncode == 0, result.stdout + result.stderr + + +def _assert_equivalent(left_output: Path, right_output: Path): + for file_name in RESULT_TABLES: + left = pd.read_csv(left_output / file_name).sort_index(axis=1) + right = pd.read_csv(right_output / file_name).sort_index(axis=1) + pdt.assert_frame_equal(left, right, check_dtype=False) + + left_coefficients = pd.read_csv( + left_output / "calibration" / "final_calibrated_coefficients.csv" + ).sort_values(["component", "coefficient_name"]) + right_coefficients = pd.read_csv( + right_output / "calibration" / "final_calibrated_coefficients.csv" + ).sort_values(["component", "coefficient_name"]) + pdt.assert_frame_equal( + left_coefficients.reset_index(drop=True), + right_coefficients.reset_index(drop=True), + check_dtype=False, + check_exact=False, + rtol=1e-12, + atol=1e-12, + ) + + +def _run_reference(root: Path, name: str, multiprocess: bool) -> Path: + configs, output = _prepare_run(root, name, multiprocess, failing=False) + _assert_success(_run(configs, output, multiprocess)) + return output + + +def _run_resumed(root: Path, name: str, multiprocess: bool) -> Path: + configs, output = _prepare_run(root, name, multiprocess, failing=True) + failed = _run(configs, output, multiprocess) + assert failed.returncode != 0 + + progress_path = output / "calibration" / "calibration_progress.json" + with open(progress_path, encoding="utf-8") as stream: + progress = json.load(stream) + assert progress["in_progress_iteration"] == 1 + + upstream_coefficients = { + file_name: (configs / file_name).read_bytes() + for file_name in COEFFICIENT_FILES[:2] + } + + _resume(configs) + _assert_success(_run(configs, output, multiprocess)) + + for file_name, contents in upstream_coefficients.items(): + assert (configs / file_name).read_bytes() == contents + return output + + +@pytest.fixture(scope="module") +def run_root(tmp_path_factory) -> Path: + return tmp_path_factory.mktemp("calibration_run_modes") + + +@pytest.fixture(scope="module") +def single_output(run_root: Path) -> Path: + return _run_reference(run_root, "single", multiprocess=False) + + +@pytest.fixture(scope="module") +def multiprocess_output(run_root: Path) -> Path: + return _run_reference(run_root, "multiprocess", multiprocess=True) + + +@pytest.fixture(scope="module") +def single_resumed_output(run_root: Path) -> Path: + return _run_resumed(run_root, "single_resumed", multiprocess=False) + + +@pytest.fixture(scope="module") +def multiprocess_resumed_output(run_root: Path) -> Path: + return _run_resumed(run_root, "multiprocess_resumed", multiprocess=True) + + +def test_single_and_multiprocess_are_equivalent( + single_output: Path, + multiprocess_output: Path, +): + _assert_equivalent(single_output, multiprocess_output) + + +def test_single_resume_matches_uninterrupted( + single_output: Path, + single_resumed_output: Path, +): + _assert_equivalent(single_output, single_resumed_output) + + +def test_multiprocess_resume_matches_uninterrupted( + multiprocess_output: Path, + multiprocess_resumed_output: Path, +): + _assert_equivalent(multiprocess_output, multiprocess_resumed_output) + + +def test_resumed_single_and_multiprocess_are_equivalent( + single_resumed_output: Path, + multiprocess_resumed_output: Path, +): + _assert_equivalent(single_resumed_output, multiprocess_resumed_output) From 97fbf166d684c7f30f205ba9bb449922fddcd13a Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:58:21 -0700 Subject: [PATCH 83/90] nailing down resume_after and iteration counting plus some bug fixes including in RNG and odds ratio --- activitysim/core/calibration/README.md | 69 ++++++- activitysim/core/calibration/component.py | 97 +++++++--- activitysim/core/calibration/expressions.py | 20 +- activitysim/core/calibration/multiprocess.py | 15 +- activitysim/core/calibration/orchestrator.py | 182 +++++++++++++++--- activitysim/core/calibration/recovery.py | 10 + activitysim/core/calibration/reporting.py | 91 +++++++-- activitysim/core/test/test_calibration.py | 155 +++++++++++++++ .../core/test/test_calibration_reporting.py | 59 ++++++ .../prototype_mtc/test/calibration/README.md | 4 + .../test/calibration/test_run_modes.py | 150 ++++++++++++++- 11 files changed, 752 insertions(+), 100 deletions(-) create mode 100644 activitysim/core/test/test_calibration.py create mode 100644 activitysim/core/test/test_calibration_reporting.py diff --git a/activitysim/core/calibration/README.md b/activitysim/core/calibration/README.md index 6e7028f8c2..ae87db4ae0 100644 --- a/activitysim/core/calibration/README.md +++ b/activitysim/core/calibration/README.md @@ -25,11 +25,70 @@ Component settings may specify `model_settings_file` when a workflow step does not follow the conventional `.yaml` naming pattern. Components that share model settings may point to the same coefficient file. -When a run is interrupted, calibration continues the active global iteration. -Any coefficient updates written before the interruption, including subsequent -manual edits, are preserved. On the first resumed iteration, top-level -`settings.yaml` `resume_after` uses standard ActivitySim semantics: the named -model is treated as complete and execution begins with the following model. +## Global iterations, recovery attempts, and `resume_after` + +`calibration.yaml` `run.global_iterations` is the maximum total number of +logical global calibration iterations, not the number to execute on each +ActivitySim invocation. Calibration records durable progress in +`output/calibration/calibration_progress.json`: + +- a new output directory starts at global iteration 1; +- a crash re-enters the interrupted global iteration; +- a cleanly completed iteration advances to the next global iteration; +- convergence can finish the run before the configured maximum; and +- after a completed run, increasing `global_iterations` continues at the next + unfinished global iteration. An unchanged or lower value remains a no-op. + +Recovery attempts are distinct from global iterations. The first execution of a +global iteration is attempt 1. Restarting an interrupted global iteration creates +attempt 2, then attempt 3 if another restart is needed. A recovery attempt does +not consume an additional global iteration. + +Calibration uses the top-level `settings.yaml` `resume_after` setting. It has +strict ActivitySim semantics: the named model is treated as complete, its +checkpoint is restored, and execution begins with the following model. The name +must occur in the top-level `models` list and must be a model-level checkpoint; +the `_` shorthand for the last checkpoint is not accepted in calibration mode. +The similarly named `calibration.yaml` `run.resume_after` compatibility field is +not used by the orchestrator. + +On the first global iteration entered by an invocation, calibrated models at or +before `resume_after` are skipped. Later global iterations in the same invocation +restart immediately before the first calibrated model and run the normal complete +calibration sequence. For example, if global iteration 3 was interrupted after +calibrated `model_a` completed: + +- `resume_after: model_a` preserves model A's attempt-1 result and resumes with + the following model under attempt 2; +- `resume_after: initialize` rewinds pipeline state and reruns model A under + attempt 2; and +- no `resume_after` replays the interrupted iteration from the beginning under + attempt 2. + +Coefficient files are the authoritative current state and are never rolled back +during recovery. Updates written before the interruption, along with subsequent +manual edits, become the starting coefficients for the new attempt. Rewinding +across a completed calibrated model therefore applies another calibration update +to its current coefficients. Calibration logs a warning when `resume_after` +causes this kind of rewind. + +Iteration histories are append-only across attempts. Standard record, summary, +and generic-report rows include `global_iter`, `attempt`, and `component_iter`, +so a rerun does not replace the coefficient transition written by an earlier +attempt. The coefficient trajectory plots show the complete sequence, including +the initial value and every subsequent update. X-axis labels use the compact +form `G-A-C`, for example `G3-A2-C1`. + +After each calibrated component completes, the progress file records its attempt, +component-iteration count, and convergence result. A restart that skips that +component retains this state. This allows a run that crashed in downstream, +non-calibrated models to finish the same terminal global iteration without losing +the calibrated components' convergence decision. + +Calibration is marked complete only after the terminal global iteration has run +all remaining models in the top-level `models` list and the final coefficient +snapshot has been written. A crash after the last calibrated model but before the +last production model therefore leaves the global iteration in progress. Dependencies should flow from `orchestrator.py` into the focused modules. Lower level modules should not import the orchestrator. Multiprocess code imports back diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index caa6b1d8d1..6683b670a6 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -50,12 +50,40 @@ ] +def _run_component_model( + state: workflow.State, + component_name: str, + run_model_name: str, + prior_step: str | None, + mp_restore_checkpoint: str | None, + shared_data_buffers: dict | None, +) -> None: + """Run one component simulation from its fixed pre-component state.""" + if state.settings.multiprocess and shared_data_buffers is not None: + _run_mp_single_component( + state, + component_name=component_name, + run_label=run_model_name.replace(";", "_").replace(".", "_"), + restore_checkpoint=mp_restore_checkpoint, + shared_data_buffers=shared_data_buffers, + ) + return + + extra_models = _prep_model_data(state, resume_after=prior_step) + if extra_models: + for model_name in extra_models: + state.run.by_name(model_name) + state.checkpoint.add(prior_step) + state.run.by_name(run_model_name) + + def _calibrate_component( state: workflow.State, component_name: str, component_settings: CalibrationComponentSettings, prior_step: str, global_iter: int, + attempt: int, shared_data_buffers: dict | None = None, ) -> CalibrationComponentResult: """Run iterative coefficient calibration for one component.""" @@ -116,41 +144,25 @@ def _calibrate_component( for component_iter in range(1, component_settings.submodel_max_iterations + 1): component_iterations = component_iter - run_model_name = f"{component_name}.c_i{component_iter};" f"g_i{global_iter}" - # Re-run only this component from its prior checkpoint so model values - # reflect the current candidate coefficients for this component. - if state.settings.multiprocess and shared_data_buffers is not None: - # Use direct MP orchestration with explicit checkpoint control. - # This ensures we always apportion from prior_step's state, - # even after multiple component iterations. - _run_mp_single_component( - state, - component_name=component_name, - # Always restore from the same immutable pre-component - # checkpoint. LAST_CHECKPOINT may point to the prior iteration's - # coalesced component output and is therefore not a safe baseline. - restore_checkpoint=mp_restore_checkpoint, - shared_data_buffers=shared_data_buffers, - ) - else: - # Restore to prior_step ourselves then run the model directly. - # state.run(resume_after=prior_step) would trigger - # checkpoint.restore → init_state which creates a fresh RNG. - # _prep_model_data also performs an exact calibration rewind, - # removing tables and RNG channels that do not exist at prior_step - # so normal model execution can recreate them at the right point. - extra_models = _prep_model_data(state, resume_after=prior_step) - if extra_models: - # prior_step checkpoint not found directly; run intermediate - # models (e.g. annotators) to recreate the correct state. - for m in extra_models: - state.run.by_name(m) - state.checkpoint.add(prior_step) - state.run.by_name(run_model_name) + run_model_name = ( + f"{component_name}.c_i{component_iter};" + f"g_i{global_iter};a_i{attempt}" + ) + _run_component_model( + state=state, + component_name=component_name, + run_model_name=run_model_name, + prior_step=prior_step, + mp_restore_checkpoint=mp_restore_checkpoint, + shared_data_buffers=shared_data_buffers, + ) eval_context = _build_expression_context( state, helper_symbols, component_name, component_settings ) + eval_context["calibration_global_iteration"] = global_iter + eval_context["calibration_attempt"] = attempt + eval_context["calibration_component_iteration"] = component_iter ( row_records, @@ -164,6 +176,7 @@ def _calibrate_component( eval_context=eval_context, global_iter=global_iter, component_iter=component_iter, + attempt=attempt, ) coefficients_df = new_coefficients_df @@ -181,6 +194,21 @@ def _calibrate_component( if component_converged: break + if component_iter == component_settings.submodel_max_iterations: + # The update just persisted above has not yet been simulated. Run + # the component once more so final pipeline tables and downstream + # models use the coefficient values left in the config file. + _run_component_model( + state=state, + component_name=component_name, + run_model_name=( + f"{component_name}.c_final;g_i{global_iter};a_i{attempt}" + ), + prior_step=prior_step, + mp_restore_checkpoint=mp_restore_checkpoint, + shared_data_buffers=shared_data_buffers, + ) + state.checkpoint.add(component_name) return CalibrationComponentResult( @@ -371,6 +399,7 @@ def _evaluate_and_update( eval_context: dict[str, Any], global_iter: int, component_iter: int, + attempt: int = 1, ) -> tuple[list[dict[str, Any]], dict[str, Any], pd.DataFrame, bool]: """Evaluate spec rows, update coefficients, and return detailed records.""" updated = coefficients_df.copy() @@ -429,7 +458,9 @@ def _evaluate_and_update( default_increment=default_increment, ) - candidate_value = prev_value if hold_fast else prev_value + raw_delta + candidate_value = ( + prev_value if hold_fast or converged else prev_value + raw_delta + ) at_min = False at_max = False @@ -468,6 +499,7 @@ def _evaluate_and_update( records.append( { "global_iter": global_iter, + "attempt": attempt, "component_iter": component_iter, "description": description, "component": component_name, @@ -492,6 +524,7 @@ def _evaluate_and_update( summary_record = { "global_iter": global_iter, + "attempt": attempt, "component_iter": component_iter, "component": component_name, "max_difference": max_difference if max_difference != -math.inf else 0.0, diff --git a/activitysim/core/calibration/expressions.py b/activitysim/core/calibration/expressions.py index 72263c9c96..e65b0c194e 100644 --- a/activitysim/core/calibration/expressions.py +++ b/activitysim/core/calibration/expressions.py @@ -107,7 +107,9 @@ def _compute_delta( if method == "log_ratio": if model_value <= 0 or target_value <= 0: logger.warning( - f"log_ratio requires positive model and target values for {component_name} / {description}. Falling back to default increment {default_increment}" + f"log_ratio requires positive model and target values for " + f"{component_name} / {description}. Falling back to default " + f"increment {default_increment}" ) if model_value <= 0 and target_value > 0: return default_increment @@ -118,22 +120,20 @@ def _compute_delta( delta = math.log(target_value / model_value) * damping elif method == "odds_ratio": - # Formula requested by the calibration outline. - numerator = (target_value * model_value) - target_value - denominator = (target_value * model_value) - model_value - - if numerator <= 0 or denominator <= 0: + if not (0 < model_value < 1 and 0 < target_value < 1): logger.warning( - f"odds_ratio produced invalid numerator/denominator for {component_name} / {description}. Falling back to default increment {default_increment}" + f"odds_ratio requires model and target values strictly between " + f"zero and one for {component_name} / {description}. Falling " + f"back to default increment {default_increment}" ) - if model_value <= 0 and target_value > 0: + if target_value > model_value: return default_increment - elif model_value > 0 and target_value <= 0: + elif target_value < model_value: return -default_increment else: return 0 - ratio = numerator / denominator + ratio = (target_value * (1 - model_value)) / (model_value * (1 - target_value)) if ratio <= 0 or not np.isfinite(ratio): raise RuntimeError( f"odds_ratio produced invalid ratio for {component_name} / {description}" diff --git a/activitysim/core/calibration/multiprocess.py b/activitysim/core/calibration/multiprocess.py index 9408787ffb..6815ae604b 100644 --- a/activitysim/core/calibration/multiprocess.py +++ b/activitysim/core/calibration/multiprocess.py @@ -31,6 +31,7 @@ def _run_mp_single_component( state: workflow.State, component_name: str, + run_label: str, restore_checkpoint: str, shared_data_buffers: dict, ) -> None: @@ -47,6 +48,10 @@ def _run_mp_single_component( state : workflow.State component_name : str The model component to run. + run_label : str + Unique name for this calibration execution's subprocess pipelines and + checkpoint. It must differ from ``component_name`` so restoring the + apportioned checkpoint does not cause ActivitySim to skip the model. restore_checkpoint : str The checkpoint name to restore from before apportioning. This should be the checkpoint representing prior_step's state. @@ -84,7 +89,7 @@ def _run_mp_single_component( # Build step_info dict matching what mp_tasks functions expect step_info = { - "name": component_name, + "name": run_label, "models": [component_name], "num_processes": num_processes, "chunk_size": chunk_size, @@ -96,9 +101,9 @@ def _run_mp_single_component( injectables = _build_calibration_injectables(state) if num_processes == 1: - sub_proc_names = [component_name] + sub_proc_names = [run_label] else: - sub_proc_names = [f"{component_name}_{i}" for i in range(num_processes)] + sub_proc_names = [f"{run_label}_{i}" for i in range(num_processes)] fail_fast = state.settings.fail_fast @@ -108,7 +113,7 @@ def _run_mp_single_component( state, multiprocessing.Process( target=mp_tasks.mp_apportion_pipeline, - name=f"{component_name}_apportion", + name=f"{run_label}_apportion", args=(injectables, sub_proc_names, step_info), ), ) @@ -149,7 +154,7 @@ def _run_mp_single_component( state, multiprocessing.Process( target=mp_tasks.mp_coalesce_pipelines, - name=f"{component_name}_coalesce", + name=f"{run_label}_coalesce", args=(injectables, sub_proc_names, slice_info), ), ) diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index facfa88eea..378b27702b 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -108,36 +108,62 @@ def run_calibration_loop( progress = _read_progress(state) if progress and progress.get("complete"): - if resume_after is not None: - raise RuntimeError( - f"settings.yaml resume_after={resume_after!r} cannot be honored " - "because calibration progress is already complete. Remove " - f"{CALIBRATION_PROGRESS_FILE} or use a new output directory to " - "start a new calibration run; current coefficient values will " - "be preserved." + completed_global_iterations = int( + progress.get("last_completed_global_iteration", 0) + ) + completed_for_global_iterations = int( + progress.get( + "configured_global_iterations", completed_global_iterations ) - logger.info( - "calibration progress is already complete; remove %s to start a " - "fresh calibration run", - CALIBRATION_PROGRESS_FILE, ) - return CalibrationRunResult( - converged=bool(progress.get("converged", False)), - completed_global_iterations=int( - progress.get( - "last_completed_global_iteration", - calibration_settings.run.global_iterations, + if calibration_settings.run.global_iterations > completed_for_global_iterations: + logger.info( + "calibration global_iterations increased from %s to %s; " + "continuing with global iteration %s", + completed_for_global_iterations, + calibration_settings.run.global_iterations, + completed_global_iterations + 1, + ) + progress = { + "complete": False, + "in_progress_iteration": None, + "next_global_iteration": completed_global_iterations + 1, + "last_completed_global_iteration": completed_global_iterations, + "converged": bool(progress.get("converged", False)), + "configured_global_iterations": calibration_settings.run.global_iterations, + "attempt": 0, + "completed_components": {}, + } + _write_progress(state, progress) + else: + if resume_after is not None: + raise RuntimeError( + f"settings.yaml resume_after={resume_after!r} cannot be honored " + "because calibration progress is already complete. Remove " + f"{CALIBRATION_PROGRESS_FILE} or use a new output directory to " + "start a new calibration run; current coefficient values will " + "be preserved." ) - ), - ) + logger.info( + "calibration progress is already complete; remove %s to start a " + "fresh calibration run", + CALIBRATION_PROGRESS_FILE, + ) + return CalibrationRunResult( + converged=bool(progress.get("converged", False)), + completed_global_iterations=completed_global_iterations, + ) interrupted_iteration = progress.get("in_progress_iteration") if progress else None if interrupted_iteration is not None: start_global_iter = int(interrupted_iteration) + start_attempt = int(progress.get("attempt", 1)) + 1 + start_completed_components = dict(progress.get("completed_components", {})) logger.warning( - "continuing interrupted calibration global iteration %s using the " - "current coefficient files", + "continuing interrupted calibration global iteration %s as attempt %s " + "using the current coefficient files", start_global_iter, + start_attempt, ) else: # Progress files from earlier versions contain next_global_iteration, so @@ -145,8 +171,28 @@ def run_calibration_loop( start_global_iter = ( int(progress.get("next_global_iteration", 1)) if progress else 1 ) + start_attempt = 1 + start_completed_components = {} completed_global_iterations = start_global_iter - 1 + if interrupted_iteration is not None and resume_after is not None: + rerun_completed_components = [ + component + for component in start_completed_components + if component in calibration_settings.run.calibrate_models + and models.index(component) > models.index(resume_after) + ] + if rerun_completed_components: + logger.warning( + "resume_after=%r rewinds across completed calibrated component(s) " + "%s. Their coefficient values will not be rolled back; rerun " + "results will be appended as attempt %s of global iteration %s.", + resume_after, + rerun_completed_components, + start_attempt, + start_global_iter, + ) + if start_global_iter > calibration_settings.run.global_iterations: logger.info( "calibration progress already reached configured global_iterations=%s", @@ -158,6 +204,11 @@ def run_calibration_loop( state, completed_global_iterations, converged, + calibration_settings.run.global_iterations, + attempt=int(progress.get("attempt", 1)) if progress else 1, + completed_components=( + dict(progress.get("completed_components", {})) if progress else {} + ), ) return CalibrationRunResult( converged=converged, @@ -241,9 +292,17 @@ def run_calibration_loop( start_global_iter, calibration_settings.run.global_iterations + 1, ): + attempt = start_attempt if global_iter == start_global_iter else 1 + completed_components = ( + dict(start_completed_components) + if global_iter == start_global_iter + else {} + ) _mark_global_iteration_in_progress( state, global_iter, + attempt, + completed_components, ) # Every global iteration after the first begins from the immutable @@ -271,24 +330,36 @@ def run_calibration_loop( state.checkpoint.close_store() logger.info( - "calibration global iteration %s/%s", + "calibration global iteration %s/%s attempt %s", global_iter, calibration_settings.run.global_iterations, + attempt, ) - # suppress early termination on first iteration if resume_after is after all calibrated models - all_converged = ( - first_model_idx is not None and first_model_idx <= last_calib_model_idx - ) or global_iter > start_global_iter + skipped_components = _skipped_calibration_components( + calibration_models=calibration_settings.run.calibrate_models, + models=models, + first_model_idx=first_model_idx, + global_iter=global_iter, + start_global_iter=start_global_iter, + ) + if skipped_components: + all_converged = all( + bool(completed_components.get(component, {}).get("converged")) + for component in skipped_components + ) + else: + all_converged = _components_ran_for_convergence( + first_model_idx=first_model_idx, + last_calib_model_idx=last_calib_model_idx, + global_iter=global_iter, + start_global_iter=start_global_iter, + ) last_calibrated_component = None for component in calibration_settings.run.calibrate_models: # on the first global iter, skip model if it's before or == resume_after - if ( - global_iter == start_global_iter - and first_model_idx is not None - and first_model_idx > models.index(component) - ): + if component in skipped_components: continue component_settings = calibration_settings.model_settings[component] @@ -313,12 +384,25 @@ def run_calibration_loop( component_settings=component_settings, prior_step=prior_step, global_iter=global_iter, + attempt=attempt, shared_data_buffers=shared_data_buffers, ) _write_component_plots(state, component) all_converged = all_converged and component_result.converged + completed_components[component] = { + "attempt": attempt, + "converged": component_result.converged, + "component_iterations": component_result.component_iterations, + } + _mark_global_iteration_in_progress( + state, + global_iter, + attempt, + completed_components, + ) + last_calibrated_component = component iteration_is_complete = ( @@ -360,6 +444,8 @@ def run_calibration_loop( "next_global_iteration": global_iter + 1, "last_completed_global_iteration": global_iter, "converged": all_converged, + "attempt": 0, + "completed_components": {}, }, ) @@ -376,6 +462,9 @@ def run_calibration_loop( state, completed_global_iterations, all_converged, + calibration_settings.run.global_iterations, + attempt=attempt, + completed_components=completed_components, ) return CalibrationRunResult( @@ -450,3 +539,34 @@ def _prior_step_name(models: list[str], component_name: str) -> str | None: if idx == 0: return None return models[idx - 1] + + +def _components_ran_for_convergence( + first_model_idx: int | None, + last_calib_model_idx: int, + global_iter: int, + start_global_iter: int, +) -> bool: + """Return whether component results can establish convergence this iteration.""" + return ( + first_model_idx is None + or first_model_idx <= last_calib_model_idx + or global_iter > start_global_iter + ) + + +def _skipped_calibration_components( + calibration_models: list[str], + models: list[str], + first_model_idx: int | None, + global_iter: int, + start_global_iter: int, +) -> list[str]: + """Return calibrated components skipped by resume_after this iteration.""" + if global_iter != start_global_iter or first_model_idx is None: + return [] + return [ + component + for component in calibration_models + if first_model_idx > models.index(component) + ] diff --git a/activitysim/core/calibration/recovery.py b/activitysim/core/calibration/recovery.py index c7a4dc28dd..7abd5f5421 100644 --- a/activitysim/core/calibration/recovery.py +++ b/activitysim/core/calibration/recovery.py @@ -14,6 +14,8 @@ def _mark_global_iteration_in_progress( state: workflow.State, global_iteration: int, + attempt: int, + completed_components: dict[str, dict[str, Any]] | None = None, ) -> None: """Durably mark a global iteration in progress.""" _write_progress( @@ -22,6 +24,8 @@ def _mark_global_iteration_in_progress( "in_progress_iteration": global_iteration, "next_global_iteration": global_iteration, "last_completed_global_iteration": global_iteration - 1, + "attempt": attempt, + "completed_components": completed_components or {}, }, ) @@ -49,6 +53,9 @@ def _write_completed_progress( state: workflow.State, completed_global_iterations: int, converged: bool, + configured_global_iterations: int, + attempt: int = 1, + completed_components: dict[str, dict[str, Any]] | None = None, ) -> None: """Mark calibration complete after all final output has been written.""" _write_progress( @@ -59,5 +66,8 @@ def _write_completed_progress( "next_global_iteration": completed_global_iterations + 1, "last_completed_global_iteration": completed_global_iterations, "converged": converged, + "configured_global_iterations": configured_global_iterations, + "attempt": attempt, + "completed_components": completed_components or {}, }, ) diff --git a/activitysim/core/calibration/reporting.py b/activitysim/core/calibration/reporting.py index a4fbd60010..6db1490b1d 100644 --- a/activitysim/core/calibration/reporting.py +++ b/activitysim/core/calibration/reporting.py @@ -39,7 +39,13 @@ def _append_iteration_records( _append_csv( df, global_path, - unique_on=["global_iter", "component_iter", "component", "coefficient"], + unique_on=[ + "global_iter", + "attempt", + "component_iter", + "component", + "coefficient", + ], ) # Also write component-local iteration history @@ -50,7 +56,13 @@ def _append_iteration_records( _append_csv( df, component_path, - unique_on=["global_iter", "component_iter", "component", "coefficient"], + unique_on=[ + "global_iter", + "attempt", + "component_iter", + "component", + "coefficient", + ], ) @@ -65,7 +77,7 @@ def _append_summary_records( _append_csv( df, path, - unique_on=["global_iter", "component_iter", "component"], + unique_on=["global_iter", "attempt", "component_iter", "component"], ) @@ -76,6 +88,10 @@ def _append_csv( os.makedirs(path.parent, exist_ok=True) if unique_on and path.exists(): existing = pd.read_csv(path) + if "attempt" in unique_on and "attempt" not in existing.columns: + # Histories created before recovery attempts were introduced belong + # to the first attempt of their logical global iteration. + existing["attempt"] = 1 df = pd.concat([existing, df], ignore_index=True).drop_duplicates( subset=unique_on, keep="last" ) @@ -125,9 +141,12 @@ def _read_component_iteration_records( iteration_records = ( pd.read_csv(path) - .set_index(["global_iter", "component_iter", "coefficient"]) - .sort_index() ) + if "attempt" not in iteration_records.columns: + iteration_records["attempt"] = 1 + iteration_records = iteration_records.set_index( + ["global_iter", "attempt", "component_iter", "coefficient"] + ).sort_index() return iteration_records.loc[iteration_records.component == component_name] @@ -140,12 +159,11 @@ def _plot_coefficient_progress( ) -> None: """Plot coefficient value progression for one coefficient subset.""" component_dir = _component_output_dir(state, component_name) - ax = ( - recs[recs.index.get_level_values("coefficient").isin(set_coefs)] - .next_coefficient.unstack("coefficient") - .plot(figsize=(10, 5)) - ) - ax.xaxis.set_label_text("Component iteration") + trajectory, step_labels = _coefficient_trajectory(recs, set_coefs) + ax = trajectory.plot(figsize=(10, 5)) + ax.set_xticks(range(len(step_labels))) + ax.set_xticklabels(step_labels, rotation=45, ha="right") + ax.xaxis.set_label_text("Calibration update (global-attempt-component)") ax.yaxis.set_label_text("Coefficient value") ax.legend(title="Coefficient label", loc="center left", bbox_to_anchor=(1.02, 0.5)) plt.tight_layout() @@ -156,13 +174,47 @@ def _plot_coefficient_progress( plt.close(ax.figure) +def _coefficient_trajectory( + recs: pd.DataFrame, + set_coefs: list[str], +) -> tuple[pd.DataFrame, list[str]]: + """Build the complete ordered coefficient path and compact step labels.""" + filtered = recs[ + recs.index.get_level_values("coefficient").isin(set_coefs) + ].reset_index() + history = filtered.pivot( + index=["global_iter", "attempt", "component_iter"], + columns="coefficient", + values="next_coefficient", + ).sort_index() + initial_values = ( + filtered.sort_values(["global_iter", "attempt", "component_iter"]) + .groupby("coefficient", sort=False) + .first()["prev_coefficient"] + .reindex(history.columns) + ) + trajectory = pd.concat( + [pd.DataFrame([initial_values], index=["Start"]), history.reset_index(drop=True)] + ) + step_labels = ["Start"] + [ + f"G{global_iter}-A{attempt}-C{component_iter}" + for global_iter, attempt, component_iter in history.index + ] + return trajectory, step_labels + + def _component_last_records(recs: pd.DataFrame, set_coefs: list[str]) -> pd.DataFrame: """Select target/model values for the latest iteration and coefficient subset.""" filtered = recs[recs.index.get_level_values("coefficient").isin(set_coefs)] last_global = filtered.index.get_level_values("global_iter")[-1] - last_comp = filtered.loc[last_global].index.get_level_values("component_iter")[-1] + last_attempt = filtered.loc[last_global].index.get_level_values("attempt")[-1] + last_comp = ( + filtered.loc[(last_global, last_attempt)] + .index.get_level_values("component_iter")[-1] + ) return filtered.xs( - (last_global, last_comp), level=("global_iter", "component_iter") + (last_global, last_attempt, last_comp), + level=("global_iter", "attempt", "component_iter"), )[["target_value", "model_value"]] @@ -215,6 +267,7 @@ def _write_generic_report( df[ [ "global_iter", + "attempt", "component_iter", "component", "description", @@ -224,14 +277,22 @@ def _write_generic_report( ] ] .copy() - .sort_values(["global_iter", "component_iter", "description"]) + .sort_values( + ["global_iter", "attempt", "component_iter", "description"] + ) ) path = _component_output_dir(state, component_name) / "generic_report.csv" _append_csv( report, path, - unique_on=["global_iter", "component_iter", "component", "description"], + unique_on=[ + "global_iter", + "attempt", + "component_iter", + "component", + "description", + ], ) diff --git a/activitysim/core/test/test_calibration.py b/activitysim/core/test/test_calibration.py new file mode 100644 index 0000000000..e8b50bc37a --- /dev/null +++ b/activitysim/core/test/test_calibration.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +import math + +import pandas as pd +import pytest + +from activitysim.core.calibration.component import _evaluate_and_update +from activitysim.core.calibration.expressions import _compute_delta +from activitysim.core.calibration.orchestrator import ( + _components_ran_for_convergence, +) + + +@pytest.mark.parametrize( + ("method", "model_value", "target_value", "damping", "expected"), + [ + ("log_ratio", 0.25, 0.5, 0.5, math.log(2) * 0.5), + ( + "odds_ratio", + 0.4, + 0.6, + 0.5, + math.log((0.6 * (1 - 0.4)) / (0.4 * (1 - 0.6))) * 0.5, + ), + ], +) +def test_compute_delta(method, model_value, target_value, damping, expected): + delta = _compute_delta( + method=method, + model_value=model_value, + target_value=target_value, + damping=damping, + component_name="test_component", + description="test target", + default_increment=2.0, + ) + + assert delta == pytest.approx(expected) + + +@pytest.mark.parametrize( + ("model_value", "target_value", "expected"), + [(0.0, 0.5, 2.0), (1.0, 0.5, -2.0), (0.0, 0.0, 0.0)], +) +def test_odds_ratio_boundary_fallback_has_correct_direction( + model_value, target_value, expected +): + delta = _compute_delta( + method="odds_ratio", + model_value=model_value, + target_value=target_value, + damping=1.0, + component_name="test_component", + description="test target", + default_increment=2.0, + ) + + assert delta == expected + + +def test_evaluate_and_update_only_changes_eligible_coefficients(): + calibration_spec = pd.DataFrame( + [ + { + "description": "unconverged", + "coefficient": "coef_update", + "model_value": 0.25, + "target_value": 0.5, + "hold_fast": False, + "min": -10.0, + "max": 10.0, + "damping": 0.5, + "method": "log_ratio", + "tolerance": 0.01, + }, + { + "description": "within tolerance", + "coefficient": "coef_converged", + "model_value": 0.49, + "target_value": 0.5, + "hold_fast": False, + "min": -10.0, + "max": 10.0, + "damping": 1.0, + "method": "log_ratio", + "tolerance": 0.02, + }, + { + "description": "held fixed", + "coefficient": "coef_held", + "model_value": 0.25, + "target_value": 0.5, + "hold_fast": True, + "min": -10.0, + "max": 10.0, + "damping": 1.0, + "method": "log_ratio", + "tolerance": 0.01, + }, + ] + ) + coefficients = pd.DataFrame( + {"value": [1.0, 2.0, 3.0]}, + index=["coef_update", "coef_converged", "coef_held"], + ) + + records, _, updated, component_converged = _evaluate_and_update( + component_name="test_component", + calibration_spec_df=calibration_spec, + coefficients_df=coefficients, + eval_context={}, + global_iter=1, + component_iter=1, + ) + + assert updated.loc["coef_update", "value"] == pytest.approx(1.0 + math.log(2) * 0.5) + assert updated.loc["coef_converged", "value"] == 2.0 + assert updated.loc["coef_held", "value"] == 3.0 + assert records[1]["coef_delta"] == 0.0 + assert records[2]["coef_delta"] == 0.0 + assert component_converged is False + + +@pytest.mark.parametrize( + ( + "first_model_idx", + "last_calib_model_idx", + "global_iter", + "start_global_iter", + "expected", + ), + [ + (None, 10, 1, 1, True), + (5, 10, 1, 1, True), + (11, 10, 1, 1, False), + (11, 10, 2, 1, True), + ], +) +def test_components_ran_for_convergence( + first_model_idx, + last_calib_model_idx, + global_iter, + start_global_iter, + expected, +): + assert ( + _components_ran_for_convergence( + first_model_idx=first_model_idx, + last_calib_model_idx=last_calib_model_idx, + global_iter=global_iter, + start_global_iter=start_global_iter, + ) + is expected + ) diff --git a/activitysim/core/test/test_calibration_reporting.py b/activitysim/core/test/test_calibration_reporting.py new file mode 100644 index 0000000000..c1e4c684c7 --- /dev/null +++ b/activitysim/core/test/test_calibration_reporting.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from pathlib import Path + +import pandas as pd + +from activitysim.core.calibration.reporting import ( + _append_iteration_records, + _coefficient_trajectory, + _read_component_iteration_records, +) + + +class _State: + def __init__(self, output_dir: Path): + self.output_dir = output_dir + + def get_output_file_path(self, file_name: str) -> Path: + return self.output_dir / file_name + + +def _record(attempt: int, previous: float, next_value: float) -> dict: + return { + "global_iter": 1, + "attempt": attempt, + "component_iter": 1, + "description": "test target", + "component": "model_a", + "coefficient": "coef_a", + "target_value": 0.5, + "model_value": 0.25, + "difference": 0.25, + "pct_difference": 50.0, + "hold_fast": False, + "prev_coefficient": previous, + "coef_delta": next_value - previous, + "next_coefficient": next_value, + "converged": False, + "at_min": False, + "at_max": False, + } + + +def test_recovery_attempts_preserve_complete_coefficient_trajectory(tmp_path): + state = _State(tmp_path) + _append_iteration_records(state, "model_a", [_record(1, 1.0, 1.5)]) + _append_iteration_records(state, "model_a", [_record(2, 1.5, 1.75)]) + + stored = pd.read_csv( + tmp_path / "calibration" / "calibration_iteration_records.csv" + ) + assert list(stored["attempt"]) == [1, 2] + assert stored.loc[1, "prev_coefficient"] == stored.loc[0, "next_coefficient"] + + records = _read_component_iteration_records(state, "model_a") + trajectory, labels = _coefficient_trajectory(records, ["coef_a"]) + + assert labels == ["Start", "G1-A1-C1", "G1-A2-C1"] + assert list(trajectory["coef_a"]) == [1.0, 1.5, 1.75] diff --git a/activitysim/examples/prototype_mtc/test/calibration/README.md b/activitysim/examples/prototype_mtc/test/calibration/README.md index f8288ef562..9054f5937e 100644 --- a/activitysim/examples/prototype_mtc/test/calibration/README.md +++ b/activitysim/examples/prototype_mtc/test/calibration/README.md @@ -12,6 +12,10 @@ that global iteration 1 remains in progress, replaces the expression, and restarts with `resume_after: non_mandatory_tour_scheduling`. The resumed runs must preserve the workplace and auto-ownership coefficient files and match the uninterrupted final tables and calibrated coefficients across both modes. +The fixture also rewinds a failed run to `initialize_landuse`, verifies that the +completed calibrated models rerun as attempt 2 of the same global iteration, +and confirms that attempt-1 and attempt-2 coefficient transitions are both +retained with a continuous `next_coefficient` to `prev_coefficient` chain. Each run copies mutable coefficient files into its temporary config directory; the shared prototype MTC configs and data remain unchanged. The 25-household diff --git a/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py b/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py index 607efdcd24..f48fb9ca2a 100644 --- a/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py +++ b/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py @@ -81,7 +81,10 @@ def _run(configs_dir: Path, output_dir: Path, multiprocess: bool): return subprocess.run(args, check=False, capture_output=True, text=True) -def _resume(configs_dir: Path): +def _resume( + configs_dir: Path, + resume_after: str = "non_mandatory_tour_scheduling", +): shutil.copyfile( HERE / "configs" / "tour_mode_choice_calibration.csv", configs_dir / "tour_mode_choice_calibration.csv", @@ -89,11 +92,20 @@ def _resume(configs_dir: Path): settings_path = configs_dir / "settings.yaml" with open(settings_path, encoding="utf-8") as stream: settings = yaml.safe_load(stream) - settings["resume_after"] = "non_mandatory_tour_scheduling" + settings["resume_after"] = resume_after with open(settings_path, "w", encoding="utf-8") as stream: yaml.safe_dump(settings, stream, sort_keys=False) +def _set_global_iterations(configs_dir: Path, global_iterations: int): + calibration_path = configs_dir / "calibration.yaml" + with open(calibration_path, encoding="utf-8") as stream: + calibration = yaml.safe_load(stream) + calibration["run"]["global_iterations"] = global_iterations + with open(calibration_path, "w", encoding="utf-8") as stream: + yaml.safe_dump(calibration, stream, sort_keys=False) + + def _assert_success(result: subprocess.CompletedProcess): assert result.returncode == 0, result.stdout + result.stderr @@ -149,6 +161,89 @@ def _run_resumed(root: Path, name: str, multiprocess: bool) -> Path: return output +def _run_with_increased_global_iterations( + root: Path, name: str, completed_output: Path, multiprocess: bool +) -> Path: + run_dir = root / name + configs = run_dir / "configs" + output = run_dir / "output" + shutil.copytree(completed_output.parent / "configs", configs) + shutil.copytree(completed_output, output) + + _set_global_iterations(configs, 2) + _assert_success(_run(configs, output, multiprocess)) + + with open( + output / "calibration" / "calibration_progress.json", encoding="utf-8" + ) as stream: + progress = json.load(stream) + assert progress["complete"] is True + assert progress["last_completed_global_iteration"] == 2 + assert progress["configured_global_iterations"] == 2 + + records = pd.read_csv( + output / "calibration" / "calibration_iteration_records.csv" + ) + assert set(records["global_iter"]) == {1, 2} + return output + + +def _run_rewound_attempt(root: Path, name: str, multiprocess: bool) -> Path: + configs, output = _prepare_run(root, name, multiprocess, failing=True) + failed = _run(configs, output, multiprocess) + assert failed.returncode != 0 + + progress_path = output / "calibration" / "calibration_progress.json" + with open(progress_path, encoding="utf-8") as stream: + progress = json.load(stream) + assert progress["in_progress_iteration"] == 1 + assert progress["attempt"] == 1 + assert set(progress["completed_components"]) == { + "workplace_location", + "auto_ownership_simulate", + } + + records_path = output / "calibration" / "calibration_iteration_records.csv" + first_attempt = pd.read_csv(records_path) + workplace_first = first_attempt[ + first_attempt["component"] == "workplace_location" + ] + assert set(workplace_first["attempt"]) == {1} + + _resume(configs, resume_after="initialize_landuse") + _assert_success(_run(configs, output, multiprocess)) + + records = pd.read_csv(records_path) + workplace = records[records["component"] == "workplace_location"].sort_values( + ["global_iter", "attempt", "component_iter"] + ) + assert set(workplace["global_iter"]) == {1} + assert set(workplace["attempt"]) == {1, 2} + + attempt_1 = workplace[workplace["attempt"] == 1].iloc[-1] + attempt_2 = workplace[workplace["attempt"] == 2].iloc[0] + assert attempt_2["prev_coefficient"] == pytest.approx( + attempt_1["next_coefficient"] + ) + + assert ( + output + / "calibration" + / "workplace_location" + / "coefficient_progress_set_0.png" + ).exists() + + with open(progress_path, encoding="utf-8") as stream: + progress = json.load(stream) + assert progress["complete"] is True + assert progress["attempt"] == 2 + assert all( + component["attempt"] == 2 + for component in progress["completed_components"].values() + ) + return output + + @pytest.fixture(scope="module") def run_root(tmp_path_factory) -> Path: return tmp_path_factory.mktemp("calibration_run_modes") @@ -174,6 +269,40 @@ def multiprocess_resumed_output(run_root: Path) -> Path: return _run_resumed(run_root, "multiprocess_resumed", multiprocess=True) +@pytest.fixture(scope="module") +def single_increased_iterations_output( + run_root: Path, single_output: Path +) -> Path: + return _run_with_increased_global_iterations( + run_root, + "single_increased_iterations", + completed_output=single_output, + multiprocess=False, + ) + + +@pytest.fixture(scope="module") +def multiprocess_increased_iterations_output( + run_root: Path, multiprocess_output: Path +) -> Path: + return _run_with_increased_global_iterations( + run_root, + "multiprocess_increased_iterations", + completed_output=multiprocess_output, + multiprocess=True, + ) + + +@pytest.fixture(scope="module") +def single_rewound_output(run_root: Path) -> Path: + return _run_rewound_attempt(run_root, "single_rewound", multiprocess=False) + + +@pytest.fixture(scope="module") +def multiprocess_rewound_output(run_root: Path) -> Path: + return _run_rewound_attempt(run_root, "multiprocess_rewound", multiprocess=True) + + def test_single_and_multiprocess_are_equivalent( single_output: Path, multiprocess_output: Path, @@ -200,3 +329,20 @@ def test_resumed_single_and_multiprocess_are_equivalent( multiprocess_resumed_output: Path, ): _assert_equivalent(single_resumed_output, multiprocess_resumed_output) + + +def test_increased_global_iterations_continue_completed_run_in_all_modes( + single_increased_iterations_output: Path, + multiprocess_increased_iterations_output: Path, +): + _assert_equivalent( + single_increased_iterations_output, + multiprocess_increased_iterations_output, + ) + + +def test_rewinding_completed_calibrated_model_appends_attempt_in_all_modes( + single_rewound_output: Path, + multiprocess_rewound_output: Path, +): + _assert_equivalent(single_rewound_output, multiprocess_rewound_output) From 910a0e004e14529c252ad53d26bc013ef9d8b1de Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:59:03 -0700 Subject: [PATCH 84/90] blacken --- activitysim/core/calibration/component.py | 3 +-- activitysim/core/calibration/orchestrator.py | 4 +--- activitysim/core/calibration/reporting.py | 20 ++++++++---------- .../core/test/test_calibration_reporting.py | 4 +--- .../test/calibration/test_run_modes.py | 21 +++++-------------- 5 files changed, 17 insertions(+), 35 deletions(-) diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index 6683b670a6..1f470c5fcd 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -145,8 +145,7 @@ def _calibrate_component( for component_iter in range(1, component_settings.submodel_max_iterations + 1): component_iterations = component_iter run_model_name = ( - f"{component_name}.c_i{component_iter};" - f"g_i{global_iter};a_i{attempt}" + f"{component_name}.c_i{component_iter};" f"g_i{global_iter};a_i{attempt}" ) _run_component_model( state=state, diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index 378b27702b..c8281d27c2 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -112,9 +112,7 @@ def run_calibration_loop( progress.get("last_completed_global_iteration", 0) ) completed_for_global_iterations = int( - progress.get( - "configured_global_iterations", completed_global_iterations - ) + progress.get("configured_global_iterations", completed_global_iterations) ) if calibration_settings.run.global_iterations > completed_for_global_iterations: logger.info( diff --git a/activitysim/core/calibration/reporting.py b/activitysim/core/calibration/reporting.py index 6db1490b1d..c4dee4ca0f 100644 --- a/activitysim/core/calibration/reporting.py +++ b/activitysim/core/calibration/reporting.py @@ -139,9 +139,7 @@ def _read_component_iteration_records( if not path.exists(): return None - iteration_records = ( - pd.read_csv(path) - ) + iteration_records = pd.read_csv(path) if "attempt" not in iteration_records.columns: iteration_records["attempt"] = 1 iteration_records = iteration_records.set_index( @@ -194,7 +192,10 @@ def _coefficient_trajectory( .reindex(history.columns) ) trajectory = pd.concat( - [pd.DataFrame([initial_values], index=["Start"]), history.reset_index(drop=True)] + [ + pd.DataFrame([initial_values], index=["Start"]), + history.reset_index(drop=True), + ] ) step_labels = ["Start"] + [ f"G{global_iter}-A{attempt}-C{component_iter}" @@ -208,10 +209,9 @@ def _component_last_records(recs: pd.DataFrame, set_coefs: list[str]) -> pd.Data filtered = recs[recs.index.get_level_values("coefficient").isin(set_coefs)] last_global = filtered.index.get_level_values("global_iter")[-1] last_attempt = filtered.loc[last_global].index.get_level_values("attempt")[-1] - last_comp = ( - filtered.loc[(last_global, last_attempt)] - .index.get_level_values("component_iter")[-1] - ) + last_comp = filtered.loc[(last_global, last_attempt)].index.get_level_values( + "component_iter" + )[-1] return filtered.xs( (last_global, last_attempt, last_comp), level=("global_iter", "attempt", "component_iter"), @@ -277,9 +277,7 @@ def _write_generic_report( ] ] .copy() - .sort_values( - ["global_iter", "attempt", "component_iter", "description"] - ) + .sort_values(["global_iter", "attempt", "component_iter", "description"]) ) path = _component_output_dir(state, component_name) / "generic_report.csv" diff --git a/activitysim/core/test/test_calibration_reporting.py b/activitysim/core/test/test_calibration_reporting.py index c1e4c684c7..d0d459cec2 100644 --- a/activitysim/core/test/test_calibration_reporting.py +++ b/activitysim/core/test/test_calibration_reporting.py @@ -46,9 +46,7 @@ def test_recovery_attempts_preserve_complete_coefficient_trajectory(tmp_path): _append_iteration_records(state, "model_a", [_record(1, 1.0, 1.5)]) _append_iteration_records(state, "model_a", [_record(2, 1.5, 1.75)]) - stored = pd.read_csv( - tmp_path / "calibration" / "calibration_iteration_records.csv" - ) + stored = pd.read_csv(tmp_path / "calibration" / "calibration_iteration_records.csv") assert list(stored["attempt"]) == [1, 2] assert stored.loc[1, "prev_coefficient"] == stored.loc[0, "next_coefficient"] diff --git a/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py b/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py index f48fb9ca2a..6daeb44ac7 100644 --- a/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py +++ b/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py @@ -181,9 +181,7 @@ def _run_with_increased_global_iterations( assert progress["last_completed_global_iteration"] == 2 assert progress["configured_global_iterations"] == 2 - records = pd.read_csv( - output / "calibration" / "calibration_iteration_records.csv" - ) + records = pd.read_csv(output / "calibration" / "calibration_iteration_records.csv") assert set(records["global_iter"]) == {1, 2} return output @@ -205,9 +203,7 @@ def _run_rewound_attempt(root: Path, name: str, multiprocess: bool) -> Path: records_path = output / "calibration" / "calibration_iteration_records.csv" first_attempt = pd.read_csv(records_path) - workplace_first = first_attempt[ - first_attempt["component"] == "workplace_location" - ] + workplace_first = first_attempt[first_attempt["component"] == "workplace_location"] assert set(workplace_first["attempt"]) == {1} _resume(configs, resume_after="initialize_landuse") @@ -222,15 +218,10 @@ def _run_rewound_attempt(root: Path, name: str, multiprocess: bool) -> Path: attempt_1 = workplace[workplace["attempt"] == 1].iloc[-1] attempt_2 = workplace[workplace["attempt"] == 2].iloc[0] - assert attempt_2["prev_coefficient"] == pytest.approx( - attempt_1["next_coefficient"] - ) + assert attempt_2["prev_coefficient"] == pytest.approx(attempt_1["next_coefficient"]) assert ( - output - / "calibration" - / "workplace_location" - / "coefficient_progress_set_0.png" + output / "calibration" / "workplace_location" / "coefficient_progress_set_0.png" ).exists() with open(progress_path, encoding="utf-8") as stream: @@ -270,9 +261,7 @@ def multiprocess_resumed_output(run_root: Path) -> Path: @pytest.fixture(scope="module") -def single_increased_iterations_output( - run_root: Path, single_output: Path -) -> Path: +def single_increased_iterations_output(run_root: Path, single_output: Path) -> Path: return _run_with_increased_global_iterations( run_root, "single_increased_iterations", From 2ef98285e3221d7db88875e59b5e8b53aa76a11b Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 24 Aug 2026 22:07:34 -0700 Subject: [PATCH 85/90] removing unused calibration.yaml settings --- activitysim/core/calibration/README.md | 2 -- activitysim/core/calibration/settings.py | 3 --- .../configs_calibration/calibration.yaml | 3 --- docs/users-guide/calibration.rst | 21 +++---------------- 4 files changed, 3 insertions(+), 26 deletions(-) diff --git a/activitysim/core/calibration/README.md b/activitysim/core/calibration/README.md index ae87db4ae0..f6c78cda00 100644 --- a/activitysim/core/calibration/README.md +++ b/activitysim/core/calibration/README.md @@ -49,8 +49,6 @@ strict ActivitySim semantics: the named model is treated as complete, its checkpoint is restored, and execution begins with the following model. The name must occur in the top-level `models` list and must be a model-level checkpoint; the `_` shorthand for the last checkpoint is not accepted in calibration mode. -The similarly named `calibration.yaml` `run.resume_after` compatibility field is -not used by the orchestrator. On the first global iteration entered by an invocation, calibrated models at or before `resume_after` are skipped. Later global iterations in the same invocation diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index dbd8accaab..83f8bdd200 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -3,7 +3,6 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Optional from pydantic import model_validator @@ -17,7 +16,6 @@ class CalibrationRunSettings(PydanticBase): """Run-control settings for calibration.""" - resume_after: Optional[str] = None calibrate_models: list[str] global_iterations: int = 1 complete_steps: bool = False @@ -49,7 +47,6 @@ class CalibrationComponentSettings(PydanticBase): helper_module: str | None = None submodel_max_iterations: int = 1 reports: CalibrationReportsSettings = CalibrationReportsSettings() - survey_file: Optional[str] = None class CalibrationConfig(PydanticReadable): diff --git a/activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml b/activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml index c4949c0718..61ac34a57d 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml +++ b/activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml @@ -14,7 +14,6 @@ model_settings: reports: generic: true bespoke: report_workplace_location - survey_file: survey_persons.csv auto_ownership_simulate: calibration_spec: auto_ownership_calibration.csv @@ -23,7 +22,6 @@ model_settings: reports: generic: true bespoke: report_auto_ownership - survey_file: survey_households.csv tour_mode_choice_simulate: calibration_spec: tour_mode_choice_calibration.csv @@ -32,5 +30,4 @@ model_settings: reports: generic: true bespoke: report_tour_mode_choice - survey_file: survey_tours.csv diff --git a/docs/users-guide/calibration.rst b/docs/users-guide/calibration.rst index 2d817f32fe..ac79a1af2f 100644 --- a/docs/users-guide/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -58,7 +58,6 @@ calibration-specific overlay directory): - workplace_location - auto_ownership_simulate - tour_mode_choice_simulate - resume_after: null # checkpoint to resume from on global iteration 1 restart_after: [] # components after which to restart (advanced) global_iterations: 3 # number of full calibration passes complete_steps: false # run model steps after the last calibrated component @@ -68,7 +67,6 @@ calibration-specific overlay directory): calibration_spec: workplace_location_calibration.csv helper_module: workplace_location_calib_helper.py submodel_max_iterations: 3 - survey_file: survey_persons.csv reports: generic: true bespoke: report_workplace_location @@ -77,7 +75,6 @@ calibration-specific overlay directory): calibration_spec: auto_ownership_calibration.csv helper_module: auto_ownership_calib_helper.py submodel_max_iterations: 3 - survey_file: survey_households.csv reports: generic: true bespoke: report_auto_ownership @@ -86,7 +83,6 @@ calibration-specific overlay directory): calibration_spec: tour_mode_choice_calibration.csv helper_module: tour_mode_choice_calib_helper.py submodel_max_iterations: 3 - survey_file: survey_tours.csv reports: generic: true bespoke: report_tour_mode_choice @@ -158,12 +154,6 @@ Configuration Reference - *required* - Model component names to calibrate. Must match names in ``settings.yaml`` ``models`` list. - * - ``resume_after`` - - ``str`` or ``null`` - - ``null`` - - Checkpoint to resume from on the first global iteration. Equivalent to - ``resume_after`` in ``settings.yaml``. Use this to skip expensive - initialization steps that do not change across calibration iterations. * - ``restart_after`` - ``list[str]`` - ``[]`` @@ -208,11 +198,6 @@ Configuration Reference - ``1`` - Maximum number of inner-loop iterations per component per global iteration. The component re-runs from its prior checkpoint each iteration. - * - ``survey_file`` - - ``str`` - - *required* - - Survey data CSV filename. Made available via - ``component_settings.survey_file`` in the expression context. * - ``reports.generic`` - ``bool`` - ``True`` @@ -718,6 +703,6 @@ Tips (e.g., for trip generation downstream of mode choice). - **Version control your coefficient files** so you can diff changes and reset to initial values. -- **Use the** ``resume_after`` **setting** to skip expensive upstream steps (like - skims loading or accessibility computation) that don't change across - calibration iterations. +- **Use the top-level** ``settings.yaml`` ``resume_after`` **setting** to skip + expensive upstream steps (like skims loading or accessibility computation) + that don't change across calibration iterations. From cd8f8b58b14cb09e34c46442bc38f35cefc53614 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Tue, 25 Aug 2026 16:27:48 -0700 Subject: [PATCH 86/90] clarifying resume_after readme --- activitysim/core/calibration/README.md | 44 ++++++++++++++++---------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/activitysim/core/calibration/README.md b/activitysim/core/calibration/README.md index f6c78cda00..5863987b67 100644 --- a/activitysim/core/calibration/README.md +++ b/activitysim/core/calibration/README.md @@ -44,24 +44,36 @@ global iteration is attempt 1. Restarting an interrupted global iteration create attempt 2, then attempt 3 if another restart is needed. A recovery attempt does not consume an additional global iteration. -Calibration uses the top-level `settings.yaml` `resume_after` setting. It has -strict ActivitySim semantics: the named model is treated as complete, its -checkpoint is restored, and execution begins with the following model. The name -must occur in the top-level `models` list and must be a model-level checkpoint; -the `_` shorthand for the last checkpoint is not accepted in calibration mode. - -On the first global iteration entered by an invocation, calibrated models at or -before `resume_after` are skipped. Later global iterations in the same invocation -restart immediately before the first calibrated model and run the normal complete -calibration sequence. For example, if global iteration 3 was interrupted after +Calibration uses only the top-level `settings.yaml` `resume_after` setting. On +the first global iteration executed by an ActivitySim invocation, including a +new attempt of an interrupted iteration, it behaves like `resume_after` in a +non-calibration run: + +- when set to a model name, that model is treated as complete, its checkpoint is + restored, and execution begins with the following model; and +- when unset or `null`, execution starts at the beginning of the top-level + `models` list. Calibration does not automatically continue after the last + model completed by the preceding attempt. + +A named value must occur in the top-level `models` list and must identify a +model-level checkpoint. Calibration does not define a special `initialize` +value. A value such as `initialize_landuse` has ordinary model-name semantics: +that model is skipped and execution begins with the next model. The `_` shorthand +for the last checkpoint is not accepted in calibration mode. + +`resume_after` affects only the first global iteration entered by the current +invocation. Calibrated models at or before a named resume point are skipped in +that iteration. Any later global iterations in the same invocation ignore +`resume_after`, restart immediately before the first calibrated model, and run +the normal complete calibration sequence. + +For example, suppose global iteration 3 attempt 1 was interrupted after calibrated `model_a` completed: -- `resume_after: model_a` preserves model A's attempt-1 result and resumes with - the following model under attempt 2; -- `resume_after: initialize` rewinds pipeline state and reruns model A under - attempt 2; and -- no `resume_after` replays the interrupted iteration from the beginning under - attempt 2. +- `resume_after: model_a` starts attempt 2 after model A, preserving its + attempt-1 result; and +- `resume_after: null` starts attempt 2 at the beginning of the complete model + list, so model A runs again. Coefficient files are the authoritative current state and are never rolled back during recovery. Updates written before the interruption, along with subsequent From 896fc066373b20130ba7796996df99ef5ce4d805 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:33:19 -0700 Subject: [PATCH 87/90] aligning tests and language around restart functionality also fixing some bugs captured by AI review --- activitysim/abm/models/settings_checker.py | 28 +- activitysim/cli/run.py | 7 +- activitysim/core/calibration/README.md | 38 ++- activitysim/core/calibration/__init__.py | 3 +- activitysim/core/calibration/component.py | 22 +- activitysim/core/calibration/orchestrator.py | 308 +++++++++++++----- activitysim/core/calibration/settings.py | 8 +- .../core/calibration/test/test_calibration.py | 301 +++++++++++++++++ .../test/test_calibration_restart.py | 202 ++++++++++++ activitysim/core/test/test_calibration.py | 155 --------- .../core/test/test_calibration_reporting.py | 57 ---- ...modes.py => test_calibration_run_modes.py} | 0 docs/users-guide/calibration.rst | 66 ++-- 13 files changed, 863 insertions(+), 332 deletions(-) create mode 100644 activitysim/core/calibration/test/test_calibration.py create mode 100644 activitysim/core/calibration/test/test_calibration_restart.py delete mode 100644 activitysim/core/test/test_calibration.py delete mode 100644 activitysim/core/test/test_calibration_reporting.py rename activitysim/examples/prototype_mtc/test/calibration/{test_run_modes.py => test_calibration_run_modes.py} (100%) diff --git a/activitysim/abm/models/settings_checker.py b/activitysim/abm/models/settings_checker.py index c65d0f5772..12b6666392 100644 --- a/activitysim/abm/models/settings_checker.py +++ b/activitysim/abm/models/settings_checker.py @@ -4,24 +4,26 @@ from pydantic import BaseModel as PydanticBase from typing import Type, Optional +from activitysim.core import config +from activitysim.core.calibration.settings import ( + CALIBRATION_SETTINGS_FILE_NAME, + CalibrationConfig, +) from activitysim.core.configuration.base import PydanticReadable - -# import core settings from activitysim.core.configuration.logit import ( LogitNestSpec, TourLocationComponentSettings, TourModeComponentSettings, TemplatedLogitComponentSettings, ) -from activitysim.core import config from activitysim.core.configuration.network import NetworkSettings +from activitysim.core.exceptions import ModelConfigurationError from activitysim.core.workflow import State from activitysim.core.simulate import ( eval_coefficients, eval_nest_coefficients, read_model_coefficient_template, ) -from activitysim.core.exceptions import ModelConfigurationError # import model settings from activitysim.abm.models.accessibility import AccessibilitySettings @@ -637,6 +639,24 @@ def check_model_settings( # Collect all errors all_errors = [] + # calibration.yaml is optional and is not itself a model step, so validate + # it explicitly using the same Pydantic/error-aggregation path as model + # settings files. + try: + CalibrationConfig.read_settings_file( + state.filesystem, + CALIBRATION_SETTINGS_FILE_NAME, + mandatory=False, + ) + except Exception as error: + all_errors.append( + SettingsCheckerError( + "calibration", + error, + CALIBRATION_SETTINGS_FILE_NAME, + ) + ) + # additional logging set up formatter = logging.Formatter( "%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S" diff --git a/activitysim/cli/run.py b/activitysim/cli/run.py index 4d7a1929f4..49a048a53e 100644 --- a/activitysim/cli/run.py +++ b/activitysim/cli/run.py @@ -336,8 +336,13 @@ def run(args): resume_after = state.settings.resume_after # cleanup if not resuming - if not resume_after: + preserve_calibration_outputs = calibration.calibration_run_should_preserve_outputs( + state + ) + if not resume_after and not preserve_calibration_outputs: cleanup_output_files(state) + elif preserve_calibration_outputs: + logger.info("preserving output files based on calibration restart preflight") elif state.settings.cleanup_trace_files_on_resume: tracing.delete_trace_files(state) diff --git a/activitysim/core/calibration/README.md b/activitysim/core/calibration/README.md index 5863987b67..2d4e939e54 100644 --- a/activitysim/core/calibration/README.md +++ b/activitysim/core/calibration/README.md @@ -27,17 +27,33 @@ that share model settings may point to the same coefficient file. ## Global iterations, recovery attempts, and `resume_after` -`calibration.yaml` `run.global_iterations` is the maximum total number of -logical global calibration iterations, not the number to execute on each -ActivitySim invocation. Calibration records durable progress in -`output/calibration/calibration_progress.json`: - -- a new output directory starts at global iteration 1; -- a crash re-enters the interrupted global iteration; -- a cleanly completed iteration advances to the next global iteration; -- convergence can finish the run before the configured maximum; and -- after a completed run, increasing `global_iterations` continues at the next - unfinished global iteration. An unchanged or lower value remains a no-op. +`calibration.yaml` `run.global_iterations` is the desired total number of +completed logical calibration iterations, not the number to execute on each +ActivitySim invocation. The run-control contract is: + +1. An unchanged setting on a completed run is a no-op, detected before normal + output cleanup so the pipeline and final outputs are preserved. +2. If a completed run's setting is changed to a value greater than the number + actually completed, calibration continues until the new total. This includes + lowering a previous maximum after early convergence, such as changing 5 to 3 + after convergence completed iteration 2. +3. A changed setting less than or equal to the number already completed is a + no-op; completed coefficient updates are never undone implicitly. +4. Top-level `settings.yaml` `resume_after` has normal ActivitySim semantics for + the first global iteration entered by the current invocation. Later global + iterations ignore it and execute every calibrated component. +5. A global iteration counts only if it has at least one durable calibrated + component result, either from the current attempt or an earlier attempt of + that same logical iteration. +6. `global_iterations` cannot be lowered below an interrupted iteration because + its coefficient files may already contain updates from that iteration. The + run stops with instructions to resume the iteration or deliberately reset + progress and coefficients. +7. Startup logs report the detected completed count, requested target, selected + action, starting iteration and attempt, and `resume_after` value. + +Calibration records the state needed to apply this contract in +`output/calibration/calibration_progress.json`. Recovery attempts are distinct from global iterations. The first execution of a global iteration is attempt 1. Restarting an interrupted global iteration creates diff --git a/activitysim/core/calibration/__init__.py b/activitysim/core/calibration/__init__.py index 442046abd7..d01414fdcf 100644 --- a/activitysim/core/calibration/__init__.py +++ b/activitysim/core/calibration/__init__.py @@ -17,7 +17,7 @@ reporting, settings, ) -from .orchestrator import run_calibration_loop +from .orchestrator import calibration_run_should_preserve_outputs, run_calibration_loop from .settings import ( CalibrationComponentResult, CalibrationComponentSettings, @@ -36,6 +36,7 @@ "CalibrationReportsSettings", "CalibrationRunResult", "CalibrationRunSettings", + "calibration_run_should_preserve_outputs", "calibration_enabled", "read_calibration_settings", "run_calibration_loop", diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index 1f470c5fcd..2009cc4787 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -185,10 +185,28 @@ def _calibrate_component( _append_summary_records(state, [summary_record]) if component_settings.reports.generic: - _write_generic_report(state, component_name, row_records) + try: + _write_generic_report(state, component_name, row_records) + except Exception as e: + logger.exception( + "calibration component %s iteration %s completed, but its " + "optional generic report could not be written: %s", + component_name, + component_iter, + e, + ) if bespoke_callable is not None: - bespoke_callable(eval_context) + try: + bespoke_callable(eval_context) + except Exception as e: + logger.exception( + "calibration component %s iteration %s completed, but its " + "optional bespoke report could not be written: %s", + component_name, + component_iter, + e, + ) if component_converged: break diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index c8281d27c2..481e37f2bb 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -3,6 +3,8 @@ from __future__ import annotations import logging +from dataclasses import dataclass +from typing import Literal from activitysim.core import workflow @@ -25,6 +27,7 @@ _write_final_coefficients_snapshot, ) from .settings import ( + CALIBRATION_SETTINGS_FILE_NAME, CalibrationRunResult, read_calibration_settings, ) @@ -32,6 +35,126 @@ logger = logging.getLogger("calibration") +@dataclass(frozen=True) +class CalibrationRestartPlan: + """Side-effect-free decision about how persisted calibration should continue.""" + + action: Literal["run", "finalize", "noop", "error"] + start_global_iteration: int | None + attempt: int | None + completed_global_iterations: int + completed_components: dict + message: str | None = None + + +def _plan_calibration_restart( + progress: dict | None, + global_iterations: int, +) -> CalibrationRestartPlan: + """Apply the user-facing global-iteration restart contract. + + The contract is intentionally based on completed logical iterations: + + 1. An unchanged target on a completed run is a no-op. + 2. A changed target above the completed count extends to that new total, + even when it is below the previous maximum after early convergence. + 3. A changed target already satisfied by the completed count is a no-op. + 4. ``resume_after`` is applied separately by the orchestrator, only to the + first global iteration entered by the current invocation. + 5. The orchestrator counts an iteration only when it has a durable + calibrated-component result from some attempt of that iteration. + 6. An active iteration cannot be discarded by lowering the target. + 7. A lower target at a clean boundary is finalized without another + coefficient update so downstream outputs reflect current coefficients. + """ + if not progress: + return CalibrationRestartPlan("run", 1, 1, 0, {}) + + completed = int(progress.get("last_completed_global_iteration", 0)) + completed_components = dict(progress.get("completed_components", {})) + + if progress.get("complete"): + previous_target = int(progress.get("configured_global_iterations", completed)) + if global_iterations == previous_target or global_iterations <= completed: + return CalibrationRestartPlan( + "noop", None, None, completed, completed_components + ) + return CalibrationRestartPlan("run", completed + 1, 1, completed, {}) + + interrupted = progress.get("in_progress_iteration") + if interrupted is not None: + interrupted = int(interrupted) + if global_iterations < interrupted: + return CalibrationRestartPlan( + "error", + None, + None, + completed, + completed_components, + "Cannot set calibration run.global_iterations to " + f"{global_iterations} because global iteration {interrupted} is " + "currently in progress and coefficient files may already contain " + f"iteration {interrupted} updates. Set global_iterations to at " + f"least {interrupted} to resume, or deliberately reset calibration " + "progress and restore the desired coefficient files before starting " + "a new run.", + ) + return CalibrationRestartPlan( + "run", + interrupted, + int(progress.get("attempt", 1)) + 1, + completed, + completed_components, + ) + + next_iteration = int(progress.get("next_global_iteration", completed + 1)) + if next_iteration > global_iterations: + return CalibrationRestartPlan("finalize", next_iteration, 1, completed, {}) + return CalibrationRestartPlan("run", next_iteration, 1, completed, {}) + + +def calibration_run_should_preserve_outputs(state: workflow.State) -> bool: + """Return whether preflight must preserve outputs before orchestration.""" + # Read the small set of preflight fields without schema validation. The + # normal settings checker runs after cleanup selection and must remain the + # place that aggregates calibration.yaml validation errors. + calibration_settings = state.filesystem.read_settings_file( + CALIBRATION_SETTINGS_FILE_NAME, + mandatory=False, + ) + if not calibration_settings or not calibration_settings.get("enable", False): + return False + + progress = _read_progress(state) + if not progress or not progress.get("complete"): + if not progress: + return False + configured_global_iterations = int( + calibration_settings.get("run", {}).get("global_iterations", 1) + ) + plan = _plan_calibration_restart(progress, configured_global_iterations) + return plan.action in {"noop", "error"} + + +def _validate_counted_iteration_has_calibration( + calibration_models: list[str], + skipped_components: list[str], + completed_components: dict, +) -> None: + """Do not count a new logical iteration that has no calibration result.""" + if set(skipped_components) != set(calibration_models): + return + if any(component in completed_components for component in calibration_models): + return + raise RuntimeError( + "settings.yaml resume_after skips every calibrated model in the first " + "pending global iteration, and that iteration has no durable calibrated " + "component result from an earlier attempt. Choose a resume_after before " + "the last calibrated model, or remove resume_after, so the requested " + "global calibration iteration performs a coefficient update." + ) + + def run_calibration_loop( state: workflow.State, models: list[str], @@ -88,6 +211,7 @@ def run_calibration_loop( models, calibration_settings.run.calibrate_models[0] ) + skipped_calibration_models = [] if resume_after is not None: skipped_calibration_models = [ component @@ -107,71 +231,82 @@ def run_calibration_loop( _ensure_calibration_output_dir(state) progress = _read_progress(state) - if progress and progress.get("complete"): - completed_global_iterations = int( - progress.get("last_completed_global_iteration", 0) + restart_plan = _plan_calibration_restart( + progress, + calibration_settings.run.global_iterations, + ) + logger.info( + "calibration restart plan: action=%s completed=%s requested=%s " + "start_iteration=%s attempt=%s resume_after=%r", + restart_plan.action, + restart_plan.completed_global_iterations, + calibration_settings.run.global_iterations, + restart_plan.start_global_iteration, + restart_plan.attempt, + resume_after, + ) + if restart_plan.action == "error": + raise RuntimeError(restart_plan.message) + if restart_plan.action == "noop": + logger.info( + "calibration progress is already complete for the requested target; " + "remove %s to start a fresh calibration run", + CALIBRATION_PROGRESS_FILE, ) - completed_for_global_iterations = int( - progress.get("configured_global_iterations", completed_global_iterations) + return CalibrationRunResult( + converged=bool(progress.get("converged", False)), + completed_global_iterations=restart_plan.completed_global_iterations, ) - if calibration_settings.run.global_iterations > completed_for_global_iterations: - logger.info( - "calibration global_iterations increased from %s to %s; " - "continuing with global iteration %s", - completed_for_global_iterations, - calibration_settings.run.global_iterations, - completed_global_iterations + 1, - ) - progress = { - "complete": False, - "in_progress_iteration": None, - "next_global_iteration": completed_global_iterations + 1, - "last_completed_global_iteration": completed_global_iterations, - "converged": bool(progress.get("converged", False)), - "configured_global_iterations": calibration_settings.run.global_iterations, - "attempt": 0, - "completed_components": {}, - } - _write_progress(state, progress) - else: - if resume_after is not None: - raise RuntimeError( - f"settings.yaml resume_after={resume_after!r} cannot be honored " - "because calibration progress is already complete. Remove " - f"{CALIBRATION_PROGRESS_FILE} or use a new output directory to " - "start a new calibration run; current coefficient values will " - "be preserved." - ) - logger.info( - "calibration progress is already complete; remove %s to start a " - "fresh calibration run", - CALIBRATION_PROGRESS_FILE, - ) - return CalibrationRunResult( - converged=bool(progress.get("converged", False)), - completed_global_iterations=completed_global_iterations, + + progress_was_complete = bool(progress and progress.get("complete")) + if progress_was_complete: + previous_target = int( + progress.get( + "configured_global_iterations", + restart_plan.completed_global_iterations, ) + ) + logger.info( + "calibration global_iterations changed from %s to %s; continuing " + "with global iteration %s", + previous_target, + calibration_settings.run.global_iterations, + restart_plan.start_global_iteration, + ) + progress = { + "complete": False, + "in_progress_iteration": None, + "next_global_iteration": restart_plan.start_global_iteration, + "last_completed_global_iteration": ( + restart_plan.completed_global_iterations + ), + "converged": bool(progress.get("converged", False)), + "configured_global_iterations": calibration_settings.run.global_iterations, + "attempt": 0, + "completed_components": {}, + } + _write_progress(state, progress) interrupted_iteration = progress.get("in_progress_iteration") if progress else None + start_global_iter = restart_plan.start_global_iteration + start_attempt = restart_plan.attempt + start_completed_components = restart_plan.completed_components + completed_global_iterations = restart_plan.completed_global_iterations + if interrupted_iteration is not None: - start_global_iter = int(interrupted_iteration) - start_attempt = int(progress.get("attempt", 1)) + 1 - start_completed_components = dict(progress.get("completed_components", {})) logger.warning( "continuing interrupted calibration global iteration %s as attempt %s " "using the current coefficient files", start_global_iter, start_attempt, ) - else: - # Progress files from earlier versions contain next_global_iteration, so - # they remain compatible with the corrected total-count semantics. - start_global_iter = ( - int(progress.get("next_global_iteration", 1)) if progress else 1 + + if restart_plan.action == "run": + _validate_counted_iteration_has_calibration( + calibration_settings.run.calibrate_models, + skipped_calibration_models, + start_completed_components, ) - start_attempt = 1 - start_completed_components = {} - completed_global_iterations = start_global_iter - 1 if interrupted_iteration is not None and resume_after is not None: rerun_completed_components = [ @@ -191,28 +326,6 @@ def run_calibration_loop( start_global_iter, ) - if start_global_iter > calibration_settings.run.global_iterations: - logger.info( - "calibration progress already reached configured global_iterations=%s", - calibration_settings.run.global_iterations, - ) - converged = bool(progress.get("converged", False)) if progress else False - _write_final_coefficients_snapshot(state, calibration_settings) - _write_completed_progress( - state, - completed_global_iterations, - converged, - calibration_settings.run.global_iterations, - attempt=int(progress.get("attempt", 1)) if progress else 1, - completed_components=( - dict(progress.get("completed_components", {})) if progress else {} - ), - ) - return CalibrationRunResult( - converged=converged, - completed_global_iterations=completed_global_iterations, - ) - if state.settings.resume_after is None: # compute_accessibility requires its accessibility table to be empty; # unlike most model steps, it will not overwrite a prior result. @@ -230,6 +343,42 @@ def run_calibration_loop( shared_data_buffers = _initialize_mp_shared_resources(state) try: + if restart_plan.action == "finalize": + # The target was lowered at a clean boundary between iterations. + # No active coefficient update is being discarded, but the normal + # model sequence still runs so final outputs use current coefficients. + logger.info( + "calibration global_iterations=%s is below next global iteration " + "%s; running the final model sequence without another " + "calibration update", + calibration_settings.run.global_iterations, + start_global_iter, + ) + final_models = ( + models if first_model_idx is None else models[first_model_idx:] + ) + _run_in_configured_mode( + state, + models=final_models, + resume_after=state.settings.resume_after, + shared_data_buffers=shared_data_buffers, + ) + completed_global_iterations = restart_plan.completed_global_iterations + converged = bool(progress.get("converged", False)) if progress else False + _write_final_coefficients_snapshot(state, calibration_settings) + _write_completed_progress( + state, + completed_global_iterations, + converged, + calibration_settings.run.global_iterations, + attempt=int(progress.get("attempt", 1)) if progress else 1, + completed_components={}, + ) + return CalibrationRunResult( + converged=converged, + completed_global_iterations=completed_global_iterations, + ) + # skip precursors if, on first iter, resume_after exists and is >= first_calib_model_idx if ( state.settings.resume_after is None @@ -385,8 +534,6 @@ def run_calibration_loop( attempt=attempt, shared_data_buffers=shared_data_buffers, ) - _write_component_plots(state, component) - all_converged = all_converged and component_result.converged completed_components[component] = { @@ -401,6 +548,15 @@ def run_calibration_loop( completed_components, ) + try: + _write_component_plots(state, component) + except Exception: + logger.exception( + "calibration component %s completed, but its optional " + "standard plots could not be written", + component, + ) + last_calibrated_component = component iteration_is_complete = ( diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index 83f8bdd200..464d553a7a 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -13,7 +13,7 @@ CALIBRATION_SETTINGS_FILE_NAME = "calibration.yaml" -class CalibrationRunSettings(PydanticBase): +class CalibrationRunSettings(PydanticBase, extra="forbid"): """Run-control settings for calibration.""" calibrate_models: list[str] @@ -32,14 +32,14 @@ def validate_run_settings(self): return self -class CalibrationReportsSettings(PydanticBase): +class CalibrationReportsSettings(PydanticBase, extra="forbid"): """Reporting settings for a calibrated component.""" generic: bool = True bespoke: str | None = None -class CalibrationComponentSettings(PydanticBase): +class CalibrationComponentSettings(PydanticBase, extra="forbid"): """Settings for one calibratable model component.""" calibration_spec: str @@ -49,7 +49,7 @@ class CalibrationComponentSettings(PydanticBase): reports: CalibrationReportsSettings = CalibrationReportsSettings() -class CalibrationConfig(PydanticReadable): +class CalibrationConfig(PydanticReadable, extra="forbid"): """Top-level calibration configuration.""" enable: bool = False diff --git a/activitysim/core/calibration/test/test_calibration.py b/activitysim/core/calibration/test/test_calibration.py new file mode 100644 index 0000000000..4352d007d1 --- /dev/null +++ b/activitysim/core/calibration/test/test_calibration.py @@ -0,0 +1,301 @@ +from __future__ import annotations + +import math +import copy + +import pandas as pd +import pytest +from pathlib import Path +from pydantic import ValidationError + +from activitysim.core.calibration.component import _evaluate_and_update +from activitysim.core.calibration.expressions import _compute_delta +from activitysim.core.calibration.orchestrator import ( + _components_ran_for_convergence, +) +from activitysim.core.calibration.reporting import ( + _append_iteration_records, + _coefficient_trajectory, + _read_component_iteration_records, +) +from activitysim.core.calibration.settings import CalibrationConfig + + +@pytest.mark.parametrize( + ("method", "model_value", "target_value", "damping", "expected"), + [ + # log_ratio: delta = log(target / model) * damping = log(2) * 0.5 + ("log_ratio", 0.25, 0.5, 0.5, math.log(2) * 0.5), + ( + # odds_ratio: delta = log((t*(1-m)) / (m*(1-t))) * damping + "odds_ratio", + 0.4, + 0.6, + 0.5, + math.log((0.6 * (1 - 0.4)) / (0.4 * (1 - 0.6))) * 0.5, + ), + ], +) +def test_compute_delta(method, model_value, target_value, damping, expected): + delta = _compute_delta( + method=method, + model_value=model_value, + target_value=target_value, + damping=damping, + component_name="test_component", + description="test target", + default_increment=2.0, + ) + + assert delta == pytest.approx(expected) + + +@pytest.mark.parametrize( + ("model_value", "target_value", "expected"), + [ + ( + 0.0, + 0.5, + 2.0, + ), # model is zero but target is not: fallback must nudge coefficient upward + ( + 1.0, + 0.5, + -2.0, + ), # model is one but target is not: fallback must nudge coefficient downward + (0.0, 0.0, 0.0), # both at boundary zero: no change needed + ], +) +def test_odds_ratio_boundary_fallback_has_correct_direction( + model_value, target_value, expected +): + delta = _compute_delta( + method="odds_ratio", + model_value=model_value, + target_value=target_value, + damping=1.0, + component_name="test_component", + description="test target", + default_increment=2.0, + ) + + assert delta == expected + + +def test_evaluate_and_update_only_changes_eligible_coefficients(): + calibration_spec = pd.DataFrame( + [ + { + "description": "unconverged", + "coefficient": "coef_update", + "model_value": 0.25, + "target_value": 0.5, + "hold_fast": False, + "min": -10.0, + "max": 10.0, + "damping": 0.5, + "method": "log_ratio", + "tolerance": 0.01, + }, + { + "description": "within tolerance", + "coefficient": "coef_converged", + "model_value": 0.49, + "target_value": 0.5, + "hold_fast": False, + "min": -10.0, + "max": 10.0, + "damping": 1.0, + "method": "log_ratio", + "tolerance": 0.02, + }, + { + "description": "held fixed", + "coefficient": "coef_held", + "model_value": 0.25, + "target_value": 0.5, + "hold_fast": True, + "min": -10.0, + "max": 10.0, + "damping": 1.0, + "method": "log_ratio", + "tolerance": 0.01, + }, + ] + ) + coefficients = pd.DataFrame( + {"value": [1.0, 2.0, 3.0]}, + index=["coef_update", "coef_converged", "coef_held"], + ) + + records, _, updated, component_converged = _evaluate_and_update( + component_name="test_component", + calibration_spec_df=calibration_spec, + coefficients_df=coefficients, + eval_context={}, + global_iter=1, + component_iter=1, + ) + + assert updated.loc["coef_update", "value"] == pytest.approx( + 1.0 + math.log(2) * 0.5 + ) # unconverged: full log-ratio delta applied + assert ( + updated.loc["coef_converged", "value"] == 2.0 + ) # within tolerance: value must not change + assert ( + updated.loc["coef_held", "value"] == 3.0 + ) # hold_fast=True: value must not change + assert records[1]["coef_delta"] == 0.0 # converged row records zero change + assert records[2]["coef_delta"] == 0.0 # held row records zero change + assert ( + component_converged is False + ) # one unconverged row means the whole component is not done + + +@pytest.mark.parametrize( + ( + "first_model_idx", + "last_calib_model_idx", + "global_iter", + "start_global_iter", + "expected", + ), + [ + ( + None, + 10, + 1, + 1, + True, + ), # no pre-calibration model ran: always counts toward convergence + ( + 5, + 10, + 1, + 1, + True, + ), # model ran before last calibration model in pipeline: counts + ( + 11, + 10, + 1, + 1, + False, + ), # model ran after last calibration model: skipped on first global iter + ( + 11, + 10, + 2, + 1, + True, + ), # same position but later global iter: skip only applies to first pass + ], +) +def test_components_ran_for_convergence( + first_model_idx, + last_calib_model_idx, + global_iter, + start_global_iter, + expected, +): + assert ( + _components_ran_for_convergence( + first_model_idx=first_model_idx, + last_calib_model_idx=last_calib_model_idx, + global_iter=global_iter, + start_global_iter=start_global_iter, + ) + is expected + ) + + +@pytest.mark.parametrize( + ("location", "unknown_setting"), + [ + ((), "cleanup_pipeline_after_run"), # unknown at top-level CalibrationConfig + (("run",), "restart_after"), # unknown inside CalibrationRunSettings + ( + ("model_settings", "test_component"), + "survey_file", + ), # unknown inside CalibrationComponentSettings + ( + ("model_settings", "test_component", "reports"), + "unexpected_report", + ), # unknown inside CalibrationReportSettings + ], +) +def test_calibration_settings_reject_unknown_fields(location, unknown_setting): + settings = { + "enable": True, + "run": { + "calibrate_models": ["test_component"], + "global_iterations": 1, + }, + "model_settings": { + "test_component": { + "calibration_spec": "test_calibration.csv", + "reports": {"generic": True}, + } + }, + } + invalid_settings = copy.deepcopy(settings) + container = invalid_settings + for key in location: + container = container[key] + container[unknown_setting] = True + + with pytest.raises(ValidationError) as error: + CalibrationConfig.model_validate(invalid_settings) + + assert error.value.errors()[0]["loc"] == (*location, unknown_setting) + + +class _State: + def __init__(self, output_dir: Path): + self.output_dir = output_dir + + def get_output_file_path(self, file_name: str) -> Path: + return self.output_dir / file_name + + +def _record(attempt: int, previous: float, next_value: float) -> dict: + return { + "global_iter": 1, + "attempt": attempt, + "component_iter": 1, + "description": "test target", + "component": "model_a", + "coefficient": "coef_a", + "target_value": 0.5, + "model_value": 0.25, + "difference": 0.25, + "pct_difference": 50.0, + "hold_fast": False, + "prev_coefficient": previous, + "coef_delta": next_value - previous, + "next_coefficient": next_value, + "converged": False, + "at_min": False, + "at_max": False, + } + + +def test_recovery_attempts_preserve_complete_coefficient_trajectory(tmp_path): + state = _State(tmp_path) + # attempt 1: coefficient moves 1.0 → 1.5 + _append_iteration_records(state, "model_a", [_record(1, 1.0, 1.5)]) + # attempt 2: picks up exactly where attempt 1 ended, 1.5 → 1.75 + _append_iteration_records(state, "model_a", [_record(2, 1.5, 1.75)]) + + stored = pd.read_csv(tmp_path / "calibration" / "calibration_iteration_records.csv") + assert list(stored["attempt"]) == [1, 2] + # the chain must be unbroken: attempt 2 prev_coefficient must equal attempt 1 next_coefficient + assert stored.loc[1, "prev_coefficient"] == stored.loc[0, "next_coefficient"] + + records = _read_component_iteration_records(state, "model_a") + trajectory, labels = _coefficient_trajectory(records, ["coef_a"]) + + # labels: one "Start" entry (initial value) plus one label per recorded iteration + assert labels == ["Start", "G1-A1-C1", "G1-A2-C1"] + assert list(trajectory["coef_a"]) == [1.0, 1.5, 1.75] diff --git a/activitysim/core/calibration/test/test_calibration_restart.py b/activitysim/core/calibration/test/test_calibration_restart.py new file mode 100644 index 0000000000..1e907f67a7 --- /dev/null +++ b/activitysim/core/calibration/test/test_calibration_restart.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import pytest + +from activitysim.core.calibration.orchestrator import ( + _plan_calibration_restart, + _skipped_calibration_components, + _validate_counted_iteration_has_calibration, +) + + +@pytest.mark.parametrize( + ( + "progress", + "global_iterations", + "expected_action", + "expected_start", + "expected_attempt", + ), + [ + # no prior progress file: start fresh from iteration 1, attempt 1 + (None, 2, "run", 1, 1), + ( + # interrupted mid-run at iteration 3; resume with an incremented attempt number + { + "in_progress_iteration": 3, + "last_completed_global_iteration": 2, + "attempt": 1, + "completed_components": {}, + }, + 3, + "run", + 3, + 2, + ), + ( + # calibration iterations done but downstream finalization models still pending + { + "last_completed_global_iteration": 2, + "next_global_iteration": 3, + }, + 2, + "finalize", + 3, + 1, + ), + ( + # fully complete and global_iterations unchanged: nothing left to do + { + "complete": True, + "last_completed_global_iteration": 2, + "configured_global_iterations": 2, + }, + 2, + "noop", + None, + None, + ), + ( + # fully complete but global_iterations increased: extend the calibration run + { + "complete": True, + "last_completed_global_iteration": 2, + "configured_global_iterations": 2, + }, + 4, + "run", + 3, + 1, + ), + # Convergence completed only two iterations against a target of five. + # Leaving the target unchanged is a no-op, while changing it to a value + # above the completed count explicitly requests more iterations. + ( + # converged early at iter 2; configured target of 5 not changed → no-op + { + "complete": True, + "last_completed_global_iteration": 2, + "configured_global_iterations": 5, + }, + 5, + "noop", + None, + None, + ), + ( + # converged early at iter 2; target lowered to 3 → run one more iteration + { + "complete": True, + "last_completed_global_iteration": 2, + "configured_global_iterations": 5, + }, + 3, + "run", + 3, + 1, + ), + ( + # converged early at iter 2; target lowered to 4 → run two more iterations starting at 3 + { + "complete": True, + "last_completed_global_iteration": 2, + "configured_global_iterations": 5, + }, + 4, + "run", + 3, + 1, + ), + ( + # converged early at iter 2; target lowered to 2 (≤ completed count) → no-op + { + "complete": True, + "last_completed_global_iteration": 2, + "configured_global_iterations": 5, + }, + 2, + "noop", + None, + None, + ), + ], +) +def test_plan_calibration_restart( + progress, + global_iterations, + expected_action, + expected_start, + expected_attempt, +): + plan = _plan_calibration_restart(progress, global_iterations) + + assert plan.action == expected_action + assert plan.start_global_iteration == expected_start + assert plan.attempt == expected_attempt + + +def test_cannot_lower_limit_below_interrupted_iteration(): + # iteration 3 is in progress; lowering global_iterations to 2 would abandon it mid-run + plan = _plan_calibration_restart( + { + "in_progress_iteration": 3, + "last_completed_global_iteration": 2, + "attempt": 1, + "completed_components": {"model_a": {"attempt": 1, "converged": True}}, + }, + global_iterations=2, + ) + + assert plan.action == "error" + # error message must identify the blocked iteration and the minimum required setting + assert "global iteration 3 is currently in progress" in plan.message + assert "global_iterations to at least 3" in plan.message + + +def test_resume_after_only_skips_components_in_first_entered_iteration(): + # model_a ran and completed in a prior attempt during iteration 3; model_b had not yet started + # first_model_idx is the pipeline position immediately after model_a + models = ["initialize", "model_a", "intermediate", "model_b", "output"] + calibration_models = ["model_a", "model_b"] + first_model_idx = models.index("model_a") + 1 + + # re-entering the interrupted iteration: model_a already ran, so it must be skipped + assert _skipped_calibration_components( + calibration_models, + models, + first_model_idx, + global_iter=3, + start_global_iter=3, + ) == ["model_a"] + # on a later full iteration nothing was partially skipped before, run everything normally + assert ( + _skipped_calibration_components( + calibration_models, + models, + first_model_idx, + global_iter=4, + start_global_iter=3, + ) + == [] + ) + + +def test_new_iteration_cannot_skip_every_calibrated_component(): + # with no prior-attempt results in completed_components, skipping all calibration + # models would produce an iteration with zero calibration updates — that is a bug + with pytest.raises(RuntimeError, match="skips every calibrated model"): + _validate_counted_iteration_has_calibration( + calibration_models=["model_a", "model_b"], + skipped_components=["model_a", "model_b"], + completed_components={}, + ) + + +def test_prior_attempt_result_allows_downstream_only_recovery(): + # model_a has a completed result from a prior attempt, so skipping both calibration + # models is valid: downstream models will still pick up model_a's updated coefficients + _validate_counted_iteration_has_calibration( + calibration_models=["model_a", "model_b"], + skipped_components=["model_a", "model_b"], + completed_components={"model_a": {"attempt": 1, "converged": True}}, + ) diff --git a/activitysim/core/test/test_calibration.py b/activitysim/core/test/test_calibration.py deleted file mode 100644 index e8b50bc37a..0000000000 --- a/activitysim/core/test/test_calibration.py +++ /dev/null @@ -1,155 +0,0 @@ -from __future__ import annotations - -import math - -import pandas as pd -import pytest - -from activitysim.core.calibration.component import _evaluate_and_update -from activitysim.core.calibration.expressions import _compute_delta -from activitysim.core.calibration.orchestrator import ( - _components_ran_for_convergence, -) - - -@pytest.mark.parametrize( - ("method", "model_value", "target_value", "damping", "expected"), - [ - ("log_ratio", 0.25, 0.5, 0.5, math.log(2) * 0.5), - ( - "odds_ratio", - 0.4, - 0.6, - 0.5, - math.log((0.6 * (1 - 0.4)) / (0.4 * (1 - 0.6))) * 0.5, - ), - ], -) -def test_compute_delta(method, model_value, target_value, damping, expected): - delta = _compute_delta( - method=method, - model_value=model_value, - target_value=target_value, - damping=damping, - component_name="test_component", - description="test target", - default_increment=2.0, - ) - - assert delta == pytest.approx(expected) - - -@pytest.mark.parametrize( - ("model_value", "target_value", "expected"), - [(0.0, 0.5, 2.0), (1.0, 0.5, -2.0), (0.0, 0.0, 0.0)], -) -def test_odds_ratio_boundary_fallback_has_correct_direction( - model_value, target_value, expected -): - delta = _compute_delta( - method="odds_ratio", - model_value=model_value, - target_value=target_value, - damping=1.0, - component_name="test_component", - description="test target", - default_increment=2.0, - ) - - assert delta == expected - - -def test_evaluate_and_update_only_changes_eligible_coefficients(): - calibration_spec = pd.DataFrame( - [ - { - "description": "unconverged", - "coefficient": "coef_update", - "model_value": 0.25, - "target_value": 0.5, - "hold_fast": False, - "min": -10.0, - "max": 10.0, - "damping": 0.5, - "method": "log_ratio", - "tolerance": 0.01, - }, - { - "description": "within tolerance", - "coefficient": "coef_converged", - "model_value": 0.49, - "target_value": 0.5, - "hold_fast": False, - "min": -10.0, - "max": 10.0, - "damping": 1.0, - "method": "log_ratio", - "tolerance": 0.02, - }, - { - "description": "held fixed", - "coefficient": "coef_held", - "model_value": 0.25, - "target_value": 0.5, - "hold_fast": True, - "min": -10.0, - "max": 10.0, - "damping": 1.0, - "method": "log_ratio", - "tolerance": 0.01, - }, - ] - ) - coefficients = pd.DataFrame( - {"value": [1.0, 2.0, 3.0]}, - index=["coef_update", "coef_converged", "coef_held"], - ) - - records, _, updated, component_converged = _evaluate_and_update( - component_name="test_component", - calibration_spec_df=calibration_spec, - coefficients_df=coefficients, - eval_context={}, - global_iter=1, - component_iter=1, - ) - - assert updated.loc["coef_update", "value"] == pytest.approx(1.0 + math.log(2) * 0.5) - assert updated.loc["coef_converged", "value"] == 2.0 - assert updated.loc["coef_held", "value"] == 3.0 - assert records[1]["coef_delta"] == 0.0 - assert records[2]["coef_delta"] == 0.0 - assert component_converged is False - - -@pytest.mark.parametrize( - ( - "first_model_idx", - "last_calib_model_idx", - "global_iter", - "start_global_iter", - "expected", - ), - [ - (None, 10, 1, 1, True), - (5, 10, 1, 1, True), - (11, 10, 1, 1, False), - (11, 10, 2, 1, True), - ], -) -def test_components_ran_for_convergence( - first_model_idx, - last_calib_model_idx, - global_iter, - start_global_iter, - expected, -): - assert ( - _components_ran_for_convergence( - first_model_idx=first_model_idx, - last_calib_model_idx=last_calib_model_idx, - global_iter=global_iter, - start_global_iter=start_global_iter, - ) - is expected - ) diff --git a/activitysim/core/test/test_calibration_reporting.py b/activitysim/core/test/test_calibration_reporting.py deleted file mode 100644 index d0d459cec2..0000000000 --- a/activitysim/core/test/test_calibration_reporting.py +++ /dev/null @@ -1,57 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pandas as pd - -from activitysim.core.calibration.reporting import ( - _append_iteration_records, - _coefficient_trajectory, - _read_component_iteration_records, -) - - -class _State: - def __init__(self, output_dir: Path): - self.output_dir = output_dir - - def get_output_file_path(self, file_name: str) -> Path: - return self.output_dir / file_name - - -def _record(attempt: int, previous: float, next_value: float) -> dict: - return { - "global_iter": 1, - "attempt": attempt, - "component_iter": 1, - "description": "test target", - "component": "model_a", - "coefficient": "coef_a", - "target_value": 0.5, - "model_value": 0.25, - "difference": 0.25, - "pct_difference": 50.0, - "hold_fast": False, - "prev_coefficient": previous, - "coef_delta": next_value - previous, - "next_coefficient": next_value, - "converged": False, - "at_min": False, - "at_max": False, - } - - -def test_recovery_attempts_preserve_complete_coefficient_trajectory(tmp_path): - state = _State(tmp_path) - _append_iteration_records(state, "model_a", [_record(1, 1.0, 1.5)]) - _append_iteration_records(state, "model_a", [_record(2, 1.5, 1.75)]) - - stored = pd.read_csv(tmp_path / "calibration" / "calibration_iteration_records.csv") - assert list(stored["attempt"]) == [1, 2] - assert stored.loc[1, "prev_coefficient"] == stored.loc[0, "next_coefficient"] - - records = _read_component_iteration_records(state, "model_a") - trajectory, labels = _coefficient_trajectory(records, ["coef_a"]) - - assert labels == ["Start", "G1-A1-C1", "G1-A2-C1"] - assert list(trajectory["coef_a"]) == [1.0, 1.5, 1.75] diff --git a/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py b/activitysim/examples/prototype_mtc/test/calibration/test_calibration_run_modes.py similarity index 100% rename from activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py rename to activitysim/examples/prototype_mtc/test/calibration/test_calibration_run_modes.py diff --git a/docs/users-guide/calibration.rst b/docs/users-guide/calibration.rst index ac79a1af2f..ede52d8cea 100644 --- a/docs/users-guide/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -58,7 +58,6 @@ calibration-specific overlay directory): - workplace_location - auto_ownership_simulate - tour_mode_choice_simulate - restart_after: [] # components after which to restart (advanced) global_iterations: 3 # number of full calibration passes complete_steps: false # run model steps after the last calibrated component @@ -135,8 +134,13 @@ Configuration Reference - Every component listed in ``run.calibrate_models`` must have a corresponding entry in ``model_settings``. -- Every component in ``run.restart_after`` must also appear in ``run.calibrate_models``. - ``run.global_iterations`` must be ≥ 1. +- Unknown fields are rejected at every level of ``calibration.yaml``, following + the same strict settings-model convention used by checked ActivitySim model + settings. When the settings checker is enabled, calibration validation errors + are included in its normal aggregated error report. General run settings such + as ``cleanup_pipeline_after_run`` belong in the top-level ``settings.yaml`` + file. ``run`` — Run Control Settings ------------------------------- @@ -154,10 +158,6 @@ Configuration Reference - *required* - Model component names to calibrate. Must match names in ``settings.yaml`` ``models`` list. - * - ``restart_after`` - - ``list[str]`` - - ``[]`` - - Components after which to restart. * - ``global_iterations`` - ``int`` - ``1`` @@ -581,23 +581,47 @@ each component iteration. This means: Crash Recovery ============== -Before each global iteration, calibration replaces the files in its recovery -directory with a copy of every coefficient file that it may modify, then records -the active iteration in ``calibration_progress.json``. If a run is interrupted: - -1. Restarting ``activitysim run`` restores all coefficient files from the - start-of-iteration recovery snapshot. -2. The interrupted global iteration is replayed from that consistent boundary. -3. Remaining iterations run only until the configured total - ``global_iterations`` is reached. - -The progress file is written using atomic replacement. Once progress is marked -complete, rerunning with the same output directory does not apply additional -calibration iterations. +Calibration follows this run-control contract: + +1. ``global_iterations`` is the desired total number of calibration iterations. +2. If a completed run's setting is changed to a value greater than the number + actually completed, calibration continues until the new total. This applies + even when the new value is below the previous maximum after early convergence. + For example, changing 5 to 3 after convergence at iteration 2 requests one + additional iteration. +3. A changed setting less than or equal to the number already completed will not + run and an error message is thrown; completed coefficient updates are not + implicitly undone. +4. Top-level ``settings.yaml`` ``resume_after`` follows normal ActivitySim + semantics for the first global iteration entered by the current invocation. + Later global iterations ignore it and run every calibrated component. +5. A logical global iteration counts only if it has at least one durable + calibrated-component result from the current attempt or an earlier attempt + of that iteration. A new iteration cannot be consumed by a ``resume_after`` + value that skips every calibrated model. +6. ``global_iterations`` cannot be lowered below an interrupted iteration. + ActivitySim raises an error explaining that coefficient files may already + contain updates from the active iteration and instructs the user either to + resume it or deliberately reset progress and coefficient files. +7. At startup, ActivitySim logs the detected completed count, requested target, + selected action, starting iteration and attempt, and ``resume_after`` value. +8. Coefficient updates are stored in the configs directory and are **not** + rolled back after an interruption. This allows a new run to resume from the + last known coefficients and allows the user to update coefficients manually. +9. Coefficient updates are uniquely numbered by global iteration, calibration + iteration, and attempt. Attempt number is typically only 1 unless the model + crashes after coefficient updates are made but before component fully + completes (e.g. plotting functions cause crash). + +Calibration records the active global iteration and recovery attempt in +``calibration_progress.json``. Coefficient files are the authoritative current +state and are not rolled back after an interruption. Restarting re-enters the +interrupted global iteration using those current coefficients; the top-level +``settings.yaml`` ``resume_after`` value determines where model execution +resumes. To force a fresh start, delete ``output/calibration/calibration_progress.json`` -and restore original coefficient files. Recovery snapshots can also be removed -after a completed run if they are no longer needed. +and restore the desired starting coefficient files. Multiprocess Mode From f414f230632e176052fd6dda13c0df0d1e89c343 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:34:16 -0700 Subject: [PATCH 88/90] setting checking cleanup --- activitysim/core/calibration/component.py | 8 +-- activitysim/core/calibration/orchestrator.py | 59 +++++++++++++------ activitysim/core/calibration/settings.py | 21 +++++-- .../core/calibration/test/test_calibration.py | 46 +++++++++++++++ .../test/test_calibration_restart.py | 42 +++++++++++++ docs/users-guide/calibration.rst | 12 ++-- 6 files changed, 155 insertions(+), 33 deletions(-) diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index 2009cc4787..c82577c351 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -190,11 +190,11 @@ def _calibrate_component( except Exception as e: logger.exception( "calibration component %s iteration %s completed, but its " - "optional generic report could not be written: %s", + "generic report could not be written.", component_name, component_iter, - e, ) + raise RuntimeError(e) if bespoke_callable is not None: try: @@ -202,11 +202,11 @@ def _calibrate_component( except Exception as e: logger.exception( "calibration component %s iteration %s completed, but its " - "optional bespoke report could not be written: %s", + "bespoke report could not be written.", component_name, component_iter, - e, ) + raise RuntimeError(e) if component_converged: break diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index 481e37f2bb..a7a8196a34 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -115,23 +115,43 @@ def _plan_calibration_restart( def calibration_run_should_preserve_outputs(state: workflow.State) -> bool: """Return whether preflight must preserve outputs before orchestration.""" - # Read the small set of preflight fields without schema validation. The - # normal settings checker runs after cleanup selection and must remain the - # place that aggregates calibration.yaml validation errors. - calibration_settings = state.filesystem.read_settings_file( - CALIBRATION_SETTINGS_FILE_NAME, - mandatory=False, - ) - if not calibration_settings or not calibration_settings.get("enable", False): + # Keep missing and disabled calibration files on the ordinary ActivitySim + # cleanup path. Only enabled calibration runs get calibration-specific + # protection during preflight. + try: + raw_settings = state.filesystem.read_settings_file( + CALIBRATION_SETTINGS_FILE_NAME, + mandatory=False, + ) + except Exception: + logger.debug( + "preserving outputs because calibration settings could not be read", + exc_info=True, + ) + return True + if not raw_settings or not raw_settings.get("enable", False): + return False + + # Calibration validation normally runs after output cleanup. Preserve the + # outputs if enabled settings are invalid, then let the normal settings + # checker aggregate and report the validation error later. + try: + calibration_settings = read_calibration_settings(state) + except Exception: + logger.debug( + "preserving outputs because calibration settings validation failed", + exc_info=True, + ) + return True + + if not calibration_settings or not calibration_settings.enable: return False progress = _read_progress(state) if not progress or not progress.get("complete"): if not progress: return False - configured_global_iterations = int( - calibration_settings.get("run", {}).get("global_iterations", 1) - ) + configured_global_iterations = calibration_settings.run.global_iterations plan = _plan_calibration_restart(progress, configured_global_iterations) return plan.action in {"noop", "error"} @@ -258,6 +278,16 @@ def run_calibration_loop( completed_global_iterations=restart_plan.completed_global_iterations, ) + # Validate the requested restart before changing a formerly complete + # progress record. A rejected resume_after must leave durable progress in + # exactly the state in which it was found. + if restart_plan.action == "run": + _validate_counted_iteration_has_calibration( + calibration_settings.run.calibrate_models, + skipped_calibration_models, + restart_plan.completed_components, + ) + progress_was_complete = bool(progress and progress.get("complete")) if progress_was_complete: previous_target = int( @@ -301,13 +331,6 @@ def run_calibration_loop( start_attempt, ) - if restart_plan.action == "run": - _validate_counted_iteration_has_calibration( - calibration_settings.run.calibrate_models, - skipped_calibration_models, - start_completed_components, - ) - if interrupted_iteration is not None and resume_after is not None: rerun_completed_components = [ component diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index 464d553a7a..d35c4cd50e 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -4,7 +4,7 @@ from dataclasses import dataclass -from pydantic import model_validator +from pydantic import Field, model_validator from activitysim.core import workflow from activitysim.core.configuration import PydanticReadable @@ -17,7 +17,7 @@ class CalibrationRunSettings(PydanticBase, extra="forbid"): """Run-control settings for calibration.""" calibrate_models: list[str] - global_iterations: int = 1 + global_iterations: int = Field(default=1, ge=1) complete_steps: bool = False # Deprecated compatibility setting retained so existing configurations # continue to parse. Exact calibration restores supersede this setting. @@ -29,6 +29,18 @@ def validate_run_settings(self): raise ValueError( "calibration.run.calibrate_models must contain at least one model name" ) + duplicate_models = sorted( + { + model + for model in self.calibrate_models + if self.calibrate_models.count(model) > 1 + } + ) + if duplicate_models: + raise ValueError( + "calibration.run.calibrate_models contains duplicate model " + f"name(s): {duplicate_models}" + ) return self @@ -45,7 +57,7 @@ class CalibrationComponentSettings(PydanticBase, extra="forbid"): calibration_spec: str model_settings_file: str | None = None helper_module: str | None = None - submodel_max_iterations: int = 1 + submodel_max_iterations: int = Field(default=1, ge=1) reports: CalibrationReportsSettings = CalibrationReportsSettings() @@ -65,9 +77,6 @@ def validate_model_settings(self): f"calibration model '{component}' is not in model_settings" ) - if self.run.global_iterations < 1: - raise ValueError("max_iterations must be >= 1") - return self diff --git a/activitysim/core/calibration/test/test_calibration.py b/activitysim/core/calibration/test/test_calibration.py index 4352d007d1..6cd47b9885 100644 --- a/activitysim/core/calibration/test/test_calibration.py +++ b/activitysim/core/calibration/test/test_calibration.py @@ -251,6 +251,52 @@ def test_calibration_settings_reject_unknown_fields(location, unknown_setting): assert error.value.errors()[0]["loc"] == (*location, unknown_setting) +@pytest.mark.parametrize( + ("field", "value"), + [ + ("global_iterations", 0), + ("submodel_max_iterations", 0), + ], +) +def test_calibration_iteration_limits_must_be_positive(field, value): + settings = { + "enable": True, + "run": { + "calibrate_models": ["test_component"], + "global_iterations": value if field == "global_iterations" else 1, + }, + "model_settings": { + "test_component": { + "calibration_spec": "test_calibration.csv", + "submodel_max_iterations": ( + value if field == "submodel_max_iterations" else 1 + ), + } + }, + } + + with pytest.raises(ValidationError) as error: + CalibrationConfig.model_validate(settings) + + assert error.value.errors()[0]["type"] == "greater_than_equal" + + +def test_calibration_models_must_be_unique(): + settings = { + "enable": True, + "run": { + "calibrate_models": ["test_component", "test_component"], + "global_iterations": 1, + }, + "model_settings": { + "test_component": {"calibration_spec": "test_calibration.csv"} + }, + } + + with pytest.raises(ValidationError, match="duplicate model name"): + CalibrationConfig.model_validate(settings) + + class _State: def __init__(self, output_dir: Path): self.output_dir = output_dir diff --git a/activitysim/core/calibration/test/test_calibration_restart.py b/activitysim/core/calibration/test/test_calibration_restart.py index 1e907f67a7..b1809115e5 100644 --- a/activitysim/core/calibration/test/test_calibration_restart.py +++ b/activitysim/core/calibration/test/test_calibration_restart.py @@ -3,12 +3,54 @@ import pytest from activitysim.core.calibration.orchestrator import ( + calibration_run_should_preserve_outputs, _plan_calibration_restart, _skipped_calibration_components, _validate_counted_iteration_has_calibration, ) +class _PreflightFilesystem: + def __init__(self, calibration_settings): + self.calibration_settings = calibration_settings + + def read_settings_file(self, _file_name, mandatory=False): + assert mandatory is False + return self.calibration_settings + + +class _PreflightState: + def __init__(self, calibration_settings): + self.filesystem = _PreflightFilesystem(calibration_settings) + + +def test_preflight_preserves_outputs_when_calibration_settings_are_invalid(monkeypatch): + def invalid_settings(_state): + raise ValueError("invalid calibration settings") + + monkeypatch.setattr( + "activitysim.core.calibration.orchestrator.read_calibration_settings", + invalid_settings, + ) + + state = _PreflightState({"enable": True}) + + assert calibration_run_should_preserve_outputs(state) is True + + +@pytest.mark.parametrize("calibration_settings", [None, {"enable": False}]) +def test_preflight_leaves_non_calibration_cleanup_unchanged( + monkeypatch, calibration_settings +): + monkeypatch.setattr( + "activitysim.core.calibration.orchestrator.read_calibration_settings", + lambda _state: pytest.fail("non-calibration settings must not be validated"), + ) + state = _PreflightState(calibration_settings) + + assert calibration_run_should_preserve_outputs(state) is False + + @pytest.mark.parametrize( ( "progress", diff --git a/docs/users-guide/calibration.rst b/docs/users-guide/calibration.rst index ede52d8cea..d8ec7bce43 100644 --- a/docs/users-guide/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -134,7 +134,9 @@ Configuration Reference - Every component listed in ``run.calibrate_models`` must have a corresponding entry in ``model_settings``. +- Component names in ``run.calibrate_models`` must be unique. - ``run.global_iterations`` must be ≥ 1. +- Each component's ``submodel_max_iterations`` must be ≥ 1. - Unknown fields are rejected at every level of ``calibration.yaml``, following the same strict settings-model convention used by checked ActivitySim model settings. When the settings checker is enabled, calibration validation errors @@ -653,15 +655,15 @@ A coefficient is considered **converged** when: .. math:: - \text{target_value} - \text{model_value} \leq \text{tolerance} + \left|\text{target_value} - \text{model_value}\right| \leq \text{tolerance} A component is converged when **all** of its coefficients are converged. The component inner loop stops early upon convergence. -The overall calibration run completes after all ``global_iterations`` have -executed. Global convergence is tracked but does not currently trigger early -termination of the outer loop — use ``global_iterations`` to control the total -number of passes. +The outer loop also stops early when every calibrated component converges in +the same global iteration. Otherwise, it runs until ``global_iterations`` have +completed. In either case, the remaining downstream models run once with the +final coefficient values before the calibration run is marked complete. Coefficient Requirements From 6c59ee2b2bab16ef58b6be1ef4bf1640266ff788 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:52:37 -0700 Subject: [PATCH 89/90] addressing RNG and resume review comments --- activitysim/core/calibration/component.py | 5 +- activitysim/core/calibration/multiprocess.py | 67 ++++++++--- activitysim/core/calibration/orchestrator.py | 10 +- .../core/calibration/test/test_calibration.py | 109 +++++++++++++++++- activitysim/core/random.py | 2 +- activitysim/core/test/extensions/steps.py | 8 ++ activitysim/core/test/test_pipeline.py | 26 +++++ activitysim/core/workflow/runner.py | 33 ++++-- 8 files changed, 226 insertions(+), 34 deletions(-) diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index c82577c351..9d32d61b1c 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -74,7 +74,10 @@ def _run_component_model( for model_name in extra_models: state.run.by_name(model_name) state.checkpoint.add(prior_step) - state.run.by_name(run_model_name) + # The labeled invocation keeps calibration checkpoints unique, while the + # canonical component RNG name supplies common random numbers across + # component iterations, global iterations, and recovery attempts. + state.run.by_name_with_rng(run_model_name, rng_step_name=component_name) def _calibrate_component( diff --git a/activitysim/core/calibration/multiprocess.py b/activitysim/core/calibration/multiprocess.py index 6815ae604b..c188d1bb13 100644 --- a/activitysim/core/calibration/multiprocess.py +++ b/activitysim/core/calibration/multiprocess.py @@ -188,8 +188,8 @@ def _restore_from_subprocess_pipelines( from activitysim.core.workflow.checkpoint import ( CHECKPOINT_NAME, CHECKPOINT_TABLE_NAME, - HdfStore, NON_TABLE_COLUMNS, + HdfStore, ParquetStore, ) @@ -313,35 +313,64 @@ def _subprocess_path(proc_name): for table_name, dfs in omnibus.items(): tables[table_name] = pd.concat(dfs, sort=False) - # Load into parent state - prior_rng_channels = list(state.get_injectable("rng_channels", [])) - prior_index_to_channel = ( - dict(state.rng().index_to_channel) - if hasattr(state.rng(), "index_to_channel") - else {} + _install_restored_subprocess_state(state, tables, resume_after) + + logger.info( + "calibration: restored %d tables from subprocess pipelines at " + "checkpoint '%s'", + len(tables), + resume_after, ) + return True - state.init_state() + +def _install_restored_subprocess_state( + state: workflow.State, + tables: dict[str, pd.DataFrame], + checkpoint_name: str, +) -> None: + """Install an exact subprocess snapshot and make it a parent checkpoint.""" + restored_table_names = set(tables) + + # init_state() resets salient-table and RNG bookkeeping but deliberately + # retains cached context values. Remove later tables before that reset so + # they cannot be returned from the context after this rewind. + stale_table_names = set(state.registered_tables()) - restored_table_names + prior_rng_channels = [ + channel_name + for channel_name in state.get_injectable("rng_channels", []) + if channel_name in restored_table_names + ] + prior_index_to_channel = { + index_name: channel_name + for index_name, channel_name in getattr( + state.rng(), "index_to_channel", {} + ).items() + if channel_name in restored_table_names + } + + for table_name in stale_table_names: + if table_name in state: + state.drop(table_name) + if table_name in state.rng().channels: + state.rng().drop_channel(table_name) + + # Close the current handle before init_state() forgets it. if state.checkpoint.store_is_open(): state.checkpoint.close_store() + state.init_state() state.checkpoint.open_store(overwrite=False) for table_name, df in tables.items(): state.add_table(table_name, df) + state.add_injectable("rng_channels", prior_rng_channels) _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) - # Mark all tables dirty for subsequent checkpoint.add - for table_name in list(state.existing_table_names): - state.existing_table_status[table_name] = True - - logger.info( - "calibration: restored %d tables from subprocess pipelines at " - "checkpoint '%s'", - len(tables), - resume_after, - ) - return True + # Persist the reconstructed model-level state in the parent pipeline. This + # both makes resume_after observable to the caller and avoids repeating the + # subprocess coalesce on later restores. + state.checkpoint.add(checkpoint_name) def _run_multiprocess_with_overrides( diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index a7a8196a34..5c381912d1 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -455,7 +455,15 @@ def run_calibration_loop( shared_data_buffers=shared_data_buffers, ) else: - state.checkpoint.add(state.settings.resume_after) + # Direct restores already end at this checkpoint, while a + # subprocess restore now persists it in the parent pipeline. + # Only add it here for an alternate restore implementation + # that made the checkpoint visible but did not select it. + if ( + state.checkpoint.last_checkpoint.get("checkpoint_name") + != state.settings.resume_after + ): + state.checkpoint.add(state.settings.resume_after) state.checkpoint.close_store() for global_iter in range( diff --git a/activitysim/core/calibration/test/test_calibration.py b/activitysim/core/calibration/test/test_calibration.py index 6cd47b9885..a3abb7d096 100644 --- a/activitysim/core/calibration/test/test_calibration.py +++ b/activitysim/core/calibration/test/test_calibration.py @@ -1,15 +1,18 @@ from __future__ import annotations -import math import copy +import math +from pathlib import Path import pandas as pd import pytest -from pathlib import Path from pydantic import ValidationError from activitysim.core.calibration.component import _evaluate_and_update from activitysim.core.calibration.expressions import _compute_delta +from activitysim.core.calibration.multiprocess import ( + _install_restored_subprocess_state, +) from activitysim.core.calibration.orchestrator import ( _components_ran_for_convergence, ) @@ -19,6 +22,7 @@ _read_component_iteration_records, ) from activitysim.core.calibration.settings import CalibrationConfig +from activitysim.core.random import Random @pytest.mark.parametrize( @@ -297,6 +301,107 @@ def test_calibration_models_must_be_unique(): CalibrationConfig.model_validate(settings) +class _RestoreCheckpoint: + def __init__(self, state): + self.state = state + self.is_open = True + self.added = [] + self.last_checkpoint = {} + + def initialize(self): + self.is_open = False + self.last_checkpoint = {} + + def store_is_open(self): + return self.is_open + + def close_store(self): + self.is_open = False + + def open_store(self, overwrite=False): + assert overwrite is False + self.is_open = True + + def add(self, checkpoint_name): + self.added.append(checkpoint_name) + self.last_checkpoint = {"checkpoint_name": checkpoint_name} + for table_name in self.state.existing_table_status: + self.state.existing_table_status[table_name] = False + + +class _RestoreState: + def __init__(self): + households = pd.DataFrame(index=pd.Index([1], name="household_id")) + tours = pd.DataFrame(index=pd.Index([10], name="tour_id")) + self.context = { + "households": households, + "tours": tours, + "rng_channels": ["households", "tours"], + } + self.existing_table_status = {"households": False, "tours": False} + self._rng = Random() + self._rng.add_channel("households", households) + self._rng.add_channel("tours", tours) + self.checkpoint = _RestoreCheckpoint(self) + + def __contains__(self, key): + return key in self.context + + def registered_tables(self): + return [name for name in self.existing_table_status if name in self.context] + + def get_injectable(self, name, default=None): + return self.context.get(name, default) + + def add_injectable(self, name, value): + self.context[name] = value + + def rng(self): + return self._rng + + def drop(self, name): + del self.context[name] + + def init_state(self): + # Match workflow.State: reset bookkeeping and RNG, retain context. + self.checkpoint.initialize() + self._rng = Random() + self.existing_table_status = {} + + def add_table(self, name, table): + self.context[name] = table + self.existing_table_status[name] = True + + def is_table(self, name): + return name in self.existing_table_status + + def get_dataframe(self, name): + return self.context[name].copy() + + +def test_subprocess_restore_is_exact_and_durable(): + state = _RestoreState() + restored_households = pd.DataFrame( + {"value": [2]}, index=pd.Index([1], name="household_id") + ) + + _install_restored_subprocess_state( + state, + tables={"households": restored_households}, + checkpoint_name="model_a", + ) + + assert "tours" not in state.context + assert "tours" not in state.rng().channels + assert "tour_id" not in state.rng().index_to_channel + assert state.get_injectable("rng_channels") == ["households"] + pd.testing.assert_frame_equal( + state.get_dataframe("households"), restored_households + ) + assert state.checkpoint.last_checkpoint == {"checkpoint_name": "model_a"} + assert state.checkpoint.added == ["model_a"] + + class _State: def __init__(self, output_dir: Path): self.output_dir = output_dir diff --git a/activitysim/core/random.py b/activitysim/core/random.py index 9c87b7abd3..3a71acd177 100644 --- a/activitysim/core/random.py +++ b/activitysim/core/random.py @@ -816,7 +816,7 @@ def drop_channel(self, channel_name): for index_name, mapped_channel_name in self.index_to_channel.items() if mapped_channel_name != channel_name } - + # Also clear any index_to_channel entries that pointed at the # dropped channel; a stale mapping would otherwise survive and # could mis-route a subsequent channel registered against the diff --git a/activitysim/core/test/extensions/steps.py b/activitysim/core/test/extensions/steps.py index 0ac63f0952..cf59fb5d19 100644 --- a/activitysim/core/test/extensions/steps.py +++ b/activitysim/core/test/extensions/steps.py @@ -58,3 +58,11 @@ def create_households(state: workflow.State) -> None: state.get_rn_generator().add_channel("households", df) state.tracing.register_traceable_table("households", df) + + +@workflow.step +def record_random_draw(state: workflow.State) -> None: + """Record one global RNG draw for runner stream-name tests.""" + draws = list(state.get_injectable("recorded_random_draws", [])) + draws.append(state.get_rn_generator().get_global_rng().rand()) + state.add_injectable("recorded_random_draws", draws) diff --git a/activitysim/core/test/test_pipeline.py b/activitysim/core/test/test_pipeline.py index 95dca6e436..2c9471bfa6 100644 --- a/activitysim/core/test/test_pipeline.py +++ b/activitysim/core/test/test_pipeline.py @@ -144,6 +144,32 @@ def test_get_table_returns_current_table_after_recreation(state): close_handlers() +def test_runner_rng_name_override_is_explicit(state): + state.run.by_name("_record_random_draw.label=one") + state.run.by_name("_record_random_draw.label=two") + default_one, default_two = state.get_injectable("recorded_random_draws") + + # Normal parameterized invocations retain distinct ActivitySim streams. + assert default_one != default_two + + state.run.by_name_with_rng( + "_record_random_draw.calibration=one", + rng_step_name="record_random_draw", + ) + state.run.by_name_with_rng( + "_record_random_draw.calibration=two", + rng_step_name="record_random_draw", + ) + override_one, override_two = state.get_injectable("recorded_random_draws")[-2:] + + # Calibration can explicitly request common random numbers while keeping + # distinct invocation names for logging and checkpoint management. + assert override_one == override_two + + state.checkpoint.close_store() + close_handlers() + + # if __name__ == "__main__": # # print "\n\ntest_pipeline_run" diff --git a/activitysim/core/workflow/runner.py b/activitysim/core/workflow/runner.py index 14ef972049..e144d9e4b7 100644 --- a/activitysim/core/workflow/runner.py +++ b/activitysim/core/workflow/runner.py @@ -5,6 +5,7 @@ import time from collections.abc import Callable, Iterable from datetime import timedelta +from typing import Any from activitysim.core import tracing from activitysim.core.exceptions import DuplicateWorkflowNameError @@ -246,7 +247,9 @@ def log_runtime(self, model_name, start_time=None, timing=None, force=False): self.timing_notes.clear() - def _pre_run_step(self, model_name: str) -> bool | None: + def _pre_run_step( + self, model_name: str, rng_step_name: str | None = None + ) -> bool | None: """ Parameters @@ -270,11 +273,6 @@ def _pre_run_step(self, model_name: str) -> bool | None: f"Cannot run model '{model_name}' more than once" ) - # Parse the canonical workflow step before initializing the RNG. - # Arguments appended to a model invocation (for example calibration - # iteration labels) may affect logging and checkpoint names, but must - # never affect the deterministic random stream for the model itself. - # check for args if "." in model_name: step_name, arg_string = model_name.split(".", 1) @@ -288,9 +286,11 @@ def _pre_run_step(self, model_name: str) -> bool | None: step_name = model_name args = {} - self.rng_step_name = ( - step_name[1:] if step_name.startswith(NO_CHECKPOINT_PREFIX) else step_name - ) + # Preserve ActivitySim's normal behavior: the complete invocation name, + # including arguments and a no-checkpoint prefix, identifies its random + # stream. Specialized callers such as calibration may explicitly reuse + # a canonical stream while retaining a unique checkpoint/logging name. + self.rng_step_name = model_name if rng_step_name is None else rng_step_name self._obj.rng().begin_step(self.rng_step_name) # check for no_checkpoint prefix @@ -321,10 +321,23 @@ def by_name(self, model_name, **kwargs): model_name : str model_name is assumed to be the name of a registered workflow step """ + return self._by_name(model_name, rng_step_name=None, **kwargs) + + def by_name_with_rng(self, model_name: str, rng_step_name: str, **kwargs) -> None: + """Run a model using an explicit deterministic random-stream name. + + This specialized entry point lets calibration retain a unique labeled + invocation for checkpointing and logging while reusing the canonical + component's random stream. Ordinary ``by_name`` callers retain the + complete invocation name as their stream identifier. + """ + return self._by_name(model_name, rng_step_name=rng_step_name, **kwargs) + + def _by_name(self, model_name: str, rng_step_name: str | None, **kwargs) -> None: self.t0 = time.time() self.rng_step_name = None try: - should_skip = self._pre_run_step(model_name) + should_skip = self._pre_run_step(model_name, rng_step_name=rng_step_name) if should_skip: return From 0b953b2bc35c6fab394f87fcfcfb7992f3ebc295 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:10:13 -0700 Subject: [PATCH 90/90] clearer logging at max iterations --- activitysim/cli/run.py | 40 +++++++++++++++++--- activitysim/core/calibration/orchestrator.py | 6 +++ activitysim/core/calibration/settings.py | 2 + 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/activitysim/cli/run.py b/activitysim/cli/run.py index 49a048a53e..c6795a567c 100644 --- a/activitysim/cli/run.py +++ b/activitysim/cli/run.py @@ -424,18 +424,46 @@ def run(args): try: if calibration.calibration_enabled(state): - logger.info("run calibration workflow") + logger.info("evaluate calibration workflow") calibration_result = calibration.run_calibration_loop( state=state, models=state.settings.models, ) - logger.info( - "calibration workflow complete converged=%s completed_global_iterations=%s", - calibration_result.converged, - calibration_result.completed_global_iterations, - ) + if calibration_result.model_system_ran: + logger.info( + "calibration workflow complete converged=%s " + "completed_global_iterations=%s configured_global_iterations=%s", + calibration_result.converged, + calibration_result.completed_global_iterations, + calibration_result.configured_global_iterations, + ) + else: + if ( + calibration_result.completed_global_iterations + >= calibration_result.configured_global_iterations + ): + logger.info( + "calibration workflow skipped: no model system steps were run " + "because completed_global_iterations=%s has reached the " + "configured run.global_iterations=%s limit. Increase " + "run.global_iterations to request additional iterations, or " + "remove output/calibration/calibration_progress.json to start a fresh run.", + calibration_result.completed_global_iterations, + calibration_result.configured_global_iterations, + ) + else: + logger.info( + "calibration workflow skipped: no model system steps were run " + "because the prior calibration run is already complete " + "(completed_global_iterations=%s, run.global_iterations=%s). " + "Change run.global_iterations to a new value above the completed " + "count to request additional iterations, or remove " + "output/calibration/calibration_progress.json to start a fresh run.", + calibration_result.completed_global_iterations, + calibration_result.configured_global_iterations, + ) if state.settings.cleanup_pipeline_after_run: state.checkpoint.cleanup() diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index 5c381912d1..6b88ceb19b 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -276,6 +276,8 @@ def run_calibration_loop( return CalibrationRunResult( converged=bool(progress.get("converged", False)), completed_global_iterations=restart_plan.completed_global_iterations, + configured_global_iterations=(calibration_settings.run.global_iterations), + model_system_ran=False, ) # Validate the requested restart before changing a formerly complete @@ -400,6 +402,9 @@ def run_calibration_loop( return CalibrationRunResult( converged=converged, completed_global_iterations=completed_global_iterations, + configured_global_iterations=( + calibration_settings.run.global_iterations + ), ) # skip precursors if, on first iter, resume_after exists and is >= first_calib_model_idx @@ -655,6 +660,7 @@ def run_calibration_loop( return CalibrationRunResult( converged=all_converged, completed_global_iterations=completed_global_iterations, + configured_global_iterations=calibration_settings.run.global_iterations, ) finally: state.filesystem.pipeline_file_name = original_pipeline_name diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index d35c4cd50e..15e40c78ad 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -95,6 +95,8 @@ class CalibrationRunResult: converged: bool completed_global_iterations: int + configured_global_iterations: int + model_system_ran: bool = True def read_calibration_settings(state: workflow.State) -> CalibrationConfig | None: