Location model patch#1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bwentl merged 5 commits into from
Nov 2, 2023
Merged

Location model patch #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7becd2d
add chunking and caching of alt_values
bwentl May 11, 2023
0e5d3bf
add alt_values_to_feather option
bwentl Jun 23, 2023
425467e
format code with black
bwentl Jun 23, 2023
b8beb66
merging location choice model with BayDAG version
bwentl Nov 1, 2023
4d2bb4b
fix new line
bwentl Nov 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 79 additions & 17 deletions activitysim/abm/models/location_choice.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
)
from activitysim.core.interaction_sample import interaction_sample
from activitysim.core.interaction_sample_simulate import interaction_sample_simulate
from activitysim.core.util import reindex

from .util import estimation
from .util import logsums as logsum
Expand DownExpand Up@@ -138,22 +139,17 @@ def _location_sample(
logger.info("Running %s with %d persons" % (trace_label, len(choosers.index)))

sample_size = model_settings["SAMPLE_SIZE"]
if config.setting("disable_destination_sampling", False) or (
estimator and estimator.want_unsampled_alternatives
):
# FIXME interaction_sample will return unsampled complete alternatives with probs and pick_count
logger.info(
"Estimation mode for %s using unsampled alternatives short_circuit_choices"
% (trace_label,)
)
sample_size = 0
if estimator:
sample_size = model_settings.get("ESTIMATION_SAMPLE_SIZE", 0)

locals_d = {
"skims": skims,
"segment_size": segment_name,
"orig_col_name": skims.orig_key, # added for sharrow flows
"dest_col_name": skims.dest_key, # added for sharrow flows
"timeframe": "timeless",
"reindex": reindex,
"land_use": inject.get_table("land_use").to_frame(),
}
constants = config.get_model_constants(model_settings)
locals_d.update(constants)
Expand DownExpand Up@@ -470,6 +466,38 @@ def run_location_sample(
trace_label=trace_label,
)

# FIXME temporary code to ensure sampled alternative is in choices for estimation
# Hack to get shorter run times when you don't care about creating EDB for location choice models
if estimator:
# grabbing survey values
survey_persons = estimation.manager.get_survey_table("persons")
if "school_location" in trace_label:
survey_choices = survey_persons["school_zone_id"].reset_index()
elif ("workplace_location" in trace_label) and ("external" not in trace_label):
survey_choices = survey_persons["workplace_zone_id"].reset_index()
else:
return choices
survey_choices.columns = ["person_id", "alt_dest"]
survey_choices = survey_choices[
survey_choices["person_id"].isin(choices.index)
& (survey_choices.alt_dest > 0)
]
# merging survey destination into table if not available
joined_data = survey_choices.merge(
choices, on=["person_id", "alt_dest"], how="left", indicator=True
)
missing_rows = joined_data[joined_data["_merge"] == "left_only"]
missing_rows["pick_count"] = 1
if len(missing_rows) > 0:
new_choices = missing_rows[
["person_id", "alt_dest", "prob", "pick_count"]
].set_index("person_id")
choices = choices.append(new_choices, ignore_index=False).sort_index()
# making probability the mean of all other sampled destinations by person
choices["prob"] = choices["prob"].fillna(
choices.groupby("person_id")["prob"].transform("mean")
)

return choices


Expand DownExpand Up@@ -601,6 +629,8 @@ def run_location_simulate(
"orig_col_name": skims.orig_key, # added for sharrow flows
"dest_col_name": skims.dest_key, # added for sharrow flows
"timeframe": "timeless",
"reindex": reindex,
"land_use": inject.get_table("land_use").to_frame(),
}
constants = config.get_model_constants(model_settings)
if constants is not None:
Expand DownExpand Up@@ -808,6 +838,24 @@ def run_location_choice(
)
tracing.trace_df(choices_df, estimation_trace_label)

if want_logsums & (not skip_choice):
# grabbing index, could be person_id or proto_person_id
index_name = choices_df.index.name
# merging mode choice logsum of chosen alternative to choices
choices_df = (
pd.merge(
choices_df.reset_index(),
location_sample_df.reset_index()[
[index_name, model_settings["ALT_DEST_COL_NAME"], ALT_LOGSUM]
],
how="left",
left_on=[index_name, "choice"],
right_on=[index_name, model_settings["ALT_DEST_COL_NAME"]],
)
.drop(columns=model_settings["ALT_DEST_COL_NAME"])
.set_index(index_name)
)

choices_list.append(choices_df)

if want_sample_table:
Expand All@@ -825,7 +873,7 @@ def run_location_choice(
else:
# this will only happen with small samples (e.g. singleton) with no (e.g.) school segs
logger.warning("%s no choices", trace_label)
choices_df = pd.DataFrame(columns=["choice", "logsum"])
choices_df = pd.DataFrame(columns=["choice", "logsum", ALT_LOGSUM])

if len(sample_list) > 0:
save_sample_df = pd.concat(sample_list)
Expand DownExpand Up@@ -869,7 +917,8 @@ def iterate_location_choice(
Returns
-------
adds choice column model_settings['DEST_CHOICE_COLUMN_NAME']
adds logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
adds destination choice logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
adds mode choice logsum to selected destination column model_settings['MODE_CHOICE_LOGSUM_COLUMN_NAME']- if provided
adds annotations to persons table
"""

Expand All@@ -879,7 +928,11 @@ def iterate_location_choice(
chooser_filter_column = model_settings["CHOOSER_FILTER_COLUMN_NAME"]

dest_choice_column_name = model_settings["DEST_CHOICE_COLUMN_NAME"]
logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
dc_logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
mc_logsum_column_name = model_settings.get("MODE_CHOICE_LOGSUM_COLUMN_NAME")
want_logsums = (dc_logsum_column_name is not None) | (
mc_logsum_column_name is not None
)

sample_table_name = model_settings.get("DEST_CHOICE_SAMPLE_TABLE_NAME")
want_sample_table = (
Expand DownExpand Up@@ -929,7 +982,7 @@ def iterate_location_choice(
persons_merged_df_,
network_los,
shadow_price_calculator=spc,
want_logsums=logsum_column_name is not None,
want_logsums=want_logsums,
want_sample_table=want_sample_table,
estimator=estimator,
model_settings=model_settings,
Expand DownExpand Up@@ -1005,10 +1058,15 @@ def iterate_location_choice(
)

# add the dest_choice_logsum column to persons dataframe
if logsum_column_name:
persons_df[logsum_column_name] = (
if dc_logsum_column_name:
persons_df[dc_logsum_column_name] = (
choices_df["logsum"].reindex(persons_df.index).astype("float")
)
# add the mode choice logsum column to persons dataframe
if mc_logsum_column_name:
persons_df[mc_logsum_column_name] = (
choices_df[ALT_LOGSUM].reindex(persons_df.index).astype("float")
)

if save_sample_df is not None:
# might be None for tiny samples even if sample_table_name was specified
Expand DownExpand Up@@ -1047,9 +1105,13 @@ def iterate_location_choice(
if trace_hh_id:
tracing.trace_df(households_df, label=trace_label, warn_if_empty=True)

if logsum_column_name:
if dc_logsum_column_name:
tracing.print_summary(
dc_logsum_column_name, choices_df["logsum"], value_counts=True
)
if mc_logsum_column_name:
tracing.print_summary(
logsum_column_name, choices_df["logsum"], value_counts=True
mc_logsum_column_name, choices_df[ALT_LOGSUM], value_counts=True
)

return persons_df
Expand Down
101 changes: 99 additions & 2 deletions activitysim/estimation/larch/location_choice.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
import os
from pathlib import Path
from typing import Collection
import pickle
from datetime import datetime

import numpy as np
import pandas as pd
Expand DownExpand Up@@ -44,6 +46,8 @@ def location_choice_model(
settings_file="{name}_model_settings.yaml",
landuse_file="{name}_landuse.csv",
return_data=False,
alt_values_to_feather=False,
chunking_size=None,
):
model_selector = name.replace("_location", "")
model_selector = model_selector.replace("_destination", "")
Expand All@@ -57,12 +61,42 @@ def _read_csv(filename, **kwargs):
filename = filename.format(name=name)
return pd.read_csv(os.path.join(edb_directory, filename), **kwargs)

def _read_feather(filename, **kwargs):
filename = filename.format(name=name)
return pd.read_feather(os.path.join(edb_directory, filename), **kwargs)

def _to_feather(df, filename, **kwargs):
filename = filename.format(name=name)
return df.to_feather(os.path.join(edb_directory, filename), **kwargs)

def _read_pickle(filename, **kwargs):
filename = filename.format(name=name)
return pd.read_pickle(os.path.join(edb_directory, filename))

def _to_pickle(df, filename, **kwargs):
filename = filename.format(name=name)
return df.to_pickle(os.path.join(edb_directory, filename))

def _file_exists(filename):
filename = filename.format(name=name)
return os.path.exists(os.path.join(edb_directory, filename))

coefficients = _read_csv(
coefficients_file,
index_col="coefficient_name",
)
spec = _read_csv(spec_file, comment="#")
alt_values = _read_csv(alt_values_file)

# read alternative values either as csv or feather file
alt_values_fea_file = alt_values_file.replace(".csv", ".fea")
if os.path.exists(
os.path.join(edb_directory, alt_values_fea_file.format(name=name))
):
alt_values = _read_feather(alt_values_fea_file)
else:
alt_values = _read_csv(alt_values_file)
if alt_values_to_feather:
_to_feather(df=alt_values, filename=alt_values_fea_file)
chooser_data = _read_csv(chooser_file)
landuse = _read_csv(landuse_file, index_col="zone_id")
master_size_spec = _read_csv(size_spec_file)
Expand DownExpand Up@@ -106,6 +140,9 @@ def _read_csv(filename, **kwargs):
.set_index("segment")
)
size_spec = size_spec.loc[:, size_spec.max() > 0]
assert (
len(size_spec) > 0
), f"Empty size_spec, is model_selector {SIZE_TERM_SELECTOR} in your size term file?"

size_coef = size_coefficients_from_spec(size_spec)

Expand DownExpand Up@@ -148,7 +185,48 @@ def _read_csv(filename, **kwargs):

chooser_index_name = chooser_data.columns[0]
x_co = chooser_data.set_index(chooser_index_name)
x_ca = cv_to_ca(alt_values.set_index([chooser_index_name, alt_values.columns[1]]))

def split(a, n):
k, m = divmod(len(a), n)
return (a[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n))

# process x_ca with cv_to_ca with or without chunking
x_ca_pickle_file = "{name}_x_ca.pkl"
if chunking_size == None:
x_ca = cv_to_ca(
alt_values.set_index([chooser_index_name, alt_values.columns[1]])
)
elif _file_exists(x_ca_pickle_file):
# if pickle file from previous x_ca processing exist, load it to save time
time_start = datetime.now()
x_ca = _read_pickle(x_ca_pickle_file)
print(
f"x_ca data loaded from {name}_x_ca.fea - time elapsed {(datetime.now() - time_start).total_seconds()}"
)
else:
time_start = datetime.now()
# calculate num_chunks based on chunking_size (or max number of rows per chunk)
num_chunks = int(len(alt_values) / chunking_size)
all_person_ids = list(alt_values["person_id"].unique())
split_ids = list(split(all_person_ids, num_chunks))
x_ca_list = []
i = 0
for chunk_ids in split_ids:
alt_values_i = alt_values[alt_values["person_id"].isin(chunk_ids)]
x_ca_i = cv_to_ca(
alt_values_i.set_index([chooser_index_name, alt_values_i.columns[1]])
)
x_ca_list.append(x_ca_i)
print(
f"\rx_ca_i compute done for chunk {i}/{num_chunks} - time elapsed {(datetime.now() - time_start).total_seconds()}"
)
i = i + 1
x_ca = pd.concat(x_ca_list, axis=0)
# save final x_ca result as pickle file to save time for future data loading
_to_pickle(df=x_ca, filename=x_ca_pickle_file)
print(
f"x_ca compute done - time elapsed {(datetime.now() - time_start).total_seconds()}"
)

if CHOOSER_SEGMENT_COLUMN_NAME is not None:
# label segments with names
Expand DownExpand Up@@ -214,6 +292,9 @@ def _read_csv(filename, **kwargs):
else:
av = 1

assert len(x_co) > 0, "Empty chooser dataframe"
assert len(x_ca_1) > 0, "Empty alternatives dataframe"

d = DataFrames(co=x_co, ca=x_ca_1, av=av)

m = Model(dataservice=d)
Expand DownExpand Up@@ -331,6 +412,14 @@ def workplace_location_model(**kwargs):
)


def external_workplace_location_model(**kwargs):
unused = kwargs.pop("name", None)
return location_choice_model(
name="external_workplace_location",
**kwargs,
)


def school_location_model(**kwargs):
unused = kwargs.pop("name", None)
return location_choice_model(
Expand DownExpand Up@@ -367,6 +456,14 @@ def non_mandatory_tour_destination_model(**kwargs):
)


def external_non_mandatory_destination_model(**kwargs):
unused = kwargs.pop("name", None)
return location_choice_model(
name="external_non_mandatory_destination",
**kwargs,
)


def trip_destination_model(**kwargs):
unused = kwargs.pop("name", None)
return location_choice_model(
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Location model patch#1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bwentl
merged 5 commits into

from

Nov 2, 2023
 Merged


Location model patch
#1
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
7becd2d

add chunking and caching of alt_values


bwentl May 11, 2023
0e5d3bf

add alt_values_to_feather option


bwentl Jun 23, 2023
425467e

format code with black


bwentl Jun 23, 2023
b8beb66

merging location choice model with BayDAG version


bwentl Nov 1, 2023
4d2bb4b

fix new line


bwentl Nov 2, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


96 changes: 79 additions & 17 deletions
activitysim/abm/models/location_choice.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
)
from activitysim.core.interaction_sample import interaction_sample
from activitysim.core.interaction_sample_simulate import interaction_sample_simulate
from activitysim.core.util import reindex

from .util import estimation
from .util import logsums as logsum
Expand DownExpand Up@@ -138,22 +139,17 @@ def _location_sample(
 logger.info("Running %s with %d persons" % (trace_label, len(choosers.index)))

 sample_size = model_settings["SAMPLE_SIZE"]
 if config.setting("disable_destination_sampling", False) or (
 estimator and estimator.want_unsampled_alternatives
 ):
 # FIXME interaction_sample will return unsampled complete alternatives with probs and pick_count
 logger.info(
 "Estimation mode for %s using unsampled alternatives short_circuit_choices"
 % (trace_label,)
 )
 sample_size = 0
 if estimator:
 sample_size = model_settings.get("ESTIMATION_SAMPLE_SIZE", 0)

 locals_d = {
 "skims": skims,
 "segment_size": segment_name,
 "orig_col_name": skims.orig_key, # added for sharrow flows
 "dest_col_name": skims.dest_key, # added for sharrow flows
 "timeframe": "timeless",
 "reindex": reindex,
 "land_use": inject.get_table("land_use").to_frame(),
 }
 constants = config.get_model_constants(model_settings)
 locals_d.update(constants)
Expand DownExpand Up@@ -470,6 +466,38 @@ def run_location_sample(
 trace_label=trace_label,
 )

 # FIXME temporary code to ensure sampled alternative is in choices for estimation
 # Hack to get shorter run times when you don't care about creating EDB for location choice models
 if estimator:
 # grabbing survey values
 survey_persons = estimation.manager.get_survey_table("persons")
 if "school_location" in trace_label:
 survey_choices = survey_persons["school_zone_id"].reset_index()
 elif ("workplace_location" in trace_label) and ("external" not in trace_label):
 survey_choices = survey_persons["workplace_zone_id"].reset_index()
 else:
 return choices
 survey_choices.columns = ["person_id", "alt_dest"]
 survey_choices = survey_choices[
 survey_choices["person_id"].isin(choices.index)
 & (survey_choices.alt_dest > 0)
 ]
 # merging survey destination into table if not available
 joined_data = survey_choices.merge(
 choices, on=["person_id", "alt_dest"], how="left", indicator=True
 )
 missing_rows = joined_data[joined_data["_merge"] == "left_only"]
 missing_rows["pick_count"] = 1
 if len(missing_rows) > 0:
 new_choices = missing_rows[
 ["person_id", "alt_dest", "prob", "pick_count"]
 ].set_index("person_id")
 choices = choices.append(new_choices, ignore_index=False).sort_index()
 # making probability the mean of all other sampled destinations by person
 choices["prob"] = choices["prob"].fillna(
 choices.groupby("person_id")["prob"].transform("mean")
 )

 return choices


Expand DownExpand Up@@ -601,6 +629,8 @@ def run_location_simulate(
 "orig_col_name": skims.orig_key, # added for sharrow flows
 "dest_col_name": skims.dest_key, # added for sharrow flows
 "timeframe": "timeless",
 "reindex": reindex,
 "land_use": inject.get_table("land_use").to_frame(),
 }
 constants = config.get_model_constants(model_settings)
 if constants is not None:
Expand DownExpand Up@@ -808,6 +838,24 @@ def run_location_choice(
 )
 tracing.trace_df(choices_df, estimation_trace_label)

 if want_logsums & (not skip_choice):
 # grabbing index, could be person_id or proto_person_id
 index_name = choices_df.index.name
 # merging mode choice logsum of chosen alternative to choices
 choices_df = (
 pd.merge(
 choices_df.reset_index(),
 location_sample_df.reset_index()[
 [index_name, model_settings["ALT_DEST_COL_NAME"], ALT_LOGSUM]
 ],
 how="left",
 left_on=[index_name, "choice"],
 right_on=[index_name, model_settings["ALT_DEST_COL_NAME"]],
 )
 .drop(columns=model_settings["ALT_DEST_COL_NAME"])
 .set_index(index_name)
 )

 choices_list.append(choices_df)

 if want_sample_table:
Expand All@@ -825,7 +873,7 @@ def run_location_choice(
 else:
 # this will only happen with small samples (e.g. singleton) with no (e.g.) school segs
 logger.warning("%s no choices", trace_label)
 choices_df = pd.DataFrame(columns=["choice", "logsum"])
 choices_df = pd.DataFrame(columns=["choice", "logsum", ALT_LOGSUM])

 if len(sample_list) > 0:
 save_sample_df = pd.concat(sample_list)
Expand DownExpand Up@@ -869,7 +917,8 @@ def iterate_location_choice(
 Returns
 -------
 adds choice column model_settings['DEST_CHOICE_COLUMN_NAME']
 adds logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds destination choice logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds mode choice logsum to selected destination column model_settings['MODE_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds annotations to persons table
 """

Expand All@@ -879,7 +928,11 @@ def iterate_location_choice(
 chooser_filter_column = model_settings["CHOOSER_FILTER_COLUMN_NAME"]

 dest_choice_column_name = model_settings["DEST_CHOICE_COLUMN_NAME"]
 logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
 dc_logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
 mc_logsum_column_name = model_settings.get("MODE_CHOICE_LOGSUM_COLUMN_NAME")
 want_logsums = (dc_logsum_column_name is not None) | (
 mc_logsum_column_name is not None
 )

 sample_table_name = model_settings.get("DEST_CHOICE_SAMPLE_TABLE_NAME")
 want_sample_table = (
Expand DownExpand Up@@ -929,7 +982,7 @@ def iterate_location_choice(
 persons_merged_df_,
 network_los,
 shadow_price_calculator=spc,
 want_logsums=logsum_column_name is not None,
 want_logsums=want_logsums,
 want_sample_table=want_sample_table,
 estimator=estimator,
 model_settings=model_settings,
Expand DownExpand Up@@ -1005,10 +1058,15 @@ def iterate_location_choice(
 )

 # add the dest_choice_logsum column to persons dataframe
 if logsum_column_name:
 persons_df[logsum_column_name] = (
 if dc_logsum_column_name:
 persons_df[dc_logsum_column_name] = (
 choices_df["logsum"].reindex(persons_df.index).astype("float")
 )
 # add the mode choice logsum column to persons dataframe
 if mc_logsum_column_name:
 persons_df[mc_logsum_column_name] = (
 choices_df[ALT_LOGSUM].reindex(persons_df.index).astype("float")
 )

 if save_sample_df is not None:
 # might be None for tiny samples even if sample_table_name was specified
Expand DownExpand Up@@ -1047,9 +1105,13 @@ def iterate_location_choice(
 if trace_hh_id:
 tracing.trace_df(households_df, label=trace_label, warn_if_empty=True)

 if logsum_column_name:
 if dc_logsum_column_name:
 tracing.print_summary(
 dc_logsum_column_name, choices_df["logsum"], value_counts=True
 )
 if mc_logsum_column_name:
 tracing.print_summary(
logsum_column_name, choices_df["logsum"], value_counts=True
mc_logsum_column_name, choices_df[ALT_LOGSUM], value_counts=True
 )

 return persons_df
Expand Down

101 changes: 99 additions & 2 deletions
activitysim/estimation/larch/location_choice.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
import os
from pathlib import Path
from typing import Collection
import pickle
from datetime import datetime

import numpy as np
import pandas as pd
Expand DownExpand Up@@ -44,6 +46,8 @@ def location_choice_model(
 settings_file="{name}_model_settings.yaml",
 landuse_file="{name}_landuse.csv",
 return_data=False,
 alt_values_to_feather=False,
 chunking_size=None,
):
 model_selector = name.replace("_location", "")
 model_selector = model_selector.replace("_destination", "")
Expand All@@ -57,12 +61,42 @@ def _read_csv(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_csv(os.path.join(edb_directory, filename), **kwargs)

 def _read_feather(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_feather(os.path.join(edb_directory, filename), **kwargs)

 def _to_feather(df, filename, **kwargs):
 filename = filename.format(name=name)
 return df.to_feather(os.path.join(edb_directory, filename), **kwargs)

 def _read_pickle(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_pickle(os.path.join(edb_directory, filename))

 def _to_pickle(df, filename, **kwargs):
 filename = filename.format(name=name)
 return df.to_pickle(os.path.join(edb_directory, filename))

 def _file_exists(filename):
 filename = filename.format(name=name)
 return os.path.exists(os.path.join(edb_directory, filename))

 coefficients = _read_csv(
 coefficients_file,
 index_col="coefficient_name",
 )
 spec = _read_csv(spec_file, comment="#")
 alt_values = _read_csv(alt_values_file)

 # read alternative values either as csv or feather file
 alt_values_fea_file = alt_values_file.replace(".csv", ".fea")
 if os.path.exists(
 os.path.join(edb_directory, alt_values_fea_file.format(name=name))
 ):
 alt_values = _read_feather(alt_values_fea_file)
 else:
 alt_values = _read_csv(alt_values_file)
 if alt_values_to_feather:
 _to_feather(df=alt_values, filename=alt_values_fea_file)
 chooser_data = _read_csv(chooser_file)
 landuse = _read_csv(landuse_file, index_col="zone_id")
 master_size_spec = _read_csv(size_spec_file)
Expand DownExpand Up@@ -106,6 +140,9 @@ def _read_csv(filename, **kwargs):
 .set_index("segment")
 )
 size_spec = size_spec.loc[:, size_spec.max() > 0]
 assert (
 len(size_spec) > 0
 ), f"Empty size_spec, is model_selector {SIZE_TERM_SELECTOR} in your size term file?"

 size_coef = size_coefficients_from_spec(size_spec)

Expand DownExpand Up@@ -148,7 +185,48 @@ def _read_csv(filename, **kwargs):

 chooser_index_name = chooser_data.columns[0]
 x_co = chooser_data.set_index(chooser_index_name)
 x_ca = cv_to_ca(alt_values.set_index([chooser_index_name, alt_values.columns[1]]))

 def split(a, n):
 k, m = divmod(len(a), n)
 return (a[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n))

 # process x_ca with cv_to_ca with or without chunking
 x_ca_pickle_file = "{name}_x_ca.pkl"
 if chunking_size == None:
 x_ca = cv_to_ca(
 alt_values.set_index([chooser_index_name, alt_values.columns[1]])
 )
 elif _file_exists(x_ca_pickle_file):
 # if pickle file from previous x_ca processing exist, load it to save time
 time_start = datetime.now()
 x_ca = _read_pickle(x_ca_pickle_file)
 print(
 f"x_ca data loaded from {name}_x_ca.fea - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )
 else:
 time_start = datetime.now()
 # calculate num_chunks based on chunking_size (or max number of rows per chunk)
 num_chunks = int(len(alt_values) / chunking_size)
 all_person_ids = list(alt_values["person_id"].unique())
 split_ids = list(split(all_person_ids, num_chunks))
 x_ca_list = []
 i = 0
 for chunk_ids in split_ids:
 alt_values_i = alt_values[alt_values["person_id"].isin(chunk_ids)]
 x_ca_i = cv_to_ca(
 alt_values_i.set_index([chooser_index_name, alt_values_i.columns[1]])
 )
 x_ca_list.append(x_ca_i)
 print(
 f"\rx_ca_i compute done for chunk {i}/{num_chunks} - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )
 i = i + 1
 x_ca = pd.concat(x_ca_list, axis=0)
 # save final x_ca result as pickle file to save time for future data loading
 _to_pickle(df=x_ca, filename=x_ca_pickle_file)
 print(
 f"x_ca compute done - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )

 if CHOOSER_SEGMENT_COLUMN_NAME is not None:
 # label segments with names
Expand DownExpand Up@@ -214,6 +292,9 @@ def _read_csv(filename, **kwargs):
 else:
 av = 1

 assert len(x_co) > 0, "Empty chooser dataframe"
 assert len(x_ca_1) > 0, "Empty alternatives dataframe"

 d = DataFrames(co=x_co, ca=x_ca_1, av=av)

 m = Model(dataservice=d)
Expand DownExpand Up@@ -331,6 +412,14 @@ def workplace_location_model(**kwargs):
 )


def external_workplace_location_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
 name="external_workplace_location",
 **kwargs,
 )


def school_location_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
Expand DownExpand Up@@ -367,6 +456,14 @@ def non_mandatory_tour_destination_model(**kwargs):
 )


def external_non_mandatory_destination_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
 name="external_non_mandatory_destination",
 **kwargs,
 )


def trip_destination_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Location model patch#1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bwentl
merged 5 commits into

from

Nov 2, 2023
 Merged


Location model patch
#1
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
7becd2d

add chunking and caching of alt_values


bwentl May 11, 2023
0e5d3bf

add alt_values_to_feather option


bwentl Jun 23, 2023
425467e

format code with black


bwentl Jun 23, 2023
b8beb66

merging location choice model with BayDAG version


bwentl Nov 1, 2023
4d2bb4b

fix new line


bwentl Nov 2, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


96 changes: 79 additions & 17 deletions
activitysim/abm/models/location_choice.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
)
from activitysim.core.interaction_sample import interaction_sample
from activitysim.core.interaction_sample_simulate import interaction_sample_simulate
from activitysim.core.util import reindex

from .util import estimation
from .util import logsums as logsum
Expand DownExpand Up@@ -138,22 +139,17 @@ def _location_sample(
 logger.info("Running %s with %d persons" % (trace_label, len(choosers.index)))

 sample_size = model_settings["SAMPLE_SIZE"]
 if config.setting("disable_destination_sampling", False) or (
 estimator and estimator.want_unsampled_alternatives
 ):
 # FIXME interaction_sample will return unsampled complete alternatives with probs and pick_count
 logger.info(
 "Estimation mode for %s using unsampled alternatives short_circuit_choices"
 % (trace_label,)
 )
 sample_size = 0
 if estimator:
 sample_size = model_settings.get("ESTIMATION_SAMPLE_SIZE", 0)

 locals_d = {
 "skims": skims,
 "segment_size": segment_name,
 "orig_col_name": skims.orig_key, # added for sharrow flows
 "dest_col_name": skims.dest_key, # added for sharrow flows
 "timeframe": "timeless",
 "reindex": reindex,
 "land_use": inject.get_table("land_use").to_frame(),
 }
 constants = config.get_model_constants(model_settings)
 locals_d.update(constants)
Expand DownExpand Up@@ -470,6 +466,38 @@ def run_location_sample(
 trace_label=trace_label,
 )

 # FIXME temporary code to ensure sampled alternative is in choices for estimation
 # Hack to get shorter run times when you don't care about creating EDB for location choice models
 if estimator:
 # grabbing survey values
 survey_persons = estimation.manager.get_survey_table("persons")
 if "school_location" in trace_label:
 survey_choices = survey_persons["school_zone_id"].reset_index()
 elif ("workplace_location" in trace_label) and ("external" not in trace_label):
 survey_choices = survey_persons["workplace_zone_id"].reset_index()
 else:
 return choices
 survey_choices.columns = ["person_id", "alt_dest"]
 survey_choices = survey_choices[
 survey_choices["person_id"].isin(choices.index)
 & (survey_choices.alt_dest > 0)
 ]
 # merging survey destination into table if not available
 joined_data = survey_choices.merge(
 choices, on=["person_id", "alt_dest"], how="left", indicator=True
 )
 missing_rows = joined_data[joined_data["_merge"] == "left_only"]
 missing_rows["pick_count"] = 1
 if len(missing_rows) > 0:
 new_choices = missing_rows[
 ["person_id", "alt_dest", "prob", "pick_count"]
 ].set_index("person_id")
 choices = choices.append(new_choices, ignore_index=False).sort_index()
 # making probability the mean of all other sampled destinations by person
 choices["prob"] = choices["prob"].fillna(
 choices.groupby("person_id")["prob"].transform("mean")
 )

 return choices


Expand DownExpand Up@@ -601,6 +629,8 @@ def run_location_simulate(
 "orig_col_name": skims.orig_key, # added for sharrow flows
 "dest_col_name": skims.dest_key, # added for sharrow flows
 "timeframe": "timeless",
 "reindex": reindex,
 "land_use": inject.get_table("land_use").to_frame(),
 }
 constants = config.get_model_constants(model_settings)
 if constants is not None:
Expand DownExpand Up@@ -808,6 +838,24 @@ def run_location_choice(
 )
 tracing.trace_df(choices_df, estimation_trace_label)

 if want_logsums & (not skip_choice):
 # grabbing index, could be person_id or proto_person_id
 index_name = choices_df.index.name
 # merging mode choice logsum of chosen alternative to choices
 choices_df = (
 pd.merge(
 choices_df.reset_index(),
 location_sample_df.reset_index()[
 [index_name, model_settings["ALT_DEST_COL_NAME"], ALT_LOGSUM]
 ],
 how="left",
 left_on=[index_name, "choice"],
 right_on=[index_name, model_settings["ALT_DEST_COL_NAME"]],
 )
 .drop(columns=model_settings["ALT_DEST_COL_NAME"])
 .set_index(index_name)
 )

 choices_list.append(choices_df)

 if want_sample_table:
Expand All@@ -825,7 +873,7 @@ def run_location_choice(
 else:
 # this will only happen with small samples (e.g. singleton) with no (e.g.) school segs
 logger.warning("%s no choices", trace_label)
 choices_df = pd.DataFrame(columns=["choice", "logsum"])
 choices_df = pd.DataFrame(columns=["choice", "logsum", ALT_LOGSUM])

 if len(sample_list) > 0:
 save_sample_df = pd.concat(sample_list)
Expand DownExpand Up@@ -869,7 +917,8 @@ def iterate_location_choice(
 Returns
 -------
 adds choice column model_settings['DEST_CHOICE_COLUMN_NAME']
 adds logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds destination choice logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds mode choice logsum to selected destination column model_settings['MODE_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds annotations to persons table
 """

Expand All@@ -879,7 +928,11 @@ def iterate_location_choice(
 chooser_filter_column = model_settings["CHOOSER_FILTER_COLUMN_NAME"]

 dest_choice_column_name = model_settings["DEST_CHOICE_COLUMN_NAME"]
 logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
 dc_logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
 mc_logsum_column_name = model_settings.get("MODE_CHOICE_LOGSUM_COLUMN_NAME")
 want_logsums = (dc_logsum_column_name is not None) | (
 mc_logsum_column_name is not None
 )

 sample_table_name = model_settings.get("DEST_CHOICE_SAMPLE_TABLE_NAME")
 want_sample_table = (
Expand DownExpand Up@@ -929,7 +982,7 @@ def iterate_location_choice(
 persons_merged_df_,
 network_los,
 shadow_price_calculator=spc,
 want_logsums=logsum_column_name is not None,
 want_logsums=want_logsums,
 want_sample_table=want_sample_table,
 estimator=estimator,
 model_settings=model_settings,
Expand DownExpand Up@@ -1005,10 +1058,15 @@ def iterate_location_choice(
 )

 # add the dest_choice_logsum column to persons dataframe
 if logsum_column_name:
 persons_df[logsum_column_name] = (
 if dc_logsum_column_name:
 persons_df[dc_logsum_column_name] = (
 choices_df["logsum"].reindex(persons_df.index).astype("float")
 )
 # add the mode choice logsum column to persons dataframe
 if mc_logsum_column_name:
 persons_df[mc_logsum_column_name] = (
 choices_df[ALT_LOGSUM].reindex(persons_df.index).astype("float")
 )

 if save_sample_df is not None:
 # might be None for tiny samples even if sample_table_name was specified
Expand DownExpand Up@@ -1047,9 +1105,13 @@ def iterate_location_choice(
 if trace_hh_id:
 tracing.trace_df(households_df, label=trace_label, warn_if_empty=True)

 if logsum_column_name:
 if dc_logsum_column_name:
 tracing.print_summary(
 dc_logsum_column_name, choices_df["logsum"], value_counts=True
 )
 if mc_logsum_column_name:
 tracing.print_summary(
logsum_column_name, choices_df["logsum"], value_counts=True
mc_logsum_column_name, choices_df[ALT_LOGSUM], value_counts=True
 )

 return persons_df
Expand Down

101 changes: 99 additions & 2 deletions
activitysim/estimation/larch/location_choice.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
import os
from pathlib import Path
from typing import Collection
import pickle
from datetime import datetime

import numpy as np
import pandas as pd
Expand DownExpand Up@@ -44,6 +46,8 @@ def location_choice_model(
 settings_file="{name}_model_settings.yaml",
 landuse_file="{name}_landuse.csv",
 return_data=False,
 alt_values_to_feather=False,
 chunking_size=None,
):
 model_selector = name.replace("_location", "")
 model_selector = model_selector.replace("_destination", "")
Expand All@@ -57,12 +61,42 @@ def _read_csv(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_csv(os.path.join(edb_directory, filename), **kwargs)

 def _read_feather(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_feather(os.path.join(edb_directory, filename), **kwargs)

 def _to_feather(df, filename, **kwargs):
 filename = filename.format(name=name)
 return df.to_feather(os.path.join(edb_directory, filename), **kwargs)

 def _read_pickle(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_pickle(os.path.join(edb_directory, filename))

 def _to_pickle(df, filename, **kwargs):
 filename = filename.format(name=name)
 return df.to_pickle(os.path.join(edb_directory, filename))

 def _file_exists(filename):
 filename = filename.format(name=name)
 return os.path.exists(os.path.join(edb_directory, filename))

 coefficients = _read_csv(
 coefficients_file,
 index_col="coefficient_name",
 )
 spec = _read_csv(spec_file, comment="#")
 alt_values = _read_csv(alt_values_file)

 # read alternative values either as csv or feather file
 alt_values_fea_file = alt_values_file.replace(".csv", ".fea")
 if os.path.exists(
 os.path.join(edb_directory, alt_values_fea_file.format(name=name))
 ):
 alt_values = _read_feather(alt_values_fea_file)
 else:
 alt_values = _read_csv(alt_values_file)
 if alt_values_to_feather:
 _to_feather(df=alt_values, filename=alt_values_fea_file)
 chooser_data = _read_csv(chooser_file)
 landuse = _read_csv(landuse_file, index_col="zone_id")
 master_size_spec = _read_csv(size_spec_file)
Expand DownExpand Up@@ -106,6 +140,9 @@ def _read_csv(filename, **kwargs):
 .set_index("segment")
 )
 size_spec = size_spec.loc[:, size_spec.max() > 0]
 assert (
 len(size_spec) > 0
 ), f"Empty size_spec, is model_selector {SIZE_TERM_SELECTOR} in your size term file?"

 size_coef = size_coefficients_from_spec(size_spec)

Expand DownExpand Up@@ -148,7 +185,48 @@ def _read_csv(filename, **kwargs):

 chooser_index_name = chooser_data.columns[0]
 x_co = chooser_data.set_index(chooser_index_name)
 x_ca = cv_to_ca(alt_values.set_index([chooser_index_name, alt_values.columns[1]]))

 def split(a, n):
 k, m = divmod(len(a), n)
 return (a[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n))

 # process x_ca with cv_to_ca with or without chunking
 x_ca_pickle_file = "{name}_x_ca.pkl"
 if chunking_size == None:
 x_ca = cv_to_ca(
 alt_values.set_index([chooser_index_name, alt_values.columns[1]])
 )
 elif _file_exists(x_ca_pickle_file):
 # if pickle file from previous x_ca processing exist, load it to save time
 time_start = datetime.now()
 x_ca = _read_pickle(x_ca_pickle_file)
 print(
 f"x_ca data loaded from {name}_x_ca.fea - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )
 else:
 time_start = datetime.now()
 # calculate num_chunks based on chunking_size (or max number of rows per chunk)
 num_chunks = int(len(alt_values) / chunking_size)
 all_person_ids = list(alt_values["person_id"].unique())
 split_ids = list(split(all_person_ids, num_chunks))
 x_ca_list = []
 i = 0
 for chunk_ids in split_ids:
 alt_values_i = alt_values[alt_values["person_id"].isin(chunk_ids)]
 x_ca_i = cv_to_ca(
 alt_values_i.set_index([chooser_index_name, alt_values_i.columns[1]])
 )
 x_ca_list.append(x_ca_i)
 print(
 f"\rx_ca_i compute done for chunk {i}/{num_chunks} - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )
 i = i + 1
 x_ca = pd.concat(x_ca_list, axis=0)
 # save final x_ca result as pickle file to save time for future data loading
 _to_pickle(df=x_ca, filename=x_ca_pickle_file)
 print(
 f"x_ca compute done - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )

 if CHOOSER_SEGMENT_COLUMN_NAME is not None:
 # label segments with names
Expand DownExpand Up@@ -214,6 +292,9 @@ def _read_csv(filename, **kwargs):
 else:
 av = 1

 assert len(x_co) > 0, "Empty chooser dataframe"
 assert len(x_ca_1) > 0, "Empty alternatives dataframe"

 d = DataFrames(co=x_co, ca=x_ca_1, av=av)

 m = Model(dataservice=d)
Expand DownExpand Up@@ -331,6 +412,14 @@ def workplace_location_model(**kwargs):
 )


def external_workplace_location_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
 name="external_workplace_location",
 **kwargs,
 )


def school_location_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
Expand DownExpand Up@@ -367,6 +456,14 @@ def non_mandatory_tour_destination_model(**kwargs):
 )


def external_non_mandatory_destination_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
 name="external_non_mandatory_destination",
 **kwargs,
 )


def trip_destination_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Location model patch#1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bwentl
merged 5 commits into

from

Nov 2, 2023
 Merged


Location model patch
#1
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
7becd2d

add chunking and caching of alt_values


bwentl May 11, 2023
0e5d3bf

add alt_values_to_feather option


bwentl Jun 23, 2023
425467e

format code with black


bwentl Jun 23, 2023
b8beb66

merging location choice model with BayDAG version


bwentl Nov 1, 2023
4d2bb4b

fix new line


bwentl Nov 2, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


96 changes: 79 additions & 17 deletions
activitysim/abm/models/location_choice.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
)
from activitysim.core.interaction_sample import interaction_sample
from activitysim.core.interaction_sample_simulate import interaction_sample_simulate
from activitysim.core.util import reindex

from .util import estimation
from .util import logsums as logsum
Expand DownExpand Up@@ -138,22 +139,17 @@ def _location_sample(
 logger.info("Running %s with %d persons" % (trace_label, len(choosers.index)))

 sample_size = model_settings["SAMPLE_SIZE"]
 if config.setting("disable_destination_sampling", False) or (
 estimator and estimator.want_unsampled_alternatives
 ):
 # FIXME interaction_sample will return unsampled complete alternatives with probs and pick_count
 logger.info(
 "Estimation mode for %s using unsampled alternatives short_circuit_choices"
 % (trace_label,)
 )
 sample_size = 0
 if estimator:
 sample_size = model_settings.get("ESTIMATION_SAMPLE_SIZE", 0)

 locals_d = {
 "skims": skims,
 "segment_size": segment_name,
 "orig_col_name": skims.orig_key, # added for sharrow flows
 "dest_col_name": skims.dest_key, # added for sharrow flows
 "timeframe": "timeless",
 "reindex": reindex,
 "land_use": inject.get_table("land_use").to_frame(),
 }
 constants = config.get_model_constants(model_settings)
 locals_d.update(constants)
Expand DownExpand Up@@ -470,6 +466,38 @@ def run_location_sample(
 trace_label=trace_label,
 )

 # FIXME temporary code to ensure sampled alternative is in choices for estimation
 # Hack to get shorter run times when you don't care about creating EDB for location choice models
 if estimator:
 # grabbing survey values
 survey_persons = estimation.manager.get_survey_table("persons")
 if "school_location" in trace_label:
 survey_choices = survey_persons["school_zone_id"].reset_index()
 elif ("workplace_location" in trace_label) and ("external" not in trace_label):
 survey_choices = survey_persons["workplace_zone_id"].reset_index()
 else:
 return choices
 survey_choices.columns = ["person_id", "alt_dest"]
 survey_choices = survey_choices[
 survey_choices["person_id"].isin(choices.index)
 & (survey_choices.alt_dest > 0)
 ]
 # merging survey destination into table if not available
 joined_data = survey_choices.merge(
 choices, on=["person_id", "alt_dest"], how="left", indicator=True
 )
 missing_rows = joined_data[joined_data["_merge"] == "left_only"]
 missing_rows["pick_count"] = 1
 if len(missing_rows) > 0:
 new_choices = missing_rows[
 ["person_id", "alt_dest", "prob", "pick_count"]
 ].set_index("person_id")
 choices = choices.append(new_choices, ignore_index=False).sort_index()
 # making probability the mean of all other sampled destinations by person
 choices["prob"] = choices["prob"].fillna(
 choices.groupby("person_id")["prob"].transform("mean")
 )

 return choices


Expand DownExpand Up@@ -601,6 +629,8 @@ def run_location_simulate(
 "orig_col_name": skims.orig_key, # added for sharrow flows
 "dest_col_name": skims.dest_key, # added for sharrow flows
 "timeframe": "timeless",
 "reindex": reindex,
 "land_use": inject.get_table("land_use").to_frame(),
 }
 constants = config.get_model_constants(model_settings)
 if constants is not None:
Expand DownExpand Up@@ -808,6 +838,24 @@ def run_location_choice(
 )
 tracing.trace_df(choices_df, estimation_trace_label)

 if want_logsums & (not skip_choice):
 # grabbing index, could be person_id or proto_person_id
 index_name = choices_df.index.name
 # merging mode choice logsum of chosen alternative to choices
 choices_df = (
 pd.merge(
 choices_df.reset_index(),
 location_sample_df.reset_index()[
 [index_name, model_settings["ALT_DEST_COL_NAME"], ALT_LOGSUM]
 ],
 how="left",
 left_on=[index_name, "choice"],
 right_on=[index_name, model_settings["ALT_DEST_COL_NAME"]],
 )
 .drop(columns=model_settings["ALT_DEST_COL_NAME"])
 .set_index(index_name)
 )

 choices_list.append(choices_df)

 if want_sample_table:
Expand All@@ -825,7 +873,7 @@ def run_location_choice(
 else:
 # this will only happen with small samples (e.g. singleton) with no (e.g.) school segs
 logger.warning("%s no choices", trace_label)
 choices_df = pd.DataFrame(columns=["choice", "logsum"])
 choices_df = pd.DataFrame(columns=["choice", "logsum", ALT_LOGSUM])

 if len(sample_list) > 0:
 save_sample_df = pd.concat(sample_list)
Expand DownExpand Up@@ -869,7 +917,8 @@ def iterate_location_choice(
 Returns
 -------
 adds choice column model_settings['DEST_CHOICE_COLUMN_NAME']
 adds logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds destination choice logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds mode choice logsum to selected destination column model_settings['MODE_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds annotations to persons table
 """

Expand All@@ -879,7 +928,11 @@ def iterate_location_choice(
 chooser_filter_column = model_settings["CHOOSER_FILTER_COLUMN_NAME"]

 dest_choice_column_name = model_settings["DEST_CHOICE_COLUMN_NAME"]
 logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
 dc_logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
 mc_logsum_column_name = model_settings.get("MODE_CHOICE_LOGSUM_COLUMN_NAME")
 want_logsums = (dc_logsum_column_name is not None) | (
 mc_logsum_column_name is not None
 )

 sample_table_name = model_settings.get("DEST_CHOICE_SAMPLE_TABLE_NAME")
 want_sample_table = (
Expand DownExpand Up@@ -929,7 +982,7 @@ def iterate_location_choice(
 persons_merged_df_,
 network_los,
 shadow_price_calculator=spc,
 want_logsums=logsum_column_name is not None,
 want_logsums=want_logsums,
 want_sample_table=want_sample_table,
 estimator=estimator,
 model_settings=model_settings,
Expand DownExpand Up@@ -1005,10 +1058,15 @@ def iterate_location_choice(
 )

 # add the dest_choice_logsum column to persons dataframe
 if logsum_column_name:
 persons_df[logsum_column_name] = (
 if dc_logsum_column_name:
 persons_df[dc_logsum_column_name] = (
 choices_df["logsum"].reindex(persons_df.index).astype("float")
 )
 # add the mode choice logsum column to persons dataframe
 if mc_logsum_column_name:
 persons_df[mc_logsum_column_name] = (
 choices_df[ALT_LOGSUM].reindex(persons_df.index).astype("float")
 )

 if save_sample_df is not None:
 # might be None for tiny samples even if sample_table_name was specified
Expand DownExpand Up@@ -1047,9 +1105,13 @@ def iterate_location_choice(
 if trace_hh_id:
 tracing.trace_df(households_df, label=trace_label, warn_if_empty=True)

 if logsum_column_name:
 if dc_logsum_column_name:
 tracing.print_summary(
 dc_logsum_column_name, choices_df["logsum"], value_counts=True
 )
 if mc_logsum_column_name:
 tracing.print_summary(
logsum_column_name, choices_df["logsum"], value_counts=True
mc_logsum_column_name, choices_df[ALT_LOGSUM], value_counts=True
 )

 return persons_df
Expand Down

101 changes: 99 additions & 2 deletions
activitysim/estimation/larch/location_choice.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
import os
from pathlib import Path
from typing import Collection
import pickle
from datetime import datetime

import numpy as np
import pandas as pd
Expand DownExpand Up@@ -44,6 +46,8 @@ def location_choice_model(
 settings_file="{name}_model_settings.yaml",
 landuse_file="{name}_landuse.csv",
 return_data=False,
 alt_values_to_feather=False,
 chunking_size=None,
):
 model_selector = name.replace("_location", "")
 model_selector = model_selector.replace("_destination", "")
Expand All@@ -57,12 +61,42 @@ def _read_csv(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_csv(os.path.join(edb_directory, filename), **kwargs)

 def _read_feather(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_feather(os.path.join(edb_directory, filename), **kwargs)

 def _to_feather(df, filename, **kwargs):
 filename = filename.format(name=name)
 return df.to_feather(os.path.join(edb_directory, filename), **kwargs)

 def _read_pickle(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_pickle(os.path.join(edb_directory, filename))

 def _to_pickle(df, filename, **kwargs):
 filename = filename.format(name=name)
 return df.to_pickle(os.path.join(edb_directory, filename))

 def _file_exists(filename):
 filename = filename.format(name=name)
 return os.path.exists(os.path.join(edb_directory, filename))

 coefficients = _read_csv(
 coefficients_file,
 index_col="coefficient_name",
 )
 spec = _read_csv(spec_file, comment="#")
 alt_values = _read_csv(alt_values_file)

 # read alternative values either as csv or feather file
 alt_values_fea_file = alt_values_file.replace(".csv", ".fea")
 if os.path.exists(
 os.path.join(edb_directory, alt_values_fea_file.format(name=name))
 ):
 alt_values = _read_feather(alt_values_fea_file)
 else:
 alt_values = _read_csv(alt_values_file)
 if alt_values_to_feather:
 _to_feather(df=alt_values, filename=alt_values_fea_file)
 chooser_data = _read_csv(chooser_file)
 landuse = _read_csv(landuse_file, index_col="zone_id")
 master_size_spec = _read_csv(size_spec_file)
Expand DownExpand Up@@ -106,6 +140,9 @@ def _read_csv(filename, **kwargs):
 .set_index("segment")
 )
 size_spec = size_spec.loc[:, size_spec.max() > 0]
 assert (
 len(size_spec) > 0
 ), f"Empty size_spec, is model_selector {SIZE_TERM_SELECTOR} in your size term file?"

 size_coef = size_coefficients_from_spec(size_spec)

Expand DownExpand Up@@ -148,7 +185,48 @@ def _read_csv(filename, **kwargs):

 chooser_index_name = chooser_data.columns[0]
 x_co = chooser_data.set_index(chooser_index_name)
 x_ca = cv_to_ca(alt_values.set_index([chooser_index_name, alt_values.columns[1]]))

 def split(a, n):
 k, m = divmod(len(a), n)
 return (a[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n))

 # process x_ca with cv_to_ca with or without chunking
 x_ca_pickle_file = "{name}_x_ca.pkl"
 if chunking_size == None:
 x_ca = cv_to_ca(
 alt_values.set_index([chooser_index_name, alt_values.columns[1]])
 )
 elif _file_exists(x_ca_pickle_file):
 # if pickle file from previous x_ca processing exist, load it to save time
 time_start = datetime.now()
 x_ca = _read_pickle(x_ca_pickle_file)
 print(
 f"x_ca data loaded from {name}_x_ca.fea - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )
 else:
 time_start = datetime.now()
 # calculate num_chunks based on chunking_size (or max number of rows per chunk)
 num_chunks = int(len(alt_values) / chunking_size)
 all_person_ids = list(alt_values["person_id"].unique())
 split_ids = list(split(all_person_ids, num_chunks))
 x_ca_list = []
 i = 0
 for chunk_ids in split_ids:
 alt_values_i = alt_values[alt_values["person_id"].isin(chunk_ids)]
 x_ca_i = cv_to_ca(
 alt_values_i.set_index([chooser_index_name, alt_values_i.columns[1]])
 )
 x_ca_list.append(x_ca_i)
 print(
 f"\rx_ca_i compute done for chunk {i}/{num_chunks} - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )
 i = i + 1
 x_ca = pd.concat(x_ca_list, axis=0)
 # save final x_ca result as pickle file to save time for future data loading
 _to_pickle(df=x_ca, filename=x_ca_pickle_file)
 print(
 f"x_ca compute done - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )

 if CHOOSER_SEGMENT_COLUMN_NAME is not None:
 # label segments with names
Expand DownExpand Up@@ -214,6 +292,9 @@ def _read_csv(filename, **kwargs):
 else:
 av = 1

 assert len(x_co) > 0, "Empty chooser dataframe"
 assert len(x_ca_1) > 0, "Empty alternatives dataframe"

 d = DataFrames(co=x_co, ca=x_ca_1, av=av)

 m = Model(dataservice=d)
Expand DownExpand Up@@ -331,6 +412,14 @@ def workplace_location_model(**kwargs):
 )


def external_workplace_location_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
 name="external_workplace_location",
 **kwargs,
 )


def school_location_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
Expand DownExpand Up@@ -367,6 +456,14 @@ def non_mandatory_tour_destination_model(**kwargs):
 )


def external_non_mandatory_destination_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
 name="external_non_mandatory_destination",
 **kwargs,
 )


def trip_destination_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Location model patch#1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bwentl
merged 5 commits into

from

Nov 2, 2023
 Merged


Location model patch
#1
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
7becd2d

add chunking and caching of alt_values


bwentl May 11, 2023
0e5d3bf

add alt_values_to_feather option


bwentl Jun 23, 2023
425467e

format code with black


bwentl Jun 23, 2023
b8beb66

merging location choice model with BayDAG version


bwentl Nov 1, 2023
4d2bb4b

fix new line


bwentl Nov 2, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


96 changes: 79 additions & 17 deletions
activitysim/abm/models/location_choice.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
)
from activitysim.core.interaction_sample import interaction_sample
from activitysim.core.interaction_sample_simulate import interaction_sample_simulate
from activitysim.core.util import reindex

from .util import estimation
from .util import logsums as logsum
Expand DownExpand Up@@ -138,22 +139,17 @@ def _location_sample(
 logger.info("Running %s with %d persons" % (trace_label, len(choosers.index)))

 sample_size = model_settings["SAMPLE_SIZE"]
 if config.setting("disable_destination_sampling", False) or (
 estimator and estimator.want_unsampled_alternatives
 ):
 # FIXME interaction_sample will return unsampled complete alternatives with probs and pick_count
 logger.info(
 "Estimation mode for %s using unsampled alternatives short_circuit_choices"
 % (trace_label,)
 )
 sample_size = 0
 if estimator:
 sample_size = model_settings.get("ESTIMATION_SAMPLE_SIZE", 0)

 locals_d = {
 "skims": skims,
 "segment_size": segment_name,
 "orig_col_name": skims.orig_key, # added for sharrow flows
 "dest_col_name": skims.dest_key, # added for sharrow flows
 "timeframe": "timeless",
 "reindex": reindex,
 "land_use": inject.get_table("land_use").to_frame(),
 }
 constants = config.get_model_constants(model_settings)
 locals_d.update(constants)
Expand DownExpand Up@@ -470,6 +466,38 @@ def run_location_sample(
 trace_label=trace_label,
 )

 # FIXME temporary code to ensure sampled alternative is in choices for estimation
 # Hack to get shorter run times when you don't care about creating EDB for location choice models
 if estimator:
 # grabbing survey values
 survey_persons = estimation.manager.get_survey_table("persons")
 if "school_location" in trace_label:
 survey_choices = survey_persons["school_zone_id"].reset_index()
 elif ("workplace_location" in trace_label) and ("external" not in trace_label):
 survey_choices = survey_persons["workplace_zone_id"].reset_index()
 else:
 return choices
 survey_choices.columns = ["person_id", "alt_dest"]
 survey_choices = survey_choices[
 survey_choices["person_id"].isin(choices.index)
 & (survey_choices.alt_dest > 0)
 ]
 # merging survey destination into table if not available
 joined_data = survey_choices.merge(
 choices, on=["person_id", "alt_dest"], how="left", indicator=True
 )
 missing_rows = joined_data[joined_data["_merge"] == "left_only"]
 missing_rows["pick_count"] = 1
 if len(missing_rows) > 0:
 new_choices = missing_rows[
 ["person_id", "alt_dest", "prob", "pick_count"]
 ].set_index("person_id")
 choices = choices.append(new_choices, ignore_index=False).sort_index()
 # making probability the mean of all other sampled destinations by person
 choices["prob"] = choices["prob"].fillna(
 choices.groupby("person_id")["prob"].transform("mean")
 )

 return choices


Expand DownExpand Up@@ -601,6 +629,8 @@ def run_location_simulate(
 "orig_col_name": skims.orig_key, # added for sharrow flows
 "dest_col_name": skims.dest_key, # added for sharrow flows
 "timeframe": "timeless",
 "reindex": reindex,
 "land_use": inject.get_table("land_use").to_frame(),
 }
 constants = config.get_model_constants(model_settings)
 if constants is not None:
Expand DownExpand Up@@ -808,6 +838,24 @@ def run_location_choice(
 )
 tracing.trace_df(choices_df, estimation_trace_label)

 if want_logsums & (not skip_choice):
 # grabbing index, could be person_id or proto_person_id
 index_name = choices_df.index.name
 # merging mode choice logsum of chosen alternative to choices
 choices_df = (
 pd.merge(
 choices_df.reset_index(),
 location_sample_df.reset_index()[
 [index_name, model_settings["ALT_DEST_COL_NAME"], ALT_LOGSUM]
 ],
 how="left",
 left_on=[index_name, "choice"],
 right_on=[index_name, model_settings["ALT_DEST_COL_NAME"]],
 )
 .drop(columns=model_settings["ALT_DEST_COL_NAME"])
 .set_index(index_name)
 )

 choices_list.append(choices_df)

 if want_sample_table:
Expand All@@ -825,7 +873,7 @@ def run_location_choice(
 else:
 # this will only happen with small samples (e.g. singleton) with no (e.g.) school segs
 logger.warning("%s no choices", trace_label)
 choices_df = pd.DataFrame(columns=["choice", "logsum"])
 choices_df = pd.DataFrame(columns=["choice", "logsum", ALT_LOGSUM])

 if len(sample_list) > 0:
 save_sample_df = pd.concat(sample_list)
Expand DownExpand Up@@ -869,7 +917,8 @@ def iterate_location_choice(
 Returns
 -------
 adds choice column model_settings['DEST_CHOICE_COLUMN_NAME']
 adds logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds destination choice logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds mode choice logsum to selected destination column model_settings['MODE_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds annotations to persons table
 """

Expand All@@ -879,7 +928,11 @@ def iterate_location_choice(
 chooser_filter_column = model_settings["CHOOSER_FILTER_COLUMN_NAME"]

 dest_choice_column_name = model_settings["DEST_CHOICE_COLUMN_NAME"]
 logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
 dc_logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
 mc_logsum_column_name = model_settings.get("MODE_CHOICE_LOGSUM_COLUMN_NAME")
 want_logsums = (dc_logsum_column_name is not None) | (
 mc_logsum_column_name is not None
 )

 sample_table_name = model_settings.get("DEST_CHOICE_SAMPLE_TABLE_NAME")
 want_sample_table = (
Expand DownExpand Up@@ -929,7 +982,7 @@ def iterate_location_choice(
 persons_merged_df_,
 network_los,
 shadow_price_calculator=spc,
 want_logsums=logsum_column_name is not None,
 want_logsums=want_logsums,
 want_sample_table=want_sample_table,
 estimator=estimator,
 model_settings=model_settings,
Expand DownExpand Up@@ -1005,10 +1058,15 @@ def iterate_location_choice(
 )

 # add the dest_choice_logsum column to persons dataframe
 if logsum_column_name:
 persons_df[logsum_column_name] = (
 if dc_logsum_column_name:
 persons_df[dc_logsum_column_name] = (
 choices_df["logsum"].reindex(persons_df.index).astype("float")
 )
 # add the mode choice logsum column to persons dataframe
 if mc_logsum_column_name:
 persons_df[mc_logsum_column_name] = (
 choices_df[ALT_LOGSUM].reindex(persons_df.index).astype("float")
 )

 if save_sample_df is not None:
 # might be None for tiny samples even if sample_table_name was specified
Expand DownExpand Up@@ -1047,9 +1105,13 @@ def iterate_location_choice(
 if trace_hh_id:
 tracing.trace_df(households_df, label=trace_label, warn_if_empty=True)

 if logsum_column_name:
 if dc_logsum_column_name:
 tracing.print_summary(
 dc_logsum_column_name, choices_df["logsum"], value_counts=True
 )
 if mc_logsum_column_name:
 tracing.print_summary(
logsum_column_name, choices_df["logsum"], value_counts=True
mc_logsum_column_name, choices_df[ALT_LOGSUM], value_counts=True
 )

 return persons_df
Expand Down

101 changes: 99 additions & 2 deletions
activitysim/estimation/larch/location_choice.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
import os
from pathlib import Path
from typing import Collection
import pickle
from datetime import datetime

import numpy as np
import pandas as pd
Expand DownExpand Up@@ -44,6 +46,8 @@ def location_choice_model(
 settings_file="{name}_model_settings.yaml",
 landuse_file="{name}_landuse.csv",
 return_data=False,
 alt_values_to_feather=False,
 chunking_size=None,
):
 model_selector = name.replace("_location", "")
 model_selector = model_selector.replace("_destination", "")
Expand All@@ -57,12 +61,42 @@ def _read_csv(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_csv(os.path.join(edb_directory, filename), **kwargs)

 def _read_feather(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_feather(os.path.join(edb_directory, filename), **kwargs)

 def _to_feather(df, filename, **kwargs):
 filename = filename.format(name=name)
 return df.to_feather(os.path.join(edb_directory, filename), **kwargs)

 def _read_pickle(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_pickle(os.path.join(edb_directory, filename))

 def _to_pickle(df, filename, **kwargs):
 filename = filename.format(name=name)
 return df.to_pickle(os.path.join(edb_directory, filename))

 def _file_exists(filename):
 filename = filename.format(name=name)
 return os.path.exists(os.path.join(edb_directory, filename))

 coefficients = _read_csv(
 coefficients_file,
 index_col="coefficient_name",
 )
 spec = _read_csv(spec_file, comment="#")
 alt_values = _read_csv(alt_values_file)

 # read alternative values either as csv or feather file
 alt_values_fea_file = alt_values_file.replace(".csv", ".fea")
 if os.path.exists(
 os.path.join(edb_directory, alt_values_fea_file.format(name=name))
 ):
 alt_values = _read_feather(alt_values_fea_file)
 else:
 alt_values = _read_csv(alt_values_file)
 if alt_values_to_feather:
 _to_feather(df=alt_values, filename=alt_values_fea_file)
 chooser_data = _read_csv(chooser_file)
 landuse = _read_csv(landuse_file, index_col="zone_id")
 master_size_spec = _read_csv(size_spec_file)
Expand DownExpand Up@@ -106,6 +140,9 @@ def _read_csv(filename, **kwargs):
 .set_index("segment")
 )
 size_spec = size_spec.loc[:, size_spec.max() > 0]
 assert (
 len(size_spec) > 0
 ), f"Empty size_spec, is model_selector {SIZE_TERM_SELECTOR} in your size term file?"

 size_coef = size_coefficients_from_spec(size_spec)

Expand DownExpand Up@@ -148,7 +185,48 @@ def _read_csv(filename, **kwargs):

 chooser_index_name = chooser_data.columns[0]
 x_co = chooser_data.set_index(chooser_index_name)
 x_ca = cv_to_ca(alt_values.set_index([chooser_index_name, alt_values.columns[1]]))

 def split(a, n):
 k, m = divmod(len(a), n)
 return (a[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n))

 # process x_ca with cv_to_ca with or without chunking
 x_ca_pickle_file = "{name}_x_ca.pkl"
 if chunking_size == None:
 x_ca = cv_to_ca(
 alt_values.set_index([chooser_index_name, alt_values.columns[1]])
 )
 elif _file_exists(x_ca_pickle_file):
 # if pickle file from previous x_ca processing exist, load it to save time
 time_start = datetime.now()
 x_ca = _read_pickle(x_ca_pickle_file)
 print(
 f"x_ca data loaded from {name}_x_ca.fea - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )
 else:
 time_start = datetime.now()
 # calculate num_chunks based on chunking_size (or max number of rows per chunk)
 num_chunks = int(len(alt_values) / chunking_size)
 all_person_ids = list(alt_values["person_id"].unique())
 split_ids = list(split(all_person_ids, num_chunks))
 x_ca_list = []
 i = 0
 for chunk_ids in split_ids:
 alt_values_i = alt_values[alt_values["person_id"].isin(chunk_ids)]
 x_ca_i = cv_to_ca(
 alt_values_i.set_index([chooser_index_name, alt_values_i.columns[1]])
 )
 x_ca_list.append(x_ca_i)
 print(
 f"\rx_ca_i compute done for chunk {i}/{num_chunks} - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )
 i = i + 1
 x_ca = pd.concat(x_ca_list, axis=0)
 # save final x_ca result as pickle file to save time for future data loading
 _to_pickle(df=x_ca, filename=x_ca_pickle_file)
 print(
 f"x_ca compute done - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )

 if CHOOSER_SEGMENT_COLUMN_NAME is not None:
 # label segments with names
Expand DownExpand Up@@ -214,6 +292,9 @@ def _read_csv(filename, **kwargs):
 else:
 av = 1

 assert len(x_co) > 0, "Empty chooser dataframe"
 assert len(x_ca_1) > 0, "Empty alternatives dataframe"

 d = DataFrames(co=x_co, ca=x_ca_1, av=av)

 m = Model(dataservice=d)
Expand DownExpand Up@@ -331,6 +412,14 @@ def workplace_location_model(**kwargs):
 )


def external_workplace_location_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
 name="external_workplace_location",
 **kwargs,
 )


def school_location_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
Expand DownExpand Up@@ -367,6 +456,14 @@ def non_mandatory_tour_destination_model(**kwargs):
 )


def external_non_mandatory_destination_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
 name="external_non_mandatory_destination",
 **kwargs,
 )


def trip_destination_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Location model patch#1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bwentl
merged 5 commits into

from

Nov 2, 2023
 Merged


Location model patch
#1
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
7becd2d

add chunking and caching of alt_values


bwentl May 11, 2023
0e5d3bf

add alt_values_to_feather option


bwentl Jun 23, 2023
425467e

format code with black


bwentl Jun 23, 2023
b8beb66

merging location choice model with BayDAG version


bwentl Nov 1, 2023
4d2bb4b

fix new line


bwentl Nov 2, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


96 changes: 79 additions & 17 deletions
activitysim/abm/models/location_choice.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
)
from activitysim.core.interaction_sample import interaction_sample
from activitysim.core.interaction_sample_simulate import interaction_sample_simulate
from activitysim.core.util import reindex

from .util import estimation
from .util import logsums as logsum
Expand DownExpand Up@@ -138,22 +139,17 @@ def _location_sample(
 logger.info("Running %s with %d persons" % (trace_label, len(choosers.index)))

 sample_size = model_settings["SAMPLE_SIZE"]
 if config.setting("disable_destination_sampling", False) or (
 estimator and estimator.want_unsampled_alternatives
 ):
 # FIXME interaction_sample will return unsampled complete alternatives with probs and pick_count
 logger.info(
 "Estimation mode for %s using unsampled alternatives short_circuit_choices"
 % (trace_label,)
 )
 sample_size = 0
 if estimator:
 sample_size = model_settings.get("ESTIMATION_SAMPLE_SIZE", 0)

 locals_d = {
 "skims": skims,
 "segment_size": segment_name,
 "orig_col_name": skims.orig_key, # added for sharrow flows
 "dest_col_name": skims.dest_key, # added for sharrow flows
 "timeframe": "timeless",
 "reindex": reindex,
 "land_use": inject.get_table("land_use").to_frame(),
 }
 constants = config.get_model_constants(model_settings)
 locals_d.update(constants)
Expand DownExpand Up@@ -470,6 +466,38 @@ def run_location_sample(
 trace_label=trace_label,
 )

 # FIXME temporary code to ensure sampled alternative is in choices for estimation
 # Hack to get shorter run times when you don't care about creating EDB for location choice models
 if estimator:
 # grabbing survey values
 survey_persons = estimation.manager.get_survey_table("persons")
 if "school_location" in trace_label:
 survey_choices = survey_persons["school_zone_id"].reset_index()
 elif ("workplace_location" in trace_label) and ("external" not in trace_label):
 survey_choices = survey_persons["workplace_zone_id"].reset_index()
 else:
 return choices
 survey_choices.columns = ["person_id", "alt_dest"]
 survey_choices = survey_choices[
 survey_choices["person_id"].isin(choices.index)
 & (survey_choices.alt_dest > 0)
 ]
 # merging survey destination into table if not available
 joined_data = survey_choices.merge(
 choices, on=["person_id", "alt_dest"], how="left", indicator=True
 )
 missing_rows = joined_data[joined_data["_merge"] == "left_only"]
 missing_rows["pick_count"] = 1
 if len(missing_rows) > 0:
 new_choices = missing_rows[
 ["person_id", "alt_dest", "prob", "pick_count"]
 ].set_index("person_id")
 choices = choices.append(new_choices, ignore_index=False).sort_index()
 # making probability the mean of all other sampled destinations by person
 choices["prob"] = choices["prob"].fillna(
 choices.groupby("person_id")["prob"].transform("mean")
 )

 return choices


Expand DownExpand Up@@ -601,6 +629,8 @@ def run_location_simulate(
 "orig_col_name": skims.orig_key, # added for sharrow flows
 "dest_col_name": skims.dest_key, # added for sharrow flows
 "timeframe": "timeless",
 "reindex": reindex,
 "land_use": inject.get_table("land_use").to_frame(),
 }
 constants = config.get_model_constants(model_settings)
 if constants is not None:
Expand DownExpand Up@@ -808,6 +838,24 @@ def run_location_choice(
 )
 tracing.trace_df(choices_df, estimation_trace_label)

 if want_logsums & (not skip_choice):
 # grabbing index, could be person_id or proto_person_id
 index_name = choices_df.index.name
 # merging mode choice logsum of chosen alternative to choices
 choices_df = (
 pd.merge(
 choices_df.reset_index(),
 location_sample_df.reset_index()[
 [index_name, model_settings["ALT_DEST_COL_NAME"], ALT_LOGSUM]
 ],
 how="left",
 left_on=[index_name, "choice"],
 right_on=[index_name, model_settings["ALT_DEST_COL_NAME"]],
 )
 .drop(columns=model_settings["ALT_DEST_COL_NAME"])
 .set_index(index_name)
 )

 choices_list.append(choices_df)

 if want_sample_table:
Expand All@@ -825,7 +873,7 @@ def run_location_choice(
 else:
 # this will only happen with small samples (e.g. singleton) with no (e.g.) school segs
 logger.warning("%s no choices", trace_label)
 choices_df = pd.DataFrame(columns=["choice", "logsum"])
 choices_df = pd.DataFrame(columns=["choice", "logsum", ALT_LOGSUM])

 if len(sample_list) > 0:
 save_sample_df = pd.concat(sample_list)
Expand DownExpand Up@@ -869,7 +917,8 @@ def iterate_location_choice(
 Returns
 -------
 adds choice column model_settings['DEST_CHOICE_COLUMN_NAME']
 adds logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds destination choice logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds mode choice logsum to selected destination column model_settings['MODE_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds annotations to persons table
 """

Expand All@@ -879,7 +928,11 @@ def iterate_location_choice(
 chooser_filter_column = model_settings["CHOOSER_FILTER_COLUMN_NAME"]

 dest_choice_column_name = model_settings["DEST_CHOICE_COLUMN_NAME"]
 logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
 dc_logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
 mc_logsum_column_name = model_settings.get("MODE_CHOICE_LOGSUM_COLUMN_NAME")
 want_logsums = (dc_logsum_column_name is not None) | (
 mc_logsum_column_name is not None
 )

 sample_table_name = model_settings.get("DEST_CHOICE_SAMPLE_TABLE_NAME")
 want_sample_table = (
Expand DownExpand Up@@ -929,7 +982,7 @@ def iterate_location_choice(
 persons_merged_df_,
 network_los,
 shadow_price_calculator=spc,
 want_logsums=logsum_column_name is not None,
 want_logsums=want_logsums,
 want_sample_table=want_sample_table,
 estimator=estimator,
 model_settings=model_settings,
Expand DownExpand Up@@ -1005,10 +1058,15 @@ def iterate_location_choice(
 )

 # add the dest_choice_logsum column to persons dataframe
 if logsum_column_name:
 persons_df[logsum_column_name] = (
 if dc_logsum_column_name:
 persons_df[dc_logsum_column_name] = (
 choices_df["logsum"].reindex(persons_df.index).astype("float")
 )
 # add the mode choice logsum column to persons dataframe
 if mc_logsum_column_name:
 persons_df[mc_logsum_column_name] = (
 choices_df[ALT_LOGSUM].reindex(persons_df.index).astype("float")
 )

 if save_sample_df is not None:
 # might be None for tiny samples even if sample_table_name was specified
Expand DownExpand Up@@ -1047,9 +1105,13 @@ def iterate_location_choice(
 if trace_hh_id:
 tracing.trace_df(households_df, label=trace_label, warn_if_empty=True)

 if logsum_column_name:
 if dc_logsum_column_name:
 tracing.print_summary(
 dc_logsum_column_name, choices_df["logsum"], value_counts=True
 )
 if mc_logsum_column_name:
 tracing.print_summary(
logsum_column_name, choices_df["logsum"], value_counts=True
mc_logsum_column_name, choices_df[ALT_LOGSUM], value_counts=True
 )

 return persons_df
Expand Down

101 changes: 99 additions & 2 deletions
activitysim/estimation/larch/location_choice.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
import os
from pathlib import Path
from typing import Collection
import pickle
from datetime import datetime

import numpy as np
import pandas as pd
Expand DownExpand Up@@ -44,6 +46,8 @@ def location_choice_model(
 settings_file="{name}_model_settings.yaml",
 landuse_file="{name}_landuse.csv",
 return_data=False,
 alt_values_to_feather=False,
 chunking_size=None,
):
 model_selector = name.replace("_location", "")
 model_selector = model_selector.replace("_destination", "")
Expand All@@ -57,12 +61,42 @@ def _read_csv(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_csv(os.path.join(edb_directory, filename), **kwargs)

 def _read_feather(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_feather(os.path.join(edb_directory, filename), **kwargs)

 def _to_feather(df, filename, **kwargs):
 filename = filename.format(name=name)
 return df.to_feather(os.path.join(edb_directory, filename), **kwargs)

 def _read_pickle(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_pickle(os.path.join(edb_directory, filename))

 def _to_pickle(df, filename, **kwargs):
 filename = filename.format(name=name)
 return df.to_pickle(os.path.join(edb_directory, filename))

 def _file_exists(filename):
 filename = filename.format(name=name)
 return os.path.exists(os.path.join(edb_directory, filename))

 coefficients = _read_csv(
 coefficients_file,
 index_col="coefficient_name",
 )
 spec = _read_csv(spec_file, comment="#")
 alt_values = _read_csv(alt_values_file)

 # read alternative values either as csv or feather file
 alt_values_fea_file = alt_values_file.replace(".csv", ".fea")
 if os.path.exists(
 os.path.join(edb_directory, alt_values_fea_file.format(name=name))
 ):
 alt_values = _read_feather(alt_values_fea_file)
 else:
 alt_values = _read_csv(alt_values_file)
 if alt_values_to_feather:
 _to_feather(df=alt_values, filename=alt_values_fea_file)
 chooser_data = _read_csv(chooser_file)
 landuse = _read_csv(landuse_file, index_col="zone_id")
 master_size_spec = _read_csv(size_spec_file)
Expand DownExpand Up@@ -106,6 +140,9 @@ def _read_csv(filename, **kwargs):
 .set_index("segment")
 )
 size_spec = size_spec.loc[:, size_spec.max() > 0]
 assert (
 len(size_spec) > 0
 ), f"Empty size_spec, is model_selector {SIZE_TERM_SELECTOR} in your size term file?"

 size_coef = size_coefficients_from_spec(size_spec)

Expand DownExpand Up@@ -148,7 +185,48 @@ def _read_csv(filename, **kwargs):

 chooser_index_name = chooser_data.columns[0]
 x_co = chooser_data.set_index(chooser_index_name)
 x_ca = cv_to_ca(alt_values.set_index([chooser_index_name, alt_values.columns[1]]))

 def split(a, n):
 k, m = divmod(len(a), n)
 return (a[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n))

 # process x_ca with cv_to_ca with or without chunking
 x_ca_pickle_file = "{name}_x_ca.pkl"
 if chunking_size == None:
 x_ca = cv_to_ca(
 alt_values.set_index([chooser_index_name, alt_values.columns[1]])
 )
 elif _file_exists(x_ca_pickle_file):
 # if pickle file from previous x_ca processing exist, load it to save time
 time_start = datetime.now()
 x_ca = _read_pickle(x_ca_pickle_file)
 print(
 f"x_ca data loaded from {name}_x_ca.fea - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )
 else:
 time_start = datetime.now()
 # calculate num_chunks based on chunking_size (or max number of rows per chunk)
 num_chunks = int(len(alt_values) / chunking_size)
 all_person_ids = list(alt_values["person_id"].unique())
 split_ids = list(split(all_person_ids, num_chunks))
 x_ca_list = []
 i = 0
 for chunk_ids in split_ids:
 alt_values_i = alt_values[alt_values["person_id"].isin(chunk_ids)]
 x_ca_i = cv_to_ca(
 alt_values_i.set_index([chooser_index_name, alt_values_i.columns[1]])
 )
 x_ca_list.append(x_ca_i)
 print(
 f"\rx_ca_i compute done for chunk {i}/{num_chunks} - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )
 i = i + 1
 x_ca = pd.concat(x_ca_list, axis=0)
 # save final x_ca result as pickle file to save time for future data loading
 _to_pickle(df=x_ca, filename=x_ca_pickle_file)
 print(
 f"x_ca compute done - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )

 if CHOOSER_SEGMENT_COLUMN_NAME is not None:
 # label segments with names
Expand DownExpand Up@@ -214,6 +292,9 @@ def _read_csv(filename, **kwargs):
 else:
 av = 1

 assert len(x_co) > 0, "Empty chooser dataframe"
 assert len(x_ca_1) > 0, "Empty alternatives dataframe"

 d = DataFrames(co=x_co, ca=x_ca_1, av=av)

 m = Model(dataservice=d)
Expand DownExpand Up@@ -331,6 +412,14 @@ def workplace_location_model(**kwargs):
 )


def external_workplace_location_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
 name="external_workplace_location",
 **kwargs,
 )


def school_location_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
Expand DownExpand Up@@ -367,6 +456,14 @@ def non_mandatory_tour_destination_model(**kwargs):
 )


def external_non_mandatory_destination_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
 name="external_non_mandatory_destination",
 **kwargs,
 )


def trip_destination_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Location model patch#1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bwentl
merged 5 commits into

from

Nov 2, 2023
 Merged


Location model patch
#1
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
7becd2d

add chunking and caching of alt_values


bwentl May 11, 2023
0e5d3bf

add alt_values_to_feather option


bwentl Jun 23, 2023
425467e

format code with black


bwentl Jun 23, 2023
b8beb66

merging location choice model with BayDAG version


bwentl Nov 1, 2023
4d2bb4b

fix new line


bwentl Nov 2, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


96 changes: 79 additions & 17 deletions
activitysim/abm/models/location_choice.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
)
from activitysim.core.interaction_sample import interaction_sample
from activitysim.core.interaction_sample_simulate import interaction_sample_simulate
from activitysim.core.util import reindex

from .util import estimation
from .util import logsums as logsum
Expand DownExpand Up@@ -138,22 +139,17 @@ def _location_sample(
 logger.info("Running %s with %d persons" % (trace_label, len(choosers.index)))

 sample_size = model_settings["SAMPLE_SIZE"]
 if config.setting("disable_destination_sampling", False) or (
 estimator and estimator.want_unsampled_alternatives
 ):
 # FIXME interaction_sample will return unsampled complete alternatives with probs and pick_count
 logger.info(
 "Estimation mode for %s using unsampled alternatives short_circuit_choices"
 % (trace_label,)
 )
 sample_size = 0
 if estimator:
 sample_size = model_settings.get("ESTIMATION_SAMPLE_SIZE", 0)

 locals_d = {
 "skims": skims,
 "segment_size": segment_name,
 "orig_col_name": skims.orig_key, # added for sharrow flows
 "dest_col_name": skims.dest_key, # added for sharrow flows
 "timeframe": "timeless",
 "reindex": reindex,
 "land_use": inject.get_table("land_use").to_frame(),
 }
 constants = config.get_model_constants(model_settings)
 locals_d.update(constants)
Expand DownExpand Up@@ -470,6 +466,38 @@ def run_location_sample(
 trace_label=trace_label,
 )

 # FIXME temporary code to ensure sampled alternative is in choices for estimation
 # Hack to get shorter run times when you don't care about creating EDB for location choice models
 if estimator:
 # grabbing survey values
 survey_persons = estimation.manager.get_survey_table("persons")
 if "school_location" in trace_label:
 survey_choices = survey_persons["school_zone_id"].reset_index()
 elif ("workplace_location" in trace_label) and ("external" not in trace_label):
 survey_choices = survey_persons["workplace_zone_id"].reset_index()
 else:
 return choices
 survey_choices.columns = ["person_id", "alt_dest"]
 survey_choices = survey_choices[
 survey_choices["person_id"].isin(choices.index)
 & (survey_choices.alt_dest > 0)
 ]
 # merging survey destination into table if not available
 joined_data = survey_choices.merge(
 choices, on=["person_id", "alt_dest"], how="left", indicator=True
 )
 missing_rows = joined_data[joined_data["_merge"] == "left_only"]
 missing_rows["pick_count"] = 1
 if len(missing_rows) > 0:
 new_choices = missing_rows[
 ["person_id", "alt_dest", "prob", "pick_count"]
 ].set_index("person_id")
 choices = choices.append(new_choices, ignore_index=False).sort_index()
 # making probability the mean of all other sampled destinations by person
 choices["prob"] = choices["prob"].fillna(
 choices.groupby("person_id")["prob"].transform("mean")
 )

 return choices


Expand DownExpand Up@@ -601,6 +629,8 @@ def run_location_simulate(
 "orig_col_name": skims.orig_key, # added for sharrow flows
 "dest_col_name": skims.dest_key, # added for sharrow flows
 "timeframe": "timeless",
 "reindex": reindex,
 "land_use": inject.get_table("land_use").to_frame(),
 }
 constants = config.get_model_constants(model_settings)
 if constants is not None:
Expand DownExpand Up@@ -808,6 +838,24 @@ def run_location_choice(
 )
 tracing.trace_df(choices_df, estimation_trace_label)

 if want_logsums & (not skip_choice):
 # grabbing index, could be person_id or proto_person_id
 index_name = choices_df.index.name
 # merging mode choice logsum of chosen alternative to choices
 choices_df = (
 pd.merge(
 choices_df.reset_index(),
 location_sample_df.reset_index()[
 [index_name, model_settings["ALT_DEST_COL_NAME"], ALT_LOGSUM]
 ],
 how="left",
 left_on=[index_name, "choice"],
 right_on=[index_name, model_settings["ALT_DEST_COL_NAME"]],
 )
 .drop(columns=model_settings["ALT_DEST_COL_NAME"])
 .set_index(index_name)
 )

 choices_list.append(choices_df)

 if want_sample_table:
Expand All@@ -825,7 +873,7 @@ def run_location_choice(
 else:
 # this will only happen with small samples (e.g. singleton) with no (e.g.) school segs
 logger.warning("%s no choices", trace_label)
 choices_df = pd.DataFrame(columns=["choice", "logsum"])
 choices_df = pd.DataFrame(columns=["choice", "logsum", ALT_LOGSUM])

 if len(sample_list) > 0:
 save_sample_df = pd.concat(sample_list)
Expand DownExpand Up@@ -869,7 +917,8 @@ def iterate_location_choice(
 Returns
 -------
 adds choice column model_settings['DEST_CHOICE_COLUMN_NAME']
 adds logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds destination choice logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds mode choice logsum to selected destination column model_settings['MODE_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds annotations to persons table
 """

Expand All@@ -879,7 +928,11 @@ def iterate_location_choice(
 chooser_filter_column = model_settings["CHOOSER_FILTER_COLUMN_NAME"]

 dest_choice_column_name = model_settings["DEST_CHOICE_COLUMN_NAME"]
 logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
 dc_logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
 mc_logsum_column_name = model_settings.get("MODE_CHOICE_LOGSUM_COLUMN_NAME")
 want_logsums = (dc_logsum_column_name is not None) | (
 mc_logsum_column_name is not None
 )

 sample_table_name = model_settings.get("DEST_CHOICE_SAMPLE_TABLE_NAME")
 want_sample_table = (
Expand DownExpand Up@@ -929,7 +982,7 @@ def iterate_location_choice(
 persons_merged_df_,
 network_los,
 shadow_price_calculator=spc,
 want_logsums=logsum_column_name is not None,
 want_logsums=want_logsums,
 want_sample_table=want_sample_table,
 estimator=estimator,
 model_settings=model_settings,
Expand DownExpand Up@@ -1005,10 +1058,15 @@ def iterate_location_choice(
 )

 # add the dest_choice_logsum column to persons dataframe
 if logsum_column_name:
 persons_df[logsum_column_name] = (
 if dc_logsum_column_name:
 persons_df[dc_logsum_column_name] = (
 choices_df["logsum"].reindex(persons_df.index).astype("float")
 )
 # add the mode choice logsum column to persons dataframe
 if mc_logsum_column_name:
 persons_df[mc_logsum_column_name] = (
 choices_df[ALT_LOGSUM].reindex(persons_df.index).astype("float")
 )

 if save_sample_df is not None:
 # might be None for tiny samples even if sample_table_name was specified
Expand DownExpand Up@@ -1047,9 +1105,13 @@ def iterate_location_choice(
 if trace_hh_id:
 tracing.trace_df(households_df, label=trace_label, warn_if_empty=True)

 if logsum_column_name:
 if dc_logsum_column_name:
 tracing.print_summary(
 dc_logsum_column_name, choices_df["logsum"], value_counts=True
 )
 if mc_logsum_column_name:
 tracing.print_summary(
logsum_column_name, choices_df["logsum"], value_counts=True
mc_logsum_column_name, choices_df[ALT_LOGSUM], value_counts=True
 )

 return persons_df
Expand Down

101 changes: 99 additions & 2 deletions
activitysim/estimation/larch/location_choice.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
import os
from pathlib import Path
from typing import Collection
import pickle
from datetime import datetime

import numpy as np
import pandas as pd
Expand DownExpand Up@@ -44,6 +46,8 @@ def location_choice_model(
 settings_file="{name}_model_settings.yaml",
 landuse_file="{name}_landuse.csv",
 return_data=False,
 alt_values_to_feather=False,
 chunking_size=None,
):
 model_selector = name.replace("_location", "")
 model_selector = model_selector.replace("_destination", "")
Expand All@@ -57,12 +61,42 @@ def _read_csv(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_csv(os.path.join(edb_directory, filename), **kwargs)

 def _read_feather(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_feather(os.path.join(edb_directory, filename), **kwargs)

 def _to_feather(df, filename, **kwargs):
 filename = filename.format(name=name)
 return df.to_feather(os.path.join(edb_directory, filename), **kwargs)

 def _read_pickle(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_pickle(os.path.join(edb_directory, filename))

 def _to_pickle(df, filename, **kwargs):
 filename = filename.format(name=name)
 return df.to_pickle(os.path.join(edb_directory, filename))

 def _file_exists(filename):
 filename = filename.format(name=name)
 return os.path.exists(os.path.join(edb_directory, filename))

 coefficients = _read_csv(
 coefficients_file,
 index_col="coefficient_name",
 )
 spec = _read_csv(spec_file, comment="#")
 alt_values = _read_csv(alt_values_file)

 # read alternative values either as csv or feather file
 alt_values_fea_file = alt_values_file.replace(".csv", ".fea")
 if os.path.exists(
 os.path.join(edb_directory, alt_values_fea_file.format(name=name))
 ):
 alt_values = _read_feather(alt_values_fea_file)
 else:
 alt_values = _read_csv(alt_values_file)
 if alt_values_to_feather:
 _to_feather(df=alt_values, filename=alt_values_fea_file)
 chooser_data = _read_csv(chooser_file)
 landuse = _read_csv(landuse_file, index_col="zone_id")
 master_size_spec = _read_csv(size_spec_file)
Expand DownExpand Up@@ -106,6 +140,9 @@ def _read_csv(filename, **kwargs):
 .set_index("segment")
 )
 size_spec = size_spec.loc[:, size_spec.max() > 0]
 assert (
 len(size_spec) > 0
 ), f"Empty size_spec, is model_selector {SIZE_TERM_SELECTOR} in your size term file?"

 size_coef = size_coefficients_from_spec(size_spec)

Expand DownExpand Up@@ -148,7 +185,48 @@ def _read_csv(filename, **kwargs):

 chooser_index_name = chooser_data.columns[0]
 x_co = chooser_data.set_index(chooser_index_name)
 x_ca = cv_to_ca(alt_values.set_index([chooser_index_name, alt_values.columns[1]]))

 def split(a, n):
 k, m = divmod(len(a), n)
 return (a[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n))

 # process x_ca with cv_to_ca with or without chunking
 x_ca_pickle_file = "{name}_x_ca.pkl"
 if chunking_size == None:
 x_ca = cv_to_ca(
 alt_values.set_index([chooser_index_name, alt_values.columns[1]])
 )
 elif _file_exists(x_ca_pickle_file):
 # if pickle file from previous x_ca processing exist, load it to save time
 time_start = datetime.now()
 x_ca = _read_pickle(x_ca_pickle_file)
 print(
 f"x_ca data loaded from {name}_x_ca.fea - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )
 else:
 time_start = datetime.now()
 # calculate num_chunks based on chunking_size (or max number of rows per chunk)
 num_chunks = int(len(alt_values) / chunking_size)
 all_person_ids = list(alt_values["person_id"].unique())
 split_ids = list(split(all_person_ids, num_chunks))
 x_ca_list = []
 i = 0
 for chunk_ids in split_ids:
 alt_values_i = alt_values[alt_values["person_id"].isin(chunk_ids)]
 x_ca_i = cv_to_ca(
 alt_values_i.set_index([chooser_index_name, alt_values_i.columns[1]])
 )
 x_ca_list.append(x_ca_i)
 print(
 f"\rx_ca_i compute done for chunk {i}/{num_chunks} - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )
 i = i + 1
 x_ca = pd.concat(x_ca_list, axis=0)
 # save final x_ca result as pickle file to save time for future data loading
 _to_pickle(df=x_ca, filename=x_ca_pickle_file)
 print(
 f"x_ca compute done - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )

 if CHOOSER_SEGMENT_COLUMN_NAME is not None:
 # label segments with names
Expand DownExpand Up@@ -214,6 +292,9 @@ def _read_csv(filename, **kwargs):
 else:
 av = 1

 assert len(x_co) > 0, "Empty chooser dataframe"
 assert len(x_ca_1) > 0, "Empty alternatives dataframe"

 d = DataFrames(co=x_co, ca=x_ca_1, av=av)

 m = Model(dataservice=d)
Expand DownExpand Up@@ -331,6 +412,14 @@ def workplace_location_model(**kwargs):
 )


def external_workplace_location_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
 name="external_workplace_location",
 **kwargs,
 )


def school_location_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
Expand DownExpand Up@@ -367,6 +456,14 @@ def non_mandatory_tour_destination_model(**kwargs):
 )


def external_non_mandatory_destination_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
 name="external_non_mandatory_destination",
 **kwargs,
 )


def trip_destination_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Location model patch#1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bwentl
merged 5 commits into

from

Nov 2, 2023
 Merged


Location model patch
#1
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
7becd2d

add chunking and caching of alt_values


bwentl May 11, 2023
0e5d3bf

add alt_values_to_feather option


bwentl Jun 23, 2023
425467e

format code with black


bwentl Jun 23, 2023
b8beb66

merging location choice model with BayDAG version


bwentl Nov 1, 2023
4d2bb4b

fix new line


bwentl Nov 2, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


96 changes: 79 additions & 17 deletions
activitysim/abm/models/location_choice.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@
)
from activitysim.core.interaction_sample import interaction_sample
from activitysim.core.interaction_sample_simulate import interaction_sample_simulate
from activitysim.core.util import reindex

from .util import estimation
from .util import logsums as logsum
Expand DownExpand Up@@ -138,22 +139,17 @@ def _location_sample(
 logger.info("Running %s with %d persons" % (trace_label, len(choosers.index)))

 sample_size = model_settings["SAMPLE_SIZE"]
 if config.setting("disable_destination_sampling", False) or (
 estimator and estimator.want_unsampled_alternatives
 ):
 # FIXME interaction_sample will return unsampled complete alternatives with probs and pick_count
 logger.info(
 "Estimation mode for %s using unsampled alternatives short_circuit_choices"
 % (trace_label,)
 )
 sample_size = 0
 if estimator:
 sample_size = model_settings.get("ESTIMATION_SAMPLE_SIZE", 0)

 locals_d = {
 "skims": skims,
 "segment_size": segment_name,
 "orig_col_name": skims.orig_key, # added for sharrow flows
 "dest_col_name": skims.dest_key, # added for sharrow flows
 "timeframe": "timeless",
 "reindex": reindex,
 "land_use": inject.get_table("land_use").to_frame(),
 }
 constants = config.get_model_constants(model_settings)
 locals_d.update(constants)
Expand DownExpand Up@@ -470,6 +466,38 @@ def run_location_sample(
 trace_label=trace_label,
 )

 # FIXME temporary code to ensure sampled alternative is in choices for estimation
 # Hack to get shorter run times when you don't care about creating EDB for location choice models
 if estimator:
 # grabbing survey values
 survey_persons = estimation.manager.get_survey_table("persons")
 if "school_location" in trace_label:
 survey_choices = survey_persons["school_zone_id"].reset_index()
 elif ("workplace_location" in trace_label) and ("external" not in trace_label):
 survey_choices = survey_persons["workplace_zone_id"].reset_index()
 else:
 return choices
 survey_choices.columns = ["person_id", "alt_dest"]
 survey_choices = survey_choices[
 survey_choices["person_id"].isin(choices.index)
 & (survey_choices.alt_dest > 0)
 ]
 # merging survey destination into table if not available
 joined_data = survey_choices.merge(
 choices, on=["person_id", "alt_dest"], how="left", indicator=True
 )
 missing_rows = joined_data[joined_data["_merge"] == "left_only"]
 missing_rows["pick_count"] = 1
 if len(missing_rows) > 0:
 new_choices = missing_rows[
 ["person_id", "alt_dest", "prob", "pick_count"]
 ].set_index("person_id")
 choices = choices.append(new_choices, ignore_index=False).sort_index()
 # making probability the mean of all other sampled destinations by person
 choices["prob"] = choices["prob"].fillna(
 choices.groupby("person_id")["prob"].transform("mean")
 )

 return choices


Expand DownExpand Up@@ -601,6 +629,8 @@ def run_location_simulate(
 "orig_col_name": skims.orig_key, # added for sharrow flows
 "dest_col_name": skims.dest_key, # added for sharrow flows
 "timeframe": "timeless",
 "reindex": reindex,
 "land_use": inject.get_table("land_use").to_frame(),
 }
 constants = config.get_model_constants(model_settings)
 if constants is not None:
Expand DownExpand Up@@ -808,6 +838,24 @@ def run_location_choice(
 )
 tracing.trace_df(choices_df, estimation_trace_label)

 if want_logsums & (not skip_choice):
 # grabbing index, could be person_id or proto_person_id
 index_name = choices_df.index.name
 # merging mode choice logsum of chosen alternative to choices
 choices_df = (
 pd.merge(
 choices_df.reset_index(),
 location_sample_df.reset_index()[
 [index_name, model_settings["ALT_DEST_COL_NAME"], ALT_LOGSUM]
 ],
 how="left",
 left_on=[index_name, "choice"],
 right_on=[index_name, model_settings["ALT_DEST_COL_NAME"]],
 )
 .drop(columns=model_settings["ALT_DEST_COL_NAME"])
 .set_index(index_name)
 )

 choices_list.append(choices_df)

 if want_sample_table:
Expand All@@ -825,7 +873,7 @@ def run_location_choice(
 else:
 # this will only happen with small samples (e.g. singleton) with no (e.g.) school segs
 logger.warning("%s no choices", trace_label)
 choices_df = pd.DataFrame(columns=["choice", "logsum"])
 choices_df = pd.DataFrame(columns=["choice", "logsum", ALT_LOGSUM])

 if len(sample_list) > 0:
 save_sample_df = pd.concat(sample_list)
Expand DownExpand Up@@ -869,7 +917,8 @@ def iterate_location_choice(
 Returns
 -------
 adds choice column model_settings['DEST_CHOICE_COLUMN_NAME']
 adds logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds destination choice logsum column model_settings['DEST_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds mode choice logsum to selected destination column model_settings['MODE_CHOICE_LOGSUM_COLUMN_NAME']- if provided
 adds annotations to persons table
 """

Expand All@@ -879,7 +928,11 @@ def iterate_location_choice(
 chooser_filter_column = model_settings["CHOOSER_FILTER_COLUMN_NAME"]

 dest_choice_column_name = model_settings["DEST_CHOICE_COLUMN_NAME"]
 logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
 dc_logsum_column_name = model_settings.get("DEST_CHOICE_LOGSUM_COLUMN_NAME")
 mc_logsum_column_name = model_settings.get("MODE_CHOICE_LOGSUM_COLUMN_NAME")
 want_logsums = (dc_logsum_column_name is not None) | (
 mc_logsum_column_name is not None
 )

 sample_table_name = model_settings.get("DEST_CHOICE_SAMPLE_TABLE_NAME")
 want_sample_table = (
Expand DownExpand Up@@ -929,7 +982,7 @@ def iterate_location_choice(
 persons_merged_df_,
 network_los,
 shadow_price_calculator=spc,
 want_logsums=logsum_column_name is not None,
 want_logsums=want_logsums,
 want_sample_table=want_sample_table,
 estimator=estimator,
 model_settings=model_settings,
Expand DownExpand Up@@ -1005,10 +1058,15 @@ def iterate_location_choice(
 )

 # add the dest_choice_logsum column to persons dataframe
 if logsum_column_name:
 persons_df[logsum_column_name] = (
 if dc_logsum_column_name:
 persons_df[dc_logsum_column_name] = (
 choices_df["logsum"].reindex(persons_df.index).astype("float")
 )
 # add the mode choice logsum column to persons dataframe
 if mc_logsum_column_name:
 persons_df[mc_logsum_column_name] = (
 choices_df[ALT_LOGSUM].reindex(persons_df.index).astype("float")
 )

 if save_sample_df is not None:
 # might be None for tiny samples even if sample_table_name was specified
Expand DownExpand Up@@ -1047,9 +1105,13 @@ def iterate_location_choice(
 if trace_hh_id:
 tracing.trace_df(households_df, label=trace_label, warn_if_empty=True)

 if logsum_column_name:
 if dc_logsum_column_name:
 tracing.print_summary(
 dc_logsum_column_name, choices_df["logsum"], value_counts=True
 )
 if mc_logsum_column_name:
 tracing.print_summary(
logsum_column_name, choices_df["logsum"], value_counts=True
mc_logsum_column_name, choices_df[ALT_LOGSUM], value_counts=True
 )

 return persons_df
Expand Down

101 changes: 99 additions & 2 deletions
activitysim/estimation/larch/location_choice.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
import os
from pathlib import Path
from typing import Collection
import pickle
from datetime import datetime

import numpy as np
import pandas as pd
Expand DownExpand Up@@ -44,6 +46,8 @@ def location_choice_model(
 settings_file="{name}_model_settings.yaml",
 landuse_file="{name}_landuse.csv",
 return_data=False,
 alt_values_to_feather=False,
 chunking_size=None,
):
 model_selector = name.replace("_location", "")
 model_selector = model_selector.replace("_destination", "")
Expand All@@ -57,12 +61,42 @@ def _read_csv(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_csv(os.path.join(edb_directory, filename), **kwargs)

 def _read_feather(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_feather(os.path.join(edb_directory, filename), **kwargs)

 def _to_feather(df, filename, **kwargs):
 filename = filename.format(name=name)
 return df.to_feather(os.path.join(edb_directory, filename), **kwargs)

 def _read_pickle(filename, **kwargs):
 filename = filename.format(name=name)
 return pd.read_pickle(os.path.join(edb_directory, filename))

 def _to_pickle(df, filename, **kwargs):
 filename = filename.format(name=name)
 return df.to_pickle(os.path.join(edb_directory, filename))

 def _file_exists(filename):
 filename = filename.format(name=name)
 return os.path.exists(os.path.join(edb_directory, filename))

 coefficients = _read_csv(
 coefficients_file,
 index_col="coefficient_name",
 )
 spec = _read_csv(spec_file, comment="#")
 alt_values = _read_csv(alt_values_file)

 # read alternative values either as csv or feather file
 alt_values_fea_file = alt_values_file.replace(".csv", ".fea")
 if os.path.exists(
 os.path.join(edb_directory, alt_values_fea_file.format(name=name))
 ):
 alt_values = _read_feather(alt_values_fea_file)
 else:
 alt_values = _read_csv(alt_values_file)
 if alt_values_to_feather:
 _to_feather(df=alt_values, filename=alt_values_fea_file)
 chooser_data = _read_csv(chooser_file)
 landuse = _read_csv(landuse_file, index_col="zone_id")
 master_size_spec = _read_csv(size_spec_file)
Expand DownExpand Up@@ -106,6 +140,9 @@ def _read_csv(filename, **kwargs):
 .set_index("segment")
 )
 size_spec = size_spec.loc[:, size_spec.max() > 0]
 assert (
 len(size_spec) > 0
 ), f"Empty size_spec, is model_selector {SIZE_TERM_SELECTOR} in your size term file?"

 size_coef = size_coefficients_from_spec(size_spec)

Expand DownExpand Up@@ -148,7 +185,48 @@ def _read_csv(filename, **kwargs):

 chooser_index_name = chooser_data.columns[0]
 x_co = chooser_data.set_index(chooser_index_name)
 x_ca = cv_to_ca(alt_values.set_index([chooser_index_name, alt_values.columns[1]]))

 def split(a, n):
 k, m = divmod(len(a), n)
 return (a[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n))

 # process x_ca with cv_to_ca with or without chunking
 x_ca_pickle_file = "{name}_x_ca.pkl"
 if chunking_size == None:
 x_ca = cv_to_ca(
 alt_values.set_index([chooser_index_name, alt_values.columns[1]])
 )
 elif _file_exists(x_ca_pickle_file):
 # if pickle file from previous x_ca processing exist, load it to save time
 time_start = datetime.now()
 x_ca = _read_pickle(x_ca_pickle_file)
 print(
 f"x_ca data loaded from {name}_x_ca.fea - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )
 else:
 time_start = datetime.now()
 # calculate num_chunks based on chunking_size (or max number of rows per chunk)
 num_chunks = int(len(alt_values) / chunking_size)
 all_person_ids = list(alt_values["person_id"].unique())
 split_ids = list(split(all_person_ids, num_chunks))
 x_ca_list = []
 i = 0
 for chunk_ids in split_ids:
 alt_values_i = alt_values[alt_values["person_id"].isin(chunk_ids)]
 x_ca_i = cv_to_ca(
 alt_values_i.set_index([chooser_index_name, alt_values_i.columns[1]])
 )
 x_ca_list.append(x_ca_i)
 print(
 f"\rx_ca_i compute done for chunk {i}/{num_chunks} - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )
 i = i + 1
 x_ca = pd.concat(x_ca_list, axis=0)
 # save final x_ca result as pickle file to save time for future data loading
 _to_pickle(df=x_ca, filename=x_ca_pickle_file)
 print(
 f"x_ca compute done - time elapsed {(datetime.now() - time_start).total_seconds()}"
 )

 if CHOOSER_SEGMENT_COLUMN_NAME is not None:
 # label segments with names
Expand DownExpand Up@@ -214,6 +292,9 @@ def _read_csv(filename, **kwargs):
 else:
 av = 1

 assert len(x_co) > 0, "Empty chooser dataframe"
 assert len(x_ca_1) > 0, "Empty alternatives dataframe"

 d = DataFrames(co=x_co, ca=x_ca_1, av=av)

 m = Model(dataservice=d)
Expand DownExpand Up@@ -331,6 +412,14 @@ def workplace_location_model(**kwargs):
 )


def external_workplace_location_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
 name="external_workplace_location",
 **kwargs,
 )


def school_location_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
Expand DownExpand Up@@ -367,6 +456,14 @@ def non_mandatory_tour_destination_model(**kwargs):
 )


def external_non_mandatory_destination_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
 name="external_non_mandatory_destination",
 **kwargs,
 )


def trip_destination_model(**kwargs):
 unused = kwargs.pop("name", None)
 return location_choice_model(
Expand Down