BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging#768

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jpn-- merged 4 commits into from
Mar 28, 2024
Merged

BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging #768

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ea25a66
get all disaggregate accessibility values
dhensle Dec 16, 2023
1670dc9
updated settings to work with Pydantic
dhensle Mar 8, 2024
c6803bf
KEEP_COLS setting
dhensle Mar 19, 2024
ab20012
keep_cols update and returning tables
dhensle Mar 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions activitysim/abm/models/disaggregate_accessibility.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -154,6 +154,15 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
procedure work.
"""

KEEP_COLS: list[str] | None = None
"""
Disaggreate accessibility table is grouped by the "by" cols above and the KEEP_COLS are averaged
across the group. Initializing the below as NA if not in the auto ownership level, they are skipped
in the groupby mean and the values are correct.
(It's a way to avoid having to update code to reshape the table and introduce new functionality there.)
If none, will keep all of the columns with "accessibility" in the name.
"""

FROM_TEMPLATES: bool = False
annotate_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
"""
Expand All@@ -164,6 +173,11 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
"""
NEAREST_METHOD: str = "skims"

postprocess_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
"""
List of preprocessor settings to apply to the proto-population tables after generation.
"""

Comment thread
dhensle marked this conversation as resolved.

def read_disaggregate_accessibility_yaml(
state: workflow.State, file_name
Expand DownExpand Up@@ -846,6 +860,10 @@ def compute_disaggregate_accessibility(
state.tracing.register_traceable_table(tablename, df)
del df

disagg_model_settings = read_disaggregate_accessibility_yaml(
state, "disaggregate_accessibility.yaml"
)

# Run location choice
logsums = get_disaggregate_logsums(
state,
Expand DownExpand Up@@ -906,4 +924,23 @@ def compute_disaggregate_accessibility(
for k, df in logsums.items():
state.add_table(k, df)

# available post-processing
for annotations in disagg_model_settings.postprocess_proto_tables:
tablename = annotations.tablename
df = state.get_dataframe(tablename)
assert df is not None
assert annotations is not None
assign_columns(
state,
df=df,
model_settings={
**annotations.annotate.dict(),
**disagg_model_settings.suffixes.dict(),
},
trace_label=tracing.extend_trace_label(
"disaggregate_accessibility.postprocess", tablename
),
)
state.add_table(tablename, df)

return
30 changes: 15 additions & 15 deletions activitysim/abm/tables/disaggregate_accessibility.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def maz_centroids(state: workflow.State):


@workflow.table
def proto_disaggregate_accessibility(state: workflow.State):
def proto_disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
# Read existing accessibilities, but is not required to enable model compatibility
df = input.read_input_table(
state, "proto_disaggregate_accessibility", required=False
Expand All@@ -130,7 +130,7 @@ def proto_disaggregate_accessibility(state: workflow.State):


@workflow.table
def disaggregate_accessibility(state: workflow.State):
def disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
"""
This step initializes pre-computed disaggregate accessibility and merges it onto the full synthetic population.
Function adds merged all disaggregate accessibility tables to the pipeline but returns nothing.
Expand DownExpand Up@@ -169,17 +169,17 @@ def disaggregate_accessibility(state: workflow.State):
)
merging_params = model_settings.MERGE_ON
nearest_method = model_settings.NEAREST_METHOD
accessibility_cols = [
x for x in proto_accessibility_df.columns if "accessibility" in x
]

if model_settings.KEEP_COLS is None:
keep_cols = [x for x in proto_accessibility_df.columns if "accessibility" in x]
else:
keep_cols = model_settings.KEEP_COLS

# Parse the merging parameters
assert merging_params is not None

# Check if already assigned!
if set(accessibility_cols).intersection(persons_merged_df.columns) == set(
accessibility_cols
):
if set(keep_cols).intersection(persons_merged_df.columns) == set(keep_cols):
return

# Find the nearest zone (spatially) with accessibilities calculated
Expand DownExpand Up@@ -211,7 +211,7 @@ def disaggregate_accessibility(state: workflow.State):
# because it will get slightly different logsums for households in the same zone.
# This is because different destination zones were selected. To resolve, get mean by cols.
right_df = (
proto_accessibility_df.groupby(merge_cols)[accessibility_cols]
proto_accessibility_df.groupby(merge_cols)[keep_cols]
.mean()
.sort_values(nearest_cols)
.reset_index()
Expand DownExpand Up@@ -244,9 +244,9 @@ def disaggregate_accessibility(state: workflow.State):
)

# Predict the nearest person ID and pull the logsums
matched_logsums_df = right_df.loc[clf.predict(x_pop)][
accessibility_cols
].reset_index(drop=True)
matched_logsums_df = right_df.loc[clf.predict(x_pop)][keep_cols].reset_index(
drop=True
)
merge_df = pd.concat(
[left_df.reset_index(drop=False), matched_logsums_df], axis=1
).set_index("person_id")
Expand DownExpand Up@@ -278,9 +278,9 @@ def disaggregate_accessibility(state: workflow.State):

# Check that it was correctly left-joined
assert all(persons_merged_df[merge_cols] == merge_df[merge_cols])
assert any(merge_df[accessibility_cols].isnull())
assert any(merge_df[keep_cols].isnull())

# Inject merged accessibilities so that it can be included in persons_merged function
state.add_table("disaggregate_accessibility", merge_df[accessibility_cols])
state.add_table("disaggregate_accessibility", merge_df[keep_cols])

return merge_df[accessibility_cols]
return merge_df[keep_cols]
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging#768

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jpn--
merged 4 commits into

from

Mar 28, 2024
 Merged


BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging
#768
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
ea25a66

get all disaggregate accessibility values


dhensle Dec 16, 2023
1670dc9

updated settings to work with Pydantic


dhensle Mar 8, 2024
c6803bf

KEEP_COLS setting


dhensle Mar 19, 2024
ab20012

keep_cols update and returning tables


dhensle Mar 27, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


37 changes: 37 additions & 0 deletions
activitysim/abm/models/disaggregate_accessibility.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -154,6 +154,15 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
 procedure work.
 """

 KEEP_COLS: list[str] | None = None
 """
 Disaggreate accessibility table is grouped by the "by" cols above and the KEEP_COLS are averaged
 across the group. Initializing the below as NA if not in the auto ownership level, they are skipped
 in the groupby mean and the values are correct. 
 (It's a way to avoid having to update code to reshape the table and introduce new functionality there.)
 If none, will keep all of the columns with "accessibility" in the name.
 """

 FROM_TEMPLATES: bool = False
 annotate_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
 """
Expand All@@ -164,6 +173,11 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
 """
 NEAREST_METHOD: str = "skims"

 postprocess_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
 """
 List of preprocessor settings to apply to the proto-population tables after generation.
 """

Comment thread
dhensle marked this conversation as resolved.


def read_disaggregate_accessibility_yaml(
 state: workflow.State, file_name
Expand DownExpand Up@@ -846,6 +860,10 @@ def compute_disaggregate_accessibility(
 state.tracing.register_traceable_table(tablename, df)
 del df

 disagg_model_settings = read_disaggregate_accessibility_yaml(
 state, "disaggregate_accessibility.yaml"
 )

 # Run location choice
 logsums = get_disaggregate_logsums(
 state,
Expand DownExpand Up@@ -906,4 +924,23 @@ def compute_disaggregate_accessibility(
 for k, df in logsums.items():
 state.add_table(k, df)

 # available post-processing
 for annotations in disagg_model_settings.postprocess_proto_tables:
 tablename = annotations.tablename
 df = state.get_dataframe(tablename)
 assert df is not None
 assert annotations is not None
 assign_columns(
 state,
 df=df,
 model_settings={
 **annotations.annotate.dict(),
 **disagg_model_settings.suffixes.dict(),
 },
 trace_label=tracing.extend_trace_label(
 "disaggregate_accessibility.postprocess", tablename
 ),
 )
 state.add_table(tablename, df)

 return

30 changes: 15 additions & 15 deletions
activitysim/abm/tables/disaggregate_accessibility.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def maz_centroids(state: workflow.State):


@workflow.table
def proto_disaggregate_accessibility(state: workflow.State):
def proto_disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
 # Read existing accessibilities, but is not required to enable model compatibility
 df = input.read_input_table(
 state, "proto_disaggregate_accessibility", required=False
Expand All@@ -130,7 +130,7 @@ def proto_disaggregate_accessibility(state: workflow.State):


@workflow.table
def disaggregate_accessibility(state: workflow.State):
def disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
 """
 This step initializes pre-computed disaggregate accessibility and merges it onto the full synthetic population.
 Function adds merged all disaggregate accessibility tables to the pipeline but returns nothing.
Expand DownExpand Up@@ -169,17 +169,17 @@ def disaggregate_accessibility(state: workflow.State):
 )
 merging_params = model_settings.MERGE_ON
 nearest_method = model_settings.NEAREST_METHOD
 accessibility_cols = [
 x for x in proto_accessibility_df.columns if "accessibility" in x
 ]

 if model_settings.KEEP_COLS is None:
 keep_cols = [x for x in proto_accessibility_df.columns if "accessibility" in x]
 else:
 keep_cols = model_settings.KEEP_COLS

 # Parse the merging parameters
 assert merging_params is not None

 # Check if already assigned!
 if set(accessibility_cols).intersection(persons_merged_df.columns) == set(
 accessibility_cols
 ):
 if set(keep_cols).intersection(persons_merged_df.columns) == set(keep_cols):
 return

 # Find the nearest zone (spatially) with accessibilities calculated
Expand DownExpand Up@@ -211,7 +211,7 @@ def disaggregate_accessibility(state: workflow.State):
 # because it will get slightly different logsums for households in the same zone.
 # This is because different destination zones were selected. To resolve, get mean by cols.
 right_df = (
 proto_accessibility_df.groupby(merge_cols)[accessibility_cols]
 proto_accessibility_df.groupby(merge_cols)[keep_cols]
 .mean()
 .sort_values(nearest_cols)
 .reset_index()
Expand DownExpand Up@@ -244,9 +244,9 @@ def disaggregate_accessibility(state: workflow.State):
 )

 # Predict the nearest person ID and pull the logsums
 matched_logsums_df = right_df.loc[clf.predict(x_pop)][
accessibility_cols
].reset_index(drop=True)
 matched_logsums_df = right_df.loc[clf.predict(x_pop)][keep_cols].reset_index(
drop=True
 )
 merge_df = pd.concat(
 [left_df.reset_index(drop=False), matched_logsums_df], axis=1
 ).set_index("person_id")
Expand DownExpand Up@@ -278,9 +278,9 @@ def disaggregate_accessibility(state: workflow.State):

 # Check that it was correctly left-joined
 assert all(persons_merged_df[merge_cols] == merge_df[merge_cols])
 assert any(merge_df[accessibility_cols].isnull())
 assert any(merge_df[keep_cols].isnull())

 # Inject merged accessibilities so that it can be included in persons_merged function
 state.add_table("disaggregate_accessibility", merge_df[accessibility_cols])
 state.add_table("disaggregate_accessibility", merge_df[keep_cols])

 return merge_df[accessibility_cols]
 return merge_df[keep_cols]

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging#768

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jpn--
merged 4 commits into

from

Mar 28, 2024
 Merged


BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging
#768
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
ea25a66

get all disaggregate accessibility values


dhensle Dec 16, 2023
1670dc9

updated settings to work with Pydantic


dhensle Mar 8, 2024
c6803bf

KEEP_COLS setting


dhensle Mar 19, 2024
ab20012

keep_cols update and returning tables


dhensle Mar 27, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


37 changes: 37 additions & 0 deletions
activitysim/abm/models/disaggregate_accessibility.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -154,6 +154,15 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
 procedure work.
 """

 KEEP_COLS: list[str] | None = None
 """
 Disaggreate accessibility table is grouped by the "by" cols above and the KEEP_COLS are averaged
 across the group. Initializing the below as NA if not in the auto ownership level, they are skipped
 in the groupby mean and the values are correct. 
 (It's a way to avoid having to update code to reshape the table and introduce new functionality there.)
 If none, will keep all of the columns with "accessibility" in the name.
 """

 FROM_TEMPLATES: bool = False
 annotate_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
 """
Expand All@@ -164,6 +173,11 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
 """
 NEAREST_METHOD: str = "skims"

 postprocess_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
 """
 List of preprocessor settings to apply to the proto-population tables after generation.
 """

Comment thread
dhensle marked this conversation as resolved.


def read_disaggregate_accessibility_yaml(
 state: workflow.State, file_name
Expand DownExpand Up@@ -846,6 +860,10 @@ def compute_disaggregate_accessibility(
 state.tracing.register_traceable_table(tablename, df)
 del df

 disagg_model_settings = read_disaggregate_accessibility_yaml(
 state, "disaggregate_accessibility.yaml"
 )

 # Run location choice
 logsums = get_disaggregate_logsums(
 state,
Expand DownExpand Up@@ -906,4 +924,23 @@ def compute_disaggregate_accessibility(
 for k, df in logsums.items():
 state.add_table(k, df)

 # available post-processing
 for annotations in disagg_model_settings.postprocess_proto_tables:
 tablename = annotations.tablename
 df = state.get_dataframe(tablename)
 assert df is not None
 assert annotations is not None
 assign_columns(
 state,
 df=df,
 model_settings={
 **annotations.annotate.dict(),
 **disagg_model_settings.suffixes.dict(),
 },
 trace_label=tracing.extend_trace_label(
 "disaggregate_accessibility.postprocess", tablename
 ),
 )
 state.add_table(tablename, df)

 return

30 changes: 15 additions & 15 deletions
activitysim/abm/tables/disaggregate_accessibility.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def maz_centroids(state: workflow.State):


@workflow.table
def proto_disaggregate_accessibility(state: workflow.State):
def proto_disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
 # Read existing accessibilities, but is not required to enable model compatibility
 df = input.read_input_table(
 state, "proto_disaggregate_accessibility", required=False
Expand All@@ -130,7 +130,7 @@ def proto_disaggregate_accessibility(state: workflow.State):


@workflow.table
def disaggregate_accessibility(state: workflow.State):
def disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
 """
 This step initializes pre-computed disaggregate accessibility and merges it onto the full synthetic population.
 Function adds merged all disaggregate accessibility tables to the pipeline but returns nothing.
Expand DownExpand Up@@ -169,17 +169,17 @@ def disaggregate_accessibility(state: workflow.State):
 )
 merging_params = model_settings.MERGE_ON
 nearest_method = model_settings.NEAREST_METHOD
 accessibility_cols = [
 x for x in proto_accessibility_df.columns if "accessibility" in x
 ]

 if model_settings.KEEP_COLS is None:
 keep_cols = [x for x in proto_accessibility_df.columns if "accessibility" in x]
 else:
 keep_cols = model_settings.KEEP_COLS

 # Parse the merging parameters
 assert merging_params is not None

 # Check if already assigned!
 if set(accessibility_cols).intersection(persons_merged_df.columns) == set(
 accessibility_cols
 ):
 if set(keep_cols).intersection(persons_merged_df.columns) == set(keep_cols):
 return

 # Find the nearest zone (spatially) with accessibilities calculated
Expand DownExpand Up@@ -211,7 +211,7 @@ def disaggregate_accessibility(state: workflow.State):
 # because it will get slightly different logsums for households in the same zone.
 # This is because different destination zones were selected. To resolve, get mean by cols.
 right_df = (
 proto_accessibility_df.groupby(merge_cols)[accessibility_cols]
 proto_accessibility_df.groupby(merge_cols)[keep_cols]
 .mean()
 .sort_values(nearest_cols)
 .reset_index()
Expand DownExpand Up@@ -244,9 +244,9 @@ def disaggregate_accessibility(state: workflow.State):
 )

 # Predict the nearest person ID and pull the logsums
 matched_logsums_df = right_df.loc[clf.predict(x_pop)][
accessibility_cols
].reset_index(drop=True)
 matched_logsums_df = right_df.loc[clf.predict(x_pop)][keep_cols].reset_index(
drop=True
 )
 merge_df = pd.concat(
 [left_df.reset_index(drop=False), matched_logsums_df], axis=1
 ).set_index("person_id")
Expand DownExpand Up@@ -278,9 +278,9 @@ def disaggregate_accessibility(state: workflow.State):

 # Check that it was correctly left-joined
 assert all(persons_merged_df[merge_cols] == merge_df[merge_cols])
 assert any(merge_df[accessibility_cols].isnull())
 assert any(merge_df[keep_cols].isnull())

 # Inject merged accessibilities so that it can be included in persons_merged function
 state.add_table("disaggregate_accessibility", merge_df[accessibility_cols])
 state.add_table("disaggregate_accessibility", merge_df[keep_cols])

 return merge_df[accessibility_cols]
 return merge_df[keep_cols]

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging#768

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jpn--
merged 4 commits into

from

Mar 28, 2024
 Merged


BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging
#768
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
ea25a66

get all disaggregate accessibility values


dhensle Dec 16, 2023
1670dc9

updated settings to work with Pydantic


dhensle Mar 8, 2024
c6803bf

KEEP_COLS setting


dhensle Mar 19, 2024
ab20012

keep_cols update and returning tables


dhensle Mar 27, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


37 changes: 37 additions & 0 deletions
activitysim/abm/models/disaggregate_accessibility.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -154,6 +154,15 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
 procedure work.
 """

 KEEP_COLS: list[str] | None = None
 """
 Disaggreate accessibility table is grouped by the "by" cols above and the KEEP_COLS are averaged
 across the group. Initializing the below as NA if not in the auto ownership level, they are skipped
 in the groupby mean and the values are correct. 
 (It's a way to avoid having to update code to reshape the table and introduce new functionality there.)
 If none, will keep all of the columns with "accessibility" in the name.
 """

 FROM_TEMPLATES: bool = False
 annotate_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
 """
Expand All@@ -164,6 +173,11 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
 """
 NEAREST_METHOD: str = "skims"

 postprocess_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
 """
 List of preprocessor settings to apply to the proto-population tables after generation.
 """

Comment thread
dhensle marked this conversation as resolved.


def read_disaggregate_accessibility_yaml(
 state: workflow.State, file_name
Expand DownExpand Up@@ -846,6 +860,10 @@ def compute_disaggregate_accessibility(
 state.tracing.register_traceable_table(tablename, df)
 del df

 disagg_model_settings = read_disaggregate_accessibility_yaml(
 state, "disaggregate_accessibility.yaml"
 )

 # Run location choice
 logsums = get_disaggregate_logsums(
 state,
Expand DownExpand Up@@ -906,4 +924,23 @@ def compute_disaggregate_accessibility(
 for k, df in logsums.items():
 state.add_table(k, df)

 # available post-processing
 for annotations in disagg_model_settings.postprocess_proto_tables:
 tablename = annotations.tablename
 df = state.get_dataframe(tablename)
 assert df is not None
 assert annotations is not None
 assign_columns(
 state,
 df=df,
 model_settings={
 **annotations.annotate.dict(),
 **disagg_model_settings.suffixes.dict(),
 },
 trace_label=tracing.extend_trace_label(
 "disaggregate_accessibility.postprocess", tablename
 ),
 )
 state.add_table(tablename, df)

 return

30 changes: 15 additions & 15 deletions
activitysim/abm/tables/disaggregate_accessibility.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def maz_centroids(state: workflow.State):


@workflow.table
def proto_disaggregate_accessibility(state: workflow.State):
def proto_disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
 # Read existing accessibilities, but is not required to enable model compatibility
 df = input.read_input_table(
 state, "proto_disaggregate_accessibility", required=False
Expand All@@ -130,7 +130,7 @@ def proto_disaggregate_accessibility(state: workflow.State):


@workflow.table
def disaggregate_accessibility(state: workflow.State):
def disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
 """
 This step initializes pre-computed disaggregate accessibility and merges it onto the full synthetic population.
 Function adds merged all disaggregate accessibility tables to the pipeline but returns nothing.
Expand DownExpand Up@@ -169,17 +169,17 @@ def disaggregate_accessibility(state: workflow.State):
 )
 merging_params = model_settings.MERGE_ON
 nearest_method = model_settings.NEAREST_METHOD
 accessibility_cols = [
 x for x in proto_accessibility_df.columns if "accessibility" in x
 ]

 if model_settings.KEEP_COLS is None:
 keep_cols = [x for x in proto_accessibility_df.columns if "accessibility" in x]
 else:
 keep_cols = model_settings.KEEP_COLS

 # Parse the merging parameters
 assert merging_params is not None

 # Check if already assigned!
 if set(accessibility_cols).intersection(persons_merged_df.columns) == set(
 accessibility_cols
 ):
 if set(keep_cols).intersection(persons_merged_df.columns) == set(keep_cols):
 return

 # Find the nearest zone (spatially) with accessibilities calculated
Expand DownExpand Up@@ -211,7 +211,7 @@ def disaggregate_accessibility(state: workflow.State):
 # because it will get slightly different logsums for households in the same zone.
 # This is because different destination zones were selected. To resolve, get mean by cols.
 right_df = (
 proto_accessibility_df.groupby(merge_cols)[accessibility_cols]
 proto_accessibility_df.groupby(merge_cols)[keep_cols]
 .mean()
 .sort_values(nearest_cols)
 .reset_index()
Expand DownExpand Up@@ -244,9 +244,9 @@ def disaggregate_accessibility(state: workflow.State):
 )

 # Predict the nearest person ID and pull the logsums
 matched_logsums_df = right_df.loc[clf.predict(x_pop)][
accessibility_cols
].reset_index(drop=True)
 matched_logsums_df = right_df.loc[clf.predict(x_pop)][keep_cols].reset_index(
drop=True
 )
 merge_df = pd.concat(
 [left_df.reset_index(drop=False), matched_logsums_df], axis=1
 ).set_index("person_id")
Expand DownExpand Up@@ -278,9 +278,9 @@ def disaggregate_accessibility(state: workflow.State):

 # Check that it was correctly left-joined
 assert all(persons_merged_df[merge_cols] == merge_df[merge_cols])
 assert any(merge_df[accessibility_cols].isnull())
 assert any(merge_df[keep_cols].isnull())

 # Inject merged accessibilities so that it can be included in persons_merged function
 state.add_table("disaggregate_accessibility", merge_df[accessibility_cols])
 state.add_table("disaggregate_accessibility", merge_df[keep_cols])

 return merge_df[accessibility_cols]
 return merge_df[keep_cols]

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging#768

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jpn--
merged 4 commits into

from

Mar 28, 2024
 Merged


BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging
#768
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
ea25a66

get all disaggregate accessibility values


dhensle Dec 16, 2023
1670dc9

updated settings to work with Pydantic


dhensle Mar 8, 2024
c6803bf

KEEP_COLS setting


dhensle Mar 19, 2024
ab20012

keep_cols update and returning tables


dhensle Mar 27, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


37 changes: 37 additions & 0 deletions
activitysim/abm/models/disaggregate_accessibility.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -154,6 +154,15 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
 procedure work.
 """

 KEEP_COLS: list[str] | None = None
 """
 Disaggreate accessibility table is grouped by the "by" cols above and the KEEP_COLS are averaged
 across the group. Initializing the below as NA if not in the auto ownership level, they are skipped
 in the groupby mean and the values are correct. 
 (It's a way to avoid having to update code to reshape the table and introduce new functionality there.)
 If none, will keep all of the columns with "accessibility" in the name.
 """

 FROM_TEMPLATES: bool = False
 annotate_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
 """
Expand All@@ -164,6 +173,11 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
 """
 NEAREST_METHOD: str = "skims"

 postprocess_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
 """
 List of preprocessor settings to apply to the proto-population tables after generation.
 """

Comment thread
dhensle marked this conversation as resolved.


def read_disaggregate_accessibility_yaml(
 state: workflow.State, file_name
Expand DownExpand Up@@ -846,6 +860,10 @@ def compute_disaggregate_accessibility(
 state.tracing.register_traceable_table(tablename, df)
 del df

 disagg_model_settings = read_disaggregate_accessibility_yaml(
 state, "disaggregate_accessibility.yaml"
 )

 # Run location choice
 logsums = get_disaggregate_logsums(
 state,
Expand DownExpand Up@@ -906,4 +924,23 @@ def compute_disaggregate_accessibility(
 for k, df in logsums.items():
 state.add_table(k, df)

 # available post-processing
 for annotations in disagg_model_settings.postprocess_proto_tables:
 tablename = annotations.tablename
 df = state.get_dataframe(tablename)
 assert df is not None
 assert annotations is not None
 assign_columns(
 state,
 df=df,
 model_settings={
 **annotations.annotate.dict(),
 **disagg_model_settings.suffixes.dict(),
 },
 trace_label=tracing.extend_trace_label(
 "disaggregate_accessibility.postprocess", tablename
 ),
 )
 state.add_table(tablename, df)

 return

30 changes: 15 additions & 15 deletions
activitysim/abm/tables/disaggregate_accessibility.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def maz_centroids(state: workflow.State):


@workflow.table
def proto_disaggregate_accessibility(state: workflow.State):
def proto_disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
 # Read existing accessibilities, but is not required to enable model compatibility
 df = input.read_input_table(
 state, "proto_disaggregate_accessibility", required=False
Expand All@@ -130,7 +130,7 @@ def proto_disaggregate_accessibility(state: workflow.State):


@workflow.table
def disaggregate_accessibility(state: workflow.State):
def disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
 """
 This step initializes pre-computed disaggregate accessibility and merges it onto the full synthetic population.
 Function adds merged all disaggregate accessibility tables to the pipeline but returns nothing.
Expand DownExpand Up@@ -169,17 +169,17 @@ def disaggregate_accessibility(state: workflow.State):
 )
 merging_params = model_settings.MERGE_ON
 nearest_method = model_settings.NEAREST_METHOD
 accessibility_cols = [
 x for x in proto_accessibility_df.columns if "accessibility" in x
 ]

 if model_settings.KEEP_COLS is None:
 keep_cols = [x for x in proto_accessibility_df.columns if "accessibility" in x]
 else:
 keep_cols = model_settings.KEEP_COLS

 # Parse the merging parameters
 assert merging_params is not None

 # Check if already assigned!
 if set(accessibility_cols).intersection(persons_merged_df.columns) == set(
 accessibility_cols
 ):
 if set(keep_cols).intersection(persons_merged_df.columns) == set(keep_cols):
 return

 # Find the nearest zone (spatially) with accessibilities calculated
Expand DownExpand Up@@ -211,7 +211,7 @@ def disaggregate_accessibility(state: workflow.State):
 # because it will get slightly different logsums for households in the same zone.
 # This is because different destination zones were selected. To resolve, get mean by cols.
 right_df = (
 proto_accessibility_df.groupby(merge_cols)[accessibility_cols]
 proto_accessibility_df.groupby(merge_cols)[keep_cols]
 .mean()
 .sort_values(nearest_cols)
 .reset_index()
Expand DownExpand Up@@ -244,9 +244,9 @@ def disaggregate_accessibility(state: workflow.State):
 )

 # Predict the nearest person ID and pull the logsums
 matched_logsums_df = right_df.loc[clf.predict(x_pop)][
accessibility_cols
].reset_index(drop=True)
 matched_logsums_df = right_df.loc[clf.predict(x_pop)][keep_cols].reset_index(
drop=True
 )
 merge_df = pd.concat(
 [left_df.reset_index(drop=False), matched_logsums_df], axis=1
 ).set_index("person_id")
Expand DownExpand Up@@ -278,9 +278,9 @@ def disaggregate_accessibility(state: workflow.State):

 # Check that it was correctly left-joined
 assert all(persons_merged_df[merge_cols] == merge_df[merge_cols])
 assert any(merge_df[accessibility_cols].isnull())
 assert any(merge_df[keep_cols].isnull())

 # Inject merged accessibilities so that it can be included in persons_merged function
 state.add_table("disaggregate_accessibility", merge_df[accessibility_cols])
 state.add_table("disaggregate_accessibility", merge_df[keep_cols])

 return merge_df[accessibility_cols]
 return merge_df[keep_cols]

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging#768

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jpn--
merged 4 commits into

from

Mar 28, 2024
 Merged


BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging
#768
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
ea25a66

get all disaggregate accessibility values


dhensle Dec 16, 2023
1670dc9

updated settings to work with Pydantic


dhensle Mar 8, 2024
c6803bf

KEEP_COLS setting


dhensle Mar 19, 2024
ab20012

keep_cols update and returning tables


dhensle Mar 27, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


37 changes: 37 additions & 0 deletions
activitysim/abm/models/disaggregate_accessibility.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -154,6 +154,15 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
 procedure work.
 """

 KEEP_COLS: list[str] | None = None
 """
 Disaggreate accessibility table is grouped by the "by" cols above and the KEEP_COLS are averaged
 across the group. Initializing the below as NA if not in the auto ownership level, they are skipped
 in the groupby mean and the values are correct. 
 (It's a way to avoid having to update code to reshape the table and introduce new functionality there.)
 If none, will keep all of the columns with "accessibility" in the name.
 """

 FROM_TEMPLATES: bool = False
 annotate_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
 """
Expand All@@ -164,6 +173,11 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
 """
 NEAREST_METHOD: str = "skims"

 postprocess_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
 """
 List of preprocessor settings to apply to the proto-population tables after generation.
 """

Comment thread
dhensle marked this conversation as resolved.


def read_disaggregate_accessibility_yaml(
 state: workflow.State, file_name
Expand DownExpand Up@@ -846,6 +860,10 @@ def compute_disaggregate_accessibility(
 state.tracing.register_traceable_table(tablename, df)
 del df

 disagg_model_settings = read_disaggregate_accessibility_yaml(
 state, "disaggregate_accessibility.yaml"
 )

 # Run location choice
 logsums = get_disaggregate_logsums(
 state,
Expand DownExpand Up@@ -906,4 +924,23 @@ def compute_disaggregate_accessibility(
 for k, df in logsums.items():
 state.add_table(k, df)

 # available post-processing
 for annotations in disagg_model_settings.postprocess_proto_tables:
 tablename = annotations.tablename
 df = state.get_dataframe(tablename)
 assert df is not None
 assert annotations is not None
 assign_columns(
 state,
 df=df,
 model_settings={
 **annotations.annotate.dict(),
 **disagg_model_settings.suffixes.dict(),
 },
 trace_label=tracing.extend_trace_label(
 "disaggregate_accessibility.postprocess", tablename
 ),
 )
 state.add_table(tablename, df)

 return

30 changes: 15 additions & 15 deletions
activitysim/abm/tables/disaggregate_accessibility.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def maz_centroids(state: workflow.State):


@workflow.table
def proto_disaggregate_accessibility(state: workflow.State):
def proto_disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
 # Read existing accessibilities, but is not required to enable model compatibility
 df = input.read_input_table(
 state, "proto_disaggregate_accessibility", required=False
Expand All@@ -130,7 +130,7 @@ def proto_disaggregate_accessibility(state: workflow.State):


@workflow.table
def disaggregate_accessibility(state: workflow.State):
def disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
 """
 This step initializes pre-computed disaggregate accessibility and merges it onto the full synthetic population.
 Function adds merged all disaggregate accessibility tables to the pipeline but returns nothing.
Expand DownExpand Up@@ -169,17 +169,17 @@ def disaggregate_accessibility(state: workflow.State):
 )
 merging_params = model_settings.MERGE_ON
 nearest_method = model_settings.NEAREST_METHOD
 accessibility_cols = [
 x for x in proto_accessibility_df.columns if "accessibility" in x
 ]

 if model_settings.KEEP_COLS is None:
 keep_cols = [x for x in proto_accessibility_df.columns if "accessibility" in x]
 else:
 keep_cols = model_settings.KEEP_COLS

 # Parse the merging parameters
 assert merging_params is not None

 # Check if already assigned!
 if set(accessibility_cols).intersection(persons_merged_df.columns) == set(
 accessibility_cols
 ):
 if set(keep_cols).intersection(persons_merged_df.columns) == set(keep_cols):
 return

 # Find the nearest zone (spatially) with accessibilities calculated
Expand DownExpand Up@@ -211,7 +211,7 @@ def disaggregate_accessibility(state: workflow.State):
 # because it will get slightly different logsums for households in the same zone.
 # This is because different destination zones were selected. To resolve, get mean by cols.
 right_df = (
 proto_accessibility_df.groupby(merge_cols)[accessibility_cols]
 proto_accessibility_df.groupby(merge_cols)[keep_cols]
 .mean()
 .sort_values(nearest_cols)
 .reset_index()
Expand DownExpand Up@@ -244,9 +244,9 @@ def disaggregate_accessibility(state: workflow.State):
 )

 # Predict the nearest person ID and pull the logsums
 matched_logsums_df = right_df.loc[clf.predict(x_pop)][
accessibility_cols
].reset_index(drop=True)
 matched_logsums_df = right_df.loc[clf.predict(x_pop)][keep_cols].reset_index(
drop=True
 )
 merge_df = pd.concat(
 [left_df.reset_index(drop=False), matched_logsums_df], axis=1
 ).set_index("person_id")
Expand DownExpand Up@@ -278,9 +278,9 @@ def disaggregate_accessibility(state: workflow.State):

 # Check that it was correctly left-joined
 assert all(persons_merged_df[merge_cols] == merge_df[merge_cols])
 assert any(merge_df[accessibility_cols].isnull())
 assert any(merge_df[keep_cols].isnull())

 # Inject merged accessibilities so that it can be included in persons_merged function
 state.add_table("disaggregate_accessibility", merge_df[accessibility_cols])
 state.add_table("disaggregate_accessibility", merge_df[keep_cols])

 return merge_df[accessibility_cols]
 return merge_df[keep_cols]

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging#768

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jpn--
merged 4 commits into

from

Mar 28, 2024
 Merged


BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging
#768
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
ea25a66

get all disaggregate accessibility values


dhensle Dec 16, 2023
1670dc9

updated settings to work with Pydantic


dhensle Mar 8, 2024
c6803bf

KEEP_COLS setting


dhensle Mar 19, 2024
ab20012

keep_cols update and returning tables


dhensle Mar 27, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


37 changes: 37 additions & 0 deletions
activitysim/abm/models/disaggregate_accessibility.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -154,6 +154,15 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
 procedure work.
 """

 KEEP_COLS: list[str] | None = None
 """
 Disaggreate accessibility table is grouped by the "by" cols above and the KEEP_COLS are averaged
 across the group. Initializing the below as NA if not in the auto ownership level, they are skipped
 in the groupby mean and the values are correct. 
 (It's a way to avoid having to update code to reshape the table and introduce new functionality there.)
 If none, will keep all of the columns with "accessibility" in the name.
 """

 FROM_TEMPLATES: bool = False
 annotate_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
 """
Expand All@@ -164,6 +173,11 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
 """
 NEAREST_METHOD: str = "skims"

 postprocess_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
 """
 List of preprocessor settings to apply to the proto-population tables after generation.
 """

Comment thread
dhensle marked this conversation as resolved.


def read_disaggregate_accessibility_yaml(
 state: workflow.State, file_name
Expand DownExpand Up@@ -846,6 +860,10 @@ def compute_disaggregate_accessibility(
 state.tracing.register_traceable_table(tablename, df)
 del df

 disagg_model_settings = read_disaggregate_accessibility_yaml(
 state, "disaggregate_accessibility.yaml"
 )

 # Run location choice
 logsums = get_disaggregate_logsums(
 state,
Expand DownExpand Up@@ -906,4 +924,23 @@ def compute_disaggregate_accessibility(
 for k, df in logsums.items():
 state.add_table(k, df)

 # available post-processing
 for annotations in disagg_model_settings.postprocess_proto_tables:
 tablename = annotations.tablename
 df = state.get_dataframe(tablename)
 assert df is not None
 assert annotations is not None
 assign_columns(
 state,
 df=df,
 model_settings={
 **annotations.annotate.dict(),
 **disagg_model_settings.suffixes.dict(),
 },
 trace_label=tracing.extend_trace_label(
 "disaggregate_accessibility.postprocess", tablename
 ),
 )
 state.add_table(tablename, df)

 return

30 changes: 15 additions & 15 deletions
activitysim/abm/tables/disaggregate_accessibility.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def maz_centroids(state: workflow.State):


@workflow.table
def proto_disaggregate_accessibility(state: workflow.State):
def proto_disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
 # Read existing accessibilities, but is not required to enable model compatibility
 df = input.read_input_table(
 state, "proto_disaggregate_accessibility", required=False
Expand All@@ -130,7 +130,7 @@ def proto_disaggregate_accessibility(state: workflow.State):


@workflow.table
def disaggregate_accessibility(state: workflow.State):
def disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
 """
 This step initializes pre-computed disaggregate accessibility and merges it onto the full synthetic population.
 Function adds merged all disaggregate accessibility tables to the pipeline but returns nothing.
Expand DownExpand Up@@ -169,17 +169,17 @@ def disaggregate_accessibility(state: workflow.State):
 )
 merging_params = model_settings.MERGE_ON
 nearest_method = model_settings.NEAREST_METHOD
 accessibility_cols = [
 x for x in proto_accessibility_df.columns if "accessibility" in x
 ]

 if model_settings.KEEP_COLS is None:
 keep_cols = [x for x in proto_accessibility_df.columns if "accessibility" in x]
 else:
 keep_cols = model_settings.KEEP_COLS

 # Parse the merging parameters
 assert merging_params is not None

 # Check if already assigned!
 if set(accessibility_cols).intersection(persons_merged_df.columns) == set(
 accessibility_cols
 ):
 if set(keep_cols).intersection(persons_merged_df.columns) == set(keep_cols):
 return

 # Find the nearest zone (spatially) with accessibilities calculated
Expand DownExpand Up@@ -211,7 +211,7 @@ def disaggregate_accessibility(state: workflow.State):
 # because it will get slightly different logsums for households in the same zone.
 # This is because different destination zones were selected. To resolve, get mean by cols.
 right_df = (
 proto_accessibility_df.groupby(merge_cols)[accessibility_cols]
 proto_accessibility_df.groupby(merge_cols)[keep_cols]
 .mean()
 .sort_values(nearest_cols)
 .reset_index()
Expand DownExpand Up@@ -244,9 +244,9 @@ def disaggregate_accessibility(state: workflow.State):
 )

 # Predict the nearest person ID and pull the logsums
 matched_logsums_df = right_df.loc[clf.predict(x_pop)][
accessibility_cols
].reset_index(drop=True)
 matched_logsums_df = right_df.loc[clf.predict(x_pop)][keep_cols].reset_index(
drop=True
 )
 merge_df = pd.concat(
 [left_df.reset_index(drop=False), matched_logsums_df], axis=1
 ).set_index("person_id")
Expand DownExpand Up@@ -278,9 +278,9 @@ def disaggregate_accessibility(state: workflow.State):

 # Check that it was correctly left-joined
 assert all(persons_merged_df[merge_cols] == merge_df[merge_cols])
 assert any(merge_df[accessibility_cols].isnull())
 assert any(merge_df[keep_cols].isnull())

 # Inject merged accessibilities so that it can be included in persons_merged function
 state.add_table("disaggregate_accessibility", merge_df[accessibility_cols])
 state.add_table("disaggregate_accessibility", merge_df[keep_cols])

 return merge_df[accessibility_cols]
 return merge_df[keep_cols]

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging#768

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jpn--
merged 4 commits into

from

Mar 28, 2024
 Merged


BayDAG Contribution #2: Enhanced Disaggregate Accessibility Merging
#768
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
ea25a66

get all disaggregate accessibility values


dhensle Dec 16, 2023
1670dc9

updated settings to work with Pydantic


dhensle Mar 8, 2024
c6803bf

KEEP_COLS setting


dhensle Mar 19, 2024
ab20012

keep_cols update and returning tables


dhensle Mar 27, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


37 changes: 37 additions & 0 deletions
activitysim/abm/models/disaggregate_accessibility.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -154,6 +154,15 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
 procedure work.
 """

 KEEP_COLS: list[str] | None = None
 """
 Disaggreate accessibility table is grouped by the "by" cols above and the KEEP_COLS are averaged
 across the group. Initializing the below as NA if not in the auto ownership level, they are skipped
 in the groupby mean and the values are correct. 
 (It's a way to avoid having to update code to reshape the table and introduce new functionality there.)
 If none, will keep all of the columns with "accessibility" in the name.
 """

 FROM_TEMPLATES: bool = False
 annotate_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
 """
Expand All@@ -164,6 +173,11 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
 """
 NEAREST_METHOD: str = "skims"

 postprocess_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = []
 """
 List of preprocessor settings to apply to the proto-population tables after generation.
 """

Comment thread
dhensle marked this conversation as resolved.


def read_disaggregate_accessibility_yaml(
 state: workflow.State, file_name
Expand DownExpand Up@@ -846,6 +860,10 @@ def compute_disaggregate_accessibility(
 state.tracing.register_traceable_table(tablename, df)
 del df

 disagg_model_settings = read_disaggregate_accessibility_yaml(
 state, "disaggregate_accessibility.yaml"
 )

 # Run location choice
 logsums = get_disaggregate_logsums(
 state,
Expand DownExpand Up@@ -906,4 +924,23 @@ def compute_disaggregate_accessibility(
 for k, df in logsums.items():
 state.add_table(k, df)

 # available post-processing
 for annotations in disagg_model_settings.postprocess_proto_tables:
 tablename = annotations.tablename
 df = state.get_dataframe(tablename)
 assert df is not None
 assert annotations is not None
 assign_columns(
 state,
 df=df,
 model_settings={
 **annotations.annotate.dict(),
 **disagg_model_settings.suffixes.dict(),
 },
 trace_label=tracing.extend_trace_label(
 "disaggregate_accessibility.postprocess", tablename
 ),
 )
 state.add_table(tablename, df)

 return

30 changes: 15 additions & 15 deletions
activitysim/abm/tables/disaggregate_accessibility.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def maz_centroids(state: workflow.State):


@workflow.table
def proto_disaggregate_accessibility(state: workflow.State):
def proto_disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
 # Read existing accessibilities, but is not required to enable model compatibility
 df = input.read_input_table(
 state, "proto_disaggregate_accessibility", required=False
Expand All@@ -130,7 +130,7 @@ def proto_disaggregate_accessibility(state: workflow.State):


@workflow.table
def disaggregate_accessibility(state: workflow.State):
def disaggregate_accessibility(state: workflow.State) -> pd.DataFrame:
 """
 This step initializes pre-computed disaggregate accessibility and merges it onto the full synthetic population.
 Function adds merged all disaggregate accessibility tables to the pipeline but returns nothing.
Expand DownExpand Up@@ -169,17 +169,17 @@ def disaggregate_accessibility(state: workflow.State):
 )
 merging_params = model_settings.MERGE_ON
 nearest_method = model_settings.NEAREST_METHOD
 accessibility_cols = [
 x for x in proto_accessibility_df.columns if "accessibility" in x
 ]

 if model_settings.KEEP_COLS is None:
 keep_cols = [x for x in proto_accessibility_df.columns if "accessibility" in x]
 else:
 keep_cols = model_settings.KEEP_COLS

 # Parse the merging parameters
 assert merging_params is not None

 # Check if already assigned!
 if set(accessibility_cols).intersection(persons_merged_df.columns) == set(
 accessibility_cols
 ):
 if set(keep_cols).intersection(persons_merged_df.columns) == set(keep_cols):
 return

 # Find the nearest zone (spatially) with accessibilities calculated
Expand DownExpand Up@@ -211,7 +211,7 @@ def disaggregate_accessibility(state: workflow.State):
 # because it will get slightly different logsums for households in the same zone.
 # This is because different destination zones were selected. To resolve, get mean by cols.
 right_df = (
 proto_accessibility_df.groupby(merge_cols)[accessibility_cols]
 proto_accessibility_df.groupby(merge_cols)[keep_cols]
 .mean()
 .sort_values(nearest_cols)
 .reset_index()
Expand DownExpand Up@@ -244,9 +244,9 @@ def disaggregate_accessibility(state: workflow.State):
 )

 # Predict the nearest person ID and pull the logsums
 matched_logsums_df = right_df.loc[clf.predict(x_pop)][
accessibility_cols
].reset_index(drop=True)
 matched_logsums_df = right_df.loc[clf.predict(x_pop)][keep_cols].reset_index(
drop=True
 )
 merge_df = pd.concat(
 [left_df.reset_index(drop=False), matched_logsums_df], axis=1
 ).set_index("person_id")
Expand DownExpand Up@@ -278,9 +278,9 @@ def disaggregate_accessibility(state: workflow.State):

 # Check that it was correctly left-joined
 assert all(persons_merged_df[merge_cols] == merge_df[merge_cols])
 assert any(merge_df[accessibility_cols].isnull())
 assert any(merge_df[keep_cols].isnull())

 # Inject merged accessibilities so that it can be included in persons_merged function
 state.add_table("disaggregate_accessibility", merge_df[accessibility_cols])
 state.add_table("disaggregate_accessibility", merge_df[keep_cols])

 return merge_df[accessibility_cols]
 return merge_df[keep_cols]