Speeding up chunking#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bstabler merged 4 commits into from
Aug 4, 2021
Merged

Speeding up chunking #458

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1359183
#444
bstabler Jul 12, 2021
4600b4c
a bit more chunking docs
bstabler Jul 13, 2021
40d04e5
Merge remote-tracking branch 'remotes/origin/develop' into speeding-u…
bstabler Jul 28, 2021
66d904f
faster trip scheduling in production by skipping chunking memory chec…
bstabler Aug 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
250 changes: 125 additions & 125 deletions activitysim/abm/models/trip_scheduling.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -319,60 +319,57 @@ def schedule_trips_in_leg(
result_list.append(choices)
trips = trips[~no_scheduling]

with chunk.chunk_log(trace_label):

# add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
trips = trips.sort_index()
trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

# iterate over outbound trips in ascending trip_num order, skipping the initial trip
# iterate over inbound trips in descending trip_num order, skipping the finial trip
first_trip_in_leg = True
for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

if outbound:
nth_trips = trips[trips.trip_num == i]
else:
nth_trips = trips[trips.trip_num == trips.trip_count - i]

nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

with chunk.chunk_log(nth_trace_label, chunk_tag=trace_label):
choices = schedule_nth_trips(
nth_trips,
probs_spec,
model_settings,
first_trip_in_leg=first_trip_in_leg,
report_failed_trips=is_last_iteration,
trace_hh_id=trace_hh_id,
trace_label=nth_trace_label)

# if outbound, this trip's depart constrains next trip's earliest depart option
# if inbound, we are handling in reverse order, so it constrains latest depart instead
ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

# most initial departure (when no choice was made because all probs were zero)
if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
choices = choices.reindex(nth_trips.index)
logger.warning("%s coercing %s depart choices to most initial" %
(nth_trace_label, choices.isna().sum()))
choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

# adjust allowed depart range of next trip
has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
if has_next_trip.any():
next_trip_ids = nth_trips.next_trip_id[has_next_trip]
# patch choice any trips with next_trips that weren't scheduled
trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

result_list.append(choices)

chunk.log_df(trace_label, f'result_list', result_list)

first_trip_in_leg = False
# add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
trips = trips.sort_index()
trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

# iterate over outbound trips in ascending trip_num order, skipping the initial trip
# iterate over inbound trips in descending trip_num order, skipping the finial trip
first_trip_in_leg = True
for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

if outbound:
nth_trips = trips[trips.trip_num == i]
else:
nth_trips = trips[trips.trip_num == trips.trip_count - i]

nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

choices = schedule_nth_trips(
nth_trips,
probs_spec,
model_settings,
first_trip_in_leg=first_trip_in_leg,
report_failed_trips=is_last_iteration,
trace_hh_id=trace_hh_id,
trace_label=nth_trace_label)

# if outbound, this trip's depart constrains next trip's earliest depart option
# if inbound, we are handling in reverse order, so it constrains latest depart instead
ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

# most initial departure (when no choice was made because all probs were zero)
if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
choices = choices.reindex(nth_trips.index)
logger.warning("%s coercing %s depart choices to most initial" %
(nth_trace_label, choices.isna().sum()))
choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

# adjust allowed depart range of next trip
has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
if has_next_trip.any():
next_trip_ids = nth_trips.next_trip_id[has_next_trip]
# patch choice any trips with next_trips that weren't scheduled
trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

result_list.append(choices)

chunk.log_df(trace_label, f'result_list', result_list)

first_trip_in_leg = False

if len(result_list) > 1:
choices = pd.concat(result_list)
Expand All@@ -381,7 +378,7 @@ def schedule_trips_in_leg(


def run_trip_scheduling(
trips,
trips_chunk,
tours,
probs_spec,
model_settings,
Expand All@@ -398,40 +395,38 @@ def run_trip_scheduling(
# num_choosers = (is_inbound_chooser | is_outbound_chooser).sum()

result_list = []
for i, trips_chunk, chunk_trace_label \
in chunk.adaptive_chunked_choosers_by_chunk_id(trips, chunk_size, trace_label, chunk_tag):

if trips_chunk.outbound.any():
leg_chunk = trips_chunk[trips_chunk.outbound]
leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'outbound')
choices = \
schedule_trips_in_leg(
outbound=True,
trips=leg_chunk,
probs_spec=probs_spec,
model_settings=model_settings,
is_last_iteration=is_last_iteration,
trace_hh_id=trace_hh_id,
trace_label=leg_trace_label)
result_list.append(choices)

chunk.log_df(trace_label, f'result_list', result_list)

if (~trips_chunk.outbound).any():
leg_chunk = trips_chunk[~trips_chunk.outbound]
leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'inbound')
choices = \
schedule_trips_in_leg(
outbound=False,
trips=leg_chunk,
probs_spec=probs_spec,
model_settings=model_settings,
is_last_iteration=is_last_iteration,
trace_hh_id=trace_hh_id,
trace_label=leg_trace_label)
result_list.append(choices)

chunk.log_df(trace_label, f'result_list', result_list)

if trips_chunk.outbound.any():
leg_chunk = trips_chunk[trips_chunk.outbound]
leg_trace_label = tracing.extend_trace_label(trace_label, 'outbound')
choices = \
schedule_trips_in_leg(
outbound=True,
trips=leg_chunk,
probs_spec=probs_spec,
model_settings=model_settings,
is_last_iteration=is_last_iteration,
trace_hh_id=trace_hh_id,
trace_label=leg_trace_label)
result_list.append(choices)

chunk.log_df(trace_label, f'result_list', result_list)

if (~trips_chunk.outbound).any():
leg_chunk = trips_chunk[~trips_chunk.outbound]
leg_trace_label = tracing.extend_trace_label(trace_label, 'inbound')
choices = \
schedule_trips_in_leg(
outbound=False,
trips=leg_chunk,
probs_spec=probs_spec,
model_settings=model_settings,
is_last_iteration=is_last_iteration,
trace_hh_id=trace_hh_id,
trace_label=leg_trace_label)
result_list.append(choices)

chunk.log_df(trace_label, f'result_list', result_list)

choices = pd.concat(result_list)

Expand DownExpand Up@@ -526,43 +521,48 @@ def trip_scheduling(
assert max_iterations > 0

choices_list = []
i = 0
while (i < max_iterations) and not trips_df.empty:

i += 1
is_last_iteration = (i == max_iterations)

trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
logger.info("%s scheduling %s trips", trace_label_i, trips_df.shape[0])

# first iteration gets its own chunk_tag and all subsequent iterations are aggregated
# subsequent iterations on failed trips have somewhat different overhead profile than initial batch
chunk_tag = "trip_scheduling_1" if i == 1 else "trip_scheduling_n"

choices = \
run_trip_scheduling(
trips_df,
tours,
probs_spec,
model_settings,
estimator=estimator,
is_last_iteration=is_last_iteration,
trace_hh_id=trace_hh_id,
chunk_size=chunk_size,
chunk_tag=chunk_tag,
trace_label=trace_label_i)

# boolean series of trips whose individual trip scheduling failed
failed = choices.reindex(trips_df.index).isnull()
logger.info("%s %s failed", trace_label_i, failed.sum())

if not is_last_iteration:
# boolean series of trips whose leg scheduling failed
failed_cohorts = failed_trip_cohorts(trips_df, failed)
trips_df = trips_df[failed_cohorts]
choices = choices[~failed_cohorts]

choices_list.append(choices)
for chunk_i, trips_chunk, chunk_trace_label in chunk.adaptive_chunked_choosers_by_chunk_id(trips_df,
chunk_size,
trace_label,
trace_label):

i = 0
while (i < max_iterations) and not trips_chunk.empty:

# only chunk log first iteration since memory use declines with each iteration
with chunk.chunk_log(trace_label) if i == 0 else chunk.chunk_log_skip():

i += 1
is_last_iteration = (i == max_iterations)

trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
logger.info("%s scheduling %s trips within chunk %s", trace_label_i, trips_chunk.shape[0], chunk_i)

choices = \
run_trip_scheduling(
trips_chunk,
tours,
probs_spec,
model_settings,
estimator=estimator,
is_last_iteration=is_last_iteration,
trace_hh_id=trace_hh_id,
chunk_size=chunk_size,
chunk_tag=trace_label,
trace_label=trace_label_i)

# boolean series of trips whose individual trip scheduling failed
failed = choices.reindex(trips_chunk.index).isnull()
logger.info("%s %s failed", trace_label_i, failed.sum())

if not is_last_iteration:
# boolean series of trips whose leg scheduling failed
failed_cohorts = failed_trip_cohorts(trips_chunk, failed)
trips_chunk = trips_chunk[failed_cohorts]
choices = choices[~failed_cohorts]

choices_list.append(choices)

trips_df = trips.to_frame()

Expand Down
29 changes: 25 additions & 4 deletions activitysim/core/chunk.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -568,8 +568,9 @@ def log_rss(trace_label, force=False):
hwm_trace_label = f"{trace_label}.log_rss"

if chunk_training_mode() == MODE_PRODUCTION:
trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
# FIXME - this trace_memory_info call slows things down a lot so it is turned off for now
# trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
# mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
return

rss, uss = mem.trace_memory_info(hwm_trace_label)
Expand DownExpand Up@@ -624,7 +625,12 @@ class ChunkSizer(object):
def __init__(self, chunk_tag, trace_label, num_choosers=0, chunk_size=0):

self.depth = len(CHUNK_SIZERS) + 1
self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

if chunk_metric() == USS:
self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
else:
self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
self.uss = 0

if self.depth > 1:
# nested chunkers should be unchunked
Expand DownExpand Up@@ -755,7 +761,14 @@ def adaptive_rows_per_chunk(self, i):

prev_rss = self.rss
prev_uss = self.uss
self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

if chunk_training_mode() != MODE_PRODUCTION:

if chunk_metric() == USS:
self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
else:
self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
self.uss = 0

self.headroom = self.available_headroom(self.uss if chunk_metric() == USS else self.rss)

Expand DownExpand Up@@ -904,6 +917,14 @@ def chunk_log(trace_label, chunk_tag=None, base=False):
chunk_sizer.close()


@contextmanager
def chunk_log_skip():

yield

None


def adaptive_chunked_choosers(choosers, chunk_size, trace_label, chunk_tag=None):

# generator to iterate over choosers
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Speeding up chunking#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bstabler
merged 4 commits into

from

Aug 4, 2021
 Merged


Speeding up chunking
#458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
1359183

#444


bstabler Jul 12, 2021
4600b4c

a bit more chunking docs


bstabler Jul 13, 2021
40d04e5

Merge remote-tracking branch 'remotes/origin/develop' into speeding-u…


bstabler Jul 28, 2021
66d904f

faster trip scheduling in production by skipping chunking memory chec…


bstabler Aug 3, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


250 changes: 125 additions & 125 deletions
activitysim/abm/models/trip_scheduling.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -319,60 +319,57 @@ def schedule_trips_in_leg(
 result_list.append(choices)
 trips = trips[~no_scheduling]

 with chunk.chunk_log(trace_label):

 # add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
 trips = trips.sort_index()
 trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
 is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
 trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

 # iterate over outbound trips in ascending trip_num order, skipping the initial trip
 # iterate over inbound trips in descending trip_num order, skipping the finial trip
 first_trip_in_leg = True
 for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

 if outbound:
 nth_trips = trips[trips.trip_num == i]
 else:
 nth_trips = trips[trips.trip_num == trips.trip_count - i]

 nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

 with chunk.chunk_log(nth_trace_label, chunk_tag=trace_label):
 choices = schedule_nth_trips(
 nth_trips,
 probs_spec,
 model_settings,
 first_trip_in_leg=first_trip_in_leg,
 report_failed_trips=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=nth_trace_label)

 # if outbound, this trip's depart constrains next trip's earliest depart option
 # if inbound, we are handling in reverse order, so it constrains latest depart instead
 ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

 # most initial departure (when no choice was made because all probs were zero)
 if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
 choices = choices.reindex(nth_trips.index)
 logger.warning("%s coercing %s depart choices to most initial" %
 (nth_trace_label, choices.isna().sum()))
 choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

 # adjust allowed depart range of next trip
 has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
 if has_next_trip.any():
 next_trip_ids = nth_trips.next_trip_id[has_next_trip]
 # patch choice any trips with next_trips that weren't scheduled
 trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
 choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 first_trip_in_leg = False
 # add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
 trips = trips.sort_index()
 trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
 is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
 trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

 # iterate over outbound trips in ascending trip_num order, skipping the initial trip
 # iterate over inbound trips in descending trip_num order, skipping the finial trip
 first_trip_in_leg = True
 for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

 if outbound:
 nth_trips = trips[trips.trip_num == i]
 else:
 nth_trips = trips[trips.trip_num == trips.trip_count - i]

 nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

 choices = schedule_nth_trips(
 nth_trips,
 probs_spec,
 model_settings,
 first_trip_in_leg=first_trip_in_leg,
 report_failed_trips=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=nth_trace_label)

 # if outbound, this trip's depart constrains next trip's earliest depart option
 # if inbound, we are handling in reverse order, so it constrains latest depart instead
 ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

 # most initial departure (when no choice was made because all probs were zero)
 if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
 choices = choices.reindex(nth_trips.index)
 logger.warning("%s coercing %s depart choices to most initial" %
 (nth_trace_label, choices.isna().sum()))
 choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

 # adjust allowed depart range of next trip
 has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
 if has_next_trip.any():
 next_trip_ids = nth_trips.next_trip_id[has_next_trip]
 # patch choice any trips with next_trips that weren't scheduled
 trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
 choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 first_trip_in_leg = False

 if len(result_list) > 1:
 choices = pd.concat(result_list)
Expand All@@ -381,7 +378,7 @@ def schedule_trips_in_leg(


def run_trip_scheduling(
trips,
trips_chunk,
 tours,
 probs_spec,
 model_settings,
Expand All@@ -398,40 +395,38 @@ def run_trip_scheduling(
 # num_choosers = (is_inbound_chooser | is_outbound_chooser).sum()

 result_list = []
 for i, trips_chunk, chunk_trace_label \
 in chunk.adaptive_chunked_choosers_by_chunk_id(trips, chunk_size, trace_label, chunk_tag):

 if trips_chunk.outbound.any():
 leg_chunk = trips_chunk[trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'outbound')
 choices = \
 schedule_trips_in_leg(
 outbound=True,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if (~trips_chunk.outbound).any():
 leg_chunk = trips_chunk[~trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'inbound')
 choices = \
 schedule_trips_in_leg(
 outbound=False,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if trips_chunk.outbound.any():
 leg_chunk = trips_chunk[trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(trace_label, 'outbound')
 choices = \
 schedule_trips_in_leg(
 outbound=True,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if (~trips_chunk.outbound).any():
 leg_chunk = trips_chunk[~trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(trace_label, 'inbound')
 choices = \
 schedule_trips_in_leg(
 outbound=False,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 choices = pd.concat(result_list)

Expand DownExpand Up@@ -526,43 +521,48 @@ def trip_scheduling(
 assert max_iterations > 0

 choices_list = []
 i = 0
 while (i < max_iterations) and not trips_df.empty:

 i += 1
 is_last_iteration = (i == max_iterations)

 trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
 logger.info("%s scheduling %s trips", trace_label_i, trips_df.shape[0])

 # first iteration gets its own chunk_tag and all subsequent iterations are aggregated
 # subsequent iterations on failed trips have somewhat different overhead profile than initial batch
 chunk_tag = "trip_scheduling_1" if i == 1 else "trip_scheduling_n"

 choices = \
 run_trip_scheduling(
 trips_df,
 tours,
 probs_spec,
 model_settings,
 estimator=estimator,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 chunk_size=chunk_size,
 chunk_tag=chunk_tag,
 trace_label=trace_label_i)

 # boolean series of trips whose individual trip scheduling failed
 failed = choices.reindex(trips_df.index).isnull()
 logger.info("%s %s failed", trace_label_i, failed.sum())

 if not is_last_iteration:
 # boolean series of trips whose leg scheduling failed
 failed_cohorts = failed_trip_cohorts(trips_df, failed)
 trips_df = trips_df[failed_cohorts]
 choices = choices[~failed_cohorts]

 choices_list.append(choices)
 for chunk_i, trips_chunk, chunk_trace_label in chunk.adaptive_chunked_choosers_by_chunk_id(trips_df,
 chunk_size,
 trace_label,
 trace_label):

 i = 0
 while (i < max_iterations) and not trips_chunk.empty:

 # only chunk log first iteration since memory use declines with each iteration
 with chunk.chunk_log(trace_label) if i == 0 else chunk.chunk_log_skip():

 i += 1
 is_last_iteration = (i == max_iterations)

 trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
 logger.info("%s scheduling %s trips within chunk %s", trace_label_i, trips_chunk.shape[0], chunk_i)

 choices = \
 run_trip_scheduling(
 trips_chunk,
 tours,
 probs_spec,
 model_settings,
 estimator=estimator,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 chunk_size=chunk_size,
 chunk_tag=trace_label,
 trace_label=trace_label_i)

 # boolean series of trips whose individual trip scheduling failed
 failed = choices.reindex(trips_chunk.index).isnull()
 logger.info("%s %s failed", trace_label_i, failed.sum())

 if not is_last_iteration:
 # boolean series of trips whose leg scheduling failed
 failed_cohorts = failed_trip_cohorts(trips_chunk, failed)
 trips_chunk = trips_chunk[failed_cohorts]
 choices = choices[~failed_cohorts]

 choices_list.append(choices)

 trips_df = trips.to_frame()

Expand Down

29 changes: 25 additions & 4 deletions
activitysim/core/chunk.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -568,8 +568,9 @@ def log_rss(trace_label, force=False):
 hwm_trace_label = f"{trace_label}.log_rss"

 if chunk_training_mode() == MODE_PRODUCTION:
 trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
 mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
 # FIXME - this trace_memory_info call slows things down a lot so it is turned off for now
 # trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
 # mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
 return

 rss, uss = mem.trace_memory_info(hwm_trace_label)
Expand DownExpand Up@@ -624,7 +625,12 @@ class ChunkSizer(object):
 def __init__(self, chunk_tag, trace_label, num_choosers=0, chunk_size=0):

 self.depth = len(CHUNK_SIZERS) + 1
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

 if chunk_metric() == USS:
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
 else:
 self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
 self.uss = 0

 if self.depth > 1:
 # nested chunkers should be unchunked
Expand DownExpand Up@@ -755,7 +761,14 @@ def adaptive_rows_per_chunk(self, i):

 prev_rss = self.rss
 prev_uss = self.uss
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

 if chunk_training_mode() != MODE_PRODUCTION:

 if chunk_metric() == USS:
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
 else:
 self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
 self.uss = 0

 self.headroom = self.available_headroom(self.uss if chunk_metric() == USS else self.rss)

Expand DownExpand Up@@ -904,6 +917,14 @@ def chunk_log(trace_label, chunk_tag=None, base=False):
 chunk_sizer.close()


@contextmanager
def chunk_log_skip():

 yield

 None


def adaptive_chunked_choosers(choosers, chunk_size, trace_label, chunk_tag=None):

 # generator to iterate over choosers
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Speeding up chunking#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bstabler
merged 4 commits into

from

Aug 4, 2021
 Merged


Speeding up chunking
#458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
1359183

#444


bstabler Jul 12, 2021
4600b4c

a bit more chunking docs


bstabler Jul 13, 2021
40d04e5

Merge remote-tracking branch 'remotes/origin/develop' into speeding-u…


bstabler Jul 28, 2021
66d904f

faster trip scheduling in production by skipping chunking memory chec…


bstabler Aug 3, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


250 changes: 125 additions & 125 deletions
activitysim/abm/models/trip_scheduling.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -319,60 +319,57 @@ def schedule_trips_in_leg(
 result_list.append(choices)
 trips = trips[~no_scheduling]

 with chunk.chunk_log(trace_label):

 # add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
 trips = trips.sort_index()
 trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
 is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
 trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

 # iterate over outbound trips in ascending trip_num order, skipping the initial trip
 # iterate over inbound trips in descending trip_num order, skipping the finial trip
 first_trip_in_leg = True
 for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

 if outbound:
 nth_trips = trips[trips.trip_num == i]
 else:
 nth_trips = trips[trips.trip_num == trips.trip_count - i]

 nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

 with chunk.chunk_log(nth_trace_label, chunk_tag=trace_label):
 choices = schedule_nth_trips(
 nth_trips,
 probs_spec,
 model_settings,
 first_trip_in_leg=first_trip_in_leg,
 report_failed_trips=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=nth_trace_label)

 # if outbound, this trip's depart constrains next trip's earliest depart option
 # if inbound, we are handling in reverse order, so it constrains latest depart instead
 ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

 # most initial departure (when no choice was made because all probs were zero)
 if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
 choices = choices.reindex(nth_trips.index)
 logger.warning("%s coercing %s depart choices to most initial" %
 (nth_trace_label, choices.isna().sum()))
 choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

 # adjust allowed depart range of next trip
 has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
 if has_next_trip.any():
 next_trip_ids = nth_trips.next_trip_id[has_next_trip]
 # patch choice any trips with next_trips that weren't scheduled
 trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
 choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 first_trip_in_leg = False
 # add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
 trips = trips.sort_index()
 trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
 is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
 trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

 # iterate over outbound trips in ascending trip_num order, skipping the initial trip
 # iterate over inbound trips in descending trip_num order, skipping the finial trip
 first_trip_in_leg = True
 for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

 if outbound:
 nth_trips = trips[trips.trip_num == i]
 else:
 nth_trips = trips[trips.trip_num == trips.trip_count - i]

 nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

 choices = schedule_nth_trips(
 nth_trips,
 probs_spec,
 model_settings,
 first_trip_in_leg=first_trip_in_leg,
 report_failed_trips=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=nth_trace_label)

 # if outbound, this trip's depart constrains next trip's earliest depart option
 # if inbound, we are handling in reverse order, so it constrains latest depart instead
 ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

 # most initial departure (when no choice was made because all probs were zero)
 if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
 choices = choices.reindex(nth_trips.index)
 logger.warning("%s coercing %s depart choices to most initial" %
 (nth_trace_label, choices.isna().sum()))
 choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

 # adjust allowed depart range of next trip
 has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
 if has_next_trip.any():
 next_trip_ids = nth_trips.next_trip_id[has_next_trip]
 # patch choice any trips with next_trips that weren't scheduled
 trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
 choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 first_trip_in_leg = False

 if len(result_list) > 1:
 choices = pd.concat(result_list)
Expand All@@ -381,7 +378,7 @@ def schedule_trips_in_leg(


def run_trip_scheduling(
trips,
trips_chunk,
 tours,
 probs_spec,
 model_settings,
Expand All@@ -398,40 +395,38 @@ def run_trip_scheduling(
 # num_choosers = (is_inbound_chooser | is_outbound_chooser).sum()

 result_list = []
 for i, trips_chunk, chunk_trace_label \
 in chunk.adaptive_chunked_choosers_by_chunk_id(trips, chunk_size, trace_label, chunk_tag):

 if trips_chunk.outbound.any():
 leg_chunk = trips_chunk[trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'outbound')
 choices = \
 schedule_trips_in_leg(
 outbound=True,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if (~trips_chunk.outbound).any():
 leg_chunk = trips_chunk[~trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'inbound')
 choices = \
 schedule_trips_in_leg(
 outbound=False,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if trips_chunk.outbound.any():
 leg_chunk = trips_chunk[trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(trace_label, 'outbound')
 choices = \
 schedule_trips_in_leg(
 outbound=True,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if (~trips_chunk.outbound).any():
 leg_chunk = trips_chunk[~trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(trace_label, 'inbound')
 choices = \
 schedule_trips_in_leg(
 outbound=False,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 choices = pd.concat(result_list)

Expand DownExpand Up@@ -526,43 +521,48 @@ def trip_scheduling(
 assert max_iterations > 0

 choices_list = []
 i = 0
 while (i < max_iterations) and not trips_df.empty:

 i += 1
 is_last_iteration = (i == max_iterations)

 trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
 logger.info("%s scheduling %s trips", trace_label_i, trips_df.shape[0])

 # first iteration gets its own chunk_tag and all subsequent iterations are aggregated
 # subsequent iterations on failed trips have somewhat different overhead profile than initial batch
 chunk_tag = "trip_scheduling_1" if i == 1 else "trip_scheduling_n"

 choices = \
 run_trip_scheduling(
 trips_df,
 tours,
 probs_spec,
 model_settings,
 estimator=estimator,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 chunk_size=chunk_size,
 chunk_tag=chunk_tag,
 trace_label=trace_label_i)

 # boolean series of trips whose individual trip scheduling failed
 failed = choices.reindex(trips_df.index).isnull()
 logger.info("%s %s failed", trace_label_i, failed.sum())

 if not is_last_iteration:
 # boolean series of trips whose leg scheduling failed
 failed_cohorts = failed_trip_cohorts(trips_df, failed)
 trips_df = trips_df[failed_cohorts]
 choices = choices[~failed_cohorts]

 choices_list.append(choices)
 for chunk_i, trips_chunk, chunk_trace_label in chunk.adaptive_chunked_choosers_by_chunk_id(trips_df,
 chunk_size,
 trace_label,
 trace_label):

 i = 0
 while (i < max_iterations) and not trips_chunk.empty:

 # only chunk log first iteration since memory use declines with each iteration
 with chunk.chunk_log(trace_label) if i == 0 else chunk.chunk_log_skip():

 i += 1
 is_last_iteration = (i == max_iterations)

 trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
 logger.info("%s scheduling %s trips within chunk %s", trace_label_i, trips_chunk.shape[0], chunk_i)

 choices = \
 run_trip_scheduling(
 trips_chunk,
 tours,
 probs_spec,
 model_settings,
 estimator=estimator,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 chunk_size=chunk_size,
 chunk_tag=trace_label,
 trace_label=trace_label_i)

 # boolean series of trips whose individual trip scheduling failed
 failed = choices.reindex(trips_chunk.index).isnull()
 logger.info("%s %s failed", trace_label_i, failed.sum())

 if not is_last_iteration:
 # boolean series of trips whose leg scheduling failed
 failed_cohorts = failed_trip_cohorts(trips_chunk, failed)
 trips_chunk = trips_chunk[failed_cohorts]
 choices = choices[~failed_cohorts]

 choices_list.append(choices)

 trips_df = trips.to_frame()

Expand Down

29 changes: 25 additions & 4 deletions
activitysim/core/chunk.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -568,8 +568,9 @@ def log_rss(trace_label, force=False):
 hwm_trace_label = f"{trace_label}.log_rss"

 if chunk_training_mode() == MODE_PRODUCTION:
 trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
 mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
 # FIXME - this trace_memory_info call slows things down a lot so it is turned off for now
 # trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
 # mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
 return

 rss, uss = mem.trace_memory_info(hwm_trace_label)
Expand DownExpand Up@@ -624,7 +625,12 @@ class ChunkSizer(object):
 def __init__(self, chunk_tag, trace_label, num_choosers=0, chunk_size=0):

 self.depth = len(CHUNK_SIZERS) + 1
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

 if chunk_metric() == USS:
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
 else:
 self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
 self.uss = 0

 if self.depth > 1:
 # nested chunkers should be unchunked
Expand DownExpand Up@@ -755,7 +761,14 @@ def adaptive_rows_per_chunk(self, i):

 prev_rss = self.rss
 prev_uss = self.uss
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

 if chunk_training_mode() != MODE_PRODUCTION:

 if chunk_metric() == USS:
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
 else:
 self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
 self.uss = 0

 self.headroom = self.available_headroom(self.uss if chunk_metric() == USS else self.rss)

Expand DownExpand Up@@ -904,6 +917,14 @@ def chunk_log(trace_label, chunk_tag=None, base=False):
 chunk_sizer.close()


@contextmanager
def chunk_log_skip():

 yield

 None


def adaptive_chunked_choosers(choosers, chunk_size, trace_label, chunk_tag=None):

 # generator to iterate over choosers
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Speeding up chunking#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bstabler
merged 4 commits into

from

Aug 4, 2021
 Merged


Speeding up chunking
#458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
1359183

#444


bstabler Jul 12, 2021
4600b4c

a bit more chunking docs


bstabler Jul 13, 2021
40d04e5

Merge remote-tracking branch 'remotes/origin/develop' into speeding-u…


bstabler Jul 28, 2021
66d904f

faster trip scheduling in production by skipping chunking memory chec…


bstabler Aug 3, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


250 changes: 125 additions & 125 deletions
activitysim/abm/models/trip_scheduling.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -319,60 +319,57 @@ def schedule_trips_in_leg(
 result_list.append(choices)
 trips = trips[~no_scheduling]

 with chunk.chunk_log(trace_label):

 # add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
 trips = trips.sort_index()
 trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
 is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
 trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

 # iterate over outbound trips in ascending trip_num order, skipping the initial trip
 # iterate over inbound trips in descending trip_num order, skipping the finial trip
 first_trip_in_leg = True
 for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

 if outbound:
 nth_trips = trips[trips.trip_num == i]
 else:
 nth_trips = trips[trips.trip_num == trips.trip_count - i]

 nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

 with chunk.chunk_log(nth_trace_label, chunk_tag=trace_label):
 choices = schedule_nth_trips(
 nth_trips,
 probs_spec,
 model_settings,
 first_trip_in_leg=first_trip_in_leg,
 report_failed_trips=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=nth_trace_label)

 # if outbound, this trip's depart constrains next trip's earliest depart option
 # if inbound, we are handling in reverse order, so it constrains latest depart instead
 ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

 # most initial departure (when no choice was made because all probs were zero)
 if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
 choices = choices.reindex(nth_trips.index)
 logger.warning("%s coercing %s depart choices to most initial" %
 (nth_trace_label, choices.isna().sum()))
 choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

 # adjust allowed depart range of next trip
 has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
 if has_next_trip.any():
 next_trip_ids = nth_trips.next_trip_id[has_next_trip]
 # patch choice any trips with next_trips that weren't scheduled
 trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
 choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 first_trip_in_leg = False
 # add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
 trips = trips.sort_index()
 trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
 is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
 trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

 # iterate over outbound trips in ascending trip_num order, skipping the initial trip
 # iterate over inbound trips in descending trip_num order, skipping the finial trip
 first_trip_in_leg = True
 for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

 if outbound:
 nth_trips = trips[trips.trip_num == i]
 else:
 nth_trips = trips[trips.trip_num == trips.trip_count - i]

 nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

 choices = schedule_nth_trips(
 nth_trips,
 probs_spec,
 model_settings,
 first_trip_in_leg=first_trip_in_leg,
 report_failed_trips=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=nth_trace_label)

 # if outbound, this trip's depart constrains next trip's earliest depart option
 # if inbound, we are handling in reverse order, so it constrains latest depart instead
 ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

 # most initial departure (when no choice was made because all probs were zero)
 if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
 choices = choices.reindex(nth_trips.index)
 logger.warning("%s coercing %s depart choices to most initial" %
 (nth_trace_label, choices.isna().sum()))
 choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

 # adjust allowed depart range of next trip
 has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
 if has_next_trip.any():
 next_trip_ids = nth_trips.next_trip_id[has_next_trip]
 # patch choice any trips with next_trips that weren't scheduled
 trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
 choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 first_trip_in_leg = False

 if len(result_list) > 1:
 choices = pd.concat(result_list)
Expand All@@ -381,7 +378,7 @@ def schedule_trips_in_leg(


def run_trip_scheduling(
trips,
trips_chunk,
 tours,
 probs_spec,
 model_settings,
Expand All@@ -398,40 +395,38 @@ def run_trip_scheduling(
 # num_choosers = (is_inbound_chooser | is_outbound_chooser).sum()

 result_list = []
 for i, trips_chunk, chunk_trace_label \
 in chunk.adaptive_chunked_choosers_by_chunk_id(trips, chunk_size, trace_label, chunk_tag):

 if trips_chunk.outbound.any():
 leg_chunk = trips_chunk[trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'outbound')
 choices = \
 schedule_trips_in_leg(
 outbound=True,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if (~trips_chunk.outbound).any():
 leg_chunk = trips_chunk[~trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'inbound')
 choices = \
 schedule_trips_in_leg(
 outbound=False,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if trips_chunk.outbound.any():
 leg_chunk = trips_chunk[trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(trace_label, 'outbound')
 choices = \
 schedule_trips_in_leg(
 outbound=True,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if (~trips_chunk.outbound).any():
 leg_chunk = trips_chunk[~trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(trace_label, 'inbound')
 choices = \
 schedule_trips_in_leg(
 outbound=False,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 choices = pd.concat(result_list)

Expand DownExpand Up@@ -526,43 +521,48 @@ def trip_scheduling(
 assert max_iterations > 0

 choices_list = []
 i = 0
 while (i < max_iterations) and not trips_df.empty:

 i += 1
 is_last_iteration = (i == max_iterations)

 trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
 logger.info("%s scheduling %s trips", trace_label_i, trips_df.shape[0])

 # first iteration gets its own chunk_tag and all subsequent iterations are aggregated
 # subsequent iterations on failed trips have somewhat different overhead profile than initial batch
 chunk_tag = "trip_scheduling_1" if i == 1 else "trip_scheduling_n"

 choices = \
 run_trip_scheduling(
 trips_df,
 tours,
 probs_spec,
 model_settings,
 estimator=estimator,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 chunk_size=chunk_size,
 chunk_tag=chunk_tag,
 trace_label=trace_label_i)

 # boolean series of trips whose individual trip scheduling failed
 failed = choices.reindex(trips_df.index).isnull()
 logger.info("%s %s failed", trace_label_i, failed.sum())

 if not is_last_iteration:
 # boolean series of trips whose leg scheduling failed
 failed_cohorts = failed_trip_cohorts(trips_df, failed)
 trips_df = trips_df[failed_cohorts]
 choices = choices[~failed_cohorts]

 choices_list.append(choices)
 for chunk_i, trips_chunk, chunk_trace_label in chunk.adaptive_chunked_choosers_by_chunk_id(trips_df,
 chunk_size,
 trace_label,
 trace_label):

 i = 0
 while (i < max_iterations) and not trips_chunk.empty:

 # only chunk log first iteration since memory use declines with each iteration
 with chunk.chunk_log(trace_label) if i == 0 else chunk.chunk_log_skip():

 i += 1
 is_last_iteration = (i == max_iterations)

 trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
 logger.info("%s scheduling %s trips within chunk %s", trace_label_i, trips_chunk.shape[0], chunk_i)

 choices = \
 run_trip_scheduling(
 trips_chunk,
 tours,
 probs_spec,
 model_settings,
 estimator=estimator,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 chunk_size=chunk_size,
 chunk_tag=trace_label,
 trace_label=trace_label_i)

 # boolean series of trips whose individual trip scheduling failed
 failed = choices.reindex(trips_chunk.index).isnull()
 logger.info("%s %s failed", trace_label_i, failed.sum())

 if not is_last_iteration:
 # boolean series of trips whose leg scheduling failed
 failed_cohorts = failed_trip_cohorts(trips_chunk, failed)
 trips_chunk = trips_chunk[failed_cohorts]
 choices = choices[~failed_cohorts]

 choices_list.append(choices)

 trips_df = trips.to_frame()

Expand Down

29 changes: 25 additions & 4 deletions
activitysim/core/chunk.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -568,8 +568,9 @@ def log_rss(trace_label, force=False):
 hwm_trace_label = f"{trace_label}.log_rss"

 if chunk_training_mode() == MODE_PRODUCTION:
 trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
 mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
 # FIXME - this trace_memory_info call slows things down a lot so it is turned off for now
 # trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
 # mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
 return

 rss, uss = mem.trace_memory_info(hwm_trace_label)
Expand DownExpand Up@@ -624,7 +625,12 @@ class ChunkSizer(object):
 def __init__(self, chunk_tag, trace_label, num_choosers=0, chunk_size=0):

 self.depth = len(CHUNK_SIZERS) + 1
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

 if chunk_metric() == USS:
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
 else:
 self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
 self.uss = 0

 if self.depth > 1:
 # nested chunkers should be unchunked
Expand DownExpand Up@@ -755,7 +761,14 @@ def adaptive_rows_per_chunk(self, i):

 prev_rss = self.rss
 prev_uss = self.uss
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

 if chunk_training_mode() != MODE_PRODUCTION:

 if chunk_metric() == USS:
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
 else:
 self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
 self.uss = 0

 self.headroom = self.available_headroom(self.uss if chunk_metric() == USS else self.rss)

Expand DownExpand Up@@ -904,6 +917,14 @@ def chunk_log(trace_label, chunk_tag=None, base=False):
 chunk_sizer.close()


@contextmanager
def chunk_log_skip():

 yield

 None


def adaptive_chunked_choosers(choosers, chunk_size, trace_label, chunk_tag=None):

 # generator to iterate over choosers
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Speeding up chunking#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bstabler
merged 4 commits into

from

Aug 4, 2021
 Merged


Speeding up chunking
#458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
1359183

#444


bstabler Jul 12, 2021
4600b4c

a bit more chunking docs


bstabler Jul 13, 2021
40d04e5

Merge remote-tracking branch 'remotes/origin/develop' into speeding-u…


bstabler Jul 28, 2021
66d904f

faster trip scheduling in production by skipping chunking memory chec…


bstabler Aug 3, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


250 changes: 125 additions & 125 deletions
activitysim/abm/models/trip_scheduling.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -319,60 +319,57 @@ def schedule_trips_in_leg(
 result_list.append(choices)
 trips = trips[~no_scheduling]

 with chunk.chunk_log(trace_label):

 # add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
 trips = trips.sort_index()
 trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
 is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
 trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

 # iterate over outbound trips in ascending trip_num order, skipping the initial trip
 # iterate over inbound trips in descending trip_num order, skipping the finial trip
 first_trip_in_leg = True
 for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

 if outbound:
 nth_trips = trips[trips.trip_num == i]
 else:
 nth_trips = trips[trips.trip_num == trips.trip_count - i]

 nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

 with chunk.chunk_log(nth_trace_label, chunk_tag=trace_label):
 choices = schedule_nth_trips(
 nth_trips,
 probs_spec,
 model_settings,
 first_trip_in_leg=first_trip_in_leg,
 report_failed_trips=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=nth_trace_label)

 # if outbound, this trip's depart constrains next trip's earliest depart option
 # if inbound, we are handling in reverse order, so it constrains latest depart instead
 ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

 # most initial departure (when no choice was made because all probs were zero)
 if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
 choices = choices.reindex(nth_trips.index)
 logger.warning("%s coercing %s depart choices to most initial" %
 (nth_trace_label, choices.isna().sum()))
 choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

 # adjust allowed depart range of next trip
 has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
 if has_next_trip.any():
 next_trip_ids = nth_trips.next_trip_id[has_next_trip]
 # patch choice any trips with next_trips that weren't scheduled
 trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
 choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 first_trip_in_leg = False
 # add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
 trips = trips.sort_index()
 trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
 is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
 trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

 # iterate over outbound trips in ascending trip_num order, skipping the initial trip
 # iterate over inbound trips in descending trip_num order, skipping the finial trip
 first_trip_in_leg = True
 for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

 if outbound:
 nth_trips = trips[trips.trip_num == i]
 else:
 nth_trips = trips[trips.trip_num == trips.trip_count - i]

 nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

 choices = schedule_nth_trips(
 nth_trips,
 probs_spec,
 model_settings,
 first_trip_in_leg=first_trip_in_leg,
 report_failed_trips=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=nth_trace_label)

 # if outbound, this trip's depart constrains next trip's earliest depart option
 # if inbound, we are handling in reverse order, so it constrains latest depart instead
 ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

 # most initial departure (when no choice was made because all probs were zero)
 if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
 choices = choices.reindex(nth_trips.index)
 logger.warning("%s coercing %s depart choices to most initial" %
 (nth_trace_label, choices.isna().sum()))
 choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

 # adjust allowed depart range of next trip
 has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
 if has_next_trip.any():
 next_trip_ids = nth_trips.next_trip_id[has_next_trip]
 # patch choice any trips with next_trips that weren't scheduled
 trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
 choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 first_trip_in_leg = False

 if len(result_list) > 1:
 choices = pd.concat(result_list)
Expand All@@ -381,7 +378,7 @@ def schedule_trips_in_leg(


def run_trip_scheduling(
trips,
trips_chunk,
 tours,
 probs_spec,
 model_settings,
Expand All@@ -398,40 +395,38 @@ def run_trip_scheduling(
 # num_choosers = (is_inbound_chooser | is_outbound_chooser).sum()

 result_list = []
 for i, trips_chunk, chunk_trace_label \
 in chunk.adaptive_chunked_choosers_by_chunk_id(trips, chunk_size, trace_label, chunk_tag):

 if trips_chunk.outbound.any():
 leg_chunk = trips_chunk[trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'outbound')
 choices = \
 schedule_trips_in_leg(
 outbound=True,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if (~trips_chunk.outbound).any():
 leg_chunk = trips_chunk[~trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'inbound')
 choices = \
 schedule_trips_in_leg(
 outbound=False,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if trips_chunk.outbound.any():
 leg_chunk = trips_chunk[trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(trace_label, 'outbound')
 choices = \
 schedule_trips_in_leg(
 outbound=True,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if (~trips_chunk.outbound).any():
 leg_chunk = trips_chunk[~trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(trace_label, 'inbound')
 choices = \
 schedule_trips_in_leg(
 outbound=False,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 choices = pd.concat(result_list)

Expand DownExpand Up@@ -526,43 +521,48 @@ def trip_scheduling(
 assert max_iterations > 0

 choices_list = []
 i = 0
 while (i < max_iterations) and not trips_df.empty:

 i += 1
 is_last_iteration = (i == max_iterations)

 trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
 logger.info("%s scheduling %s trips", trace_label_i, trips_df.shape[0])

 # first iteration gets its own chunk_tag and all subsequent iterations are aggregated
 # subsequent iterations on failed trips have somewhat different overhead profile than initial batch
 chunk_tag = "trip_scheduling_1" if i == 1 else "trip_scheduling_n"

 choices = \
 run_trip_scheduling(
 trips_df,
 tours,
 probs_spec,
 model_settings,
 estimator=estimator,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 chunk_size=chunk_size,
 chunk_tag=chunk_tag,
 trace_label=trace_label_i)

 # boolean series of trips whose individual trip scheduling failed
 failed = choices.reindex(trips_df.index).isnull()
 logger.info("%s %s failed", trace_label_i, failed.sum())

 if not is_last_iteration:
 # boolean series of trips whose leg scheduling failed
 failed_cohorts = failed_trip_cohorts(trips_df, failed)
 trips_df = trips_df[failed_cohorts]
 choices = choices[~failed_cohorts]

 choices_list.append(choices)
 for chunk_i, trips_chunk, chunk_trace_label in chunk.adaptive_chunked_choosers_by_chunk_id(trips_df,
 chunk_size,
 trace_label,
 trace_label):

 i = 0
 while (i < max_iterations) and not trips_chunk.empty:

 # only chunk log first iteration since memory use declines with each iteration
 with chunk.chunk_log(trace_label) if i == 0 else chunk.chunk_log_skip():

 i += 1
 is_last_iteration = (i == max_iterations)

 trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
 logger.info("%s scheduling %s trips within chunk %s", trace_label_i, trips_chunk.shape[0], chunk_i)

 choices = \
 run_trip_scheduling(
 trips_chunk,
 tours,
 probs_spec,
 model_settings,
 estimator=estimator,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 chunk_size=chunk_size,
 chunk_tag=trace_label,
 trace_label=trace_label_i)

 # boolean series of trips whose individual trip scheduling failed
 failed = choices.reindex(trips_chunk.index).isnull()
 logger.info("%s %s failed", trace_label_i, failed.sum())

 if not is_last_iteration:
 # boolean series of trips whose leg scheduling failed
 failed_cohorts = failed_trip_cohorts(trips_chunk, failed)
 trips_chunk = trips_chunk[failed_cohorts]
 choices = choices[~failed_cohorts]

 choices_list.append(choices)

 trips_df = trips.to_frame()

Expand Down

29 changes: 25 additions & 4 deletions
activitysim/core/chunk.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -568,8 +568,9 @@ def log_rss(trace_label, force=False):
 hwm_trace_label = f"{trace_label}.log_rss"

 if chunk_training_mode() == MODE_PRODUCTION:
 trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
 mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
 # FIXME - this trace_memory_info call slows things down a lot so it is turned off for now
 # trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
 # mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
 return

 rss, uss = mem.trace_memory_info(hwm_trace_label)
Expand DownExpand Up@@ -624,7 +625,12 @@ class ChunkSizer(object):
 def __init__(self, chunk_tag, trace_label, num_choosers=0, chunk_size=0):

 self.depth = len(CHUNK_SIZERS) + 1
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

 if chunk_metric() == USS:
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
 else:
 self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
 self.uss = 0

 if self.depth > 1:
 # nested chunkers should be unchunked
Expand DownExpand Up@@ -755,7 +761,14 @@ def adaptive_rows_per_chunk(self, i):

 prev_rss = self.rss
 prev_uss = self.uss
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

 if chunk_training_mode() != MODE_PRODUCTION:

 if chunk_metric() == USS:
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
 else:
 self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
 self.uss = 0

 self.headroom = self.available_headroom(self.uss if chunk_metric() == USS else self.rss)

Expand DownExpand Up@@ -904,6 +917,14 @@ def chunk_log(trace_label, chunk_tag=None, base=False):
 chunk_sizer.close()


@contextmanager
def chunk_log_skip():

 yield

 None


def adaptive_chunked_choosers(choosers, chunk_size, trace_label, chunk_tag=None):

 # generator to iterate over choosers
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Speeding up chunking#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bstabler
merged 4 commits into

from

Aug 4, 2021
 Merged


Speeding up chunking
#458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
1359183

#444


bstabler Jul 12, 2021
4600b4c

a bit more chunking docs


bstabler Jul 13, 2021
40d04e5

Merge remote-tracking branch 'remotes/origin/develop' into speeding-u…


bstabler Jul 28, 2021
66d904f

faster trip scheduling in production by skipping chunking memory chec…


bstabler Aug 3, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


250 changes: 125 additions & 125 deletions
activitysim/abm/models/trip_scheduling.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -319,60 +319,57 @@ def schedule_trips_in_leg(
 result_list.append(choices)
 trips = trips[~no_scheduling]

 with chunk.chunk_log(trace_label):

 # add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
 trips = trips.sort_index()
 trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
 is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
 trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

 # iterate over outbound trips in ascending trip_num order, skipping the initial trip
 # iterate over inbound trips in descending trip_num order, skipping the finial trip
 first_trip_in_leg = True
 for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

 if outbound:
 nth_trips = trips[trips.trip_num == i]
 else:
 nth_trips = trips[trips.trip_num == trips.trip_count - i]

 nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

 with chunk.chunk_log(nth_trace_label, chunk_tag=trace_label):
 choices = schedule_nth_trips(
 nth_trips,
 probs_spec,
 model_settings,
 first_trip_in_leg=first_trip_in_leg,
 report_failed_trips=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=nth_trace_label)

 # if outbound, this trip's depart constrains next trip's earliest depart option
 # if inbound, we are handling in reverse order, so it constrains latest depart instead
 ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

 # most initial departure (when no choice was made because all probs were zero)
 if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
 choices = choices.reindex(nth_trips.index)
 logger.warning("%s coercing %s depart choices to most initial" %
 (nth_trace_label, choices.isna().sum()))
 choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

 # adjust allowed depart range of next trip
 has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
 if has_next_trip.any():
 next_trip_ids = nth_trips.next_trip_id[has_next_trip]
 # patch choice any trips with next_trips that weren't scheduled
 trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
 choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 first_trip_in_leg = False
 # add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
 trips = trips.sort_index()
 trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
 is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
 trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

 # iterate over outbound trips in ascending trip_num order, skipping the initial trip
 # iterate over inbound trips in descending trip_num order, skipping the finial trip
 first_trip_in_leg = True
 for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

 if outbound:
 nth_trips = trips[trips.trip_num == i]
 else:
 nth_trips = trips[trips.trip_num == trips.trip_count - i]

 nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

 choices = schedule_nth_trips(
 nth_trips,
 probs_spec,
 model_settings,
 first_trip_in_leg=first_trip_in_leg,
 report_failed_trips=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=nth_trace_label)

 # if outbound, this trip's depart constrains next trip's earliest depart option
 # if inbound, we are handling in reverse order, so it constrains latest depart instead
 ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

 # most initial departure (when no choice was made because all probs were zero)
 if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
 choices = choices.reindex(nth_trips.index)
 logger.warning("%s coercing %s depart choices to most initial" %
 (nth_trace_label, choices.isna().sum()))
 choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

 # adjust allowed depart range of next trip
 has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
 if has_next_trip.any():
 next_trip_ids = nth_trips.next_trip_id[has_next_trip]
 # patch choice any trips with next_trips that weren't scheduled
 trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
 choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 first_trip_in_leg = False

 if len(result_list) > 1:
 choices = pd.concat(result_list)
Expand All@@ -381,7 +378,7 @@ def schedule_trips_in_leg(


def run_trip_scheduling(
trips,
trips_chunk,
 tours,
 probs_spec,
 model_settings,
Expand All@@ -398,40 +395,38 @@ def run_trip_scheduling(
 # num_choosers = (is_inbound_chooser | is_outbound_chooser).sum()

 result_list = []
 for i, trips_chunk, chunk_trace_label \
 in chunk.adaptive_chunked_choosers_by_chunk_id(trips, chunk_size, trace_label, chunk_tag):

 if trips_chunk.outbound.any():
 leg_chunk = trips_chunk[trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'outbound')
 choices = \
 schedule_trips_in_leg(
 outbound=True,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if (~trips_chunk.outbound).any():
 leg_chunk = trips_chunk[~trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'inbound')
 choices = \
 schedule_trips_in_leg(
 outbound=False,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if trips_chunk.outbound.any():
 leg_chunk = trips_chunk[trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(trace_label, 'outbound')
 choices = \
 schedule_trips_in_leg(
 outbound=True,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if (~trips_chunk.outbound).any():
 leg_chunk = trips_chunk[~trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(trace_label, 'inbound')
 choices = \
 schedule_trips_in_leg(
 outbound=False,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 choices = pd.concat(result_list)

Expand DownExpand Up@@ -526,43 +521,48 @@ def trip_scheduling(
 assert max_iterations > 0

 choices_list = []
 i = 0
 while (i < max_iterations) and not trips_df.empty:

 i += 1
 is_last_iteration = (i == max_iterations)

 trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
 logger.info("%s scheduling %s trips", trace_label_i, trips_df.shape[0])

 # first iteration gets its own chunk_tag and all subsequent iterations are aggregated
 # subsequent iterations on failed trips have somewhat different overhead profile than initial batch
 chunk_tag = "trip_scheduling_1" if i == 1 else "trip_scheduling_n"

 choices = \
 run_trip_scheduling(
 trips_df,
 tours,
 probs_spec,
 model_settings,
 estimator=estimator,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 chunk_size=chunk_size,
 chunk_tag=chunk_tag,
 trace_label=trace_label_i)

 # boolean series of trips whose individual trip scheduling failed
 failed = choices.reindex(trips_df.index).isnull()
 logger.info("%s %s failed", trace_label_i, failed.sum())

 if not is_last_iteration:
 # boolean series of trips whose leg scheduling failed
 failed_cohorts = failed_trip_cohorts(trips_df, failed)
 trips_df = trips_df[failed_cohorts]
 choices = choices[~failed_cohorts]

 choices_list.append(choices)
 for chunk_i, trips_chunk, chunk_trace_label in chunk.adaptive_chunked_choosers_by_chunk_id(trips_df,
 chunk_size,
 trace_label,
 trace_label):

 i = 0
 while (i < max_iterations) and not trips_chunk.empty:

 # only chunk log first iteration since memory use declines with each iteration
 with chunk.chunk_log(trace_label) if i == 0 else chunk.chunk_log_skip():

 i += 1
 is_last_iteration = (i == max_iterations)

 trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
 logger.info("%s scheduling %s trips within chunk %s", trace_label_i, trips_chunk.shape[0], chunk_i)

 choices = \
 run_trip_scheduling(
 trips_chunk,
 tours,
 probs_spec,
 model_settings,
 estimator=estimator,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 chunk_size=chunk_size,
 chunk_tag=trace_label,
 trace_label=trace_label_i)

 # boolean series of trips whose individual trip scheduling failed
 failed = choices.reindex(trips_chunk.index).isnull()
 logger.info("%s %s failed", trace_label_i, failed.sum())

 if not is_last_iteration:
 # boolean series of trips whose leg scheduling failed
 failed_cohorts = failed_trip_cohorts(trips_chunk, failed)
 trips_chunk = trips_chunk[failed_cohorts]
 choices = choices[~failed_cohorts]

 choices_list.append(choices)

 trips_df = trips.to_frame()

Expand Down

29 changes: 25 additions & 4 deletions
activitysim/core/chunk.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -568,8 +568,9 @@ def log_rss(trace_label, force=False):
 hwm_trace_label = f"{trace_label}.log_rss"

 if chunk_training_mode() == MODE_PRODUCTION:
 trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
 mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
 # FIXME - this trace_memory_info call slows things down a lot so it is turned off for now
 # trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
 # mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
 return

 rss, uss = mem.trace_memory_info(hwm_trace_label)
Expand DownExpand Up@@ -624,7 +625,12 @@ class ChunkSizer(object):
 def __init__(self, chunk_tag, trace_label, num_choosers=0, chunk_size=0):

 self.depth = len(CHUNK_SIZERS) + 1
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

 if chunk_metric() == USS:
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
 else:
 self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
 self.uss = 0

 if self.depth > 1:
 # nested chunkers should be unchunked
Expand DownExpand Up@@ -755,7 +761,14 @@ def adaptive_rows_per_chunk(self, i):

 prev_rss = self.rss
 prev_uss = self.uss
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

 if chunk_training_mode() != MODE_PRODUCTION:

 if chunk_metric() == USS:
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
 else:
 self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
 self.uss = 0

 self.headroom = self.available_headroom(self.uss if chunk_metric() == USS else self.rss)

Expand DownExpand Up@@ -904,6 +917,14 @@ def chunk_log(trace_label, chunk_tag=None, base=False):
 chunk_sizer.close()


@contextmanager
def chunk_log_skip():

 yield

 None


def adaptive_chunked_choosers(choosers, chunk_size, trace_label, chunk_tag=None):

 # generator to iterate over choosers
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Speeding up chunking#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bstabler
merged 4 commits into

from

Aug 4, 2021
 Merged


Speeding up chunking
#458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
1359183

#444


bstabler Jul 12, 2021
4600b4c

a bit more chunking docs


bstabler Jul 13, 2021
40d04e5

Merge remote-tracking branch 'remotes/origin/develop' into speeding-u…


bstabler Jul 28, 2021
66d904f

faster trip scheduling in production by skipping chunking memory chec…


bstabler Aug 3, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


250 changes: 125 additions & 125 deletions
activitysim/abm/models/trip_scheduling.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -319,60 +319,57 @@ def schedule_trips_in_leg(
 result_list.append(choices)
 trips = trips[~no_scheduling]

 with chunk.chunk_log(trace_label):

 # add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
 trips = trips.sort_index()
 trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
 is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
 trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

 # iterate over outbound trips in ascending trip_num order, skipping the initial trip
 # iterate over inbound trips in descending trip_num order, skipping the finial trip
 first_trip_in_leg = True
 for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

 if outbound:
 nth_trips = trips[trips.trip_num == i]
 else:
 nth_trips = trips[trips.trip_num == trips.trip_count - i]

 nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

 with chunk.chunk_log(nth_trace_label, chunk_tag=trace_label):
 choices = schedule_nth_trips(
 nth_trips,
 probs_spec,
 model_settings,
 first_trip_in_leg=first_trip_in_leg,
 report_failed_trips=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=nth_trace_label)

 # if outbound, this trip's depart constrains next trip's earliest depart option
 # if inbound, we are handling in reverse order, so it constrains latest depart instead
 ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

 # most initial departure (when no choice was made because all probs were zero)
 if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
 choices = choices.reindex(nth_trips.index)
 logger.warning("%s coercing %s depart choices to most initial" %
 (nth_trace_label, choices.isna().sum()))
 choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

 # adjust allowed depart range of next trip
 has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
 if has_next_trip.any():
 next_trip_ids = nth_trips.next_trip_id[has_next_trip]
 # patch choice any trips with next_trips that weren't scheduled
 trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
 choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 first_trip_in_leg = False
 # add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
 trips = trips.sort_index()
 trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
 is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
 trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

 # iterate over outbound trips in ascending trip_num order, skipping the initial trip
 # iterate over inbound trips in descending trip_num order, skipping the finial trip
 first_trip_in_leg = True
 for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

 if outbound:
 nth_trips = trips[trips.trip_num == i]
 else:
 nth_trips = trips[trips.trip_num == trips.trip_count - i]

 nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

 choices = schedule_nth_trips(
 nth_trips,
 probs_spec,
 model_settings,
 first_trip_in_leg=first_trip_in_leg,
 report_failed_trips=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=nth_trace_label)

 # if outbound, this trip's depart constrains next trip's earliest depart option
 # if inbound, we are handling in reverse order, so it constrains latest depart instead
 ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

 # most initial departure (when no choice was made because all probs were zero)
 if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
 choices = choices.reindex(nth_trips.index)
 logger.warning("%s coercing %s depart choices to most initial" %
 (nth_trace_label, choices.isna().sum()))
 choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

 # adjust allowed depart range of next trip
 has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
 if has_next_trip.any():
 next_trip_ids = nth_trips.next_trip_id[has_next_trip]
 # patch choice any trips with next_trips that weren't scheduled
 trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
 choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 first_trip_in_leg = False

 if len(result_list) > 1:
 choices = pd.concat(result_list)
Expand All@@ -381,7 +378,7 @@ def schedule_trips_in_leg(


def run_trip_scheduling(
trips,
trips_chunk,
 tours,
 probs_spec,
 model_settings,
Expand All@@ -398,40 +395,38 @@ def run_trip_scheduling(
 # num_choosers = (is_inbound_chooser | is_outbound_chooser).sum()

 result_list = []
 for i, trips_chunk, chunk_trace_label \
 in chunk.adaptive_chunked_choosers_by_chunk_id(trips, chunk_size, trace_label, chunk_tag):

 if trips_chunk.outbound.any():
 leg_chunk = trips_chunk[trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'outbound')
 choices = \
 schedule_trips_in_leg(
 outbound=True,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if (~trips_chunk.outbound).any():
 leg_chunk = trips_chunk[~trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'inbound')
 choices = \
 schedule_trips_in_leg(
 outbound=False,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if trips_chunk.outbound.any():
 leg_chunk = trips_chunk[trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(trace_label, 'outbound')
 choices = \
 schedule_trips_in_leg(
 outbound=True,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if (~trips_chunk.outbound).any():
 leg_chunk = trips_chunk[~trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(trace_label, 'inbound')
 choices = \
 schedule_trips_in_leg(
 outbound=False,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 choices = pd.concat(result_list)

Expand DownExpand Up@@ -526,43 +521,48 @@ def trip_scheduling(
 assert max_iterations > 0

 choices_list = []
 i = 0
 while (i < max_iterations) and not trips_df.empty:

 i += 1
 is_last_iteration = (i == max_iterations)

 trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
 logger.info("%s scheduling %s trips", trace_label_i, trips_df.shape[0])

 # first iteration gets its own chunk_tag and all subsequent iterations are aggregated
 # subsequent iterations on failed trips have somewhat different overhead profile than initial batch
 chunk_tag = "trip_scheduling_1" if i == 1 else "trip_scheduling_n"

 choices = \
 run_trip_scheduling(
 trips_df,
 tours,
 probs_spec,
 model_settings,
 estimator=estimator,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 chunk_size=chunk_size,
 chunk_tag=chunk_tag,
 trace_label=trace_label_i)

 # boolean series of trips whose individual trip scheduling failed
 failed = choices.reindex(trips_df.index).isnull()
 logger.info("%s %s failed", trace_label_i, failed.sum())

 if not is_last_iteration:
 # boolean series of trips whose leg scheduling failed
 failed_cohorts = failed_trip_cohorts(trips_df, failed)
 trips_df = trips_df[failed_cohorts]
 choices = choices[~failed_cohorts]

 choices_list.append(choices)
 for chunk_i, trips_chunk, chunk_trace_label in chunk.adaptive_chunked_choosers_by_chunk_id(trips_df,
 chunk_size,
 trace_label,
 trace_label):

 i = 0
 while (i < max_iterations) and not trips_chunk.empty:

 # only chunk log first iteration since memory use declines with each iteration
 with chunk.chunk_log(trace_label) if i == 0 else chunk.chunk_log_skip():

 i += 1
 is_last_iteration = (i == max_iterations)

 trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
 logger.info("%s scheduling %s trips within chunk %s", trace_label_i, trips_chunk.shape[0], chunk_i)

 choices = \
 run_trip_scheduling(
 trips_chunk,
 tours,
 probs_spec,
 model_settings,
 estimator=estimator,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 chunk_size=chunk_size,
 chunk_tag=trace_label,
 trace_label=trace_label_i)

 # boolean series of trips whose individual trip scheduling failed
 failed = choices.reindex(trips_chunk.index).isnull()
 logger.info("%s %s failed", trace_label_i, failed.sum())

 if not is_last_iteration:
 # boolean series of trips whose leg scheduling failed
 failed_cohorts = failed_trip_cohorts(trips_chunk, failed)
 trips_chunk = trips_chunk[failed_cohorts]
 choices = choices[~failed_cohorts]

 choices_list.append(choices)

 trips_df = trips.to_frame()

Expand Down

29 changes: 25 additions & 4 deletions
activitysim/core/chunk.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -568,8 +568,9 @@ def log_rss(trace_label, force=False):
 hwm_trace_label = f"{trace_label}.log_rss"

 if chunk_training_mode() == MODE_PRODUCTION:
 trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
 mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
 # FIXME - this trace_memory_info call slows things down a lot so it is turned off for now
 # trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
 # mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
 return

 rss, uss = mem.trace_memory_info(hwm_trace_label)
Expand DownExpand Up@@ -624,7 +625,12 @@ class ChunkSizer(object):
 def __init__(self, chunk_tag, trace_label, num_choosers=0, chunk_size=0):

 self.depth = len(CHUNK_SIZERS) + 1
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

 if chunk_metric() == USS:
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
 else:
 self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
 self.uss = 0

 if self.depth > 1:
 # nested chunkers should be unchunked
Expand DownExpand Up@@ -755,7 +761,14 @@ def adaptive_rows_per_chunk(self, i):

 prev_rss = self.rss
 prev_uss = self.uss
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

 if chunk_training_mode() != MODE_PRODUCTION:

 if chunk_metric() == USS:
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
 else:
 self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
 self.uss = 0

 self.headroom = self.available_headroom(self.uss if chunk_metric() == USS else self.rss)

Expand DownExpand Up@@ -904,6 +917,14 @@ def chunk_log(trace_label, chunk_tag=None, base=False):
 chunk_sizer.close()


@contextmanager
def chunk_log_skip():

 yield

 None


def adaptive_chunked_choosers(choosers, chunk_size, trace_label, chunk_tag=None):

 # generator to iterate over choosers
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Speeding up chunking#458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bstabler
merged 4 commits into

from

Aug 4, 2021
 Merged


Speeding up chunking
#458
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
1359183

#444


bstabler Jul 12, 2021
4600b4c

a bit more chunking docs


bstabler Jul 13, 2021
40d04e5

Merge remote-tracking branch 'remotes/origin/develop' into speeding-u…


bstabler Jul 28, 2021
66d904f

faster trip scheduling in production by skipping chunking memory chec…


bstabler Aug 3, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


250 changes: 125 additions & 125 deletions
activitysim/abm/models/trip_scheduling.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -319,60 +319,57 @@ def schedule_trips_in_leg(
 result_list.append(choices)
 trips = trips[~no_scheduling]

 with chunk.chunk_log(trace_label):

 # add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
 trips = trips.sort_index()
 trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
 is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
 trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

 # iterate over outbound trips in ascending trip_num order, skipping the initial trip
 # iterate over inbound trips in descending trip_num order, skipping the finial trip
 first_trip_in_leg = True
 for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

 if outbound:
 nth_trips = trips[trips.trip_num == i]
 else:
 nth_trips = trips[trips.trip_num == trips.trip_count - i]

 nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

 with chunk.chunk_log(nth_trace_label, chunk_tag=trace_label):
 choices = schedule_nth_trips(
 nth_trips,
 probs_spec,
 model_settings,
 first_trip_in_leg=first_trip_in_leg,
 report_failed_trips=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=nth_trace_label)

 # if outbound, this trip's depart constrains next trip's earliest depart option
 # if inbound, we are handling in reverse order, so it constrains latest depart instead
 ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

 # most initial departure (when no choice was made because all probs were zero)
 if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
 choices = choices.reindex(nth_trips.index)
 logger.warning("%s coercing %s depart choices to most initial" %
 (nth_trace_label, choices.isna().sum()))
 choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

 # adjust allowed depart range of next trip
 has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
 if has_next_trip.any():
 next_trip_ids = nth_trips.next_trip_id[has_next_trip]
 # patch choice any trips with next_trips that weren't scheduled
 trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
 choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 first_trip_in_leg = False
 # add next_trip_id temp column (temp as trips is now a copy, as result of slicing)
 trips = trips.sort_index()
 trips['next_trip_id'] = np.roll(trips.index, -1 if outbound else 1)
 is_final = (trips.trip_num == trips.trip_count) if outbound else (trips.trip_num == 1)
 trips.next_trip_id = trips.next_trip_id.where(~is_final, NO_TRIP_ID)

 # iterate over outbound trips in ascending trip_num order, skipping the initial trip
 # iterate over inbound trips in descending trip_num order, skipping the finial trip
 first_trip_in_leg = True
 for i in range(trips.trip_num.min(), trips.trip_num.max() + 1):

 if outbound:
 nth_trips = trips[trips.trip_num == i]
 else:
 nth_trips = trips[trips.trip_num == trips.trip_count - i]

 nth_trace_label = tracing.extend_trace_label(trace_label, 'num_%s' % i)

 choices = schedule_nth_trips(
 nth_trips,
 probs_spec,
 model_settings,
 first_trip_in_leg=first_trip_in_leg,
 report_failed_trips=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=nth_trace_label)

 # if outbound, this trip's depart constrains next trip's earliest depart option
 # if inbound, we are handling in reverse order, so it constrains latest depart instead
 ADJUST_NEXT_DEPART_COL = 'earliest' if outbound else 'latest'

 # most initial departure (when no choice was made because all probs were zero)
 if is_last_iteration and (failfix == FAILFIX_CHOOSE_MOST_INITIAL):
 choices = choices.reindex(nth_trips.index)
 logger.warning("%s coercing %s depart choices to most initial" %
 (nth_trace_label, choices.isna().sum()))
 choices = choices.fillna(trips[ADJUST_NEXT_DEPART_COL])

 # adjust allowed depart range of next trip
 has_next_trip = (nth_trips.next_trip_id != NO_TRIP_ID)
 if has_next_trip.any():
 next_trip_ids = nth_trips.next_trip_id[has_next_trip]
 # patch choice any trips with next_trips that weren't scheduled
 trips.loc[next_trip_ids, ADJUST_NEXT_DEPART_COL] = \
 choices.reindex(next_trip_ids.index).fillna(trips[ADJUST_NEXT_DEPART_COL]).values

 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 first_trip_in_leg = False

 if len(result_list) > 1:
 choices = pd.concat(result_list)
Expand All@@ -381,7 +378,7 @@ def schedule_trips_in_leg(


def run_trip_scheduling(
trips,
trips_chunk,
 tours,
 probs_spec,
 model_settings,
Expand All@@ -398,40 +395,38 @@ def run_trip_scheduling(
 # num_choosers = (is_inbound_chooser | is_outbound_chooser).sum()

 result_list = []
 for i, trips_chunk, chunk_trace_label \
 in chunk.adaptive_chunked_choosers_by_chunk_id(trips, chunk_size, trace_label, chunk_tag):

 if trips_chunk.outbound.any():
 leg_chunk = trips_chunk[trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'outbound')
 choices = \
 schedule_trips_in_leg(
 outbound=True,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if (~trips_chunk.outbound).any():
 leg_chunk = trips_chunk[~trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(chunk_trace_label, 'inbound')
 choices = \
 schedule_trips_in_leg(
 outbound=False,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if trips_chunk.outbound.any():
 leg_chunk = trips_chunk[trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(trace_label, 'outbound')
 choices = \
 schedule_trips_in_leg(
 outbound=True,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 if (~trips_chunk.outbound).any():
 leg_chunk = trips_chunk[~trips_chunk.outbound]
 leg_trace_label = tracing.extend_trace_label(trace_label, 'inbound')
 choices = \
 schedule_trips_in_leg(
 outbound=False,
 trips=leg_chunk,
 probs_spec=probs_spec,
 model_settings=model_settings,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 trace_label=leg_trace_label)
 result_list.append(choices)

 chunk.log_df(trace_label, f'result_list', result_list)

 choices = pd.concat(result_list)

Expand DownExpand Up@@ -526,43 +521,48 @@ def trip_scheduling(
 assert max_iterations > 0

 choices_list = []
 i = 0
 while (i < max_iterations) and not trips_df.empty:

 i += 1
 is_last_iteration = (i == max_iterations)

 trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
 logger.info("%s scheduling %s trips", trace_label_i, trips_df.shape[0])

 # first iteration gets its own chunk_tag and all subsequent iterations are aggregated
 # subsequent iterations on failed trips have somewhat different overhead profile than initial batch
 chunk_tag = "trip_scheduling_1" if i == 1 else "trip_scheduling_n"

 choices = \
 run_trip_scheduling(
 trips_df,
 tours,
 probs_spec,
 model_settings,
 estimator=estimator,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 chunk_size=chunk_size,
 chunk_tag=chunk_tag,
 trace_label=trace_label_i)

 # boolean series of trips whose individual trip scheduling failed
 failed = choices.reindex(trips_df.index).isnull()
 logger.info("%s %s failed", trace_label_i, failed.sum())

 if not is_last_iteration:
 # boolean series of trips whose leg scheduling failed
 failed_cohorts = failed_trip_cohorts(trips_df, failed)
 trips_df = trips_df[failed_cohorts]
 choices = choices[~failed_cohorts]

 choices_list.append(choices)
 for chunk_i, trips_chunk, chunk_trace_label in chunk.adaptive_chunked_choosers_by_chunk_id(trips_df,
 chunk_size,
 trace_label,
 trace_label):

 i = 0
 while (i < max_iterations) and not trips_chunk.empty:

 # only chunk log first iteration since memory use declines with each iteration
 with chunk.chunk_log(trace_label) if i == 0 else chunk.chunk_log_skip():

 i += 1
 is_last_iteration = (i == max_iterations)

 trace_label_i = tracing.extend_trace_label(trace_label, "i%s" % i)
 logger.info("%s scheduling %s trips within chunk %s", trace_label_i, trips_chunk.shape[0], chunk_i)

 choices = \
 run_trip_scheduling(
 trips_chunk,
 tours,
 probs_spec,
 model_settings,
 estimator=estimator,
 is_last_iteration=is_last_iteration,
 trace_hh_id=trace_hh_id,
 chunk_size=chunk_size,
 chunk_tag=trace_label,
 trace_label=trace_label_i)

 # boolean series of trips whose individual trip scheduling failed
 failed = choices.reindex(trips_chunk.index).isnull()
 logger.info("%s %s failed", trace_label_i, failed.sum())

 if not is_last_iteration:
 # boolean series of trips whose leg scheduling failed
 failed_cohorts = failed_trip_cohorts(trips_chunk, failed)
 trips_chunk = trips_chunk[failed_cohorts]
 choices = choices[~failed_cohorts]

 choices_list.append(choices)

 trips_df = trips.to_frame()

Expand Down

29 changes: 25 additions & 4 deletions
activitysim/core/chunk.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -568,8 +568,9 @@ def log_rss(trace_label, force=False):
 hwm_trace_label = f"{trace_label}.log_rss"

 if chunk_training_mode() == MODE_PRODUCTION:
 trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
 mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
 # FIXME - this trace_memory_info call slows things down a lot so it is turned off for now
 # trace_ticks = 0 if force else mem.MEM_TRACE_TICK_LEN
 # mem.trace_memory_info(hwm_trace_label, trace_ticks=trace_ticks)
 return

 rss, uss = mem.trace_memory_info(hwm_trace_label)
Expand DownExpand Up@@ -624,7 +625,12 @@ class ChunkSizer(object):
 def __init__(self, chunk_tag, trace_label, num_choosers=0, chunk_size=0):

 self.depth = len(CHUNK_SIZERS) + 1
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

 if chunk_metric() == USS:
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
 else:
 self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
 self.uss = 0

 if self.depth > 1:
 # nested chunkers should be unchunked
Expand DownExpand Up@@ -755,7 +761,14 @@ def adaptive_rows_per_chunk(self, i):

 prev_rss = self.rss
 prev_uss = self.uss
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)

 if chunk_training_mode() != MODE_PRODUCTION:

 if chunk_metric() == USS:
 self.rss, self.uss = mem.get_rss(force_garbage_collect=True, uss=True)
 else:
 self.rss, _ = mem.get_rss(force_garbage_collect=True, uss=False)
 self.uss = 0

 self.headroom = self.available_headroom(self.uss if chunk_metric() == USS else self.rss)

Expand DownExpand Up@@ -904,6 +917,14 @@ def chunk_log(trace_label, chunk_tag=None, base=False):
 chunk_sizer.close()


@contextmanager
def chunk_log_skip():

 yield

 None


def adaptive_chunked_choosers(choosers, chunk_size, trace_label, chunk_tag=None):

 # generator to iterate over choosers
Expand Down
Loading