fix(scheduler): preserve worker context when resizing#1044

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Muhtasim-Munif-Fahim wants to merge 3 commits into
base:main
Choose a base branch
from
Open

fix(scheduler): preserve worker context when resizing #1044

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c049d07
fix(scheduler): preserve worker context when resizing
Muhtasim-Munif-Fahim Aug 9, 2026
2c5a622
fix(scheduler): prevent zombie threads on scheduler resize and shutdown
Muhtasim-Munif-Fahim Aug 11, 2026
0916e74
Merge branch 'main' into codex/executorlib-resize-worker-context
jan-janssen Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/executorlib/standalone/interactive/communication.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -152,7 +152,7 @@ def shutdown(self, wait: bool = True):
if self._spawner.poll():
result = self.send_and_receive_dict(
input_dict={"shutdown": True, "wait": wait}
)["result"]
).get("result")
self._spawner.shutdown(wait=wait)
self._reset_socket()
return result
Expand Down
51 changes: 30 additions & 21 deletions src/executorlib/task_scheduler/interactive/blockallocation.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -80,35 +80,37 @@ def __init__(
executor_kwargs["restart_limit"] = restart_limit
self._process_kwargs = executor_kwargs
self._max_workers = max_workers
self_id = random.getrandbits(128)
self._self_id = self_id
self._self_id = random.getrandbits(128)
_interrupt_bootup_dict[self._self_id] = False
alive_workers = [max_workers]
alive_workers_lock = Lock()
bootup_events = [Event() for _ in range(self._max_workers)]
bootup_events[0].set()
self._alive_workers = [max_workers]
self._alive_workers_lock = Lock()
self._bootup_events = [Event() for _ in range(self._max_workers)]
self._bootup_events[0].set()
self._set_process(
process=[
Thread(
target=_execute_multiple_tasks,
kwargs=executor_kwargs
| {
"worker_id": worker_id,
"stop_function": lambda: _interrupt_bootup_dict[self_id],
"bootup_event": bootup_events[worker_id],
"next_bootup_event": (
bootup_events[worker_id + 1]
if worker_id + 1 < self._max_workers
else None
),
"alive_workers": alive_workers,
"alive_workers_lock": alive_workers_lock,
},
kwargs=self._worker_kwargs(worker_id),
)
for worker_id in range(self._max_workers)
],
)

def _worker_kwargs(self, worker_id: int) -> dict:
self_id = self._self_id
return self._process_kwargs | {
"worker_id": worker_id,
"stop_function": lambda: _interrupt_bootup_dict[self_id],
"bootup_event": self._bootup_events[worker_id],
"next_bootup_event": (
self._bootup_events[worker_id + 1]
if worker_id + 1 < len(self._bootup_events)
else None
),
Comment on lines +99 to +109

@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win

Release the first added worker from bootup.

_worker_kwargs() snapshots the original tail worker's next_bootup_event as None. After resize, the first added worker waits on the new unset event, but no existing worker can signal it. The added worker and later added workers remain blocked in bootup_event.wait().

Signal the first new bootup event when resizing. If strict worker-ID boot order must also apply during concurrent startup, use a resizable handoff instead of a static next_bootup_event.

Proposed fix
 self._bootup_events.extend(
Event() for _ in range(max_workers - old_max_workers)
)
+ self._bootup_events[old_max_workers].set()
self._alive_workers[0] += max_workers - old_max_workers

Also applies to: 130-140

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` around lines
99 - 108, Update the resize path and _worker_kwargs so the first newly added
worker’s bootup event is explicitly released, while preserving sequential
handoff for later workers. Ensure concurrent startup cannot leave added workers
waiting on a statically captured next_bootup_event; use the existing
resize/startup synchronization mechanism or a resizable handoff, and apply the
same correction to the corresponding logic around the later referenced block.

"alive_workers": self._alive_workers,
"alive_workers_lock": self._alive_workers_lock,
}

@property
def max_workers(self) -> int:
return self._max_workers
Expand All@@ -126,12 +128,19 @@ def max_workers(self, max_workers: int):
process for process in self._process if process.is_alive()
]
elif self._max_workers < max_workers:
old_max_workers = self._max_workers
self._bootup_events.extend(
Event() for _ in range(max_workers - old_max_workers)
)
for idx in range(old_max_workers, max_workers):
self._bootup_events[idx].set()
self._alive_workers[0] += max_workers - old_max_workers

@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win

Synchronize the live-worker count update.

_drain_dead_worker() decrements self._alive_workers[0] under self._alive_workers_lock. A worker can fail while this resize increments the same value. An unsynchronized increment can lose either update and make the scheduler treat live workers as dead.

Proposed fix
- self._alive_workers[0] += max_workers - old_max_workers+ with self._alive_workers_lock:+ self._alive_workers[0] += max_workers - old_max_workers
📝 Committable suggestion

‼️IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
self._alive_workers[0] +=max_workers-old_max_workers
withself._alive_workers_lock:
self._alive_workers[0] +=max_workers-old_max_workers
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` at line 134,
Protect the live-worker count increment in the resize logic with
self._alive_workers_lock, matching the locking used by _drain_dead_worker().
Keep the adjustment of self._alive_workers[0] atomic with respect to concurrent
worker-failure decrements.

new_process_lst = [
Thread(
target=_execute_multiple_tasks,
kwargs=self._process_kwargs,
kwargs=self._worker_kwargs(worker_id),
)
for _ in range(max_workers - self._max_workers)
for worker_id in range(old_max_workers, max_workers)
]
for process_instance in new_process_lst:
process_instance.start()
Expand Down
49 changes: 46 additions & 3 deletions tests/unit/task_scheduler/interactive/test_blockallocation.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,54 @@
import queue
import unittest
from threading import Lock
from concurrent.futures import Future
from threading import Event, Lock
from unittest.mock import patch

from executorlib.task_scheduler.interactive.blockallocation import _drain_dead_worker
from executorlib.task_scheduler.interactive.shared import task_done
from executorlib.standalone.interactive.communication import ExecutorlibSocketError
from executorlib.task_scheduler.interactive.blockallocation import (
BlockAllocationTaskScheduler,
_drain_dead_worker,
)


class TestBlockAllocationResize(unittest.TestCase):
def test_increase_workers_passes_worker_context(self):
scheduler = object.__new__(BlockAllocationTaskScheduler)
scheduler._future_queue = queue.Queue()
scheduler._process = []
scheduler._process_kwargs = {"future_queue": scheduler._future_queue}
scheduler._max_workers = 1
scheduler._self_id = 1
scheduler._alive_workers = [1]
scheduler._alive_workers_lock = Lock()
scheduler._bootup_events = [Event()]

class FakeThread:
instances = []

def __init__(self, target, kwargs):
self.target = target
self.kwargs = kwargs
self.started = False
self.instances.append(self)

def start(self):
self.started = True

with patch(
"executorlib.task_scheduler.interactive.blockallocation.Thread",
FakeThread,
):
scheduler.max_workers = 2

worker = FakeThread.instances[-1]
self.assertEqual(worker.kwargs["worker_id"], 1)
self.assertIn("stop_function", worker.kwargs)
self.assertIn("bootup_event", worker.kwargs)
self.assertIn("next_bootup_event", worker.kwargs)
self.assertIs(worker.kwargs["alive_workers"], scheduler._alive_workers)
self.assertTrue(worker.started)
self.assertEqual(scheduler._alive_workers[0], 2)


class TestDrainDeadWorker(unittest.TestCase):
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
fix(scheduler): preserve worker context when resizing#1044

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

Muhtasim-Munif-Fahim
wants to merge
3
commits into

base:main
Choose a base branch

from

 Open


fix(scheduler): preserve worker context when resizing
#1044
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
c049d07

fix(scheduler): preserve worker context when resizing


Muhtasim-Munif-Fahim Aug 9, 2026
2c5a622

fix(scheduler): prevent zombie threads on scheduler resize and shutdown


Muhtasim-Munif-Fahim Aug 11, 2026
0916e74

Merge branch 'main' into codex/executorlib-resize-worker-context


jan-janssen Aug 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
src/executorlib/standalone/interactive/communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -152,7 +152,7 @@ def shutdown(self, wait: bool = True):
 if self._spawner.poll():
 result = self.send_and_receive_dict(
 input_dict={"shutdown": True, "wait": wait}
 )["result"]
 ).get("result")
 self._spawner.shutdown(wait=wait)
 self._reset_socket()
 return result
Expand Down

51 changes: 30 additions & 21 deletions
src/executorlib/task_scheduler/interactive/blockallocation.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -80,35 +80,37 @@ def __init__(
 executor_kwargs["restart_limit"] = restart_limit
 self._process_kwargs = executor_kwargs
 self._max_workers = max_workers
 self_id = random.getrandbits(128)
 self._self_id = self_id
 self._self_id = random.getrandbits(128)
 _interrupt_bootup_dict[self._self_id] = False
alive_workers = [max_workers]
alive_workers_lock = Lock()
bootup_events = [Event() for _ in range(self._max_workers)]
bootup_events[0].set()
self._alive_workers = [max_workers]
self._alive_workers_lock = Lock()
self._bootup_events = [Event() for _ in range(self._max_workers)]
self._bootup_events[0].set()
 self._set_process(
 process=[
 Thread(
 target=_execute_multiple_tasks,
 kwargs=executor_kwargs
 | {
 "worker_id": worker_id,
 "stop_function": lambda: _interrupt_bootup_dict[self_id],
 "bootup_event": bootup_events[worker_id],
 "next_bootup_event": (
 bootup_events[worker_id + 1]
 if worker_id + 1 < self._max_workers
 else None
 ),
 "alive_workers": alive_workers,
 "alive_workers_lock": alive_workers_lock,
 },
 kwargs=self._worker_kwargs(worker_id),
 )
 for worker_id in range(self._max_workers)
 ],
 )

 def _worker_kwargs(self, worker_id: int) -> dict:
 self_id = self._self_id
 return self._process_kwargs | {
 "worker_id": worker_id,
 "stop_function": lambda: _interrupt_bootup_dict[self_id],
 "bootup_event": self._bootup_events[worker_id],
 "next_bootup_event": (
 self._bootup_events[worker_id + 1]
 if worker_id + 1 < len(self._bootup_events)
 else None
 ),

Comment on lines

+99
 to 
+109

@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win
Release the first added worker from bootup.
_worker_kwargs() snapshots the original tail worker's next_bootup_event as None. After resize, the first added worker waits on the new unset event, but no existing worker can signal it. The added worker and later added workers remain blocked in bootup_event.wait().
Signal the first new bootup event when resizing. If strict worker-ID boot order must also apply during concurrent startup, use a resizable handoff instead of a static next_bootup_event.
Proposed fix
 self._bootup_events.extend(
Event() for _ in range(max_workers - old_max_workers)
)
+ self._bootup_events[old_max_workers].set()
self._alive_workers[0] += max_workers - old_max_workers
Also applies to: 130-140
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` around lines
99 - 108, Update the resize path and _worker_kwargs so the first newly added
worker’s bootup event is explicitly released, while preserving sequential
handoff for later workers. Ensure concurrent startup cannot leave added workers
waiting on a statically captured next_bootup_event; use the existing
resize/startup synchronization mechanism or a resizable handoff, and apply the
same correction to the corresponding logic around the later referenced block.

 "alive_workers": self._alive_workers,
 "alive_workers_lock": self._alive_workers_lock,
 }

 @property
 def max_workers(self) -> int:
 return self._max_workers
Expand All@@ -126,12 +128,19 @@ def max_workers(self, max_workers: int):
 process for process in self._process if process.is_alive()
 ]
 elif self._max_workers < max_workers:
 old_max_workers = self._max_workers
 self._bootup_events.extend(
 Event() for _ in range(max_workers - old_max_workers)
 )
 for idx in range(old_max_workers, max_workers):
 self._bootup_events[idx].set()
 self._alive_workers[0] += max_workers - old_max_workers
@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win
Synchronize the live-worker count update.
_drain_dead_worker() decrements self._alive_workers[0] under self._alive_workers_lock. A worker can fail while this resize increments the same value. An unsynchronized increment can lose either update and make the scheduler treat live workers as dead.
Proposed fix
- self._alive_workers[0] += max_workers - old_max_workers+ with self._alive_workers_lock:+ self._alive_workers[0] += max_workers - old_max_workers
📝 Committable suggestion
‼️IMPORTANT

Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change

self._alive_workers[0] +=max_workers-old_max_workers
withself._alive_workers_lock:
self._alive_workers[0] +=max_workers-old_max_workers
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` at line 134,
Protect the live-worker count increment in the resize logic with
self._alive_workers_lock, matching the locking used by _drain_dead_worker().
Keep the adjustment of self._alive_workers[0] atomic with respect to concurrent
worker-failure decrements.

 new_process_lst = [
 Thread(
 target=_execute_multiple_tasks,
 kwargs=self._process_kwargs,
 kwargs=self._worker_kwargs(worker_id),
 )
 for _ in range(max_workers - self._max_workers)
 for worker_id in range(old_max_workers, max_workers)
 ]
 for process_instance in new_process_lst:
 process_instance.start()
Expand Down

49 changes: 46 additions & 3 deletions
tests/unit/task_scheduler/interactive/test_blockallocation.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,11 +1,54 @@
import queue
import unittest
from threading import Lock
from concurrent.futures import Future
from threading import Event, Lock
from unittest.mock import patch

from executorlib.task_scheduler.interactive.blockallocation import _drain_dead_worker
from executorlib.task_scheduler.interactive.shared import task_done
from executorlib.standalone.interactive.communication import ExecutorlibSocketError
from executorlib.task_scheduler.interactive.blockallocation import (
 BlockAllocationTaskScheduler,
 _drain_dead_worker,
)


class TestBlockAllocationResize(unittest.TestCase):
 def test_increase_workers_passes_worker_context(self):
 scheduler = object.__new__(BlockAllocationTaskScheduler)
 scheduler._future_queue = queue.Queue()
 scheduler._process = []
 scheduler._process_kwargs = {"future_queue": scheduler._future_queue}
 scheduler._max_workers = 1
 scheduler._self_id = 1
 scheduler._alive_workers = [1]
 scheduler._alive_workers_lock = Lock()
 scheduler._bootup_events = [Event()]

 class FakeThread:
 instances = []

 def __init__(self, target, kwargs):
 self.target = target
 self.kwargs = kwargs
 self.started = False
 self.instances.append(self)

 def start(self):
 self.started = True

 with patch(
 "executorlib.task_scheduler.interactive.blockallocation.Thread",
 FakeThread,
 ):
 scheduler.max_workers = 2

 worker = FakeThread.instances[-1]
 self.assertEqual(worker.kwargs["worker_id"], 1)
 self.assertIn("stop_function", worker.kwargs)
 self.assertIn("bootup_event", worker.kwargs)
 self.assertIn("next_bootup_event", worker.kwargs)
 self.assertIs(worker.kwargs["alive_workers"], scheduler._alive_workers)
 self.assertTrue(worker.started)
 self.assertEqual(scheduler._alive_workers[0], 2)


class TestDrainDeadWorker(unittest.TestCase):
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix(scheduler): preserve worker context when resizing#1044

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

Muhtasim-Munif-Fahim
wants to merge
3
commits into

base:main
Choose a base branch

from

 Open


fix(scheduler): preserve worker context when resizing
#1044
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
c049d07

fix(scheduler): preserve worker context when resizing


Muhtasim-Munif-Fahim Aug 9, 2026
2c5a622

fix(scheduler): prevent zombie threads on scheduler resize and shutdown


Muhtasim-Munif-Fahim Aug 11, 2026
0916e74

Merge branch 'main' into codex/executorlib-resize-worker-context


jan-janssen Aug 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
src/executorlib/standalone/interactive/communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -152,7 +152,7 @@ def shutdown(self, wait: bool = True):
 if self._spawner.poll():
 result = self.send_and_receive_dict(
 input_dict={"shutdown": True, "wait": wait}
 )["result"]
 ).get("result")
 self._spawner.shutdown(wait=wait)
 self._reset_socket()
 return result
Expand Down

51 changes: 30 additions & 21 deletions
src/executorlib/task_scheduler/interactive/blockallocation.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -80,35 +80,37 @@ def __init__(
 executor_kwargs["restart_limit"] = restart_limit
 self._process_kwargs = executor_kwargs
 self._max_workers = max_workers
 self_id = random.getrandbits(128)
 self._self_id = self_id
 self._self_id = random.getrandbits(128)
 _interrupt_bootup_dict[self._self_id] = False
alive_workers = [max_workers]
alive_workers_lock = Lock()
bootup_events = [Event() for _ in range(self._max_workers)]
bootup_events[0].set()
self._alive_workers = [max_workers]
self._alive_workers_lock = Lock()
self._bootup_events = [Event() for _ in range(self._max_workers)]
self._bootup_events[0].set()
 self._set_process(
 process=[
 Thread(
 target=_execute_multiple_tasks,
 kwargs=executor_kwargs
 | {
 "worker_id": worker_id,
 "stop_function": lambda: _interrupt_bootup_dict[self_id],
 "bootup_event": bootup_events[worker_id],
 "next_bootup_event": (
 bootup_events[worker_id + 1]
 if worker_id + 1 < self._max_workers
 else None
 ),
 "alive_workers": alive_workers,
 "alive_workers_lock": alive_workers_lock,
 },
 kwargs=self._worker_kwargs(worker_id),
 )
 for worker_id in range(self._max_workers)
 ],
 )

 def _worker_kwargs(self, worker_id: int) -> dict:
 self_id = self._self_id
 return self._process_kwargs | {
 "worker_id": worker_id,
 "stop_function": lambda: _interrupt_bootup_dict[self_id],
 "bootup_event": self._bootup_events[worker_id],
 "next_bootup_event": (
 self._bootup_events[worker_id + 1]
 if worker_id + 1 < len(self._bootup_events)
 else None
 ),

Comment on lines

+99
 to 
+109

@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win
Release the first added worker from bootup.
_worker_kwargs() snapshots the original tail worker's next_bootup_event as None. After resize, the first added worker waits on the new unset event, but no existing worker can signal it. The added worker and later added workers remain blocked in bootup_event.wait().
Signal the first new bootup event when resizing. If strict worker-ID boot order must also apply during concurrent startup, use a resizable handoff instead of a static next_bootup_event.
Proposed fix
 self._bootup_events.extend(
Event() for _ in range(max_workers - old_max_workers)
)
+ self._bootup_events[old_max_workers].set()
self._alive_workers[0] += max_workers - old_max_workers
Also applies to: 130-140
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` around lines
99 - 108, Update the resize path and _worker_kwargs so the first newly added
worker’s bootup event is explicitly released, while preserving sequential
handoff for later workers. Ensure concurrent startup cannot leave added workers
waiting on a statically captured next_bootup_event; use the existing
resize/startup synchronization mechanism or a resizable handoff, and apply the
same correction to the corresponding logic around the later referenced block.

 "alive_workers": self._alive_workers,
 "alive_workers_lock": self._alive_workers_lock,
 }

 @property
 def max_workers(self) -> int:
 return self._max_workers
Expand All@@ -126,12 +128,19 @@ def max_workers(self, max_workers: int):
 process for process in self._process if process.is_alive()
 ]
 elif self._max_workers < max_workers:
 old_max_workers = self._max_workers
 self._bootup_events.extend(
 Event() for _ in range(max_workers - old_max_workers)
 )
 for idx in range(old_max_workers, max_workers):
 self._bootup_events[idx].set()
 self._alive_workers[0] += max_workers - old_max_workers
@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win
Synchronize the live-worker count update.
_drain_dead_worker() decrements self._alive_workers[0] under self._alive_workers_lock. A worker can fail while this resize increments the same value. An unsynchronized increment can lose either update and make the scheduler treat live workers as dead.
Proposed fix
- self._alive_workers[0] += max_workers - old_max_workers+ with self._alive_workers_lock:+ self._alive_workers[0] += max_workers - old_max_workers
📝 Committable suggestion
‼️IMPORTANT

Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change

self._alive_workers[0] +=max_workers-old_max_workers
withself._alive_workers_lock:
self._alive_workers[0] +=max_workers-old_max_workers
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` at line 134,
Protect the live-worker count increment in the resize logic with
self._alive_workers_lock, matching the locking used by _drain_dead_worker().
Keep the adjustment of self._alive_workers[0] atomic with respect to concurrent
worker-failure decrements.

 new_process_lst = [
 Thread(
 target=_execute_multiple_tasks,
 kwargs=self._process_kwargs,
 kwargs=self._worker_kwargs(worker_id),
 )
 for _ in range(max_workers - self._max_workers)
 for worker_id in range(old_max_workers, max_workers)
 ]
 for process_instance in new_process_lst:
 process_instance.start()
Expand Down

49 changes: 46 additions & 3 deletions
tests/unit/task_scheduler/interactive/test_blockallocation.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,11 +1,54 @@
import queue
import unittest
from threading import Lock
from concurrent.futures import Future
from threading import Event, Lock
from unittest.mock import patch

from executorlib.task_scheduler.interactive.blockallocation import _drain_dead_worker
from executorlib.task_scheduler.interactive.shared import task_done
from executorlib.standalone.interactive.communication import ExecutorlibSocketError
from executorlib.task_scheduler.interactive.blockallocation import (
 BlockAllocationTaskScheduler,
 _drain_dead_worker,
)


class TestBlockAllocationResize(unittest.TestCase):
 def test_increase_workers_passes_worker_context(self):
 scheduler = object.__new__(BlockAllocationTaskScheduler)
 scheduler._future_queue = queue.Queue()
 scheduler._process = []
 scheduler._process_kwargs = {"future_queue": scheduler._future_queue}
 scheduler._max_workers = 1
 scheduler._self_id = 1
 scheduler._alive_workers = [1]
 scheduler._alive_workers_lock = Lock()
 scheduler._bootup_events = [Event()]

 class FakeThread:
 instances = []

 def __init__(self, target, kwargs):
 self.target = target
 self.kwargs = kwargs
 self.started = False
 self.instances.append(self)

 def start(self):
 self.started = True

 with patch(
 "executorlib.task_scheduler.interactive.blockallocation.Thread",
 FakeThread,
 ):
 scheduler.max_workers = 2

 worker = FakeThread.instances[-1]
 self.assertEqual(worker.kwargs["worker_id"], 1)
 self.assertIn("stop_function", worker.kwargs)
 self.assertIn("bootup_event", worker.kwargs)
 self.assertIn("next_bootup_event", worker.kwargs)
 self.assertIs(worker.kwargs["alive_workers"], scheduler._alive_workers)
 self.assertTrue(worker.started)
 self.assertEqual(scheduler._alive_workers[0], 2)


class TestDrainDeadWorker(unittest.TestCase):
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix(scheduler): preserve worker context when resizing#1044

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

Muhtasim-Munif-Fahim
wants to merge
3
commits into

base:main
Choose a base branch

from

 Open


fix(scheduler): preserve worker context when resizing
#1044
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
c049d07

fix(scheduler): preserve worker context when resizing


Muhtasim-Munif-Fahim Aug 9, 2026
2c5a622

fix(scheduler): prevent zombie threads on scheduler resize and shutdown


Muhtasim-Munif-Fahim Aug 11, 2026
0916e74

Merge branch 'main' into codex/executorlib-resize-worker-context


jan-janssen Aug 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
src/executorlib/standalone/interactive/communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -152,7 +152,7 @@ def shutdown(self, wait: bool = True):
 if self._spawner.poll():
 result = self.send_and_receive_dict(
 input_dict={"shutdown": True, "wait": wait}
 )["result"]
 ).get("result")
 self._spawner.shutdown(wait=wait)
 self._reset_socket()
 return result
Expand Down

51 changes: 30 additions & 21 deletions
src/executorlib/task_scheduler/interactive/blockallocation.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -80,35 +80,37 @@ def __init__(
 executor_kwargs["restart_limit"] = restart_limit
 self._process_kwargs = executor_kwargs
 self._max_workers = max_workers
 self_id = random.getrandbits(128)
 self._self_id = self_id
 self._self_id = random.getrandbits(128)
 _interrupt_bootup_dict[self._self_id] = False
alive_workers = [max_workers]
alive_workers_lock = Lock()
bootup_events = [Event() for _ in range(self._max_workers)]
bootup_events[0].set()
self._alive_workers = [max_workers]
self._alive_workers_lock = Lock()
self._bootup_events = [Event() for _ in range(self._max_workers)]
self._bootup_events[0].set()
 self._set_process(
 process=[
 Thread(
 target=_execute_multiple_tasks,
 kwargs=executor_kwargs
 | {
 "worker_id": worker_id,
 "stop_function": lambda: _interrupt_bootup_dict[self_id],
 "bootup_event": bootup_events[worker_id],
 "next_bootup_event": (
 bootup_events[worker_id + 1]
 if worker_id + 1 < self._max_workers
 else None
 ),
 "alive_workers": alive_workers,
 "alive_workers_lock": alive_workers_lock,
 },
 kwargs=self._worker_kwargs(worker_id),
 )
 for worker_id in range(self._max_workers)
 ],
 )

 def _worker_kwargs(self, worker_id: int) -> dict:
 self_id = self._self_id
 return self._process_kwargs | {
 "worker_id": worker_id,
 "stop_function": lambda: _interrupt_bootup_dict[self_id],
 "bootup_event": self._bootup_events[worker_id],
 "next_bootup_event": (
 self._bootup_events[worker_id + 1]
 if worker_id + 1 < len(self._bootup_events)
 else None
 ),

Comment on lines

+99
 to 
+109

@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win
Release the first added worker from bootup.
_worker_kwargs() snapshots the original tail worker's next_bootup_event as None. After resize, the first added worker waits on the new unset event, but no existing worker can signal it. The added worker and later added workers remain blocked in bootup_event.wait().
Signal the first new bootup event when resizing. If strict worker-ID boot order must also apply during concurrent startup, use a resizable handoff instead of a static next_bootup_event.
Proposed fix
 self._bootup_events.extend(
Event() for _ in range(max_workers - old_max_workers)
)
+ self._bootup_events[old_max_workers].set()
self._alive_workers[0] += max_workers - old_max_workers
Also applies to: 130-140
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` around lines
99 - 108, Update the resize path and _worker_kwargs so the first newly added
worker’s bootup event is explicitly released, while preserving sequential
handoff for later workers. Ensure concurrent startup cannot leave added workers
waiting on a statically captured next_bootup_event; use the existing
resize/startup synchronization mechanism or a resizable handoff, and apply the
same correction to the corresponding logic around the later referenced block.

 "alive_workers": self._alive_workers,
 "alive_workers_lock": self._alive_workers_lock,
 }

 @property
 def max_workers(self) -> int:
 return self._max_workers
Expand All@@ -126,12 +128,19 @@ def max_workers(self, max_workers: int):
 process for process in self._process if process.is_alive()
 ]
 elif self._max_workers < max_workers:
 old_max_workers = self._max_workers
 self._bootup_events.extend(
 Event() for _ in range(max_workers - old_max_workers)
 )
 for idx in range(old_max_workers, max_workers):
 self._bootup_events[idx].set()
 self._alive_workers[0] += max_workers - old_max_workers
@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win
Synchronize the live-worker count update.
_drain_dead_worker() decrements self._alive_workers[0] under self._alive_workers_lock. A worker can fail while this resize increments the same value. An unsynchronized increment can lose either update and make the scheduler treat live workers as dead.
Proposed fix
- self._alive_workers[0] += max_workers - old_max_workers+ with self._alive_workers_lock:+ self._alive_workers[0] += max_workers - old_max_workers
📝 Committable suggestion
‼️IMPORTANT

Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change

self._alive_workers[0] +=max_workers-old_max_workers
withself._alive_workers_lock:
self._alive_workers[0] +=max_workers-old_max_workers
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` at line 134,
Protect the live-worker count increment in the resize logic with
self._alive_workers_lock, matching the locking used by _drain_dead_worker().
Keep the adjustment of self._alive_workers[0] atomic with respect to concurrent
worker-failure decrements.

 new_process_lst = [
 Thread(
 target=_execute_multiple_tasks,
 kwargs=self._process_kwargs,
 kwargs=self._worker_kwargs(worker_id),
 )
 for _ in range(max_workers - self._max_workers)
 for worker_id in range(old_max_workers, max_workers)
 ]
 for process_instance in new_process_lst:
 process_instance.start()
Expand Down

49 changes: 46 additions & 3 deletions
tests/unit/task_scheduler/interactive/test_blockallocation.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,11 +1,54 @@
import queue
import unittest
from threading import Lock
from concurrent.futures import Future
from threading import Event, Lock
from unittest.mock import patch

from executorlib.task_scheduler.interactive.blockallocation import _drain_dead_worker
from executorlib.task_scheduler.interactive.shared import task_done
from executorlib.standalone.interactive.communication import ExecutorlibSocketError
from executorlib.task_scheduler.interactive.blockallocation import (
 BlockAllocationTaskScheduler,
 _drain_dead_worker,
)


class TestBlockAllocationResize(unittest.TestCase):
 def test_increase_workers_passes_worker_context(self):
 scheduler = object.__new__(BlockAllocationTaskScheduler)
 scheduler._future_queue = queue.Queue()
 scheduler._process = []
 scheduler._process_kwargs = {"future_queue": scheduler._future_queue}
 scheduler._max_workers = 1
 scheduler._self_id = 1
 scheduler._alive_workers = [1]
 scheduler._alive_workers_lock = Lock()
 scheduler._bootup_events = [Event()]

 class FakeThread:
 instances = []

 def __init__(self, target, kwargs):
 self.target = target
 self.kwargs = kwargs
 self.started = False
 self.instances.append(self)

 def start(self):
 self.started = True

 with patch(
 "executorlib.task_scheduler.interactive.blockallocation.Thread",
 FakeThread,
 ):
 scheduler.max_workers = 2

 worker = FakeThread.instances[-1]
 self.assertEqual(worker.kwargs["worker_id"], 1)
 self.assertIn("stop_function", worker.kwargs)
 self.assertIn("bootup_event", worker.kwargs)
 self.assertIn("next_bootup_event", worker.kwargs)
 self.assertIs(worker.kwargs["alive_workers"], scheduler._alive_workers)
 self.assertTrue(worker.started)
 self.assertEqual(scheduler._alive_workers[0], 2)


class TestDrainDeadWorker(unittest.TestCase):
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
fix(scheduler): preserve worker context when resizing#1044

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

Muhtasim-Munif-Fahim
wants to merge
3
commits into

base:main
Choose a base branch

from

 Open


fix(scheduler): preserve worker context when resizing
#1044
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
c049d07

fix(scheduler): preserve worker context when resizing


Muhtasim-Munif-Fahim Aug 9, 2026
2c5a622

fix(scheduler): prevent zombie threads on scheduler resize and shutdown


Muhtasim-Munif-Fahim Aug 11, 2026
0916e74

Merge branch 'main' into codex/executorlib-resize-worker-context


jan-janssen Aug 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
src/executorlib/standalone/interactive/communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -152,7 +152,7 @@ def shutdown(self, wait: bool = True):
 if self._spawner.poll():
 result = self.send_and_receive_dict(
 input_dict={"shutdown": True, "wait": wait}
 )["result"]
 ).get("result")
 self._spawner.shutdown(wait=wait)
 self._reset_socket()
 return result
Expand Down

51 changes: 30 additions & 21 deletions
src/executorlib/task_scheduler/interactive/blockallocation.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -80,35 +80,37 @@ def __init__(
 executor_kwargs["restart_limit"] = restart_limit
 self._process_kwargs = executor_kwargs
 self._max_workers = max_workers
 self_id = random.getrandbits(128)
 self._self_id = self_id
 self._self_id = random.getrandbits(128)
 _interrupt_bootup_dict[self._self_id] = False
alive_workers = [max_workers]
alive_workers_lock = Lock()
bootup_events = [Event() for _ in range(self._max_workers)]
bootup_events[0].set()
self._alive_workers = [max_workers]
self._alive_workers_lock = Lock()
self._bootup_events = [Event() for _ in range(self._max_workers)]
self._bootup_events[0].set()
 self._set_process(
 process=[
 Thread(
 target=_execute_multiple_tasks,
 kwargs=executor_kwargs
 | {
 "worker_id": worker_id,
 "stop_function": lambda: _interrupt_bootup_dict[self_id],
 "bootup_event": bootup_events[worker_id],
 "next_bootup_event": (
 bootup_events[worker_id + 1]
 if worker_id + 1 < self._max_workers
 else None
 ),
 "alive_workers": alive_workers,
 "alive_workers_lock": alive_workers_lock,
 },
 kwargs=self._worker_kwargs(worker_id),
 )
 for worker_id in range(self._max_workers)
 ],
 )

 def _worker_kwargs(self, worker_id: int) -> dict:
 self_id = self._self_id
 return self._process_kwargs | {
 "worker_id": worker_id,
 "stop_function": lambda: _interrupt_bootup_dict[self_id],
 "bootup_event": self._bootup_events[worker_id],
 "next_bootup_event": (
 self._bootup_events[worker_id + 1]
 if worker_id + 1 < len(self._bootup_events)
 else None
 ),

Comment on lines

+99
 to 
+109

@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win
Release the first added worker from bootup.
_worker_kwargs() snapshots the original tail worker's next_bootup_event as None. After resize, the first added worker waits on the new unset event, but no existing worker can signal it. The added worker and later added workers remain blocked in bootup_event.wait().
Signal the first new bootup event when resizing. If strict worker-ID boot order must also apply during concurrent startup, use a resizable handoff instead of a static next_bootup_event.
Proposed fix
 self._bootup_events.extend(
Event() for _ in range(max_workers - old_max_workers)
)
+ self._bootup_events[old_max_workers].set()
self._alive_workers[0] += max_workers - old_max_workers
Also applies to: 130-140
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` around lines
99 - 108, Update the resize path and _worker_kwargs so the first newly added
worker’s bootup event is explicitly released, while preserving sequential
handoff for later workers. Ensure concurrent startup cannot leave added workers
waiting on a statically captured next_bootup_event; use the existing
resize/startup synchronization mechanism or a resizable handoff, and apply the
same correction to the corresponding logic around the later referenced block.

 "alive_workers": self._alive_workers,
 "alive_workers_lock": self._alive_workers_lock,
 }

 @property
 def max_workers(self) -> int:
 return self._max_workers
Expand All@@ -126,12 +128,19 @@ def max_workers(self, max_workers: int):
 process for process in self._process if process.is_alive()
 ]
 elif self._max_workers < max_workers:
 old_max_workers = self._max_workers
 self._bootup_events.extend(
 Event() for _ in range(max_workers - old_max_workers)
 )
 for idx in range(old_max_workers, max_workers):
 self._bootup_events[idx].set()
 self._alive_workers[0] += max_workers - old_max_workers
@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win
Synchronize the live-worker count update.
_drain_dead_worker() decrements self._alive_workers[0] under self._alive_workers_lock. A worker can fail while this resize increments the same value. An unsynchronized increment can lose either update and make the scheduler treat live workers as dead.
Proposed fix
- self._alive_workers[0] += max_workers - old_max_workers+ with self._alive_workers_lock:+ self._alive_workers[0] += max_workers - old_max_workers
📝 Committable suggestion
‼️IMPORTANT

Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change

self._alive_workers[0] +=max_workers-old_max_workers
withself._alive_workers_lock:
self._alive_workers[0] +=max_workers-old_max_workers
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` at line 134,
Protect the live-worker count increment in the resize logic with
self._alive_workers_lock, matching the locking used by _drain_dead_worker().
Keep the adjustment of self._alive_workers[0] atomic with respect to concurrent
worker-failure decrements.

 new_process_lst = [
 Thread(
 target=_execute_multiple_tasks,
 kwargs=self._process_kwargs,
 kwargs=self._worker_kwargs(worker_id),
 )
 for _ in range(max_workers - self._max_workers)
 for worker_id in range(old_max_workers, max_workers)
 ]
 for process_instance in new_process_lst:
 process_instance.start()
Expand Down

49 changes: 46 additions & 3 deletions
tests/unit/task_scheduler/interactive/test_blockallocation.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,11 +1,54 @@
import queue
import unittest
from threading import Lock
from concurrent.futures import Future
from threading import Event, Lock
from unittest.mock import patch

from executorlib.task_scheduler.interactive.blockallocation import _drain_dead_worker
from executorlib.task_scheduler.interactive.shared import task_done
from executorlib.standalone.interactive.communication import ExecutorlibSocketError
from executorlib.task_scheduler.interactive.blockallocation import (
 BlockAllocationTaskScheduler,
 _drain_dead_worker,
)


class TestBlockAllocationResize(unittest.TestCase):
 def test_increase_workers_passes_worker_context(self):
 scheduler = object.__new__(BlockAllocationTaskScheduler)
 scheduler._future_queue = queue.Queue()
 scheduler._process = []
 scheduler._process_kwargs = {"future_queue": scheduler._future_queue}
 scheduler._max_workers = 1
 scheduler._self_id = 1
 scheduler._alive_workers = [1]
 scheduler._alive_workers_lock = Lock()
 scheduler._bootup_events = [Event()]

 class FakeThread:
 instances = []

 def __init__(self, target, kwargs):
 self.target = target
 self.kwargs = kwargs
 self.started = False
 self.instances.append(self)

 def start(self):
 self.started = True

 with patch(
 "executorlib.task_scheduler.interactive.blockallocation.Thread",
 FakeThread,
 ):
 scheduler.max_workers = 2

 worker = FakeThread.instances[-1]
 self.assertEqual(worker.kwargs["worker_id"], 1)
 self.assertIn("stop_function", worker.kwargs)
 self.assertIn("bootup_event", worker.kwargs)
 self.assertIn("next_bootup_event", worker.kwargs)
 self.assertIs(worker.kwargs["alive_workers"], scheduler._alive_workers)
 self.assertTrue(worker.started)
 self.assertEqual(scheduler._alive_workers[0], 2)


class TestDrainDeadWorker(unittest.TestCase):
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix(scheduler): preserve worker context when resizing#1044

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

Muhtasim-Munif-Fahim
wants to merge
3
commits into

base:main
Choose a base branch

from

 Open


fix(scheduler): preserve worker context when resizing
#1044
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
c049d07

fix(scheduler): preserve worker context when resizing


Muhtasim-Munif-Fahim Aug 9, 2026
2c5a622

fix(scheduler): prevent zombie threads on scheduler resize and shutdown


Muhtasim-Munif-Fahim Aug 11, 2026
0916e74

Merge branch 'main' into codex/executorlib-resize-worker-context


jan-janssen Aug 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
src/executorlib/standalone/interactive/communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -152,7 +152,7 @@ def shutdown(self, wait: bool = True):
 if self._spawner.poll():
 result = self.send_and_receive_dict(
 input_dict={"shutdown": True, "wait": wait}
 )["result"]
 ).get("result")
 self._spawner.shutdown(wait=wait)
 self._reset_socket()
 return result
Expand Down

51 changes: 30 additions & 21 deletions
src/executorlib/task_scheduler/interactive/blockallocation.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -80,35 +80,37 @@ def __init__(
 executor_kwargs["restart_limit"] = restart_limit
 self._process_kwargs = executor_kwargs
 self._max_workers = max_workers
 self_id = random.getrandbits(128)
 self._self_id = self_id
 self._self_id = random.getrandbits(128)
 _interrupt_bootup_dict[self._self_id] = False
alive_workers = [max_workers]
alive_workers_lock = Lock()
bootup_events = [Event() for _ in range(self._max_workers)]
bootup_events[0].set()
self._alive_workers = [max_workers]
self._alive_workers_lock = Lock()
self._bootup_events = [Event() for _ in range(self._max_workers)]
self._bootup_events[0].set()
 self._set_process(
 process=[
 Thread(
 target=_execute_multiple_tasks,
 kwargs=executor_kwargs
 | {
 "worker_id": worker_id,
 "stop_function": lambda: _interrupt_bootup_dict[self_id],
 "bootup_event": bootup_events[worker_id],
 "next_bootup_event": (
 bootup_events[worker_id + 1]
 if worker_id + 1 < self._max_workers
 else None
 ),
 "alive_workers": alive_workers,
 "alive_workers_lock": alive_workers_lock,
 },
 kwargs=self._worker_kwargs(worker_id),
 )
 for worker_id in range(self._max_workers)
 ],
 )

 def _worker_kwargs(self, worker_id: int) -> dict:
 self_id = self._self_id
 return self._process_kwargs | {
 "worker_id": worker_id,
 "stop_function": lambda: _interrupt_bootup_dict[self_id],
 "bootup_event": self._bootup_events[worker_id],
 "next_bootup_event": (
 self._bootup_events[worker_id + 1]
 if worker_id + 1 < len(self._bootup_events)
 else None
 ),

Comment on lines

+99
 to 
+109

@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win
Release the first added worker from bootup.
_worker_kwargs() snapshots the original tail worker's next_bootup_event as None. After resize, the first added worker waits on the new unset event, but no existing worker can signal it. The added worker and later added workers remain blocked in bootup_event.wait().
Signal the first new bootup event when resizing. If strict worker-ID boot order must also apply during concurrent startup, use a resizable handoff instead of a static next_bootup_event.
Proposed fix
 self._bootup_events.extend(
Event() for _ in range(max_workers - old_max_workers)
)
+ self._bootup_events[old_max_workers].set()
self._alive_workers[0] += max_workers - old_max_workers
Also applies to: 130-140
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` around lines
99 - 108, Update the resize path and _worker_kwargs so the first newly added
worker’s bootup event is explicitly released, while preserving sequential
handoff for later workers. Ensure concurrent startup cannot leave added workers
waiting on a statically captured next_bootup_event; use the existing
resize/startup synchronization mechanism or a resizable handoff, and apply the
same correction to the corresponding logic around the later referenced block.

 "alive_workers": self._alive_workers,
 "alive_workers_lock": self._alive_workers_lock,
 }

 @property
 def max_workers(self) -> int:
 return self._max_workers
Expand All@@ -126,12 +128,19 @@ def max_workers(self, max_workers: int):
 process for process in self._process if process.is_alive()
 ]
 elif self._max_workers < max_workers:
 old_max_workers = self._max_workers
 self._bootup_events.extend(
 Event() for _ in range(max_workers - old_max_workers)
 )
 for idx in range(old_max_workers, max_workers):
 self._bootup_events[idx].set()
 self._alive_workers[0] += max_workers - old_max_workers
@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win
Synchronize the live-worker count update.
_drain_dead_worker() decrements self._alive_workers[0] under self._alive_workers_lock. A worker can fail while this resize increments the same value. An unsynchronized increment can lose either update and make the scheduler treat live workers as dead.
Proposed fix
- self._alive_workers[0] += max_workers - old_max_workers+ with self._alive_workers_lock:+ self._alive_workers[0] += max_workers - old_max_workers
📝 Committable suggestion
‼️IMPORTANT

Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change

self._alive_workers[0] +=max_workers-old_max_workers
withself._alive_workers_lock:
self._alive_workers[0] +=max_workers-old_max_workers
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` at line 134,
Protect the live-worker count increment in the resize logic with
self._alive_workers_lock, matching the locking used by _drain_dead_worker().
Keep the adjustment of self._alive_workers[0] atomic with respect to concurrent
worker-failure decrements.

 new_process_lst = [
 Thread(
 target=_execute_multiple_tasks,
 kwargs=self._process_kwargs,
 kwargs=self._worker_kwargs(worker_id),
 )
 for _ in range(max_workers - self._max_workers)
 for worker_id in range(old_max_workers, max_workers)
 ]
 for process_instance in new_process_lst:
 process_instance.start()
Expand Down

49 changes: 46 additions & 3 deletions
tests/unit/task_scheduler/interactive/test_blockallocation.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,11 +1,54 @@
import queue
import unittest
from threading import Lock
from concurrent.futures import Future
from threading import Event, Lock
from unittest.mock import patch

from executorlib.task_scheduler.interactive.blockallocation import _drain_dead_worker
from executorlib.task_scheduler.interactive.shared import task_done
from executorlib.standalone.interactive.communication import ExecutorlibSocketError
from executorlib.task_scheduler.interactive.blockallocation import (
 BlockAllocationTaskScheduler,
 _drain_dead_worker,
)


class TestBlockAllocationResize(unittest.TestCase):
 def test_increase_workers_passes_worker_context(self):
 scheduler = object.__new__(BlockAllocationTaskScheduler)
 scheduler._future_queue = queue.Queue()
 scheduler._process = []
 scheduler._process_kwargs = {"future_queue": scheduler._future_queue}
 scheduler._max_workers = 1
 scheduler._self_id = 1
 scheduler._alive_workers = [1]
 scheduler._alive_workers_lock = Lock()
 scheduler._bootup_events = [Event()]

 class FakeThread:
 instances = []

 def __init__(self, target, kwargs):
 self.target = target
 self.kwargs = kwargs
 self.started = False
 self.instances.append(self)

 def start(self):
 self.started = True

 with patch(
 "executorlib.task_scheduler.interactive.blockallocation.Thread",
 FakeThread,
 ):
 scheduler.max_workers = 2

 worker = FakeThread.instances[-1]
 self.assertEqual(worker.kwargs["worker_id"], 1)
 self.assertIn("stop_function", worker.kwargs)
 self.assertIn("bootup_event", worker.kwargs)
 self.assertIn("next_bootup_event", worker.kwargs)
 self.assertIs(worker.kwargs["alive_workers"], scheduler._alive_workers)
 self.assertTrue(worker.started)
 self.assertEqual(scheduler._alive_workers[0], 2)


class TestDrainDeadWorker(unittest.TestCase):
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix(scheduler): preserve worker context when resizing#1044

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

Muhtasim-Munif-Fahim
wants to merge
3
commits into

base:main
Choose a base branch

from

 Open


fix(scheduler): preserve worker context when resizing
#1044
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
c049d07

fix(scheduler): preserve worker context when resizing


Muhtasim-Munif-Fahim Aug 9, 2026
2c5a622

fix(scheduler): prevent zombie threads on scheduler resize and shutdown


Muhtasim-Munif-Fahim Aug 11, 2026
0916e74

Merge branch 'main' into codex/executorlib-resize-worker-context


jan-janssen Aug 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
src/executorlib/standalone/interactive/communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -152,7 +152,7 @@ def shutdown(self, wait: bool = True):
 if self._spawner.poll():
 result = self.send_and_receive_dict(
 input_dict={"shutdown": True, "wait": wait}
 )["result"]
 ).get("result")
 self._spawner.shutdown(wait=wait)
 self._reset_socket()
 return result
Expand Down

51 changes: 30 additions & 21 deletions
src/executorlib/task_scheduler/interactive/blockallocation.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -80,35 +80,37 @@ def __init__(
 executor_kwargs["restart_limit"] = restart_limit
 self._process_kwargs = executor_kwargs
 self._max_workers = max_workers
 self_id = random.getrandbits(128)
 self._self_id = self_id
 self._self_id = random.getrandbits(128)
 _interrupt_bootup_dict[self._self_id] = False
alive_workers = [max_workers]
alive_workers_lock = Lock()
bootup_events = [Event() for _ in range(self._max_workers)]
bootup_events[0].set()
self._alive_workers = [max_workers]
self._alive_workers_lock = Lock()
self._bootup_events = [Event() for _ in range(self._max_workers)]
self._bootup_events[0].set()
 self._set_process(
 process=[
 Thread(
 target=_execute_multiple_tasks,
 kwargs=executor_kwargs
 | {
 "worker_id": worker_id,
 "stop_function": lambda: _interrupt_bootup_dict[self_id],
 "bootup_event": bootup_events[worker_id],
 "next_bootup_event": (
 bootup_events[worker_id + 1]
 if worker_id + 1 < self._max_workers
 else None
 ),
 "alive_workers": alive_workers,
 "alive_workers_lock": alive_workers_lock,
 },
 kwargs=self._worker_kwargs(worker_id),
 )
 for worker_id in range(self._max_workers)
 ],
 )

 def _worker_kwargs(self, worker_id: int) -> dict:
 self_id = self._self_id
 return self._process_kwargs | {
 "worker_id": worker_id,
 "stop_function": lambda: _interrupt_bootup_dict[self_id],
 "bootup_event": self._bootup_events[worker_id],
 "next_bootup_event": (
 self._bootup_events[worker_id + 1]
 if worker_id + 1 < len(self._bootup_events)
 else None
 ),

Comment on lines

+99
 to 
+109

@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win
Release the first added worker from bootup.
_worker_kwargs() snapshots the original tail worker's next_bootup_event as None. After resize, the first added worker waits on the new unset event, but no existing worker can signal it. The added worker and later added workers remain blocked in bootup_event.wait().
Signal the first new bootup event when resizing. If strict worker-ID boot order must also apply during concurrent startup, use a resizable handoff instead of a static next_bootup_event.
Proposed fix
 self._bootup_events.extend(
Event() for _ in range(max_workers - old_max_workers)
)
+ self._bootup_events[old_max_workers].set()
self._alive_workers[0] += max_workers - old_max_workers
Also applies to: 130-140
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` around lines
99 - 108, Update the resize path and _worker_kwargs so the first newly added
worker’s bootup event is explicitly released, while preserving sequential
handoff for later workers. Ensure concurrent startup cannot leave added workers
waiting on a statically captured next_bootup_event; use the existing
resize/startup synchronization mechanism or a resizable handoff, and apply the
same correction to the corresponding logic around the later referenced block.

 "alive_workers": self._alive_workers,
 "alive_workers_lock": self._alive_workers_lock,
 }

 @property
 def max_workers(self) -> int:
 return self._max_workers
Expand All@@ -126,12 +128,19 @@ def max_workers(self, max_workers: int):
 process for process in self._process if process.is_alive()
 ]
 elif self._max_workers < max_workers:
 old_max_workers = self._max_workers
 self._bootup_events.extend(
 Event() for _ in range(max_workers - old_max_workers)
 )
 for idx in range(old_max_workers, max_workers):
 self._bootup_events[idx].set()
 self._alive_workers[0] += max_workers - old_max_workers
@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win
Synchronize the live-worker count update.
_drain_dead_worker() decrements self._alive_workers[0] under self._alive_workers_lock. A worker can fail while this resize increments the same value. An unsynchronized increment can lose either update and make the scheduler treat live workers as dead.
Proposed fix
- self._alive_workers[0] += max_workers - old_max_workers+ with self._alive_workers_lock:+ self._alive_workers[0] += max_workers - old_max_workers
📝 Committable suggestion
‼️IMPORTANT

Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change

self._alive_workers[0] +=max_workers-old_max_workers
withself._alive_workers_lock:
self._alive_workers[0] +=max_workers-old_max_workers
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` at line 134,
Protect the live-worker count increment in the resize logic with
self._alive_workers_lock, matching the locking used by _drain_dead_worker().
Keep the adjustment of self._alive_workers[0] atomic with respect to concurrent
worker-failure decrements.

 new_process_lst = [
 Thread(
 target=_execute_multiple_tasks,
 kwargs=self._process_kwargs,
 kwargs=self._worker_kwargs(worker_id),
 )
 for _ in range(max_workers - self._max_workers)
 for worker_id in range(old_max_workers, max_workers)
 ]
 for process_instance in new_process_lst:
 process_instance.start()
Expand Down

49 changes: 46 additions & 3 deletions
tests/unit/task_scheduler/interactive/test_blockallocation.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,11 +1,54 @@
import queue
import unittest
from threading import Lock
from concurrent.futures import Future
from threading import Event, Lock
from unittest.mock import patch

from executorlib.task_scheduler.interactive.blockallocation import _drain_dead_worker
from executorlib.task_scheduler.interactive.shared import task_done
from executorlib.standalone.interactive.communication import ExecutorlibSocketError
from executorlib.task_scheduler.interactive.blockallocation import (
 BlockAllocationTaskScheduler,
 _drain_dead_worker,
)


class TestBlockAllocationResize(unittest.TestCase):
 def test_increase_workers_passes_worker_context(self):
 scheduler = object.__new__(BlockAllocationTaskScheduler)
 scheduler._future_queue = queue.Queue()
 scheduler._process = []
 scheduler._process_kwargs = {"future_queue": scheduler._future_queue}
 scheduler._max_workers = 1
 scheduler._self_id = 1
 scheduler._alive_workers = [1]
 scheduler._alive_workers_lock = Lock()
 scheduler._bootup_events = [Event()]

 class FakeThread:
 instances = []

 def __init__(self, target, kwargs):
 self.target = target
 self.kwargs = kwargs
 self.started = False
 self.instances.append(self)

 def start(self):
 self.started = True

 with patch(
 "executorlib.task_scheduler.interactive.blockallocation.Thread",
 FakeThread,
 ):
 scheduler.max_workers = 2

 worker = FakeThread.instances[-1]
 self.assertEqual(worker.kwargs["worker_id"], 1)
 self.assertIn("stop_function", worker.kwargs)
 self.assertIn("bootup_event", worker.kwargs)
 self.assertIn("next_bootup_event", worker.kwargs)
 self.assertIs(worker.kwargs["alive_workers"], scheduler._alive_workers)
 self.assertTrue(worker.started)
 self.assertEqual(scheduler._alive_workers[0], 2)


class TestDrainDeadWorker(unittest.TestCase):
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
fix(scheduler): preserve worker context when resizing#1044

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

Muhtasim-Munif-Fahim
wants to merge
3
commits into

base:main
Choose a base branch

from

 Open


fix(scheduler): preserve worker context when resizing
#1044
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
c049d07

fix(scheduler): preserve worker context when resizing


Muhtasim-Munif-Fahim Aug 9, 2026
2c5a622

fix(scheduler): prevent zombie threads on scheduler resize and shutdown


Muhtasim-Munif-Fahim Aug 11, 2026
0916e74

Merge branch 'main' into codex/executorlib-resize-worker-context


jan-janssen Aug 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
src/executorlib/standalone/interactive/communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -152,7 +152,7 @@ def shutdown(self, wait: bool = True):
 if self._spawner.poll():
 result = self.send_and_receive_dict(
 input_dict={"shutdown": True, "wait": wait}
 )["result"]
 ).get("result")
 self._spawner.shutdown(wait=wait)
 self._reset_socket()
 return result
Expand Down

51 changes: 30 additions & 21 deletions
src/executorlib/task_scheduler/interactive/blockallocation.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -80,35 +80,37 @@ def __init__(
 executor_kwargs["restart_limit"] = restart_limit
 self._process_kwargs = executor_kwargs
 self._max_workers = max_workers
 self_id = random.getrandbits(128)
 self._self_id = self_id
 self._self_id = random.getrandbits(128)
 _interrupt_bootup_dict[self._self_id] = False
alive_workers = [max_workers]
alive_workers_lock = Lock()
bootup_events = [Event() for _ in range(self._max_workers)]
bootup_events[0].set()
self._alive_workers = [max_workers]
self._alive_workers_lock = Lock()
self._bootup_events = [Event() for _ in range(self._max_workers)]
self._bootup_events[0].set()
 self._set_process(
 process=[
 Thread(
 target=_execute_multiple_tasks,
 kwargs=executor_kwargs
 | {
 "worker_id": worker_id,
 "stop_function": lambda: _interrupt_bootup_dict[self_id],
 "bootup_event": bootup_events[worker_id],
 "next_bootup_event": (
 bootup_events[worker_id + 1]
 if worker_id + 1 < self._max_workers
 else None
 ),
 "alive_workers": alive_workers,
 "alive_workers_lock": alive_workers_lock,
 },
 kwargs=self._worker_kwargs(worker_id),
 )
 for worker_id in range(self._max_workers)
 ],
 )

 def _worker_kwargs(self, worker_id: int) -> dict:
 self_id = self._self_id
 return self._process_kwargs | {
 "worker_id": worker_id,
 "stop_function": lambda: _interrupt_bootup_dict[self_id],
 "bootup_event": self._bootup_events[worker_id],
 "next_bootup_event": (
 self._bootup_events[worker_id + 1]
 if worker_id + 1 < len(self._bootup_events)
 else None
 ),

Comment on lines

+99
 to 
+109

@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win
Release the first added worker from bootup.
_worker_kwargs() snapshots the original tail worker's next_bootup_event as None. After resize, the first added worker waits on the new unset event, but no existing worker can signal it. The added worker and later added workers remain blocked in bootup_event.wait().
Signal the first new bootup event when resizing. If strict worker-ID boot order must also apply during concurrent startup, use a resizable handoff instead of a static next_bootup_event.
Proposed fix
 self._bootup_events.extend(
Event() for _ in range(max_workers - old_max_workers)
)
+ self._bootup_events[old_max_workers].set()
self._alive_workers[0] += max_workers - old_max_workers
Also applies to: 130-140
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` around lines
99 - 108, Update the resize path and _worker_kwargs so the first newly added
worker’s bootup event is explicitly released, while preserving sequential
handoff for later workers. Ensure concurrent startup cannot leave added workers
waiting on a statically captured next_bootup_event; use the existing
resize/startup synchronization mechanism or a resizable handoff, and apply the
same correction to the corresponding logic around the later referenced block.

 "alive_workers": self._alive_workers,
 "alive_workers_lock": self._alive_workers_lock,
 }

 @property
 def max_workers(self) -> int:
 return self._max_workers
Expand All@@ -126,12 +128,19 @@ def max_workers(self, max_workers: int):
 process for process in self._process if process.is_alive()
 ]
 elif self._max_workers < max_workers:
 old_max_workers = self._max_workers
 self._bootup_events.extend(
 Event() for _ in range(max_workers - old_max_workers)
 )
 for idx in range(old_max_workers, max_workers):
 self._bootup_events[idx].set()
 self._alive_workers[0] += max_workers - old_max_workers
@coderabbitaicoderabbitaiBotAug 9, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win
Synchronize the live-worker count update.
_drain_dead_worker() decrements self._alive_workers[0] under self._alive_workers_lock. A worker can fail while this resize increments the same value. An unsynchronized increment can lose either update and make the scheduler treat live workers as dead.
Proposed fix
- self._alive_workers[0] += max_workers - old_max_workers+ with self._alive_workers_lock:+ self._alive_workers[0] += max_workers - old_max_workers
📝 Committable suggestion
‼️IMPORTANT

Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change

self._alive_workers[0] +=max_workers-old_max_workers
withself._alive_workers_lock:
self._alive_workers[0] +=max_workers-old_max_workers
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.
In `@src/executorlib/task_scheduler/interactive/blockallocation.py` at line 134,
Protect the live-worker count increment in the resize logic with
self._alive_workers_lock, matching the locking used by _drain_dead_worker().
Keep the adjustment of self._alive_workers[0] atomic with respect to concurrent
worker-failure decrements.

 new_process_lst = [
 Thread(
 target=_execute_multiple_tasks,
 kwargs=self._process_kwargs,
 kwargs=self._worker_kwargs(worker_id),
 )
 for _ in range(max_workers - self._max_workers)
 for worker_id in range(old_max_workers, max_workers)
 ]
 for process_instance in new_process_lst:
 process_instance.start()
Expand Down

49 changes: 46 additions & 3 deletions
tests/unit/task_scheduler/interactive/test_blockallocation.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,11 +1,54 @@
import queue
import unittest
from threading import Lock
from concurrent.futures import Future
from threading import Event, Lock
from unittest.mock import patch

from executorlib.task_scheduler.interactive.blockallocation import _drain_dead_worker
from executorlib.task_scheduler.interactive.shared import task_done
from executorlib.standalone.interactive.communication import ExecutorlibSocketError
from executorlib.task_scheduler.interactive.blockallocation import (
 BlockAllocationTaskScheduler,
 _drain_dead_worker,
)


class TestBlockAllocationResize(unittest.TestCase):
 def test_increase_workers_passes_worker_context(self):
 scheduler = object.__new__(BlockAllocationTaskScheduler)
 scheduler._future_queue = queue.Queue()
 scheduler._process = []
 scheduler._process_kwargs = {"future_queue": scheduler._future_queue}
 scheduler._max_workers = 1
 scheduler._self_id = 1
 scheduler._alive_workers = [1]
 scheduler._alive_workers_lock = Lock()
 scheduler._bootup_events = [Event()]

 class FakeThread:
 instances = []

 def __init__(self, target, kwargs):
 self.target = target
 self.kwargs = kwargs
 self.started = False
 self.instances.append(self)

 def start(self):
 self.started = True

 with patch(
 "executorlib.task_scheduler.interactive.blockallocation.Thread",
 FakeThread,
 ):
 scheduler.max_workers = 2

 worker = FakeThread.instances[-1]
 self.assertEqual(worker.kwargs["worker_id"], 1)
 self.assertIn("stop_function", worker.kwargs)
 self.assertIn("bootup_event", worker.kwargs)
 self.assertIn("next_bootup_event", worker.kwargs)
 self.assertIs(worker.kwargs["alive_workers"], scheduler._alive_workers)
 self.assertTrue(worker.started)
 self.assertEqual(scheduler._alive_workers[0], 2)


class TestDrainDeadWorker(unittest.TestCase):
Expand Down
Loading