Communication: Use zmq.Poller() rather than waiting infinitely#789

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jan-janssen merged 4 commits into from
Aug 30, 2025
Merged

Communication: Use zmq.Poller() rather than waiting infinitely #789

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2b6055c
Communication: Use zmq.Poller() rather than waiting infinitely
jan-janssen Aug 29, 2025
453d133
Format black
pyiron-runner Aug 29, 2025
6f205ee
fix spelling
jan-janssen Aug 29, 2025
5df5362
Add test
jan-janssen Aug 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 21 additions & 5 deletions executorlib/standalone/interactive/communication.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,46 @@
import logging
import sys
from socket import gethostname
from typing import Optional
from typing import Any, Optional

import cloudpickle
import zmq


class ExecutorlibSocketError(RuntimeError):
pass


class SocketInterface:
"""
The SocketInterface is an abstraction layer on top of the zero message queue.

Args:
spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
"""

def __init__(self, spawner=None, log_obj_size=False):
def __init__(
self, spawner=None, log_obj_size: bool = False, time_out_ms: int = 1000
):
"""
Initialize the SocketInterface.

Args:
spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
"""
self._context = zmq.Context()
self._socket = self._context.socket(zmq.PAIR)
self._poller = zmq.Poller()
self._poller.register(self._socket, zmq.POLLIN)
self._process = None
self._time_out_ms = time_out_ms
self._logger: Optional[logging.Logger] = None
if log_obj_size:
self._logger = logging.getLogger("executorlib")
else:
self._logger = None
self._spawner = spawner

def send_dict(self, input_dict: dict):
Expand All@@ -52,7 +63,12 @@ def receive_dict(self) -> dict:
Returns:
dict: dictionary with response received from the connected client
"""
data = self._socket.recv()
response_lst: list[tuple[Any, int]] = []
while len(response_lst) == 0:
response_lst = self._poller.poll(self._time_out_ms)
if not self._spawner.poll():
raise ExecutorlibSocketError()
data = self._socket.recv(zmq.NOBLOCK)
if self._logger is not None:
self._logger.warning(
"Received dictionary of size: " + str(sys.getsizeof(data))
Expand Down
30 changes: 30 additions & 0 deletions tests/test_standalone_interactive_communication.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
interface_send,
interface_receive,
SocketInterface,
ExecutorlibSocketError,
)
from executorlib.standalone.serialize import cloudpickle_register
from executorlib.standalone.interactive.spawner import MpiExecSpawner
Expand DownExpand Up@@ -114,6 +115,35 @@ def test_interface_serial_with_debug(self):
)
interface.shutdown(wait=True)

def test_interface_serial_with_stopped_process(self):
cloudpickle_register(ind=1)
task_dict = {"fn": calc, "args": (), "kwargs": {"i": 2}}
interface = SocketInterface(
spawner=MpiExecSpawner(cwd=None, cores=1, openmpi_oversubscribe=False),
log_obj_size=True,
)
interface.bootup(
command_lst=[
sys.executable,
os.path.abspath(
os.path.join(
__file__,
"..",
"..",
"executorlib",
"backend",
"interactive_serial.py",
)
),
"--zmqport",
str(interface.bind_to_random_port()),
]
)
interface.send_dict(input_dict=task_dict)
interface._spawner._process.terminate()
with self.assertRaises(ExecutorlibSocketError):
interface.receive_dict()

Comment on lines +118 to +146

@coderabbitaicoderabbitaiBotAug 30, 2025

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Verification agent

🧩 Analysis chain

Likely inverted liveness check in receive_dict; test may mask the bug.

receive_dict() currently raises when the spawner is alive and may loop forever when it exited with a non-zero code (common after terminate()). This can cause this test to hang instead of raising, or to raise for the wrong reason.

  1. Proposed fix (outside this file) in executorlib/standalone/interactive/communication.py:
 def receive_dict(self) -> dict:
"""
Receive a dictionary from a connected client process.
"""
- response_lst: list[tuple[Any, int]] = []- while len(response_lst) == 0:- response_lst = self._poller.poll(self._time_out_ms)- if not self._spawner.poll():- raise ExecutorlibSocketError()+ response_lst: list[tuple[Any, int]] = []+ while True:+ # If the spawner exited (any return code), fail fast.+ if self._spawner.poll() is not None:+ raise ExecutorlibSocketError("Spawner terminated while waiting for response")+ response_lst = self._poller.poll(self._time_out_ms)+ if response_lst:+ break
data = self._socket.recv(zmq.NOBLOCK)
  1. Verify the current logic and ensure the fix is applied:

🏁 Script executed:

#!/bin/bashset -euo pipefail
# Show the receive_dict implementation and nearby lines
rg -n -C3 $'def\\s+receive_dict\\b' executorlib/standalone/interactive/communication.py
# Detect the inverted check; this should return no matches after the fix
rg -n $'if\\s+not\\s+self\\._spawner\\.poll\\(\\)\\s*:' executorlib/standalone/interactive/communication.py ||true

Length of output: 506

Fix inverted liveness check in receive_dict
executorlib/standalone/interactive/communication.py:69 still uses

ifnotself._spawner.poll():
raiseExecutorlibSocketError()

This is inverted – it hangs when the spawner exits. Change to:

ifself._spawner.poll() isnotNone:
raiseExecutorlibSocketError("Spawner terminated while waiting for response")

to fail fast on termination.

🤖 Prompt for AI Agents
In executorlib/standalone/interactive/communication.py around line 69, the
liveness check is inverted: currently it raises when poll() is falsy and thus
hangs when the spawner exits; change the condition to check for termination
using if self._spawner.poll() is not None and raise ExecutorlibSocketError with
a clear message like "Spawner terminated while waiting for response" so the code
fails fast when the spawner has exited.


class TestZMQ(unittest.TestCase):
def test_interface_receive(self):
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Communication: Use zmq.Poller() rather than waiting infinitely#789

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jan-janssen
merged 4 commits into

from

Aug 30, 2025
 Merged


Communication: Use zmq.Poller() rather than waiting infinitely
#789
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
2b6055c

Communication: Use zmq.Poller() rather than waiting infinitely


jan-janssen Aug 29, 2025
453d133

Format black


pyiron-runner Aug 29, 2025
6f205ee

fix spelling


jan-janssen Aug 29, 2025
5df5362

Add test


jan-janssen Aug 30, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


26 changes: 21 additions & 5 deletions
executorlib/standalone/interactive/communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,35 +1,46 @@
import logging
import sys
from socket import gethostname
from typing import Optional
from typing import Any, Optional

import cloudpickle
import zmq


class ExecutorlibSocketError(RuntimeError):
 pass


class SocketInterface:
 """
 The SocketInterface is an abstraction layer on top of the zero message queue.

 Args:
 spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
 log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
 time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
 """

 def __init__(self, spawner=None, log_obj_size=False):
 def __init__(
 self, spawner=None, log_obj_size: bool = False, time_out_ms: int = 1000
 ):
 """
 Initialize the SocketInterface.

 Args:
 spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
 log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
 time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
 """
 self._context = zmq.Context()
 self._socket = self._context.socket(zmq.PAIR)
 self._poller = zmq.Poller()
 self._poller.register(self._socket, zmq.POLLIN)
 self._process = None
 self._time_out_ms = time_out_ms
 self._logger: Optional[logging.Logger] = None
 if log_obj_size:
 self._logger = logging.getLogger("executorlib")
 else:
 self._logger = None
 self._spawner = spawner

 def send_dict(self, input_dict: dict):
Expand All@@ -52,7 +63,12 @@ def receive_dict(self) -> dict:
 Returns:
 dict: dictionary with response received from the connected client
 """
 data = self._socket.recv()
 response_lst: list[tuple[Any, int]] = []
 while len(response_lst) == 0:
 response_lst = self._poller.poll(self._time_out_ms)
 if not self._spawner.poll():
 raise ExecutorlibSocketError()
 data = self._socket.recv(zmq.NOBLOCK)
 if self._logger is not None:
 self._logger.warning(
 "Received dictionary of size: " + str(sys.getsizeof(data))
Expand Down

30 changes: 30 additions & 0 deletions
tests/test_standalone_interactive_communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
 interface_send,
 interface_receive,
 SocketInterface,
 ExecutorlibSocketError,
)
from executorlib.standalone.serialize import cloudpickle_register
from executorlib.standalone.interactive.spawner import MpiExecSpawner
Expand DownExpand Up@@ -114,6 +115,35 @@ def test_interface_serial_with_debug(self):
 )
 interface.shutdown(wait=True)

 def test_interface_serial_with_stopped_process(self):
 cloudpickle_register(ind=1)
 task_dict = {"fn": calc, "args": (), "kwargs": {"i": 2}}
 interface = SocketInterface(
 spawner=MpiExecSpawner(cwd=None, cores=1, openmpi_oversubscribe=False),
 log_obj_size=True,
 )
 interface.bootup(
 command_lst=[
 sys.executable,
 os.path.abspath(
 os.path.join(
 __file__,
 "..",
 "..",
 "executorlib",
 "backend",
 "interactive_serial.py",
 )
 ),
 "--zmqport",
 str(interface.bind_to_random_port()),
 ]
 )
 interface.send_dict(input_dict=task_dict)
 interface._spawner._process.terminate()
 with self.assertRaises(ExecutorlibSocketError):
 interface.receive_dict()


Comment on lines

+118
 to 
+146

@coderabbitaicoderabbitaiBotAug 30, 2025

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Verification agent
🧩 Analysis chain
Likely inverted liveness check in receive_dict; test may mask the bug.
receive_dict() currently raises when the spawner is alive and may loop forever when it exited with a non-zero code (common after terminate()). This can cause this test to hang instead of raising, or to raise for the wrong reason.
  1. Proposed fix (outside this file) in executorlib/standalone/interactive/communication.py:
 def receive_dict(self) -> dict:
"""
Receive a dictionary from a connected client process.
"""
- response_lst: list[tuple[Any, int]] = []- while len(response_lst) == 0:- response_lst = self._poller.poll(self._time_out_ms)- if not self._spawner.poll():- raise ExecutorlibSocketError()+ response_lst: list[tuple[Any, int]] = []+ while True:+ # If the spawner exited (any return code), fail fast.+ if self._spawner.poll() is not None:+ raise ExecutorlibSocketError("Spawner terminated while waiting for response")+ response_lst = self._poller.poll(self._time_out_ms)+ if response_lst:+ break
data = self._socket.recv(zmq.NOBLOCK)
  1. Verify the current logic and ensure the fix is applied:
🏁 Script executed:
#!/bin/bashset -euo pipefail
# Show the receive_dict implementation and nearby lines
rg -n -C3 $'def\\s+receive_dict\\b' executorlib/standalone/interactive/communication.py
# Detect the inverted check; this should return no matches after the fix
rg -n $'if\\s+not\\s+self\\._spawner\\.poll\\(\\)\\s*:' executorlib/standalone/interactive/communication.py ||true
Length of output: 506
Fix inverted liveness check in receive_dict

executorlib/standalone/interactive/communication.py:69 still uses
ifnotself._spawner.poll():
raiseExecutorlibSocketError()
This is inverted – it hangs when the spawner exits. Change to:
ifself._spawner.poll() isnotNone:
raiseExecutorlibSocketError("Spawner terminated while waiting for response")
to fail fast on termination.
🤖 Prompt for AI Agents
In executorlib/standalone/interactive/communication.py around line 69, the
liveness check is inverted: currently it raises when poll() is falsy and thus
hangs when the spawner exits; change the condition to check for termination
using if self._spawner.poll() is not None and raise ExecutorlibSocketError with
a clear message like "Spawner terminated while waiting for response" so the code
fails fast when the spawner has exited.


class TestZMQ(unittest.TestCase):
 def test_interface_receive(self):
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Communication: Use zmq.Poller() rather than waiting infinitely#789

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jan-janssen
merged 4 commits into

from

Aug 30, 2025
 Merged


Communication: Use zmq.Poller() rather than waiting infinitely
#789
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
2b6055c

Communication: Use zmq.Poller() rather than waiting infinitely


jan-janssen Aug 29, 2025
453d133

Format black


pyiron-runner Aug 29, 2025
6f205ee

fix spelling


jan-janssen Aug 29, 2025
5df5362

Add test


jan-janssen Aug 30, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


26 changes: 21 additions & 5 deletions
executorlib/standalone/interactive/communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,35 +1,46 @@
import logging
import sys
from socket import gethostname
from typing import Optional
from typing import Any, Optional

import cloudpickle
import zmq


class ExecutorlibSocketError(RuntimeError):
 pass


class SocketInterface:
 """
 The SocketInterface is an abstraction layer on top of the zero message queue.

 Args:
 spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
 log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
 time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
 """

 def __init__(self, spawner=None, log_obj_size=False):
 def __init__(
 self, spawner=None, log_obj_size: bool = False, time_out_ms: int = 1000
 ):
 """
 Initialize the SocketInterface.

 Args:
 spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
 log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
 time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
 """
 self._context = zmq.Context()
 self._socket = self._context.socket(zmq.PAIR)
 self._poller = zmq.Poller()
 self._poller.register(self._socket, zmq.POLLIN)
 self._process = None
 self._time_out_ms = time_out_ms
 self._logger: Optional[logging.Logger] = None
 if log_obj_size:
 self._logger = logging.getLogger("executorlib")
 else:
 self._logger = None
 self._spawner = spawner

 def send_dict(self, input_dict: dict):
Expand All@@ -52,7 +63,12 @@ def receive_dict(self) -> dict:
 Returns:
 dict: dictionary with response received from the connected client
 """
 data = self._socket.recv()
 response_lst: list[tuple[Any, int]] = []
 while len(response_lst) == 0:
 response_lst = self._poller.poll(self._time_out_ms)
 if not self._spawner.poll():
 raise ExecutorlibSocketError()
 data = self._socket.recv(zmq.NOBLOCK)
 if self._logger is not None:
 self._logger.warning(
 "Received dictionary of size: " + str(sys.getsizeof(data))
Expand Down

30 changes: 30 additions & 0 deletions
tests/test_standalone_interactive_communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
 interface_send,
 interface_receive,
 SocketInterface,
 ExecutorlibSocketError,
)
from executorlib.standalone.serialize import cloudpickle_register
from executorlib.standalone.interactive.spawner import MpiExecSpawner
Expand DownExpand Up@@ -114,6 +115,35 @@ def test_interface_serial_with_debug(self):
 )
 interface.shutdown(wait=True)

 def test_interface_serial_with_stopped_process(self):
 cloudpickle_register(ind=1)
 task_dict = {"fn": calc, "args": (), "kwargs": {"i": 2}}
 interface = SocketInterface(
 spawner=MpiExecSpawner(cwd=None, cores=1, openmpi_oversubscribe=False),
 log_obj_size=True,
 )
 interface.bootup(
 command_lst=[
 sys.executable,
 os.path.abspath(
 os.path.join(
 __file__,
 "..",
 "..",
 "executorlib",
 "backend",
 "interactive_serial.py",
 )
 ),
 "--zmqport",
 str(interface.bind_to_random_port()),
 ]
 )
 interface.send_dict(input_dict=task_dict)
 interface._spawner._process.terminate()
 with self.assertRaises(ExecutorlibSocketError):
 interface.receive_dict()


Comment on lines

+118
 to 
+146

@coderabbitaicoderabbitaiBotAug 30, 2025

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Verification agent
🧩 Analysis chain
Likely inverted liveness check in receive_dict; test may mask the bug.
receive_dict() currently raises when the spawner is alive and may loop forever when it exited with a non-zero code (common after terminate()). This can cause this test to hang instead of raising, or to raise for the wrong reason.
  1. Proposed fix (outside this file) in executorlib/standalone/interactive/communication.py:
 def receive_dict(self) -> dict:
"""
Receive a dictionary from a connected client process.
"""
- response_lst: list[tuple[Any, int]] = []- while len(response_lst) == 0:- response_lst = self._poller.poll(self._time_out_ms)- if not self._spawner.poll():- raise ExecutorlibSocketError()+ response_lst: list[tuple[Any, int]] = []+ while True:+ # If the spawner exited (any return code), fail fast.+ if self._spawner.poll() is not None:+ raise ExecutorlibSocketError("Spawner terminated while waiting for response")+ response_lst = self._poller.poll(self._time_out_ms)+ if response_lst:+ break
data = self._socket.recv(zmq.NOBLOCK)
  1. Verify the current logic and ensure the fix is applied:
🏁 Script executed:
#!/bin/bashset -euo pipefail
# Show the receive_dict implementation and nearby lines
rg -n -C3 $'def\\s+receive_dict\\b' executorlib/standalone/interactive/communication.py
# Detect the inverted check; this should return no matches after the fix
rg -n $'if\\s+not\\s+self\\._spawner\\.poll\\(\\)\\s*:' executorlib/standalone/interactive/communication.py ||true
Length of output: 506
Fix inverted liveness check in receive_dict

executorlib/standalone/interactive/communication.py:69 still uses
ifnotself._spawner.poll():
raiseExecutorlibSocketError()
This is inverted – it hangs when the spawner exits. Change to:
ifself._spawner.poll() isnotNone:
raiseExecutorlibSocketError("Spawner terminated while waiting for response")
to fail fast on termination.
🤖 Prompt for AI Agents
In executorlib/standalone/interactive/communication.py around line 69, the
liveness check is inverted: currently it raises when poll() is falsy and thus
hangs when the spawner exits; change the condition to check for termination
using if self._spawner.poll() is not None and raise ExecutorlibSocketError with
a clear message like "Spawner terminated while waiting for response" so the code
fails fast when the spawner has exited.


class TestZMQ(unittest.TestCase):
 def test_interface_receive(self):
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Communication: Use zmq.Poller() rather than waiting infinitely#789

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jan-janssen
merged 4 commits into

from

Aug 30, 2025
 Merged


Communication: Use zmq.Poller() rather than waiting infinitely
#789
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
2b6055c

Communication: Use zmq.Poller() rather than waiting infinitely


jan-janssen Aug 29, 2025
453d133

Format black


pyiron-runner Aug 29, 2025
6f205ee

fix spelling


jan-janssen Aug 29, 2025
5df5362

Add test


jan-janssen Aug 30, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


26 changes: 21 additions & 5 deletions
executorlib/standalone/interactive/communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,35 +1,46 @@
import logging
import sys
from socket import gethostname
from typing import Optional
from typing import Any, Optional

import cloudpickle
import zmq


class ExecutorlibSocketError(RuntimeError):
 pass


class SocketInterface:
 """
 The SocketInterface is an abstraction layer on top of the zero message queue.

 Args:
 spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
 log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
 time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
 """

 def __init__(self, spawner=None, log_obj_size=False):
 def __init__(
 self, spawner=None, log_obj_size: bool = False, time_out_ms: int = 1000
 ):
 """
 Initialize the SocketInterface.

 Args:
 spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
 log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
 time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
 """
 self._context = zmq.Context()
 self._socket = self._context.socket(zmq.PAIR)
 self._poller = zmq.Poller()
 self._poller.register(self._socket, zmq.POLLIN)
 self._process = None
 self._time_out_ms = time_out_ms
 self._logger: Optional[logging.Logger] = None
 if log_obj_size:
 self._logger = logging.getLogger("executorlib")
 else:
 self._logger = None
 self._spawner = spawner

 def send_dict(self, input_dict: dict):
Expand All@@ -52,7 +63,12 @@ def receive_dict(self) -> dict:
 Returns:
 dict: dictionary with response received from the connected client
 """
 data = self._socket.recv()
 response_lst: list[tuple[Any, int]] = []
 while len(response_lst) == 0:
 response_lst = self._poller.poll(self._time_out_ms)
 if not self._spawner.poll():
 raise ExecutorlibSocketError()
 data = self._socket.recv(zmq.NOBLOCK)
 if self._logger is not None:
 self._logger.warning(
 "Received dictionary of size: " + str(sys.getsizeof(data))
Expand Down

30 changes: 30 additions & 0 deletions
tests/test_standalone_interactive_communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
 interface_send,
 interface_receive,
 SocketInterface,
 ExecutorlibSocketError,
)
from executorlib.standalone.serialize import cloudpickle_register
from executorlib.standalone.interactive.spawner import MpiExecSpawner
Expand DownExpand Up@@ -114,6 +115,35 @@ def test_interface_serial_with_debug(self):
 )
 interface.shutdown(wait=True)

 def test_interface_serial_with_stopped_process(self):
 cloudpickle_register(ind=1)
 task_dict = {"fn": calc, "args": (), "kwargs": {"i": 2}}
 interface = SocketInterface(
 spawner=MpiExecSpawner(cwd=None, cores=1, openmpi_oversubscribe=False),
 log_obj_size=True,
 )
 interface.bootup(
 command_lst=[
 sys.executable,
 os.path.abspath(
 os.path.join(
 __file__,
 "..",
 "..",
 "executorlib",
 "backend",
 "interactive_serial.py",
 )
 ),
 "--zmqport",
 str(interface.bind_to_random_port()),
 ]
 )
 interface.send_dict(input_dict=task_dict)
 interface._spawner._process.terminate()
 with self.assertRaises(ExecutorlibSocketError):
 interface.receive_dict()


Comment on lines

+118
 to 
+146

@coderabbitaicoderabbitaiBotAug 30, 2025

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Verification agent
🧩 Analysis chain
Likely inverted liveness check in receive_dict; test may mask the bug.
receive_dict() currently raises when the spawner is alive and may loop forever when it exited with a non-zero code (common after terminate()). This can cause this test to hang instead of raising, or to raise for the wrong reason.
  1. Proposed fix (outside this file) in executorlib/standalone/interactive/communication.py:
 def receive_dict(self) -> dict:
"""
Receive a dictionary from a connected client process.
"""
- response_lst: list[tuple[Any, int]] = []- while len(response_lst) == 0:- response_lst = self._poller.poll(self._time_out_ms)- if not self._spawner.poll():- raise ExecutorlibSocketError()+ response_lst: list[tuple[Any, int]] = []+ while True:+ # If the spawner exited (any return code), fail fast.+ if self._spawner.poll() is not None:+ raise ExecutorlibSocketError("Spawner terminated while waiting for response")+ response_lst = self._poller.poll(self._time_out_ms)+ if response_lst:+ break
data = self._socket.recv(zmq.NOBLOCK)
  1. Verify the current logic and ensure the fix is applied:
🏁 Script executed:
#!/bin/bashset -euo pipefail
# Show the receive_dict implementation and nearby lines
rg -n -C3 $'def\\s+receive_dict\\b' executorlib/standalone/interactive/communication.py
# Detect the inverted check; this should return no matches after the fix
rg -n $'if\\s+not\\s+self\\._spawner\\.poll\\(\\)\\s*:' executorlib/standalone/interactive/communication.py ||true
Length of output: 506
Fix inverted liveness check in receive_dict

executorlib/standalone/interactive/communication.py:69 still uses
ifnotself._spawner.poll():
raiseExecutorlibSocketError()
This is inverted – it hangs when the spawner exits. Change to:
ifself._spawner.poll() isnotNone:
raiseExecutorlibSocketError("Spawner terminated while waiting for response")
to fail fast on termination.
🤖 Prompt for AI Agents
In executorlib/standalone/interactive/communication.py around line 69, the
liveness check is inverted: currently it raises when poll() is falsy and thus
hangs when the spawner exits; change the condition to check for termination
using if self._spawner.poll() is not None and raise ExecutorlibSocketError with
a clear message like "Spawner terminated while waiting for response" so the code
fails fast when the spawner has exited.


class TestZMQ(unittest.TestCase):
 def test_interface_receive(self):
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Communication: Use zmq.Poller() rather than waiting infinitely#789

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jan-janssen
merged 4 commits into

from

Aug 30, 2025
 Merged


Communication: Use zmq.Poller() rather than waiting infinitely
#789
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
2b6055c

Communication: Use zmq.Poller() rather than waiting infinitely


jan-janssen Aug 29, 2025
453d133

Format black


pyiron-runner Aug 29, 2025
6f205ee

fix spelling


jan-janssen Aug 29, 2025
5df5362

Add test


jan-janssen Aug 30, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


26 changes: 21 additions & 5 deletions
executorlib/standalone/interactive/communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,35 +1,46 @@
import logging
import sys
from socket import gethostname
from typing import Optional
from typing import Any, Optional

import cloudpickle
import zmq


class ExecutorlibSocketError(RuntimeError):
 pass


class SocketInterface:
 """
 The SocketInterface is an abstraction layer on top of the zero message queue.

 Args:
 spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
 log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
 time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
 """

 def __init__(self, spawner=None, log_obj_size=False):
 def __init__(
 self, spawner=None, log_obj_size: bool = False, time_out_ms: int = 1000
 ):
 """
 Initialize the SocketInterface.

 Args:
 spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
 log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
 time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
 """
 self._context = zmq.Context()
 self._socket = self._context.socket(zmq.PAIR)
 self._poller = zmq.Poller()
 self._poller.register(self._socket, zmq.POLLIN)
 self._process = None
 self._time_out_ms = time_out_ms
 self._logger: Optional[logging.Logger] = None
 if log_obj_size:
 self._logger = logging.getLogger("executorlib")
 else:
 self._logger = None
 self._spawner = spawner

 def send_dict(self, input_dict: dict):
Expand All@@ -52,7 +63,12 @@ def receive_dict(self) -> dict:
 Returns:
 dict: dictionary with response received from the connected client
 """
 data = self._socket.recv()
 response_lst: list[tuple[Any, int]] = []
 while len(response_lst) == 0:
 response_lst = self._poller.poll(self._time_out_ms)
 if not self._spawner.poll():
 raise ExecutorlibSocketError()
 data = self._socket.recv(zmq.NOBLOCK)
 if self._logger is not None:
 self._logger.warning(
 "Received dictionary of size: " + str(sys.getsizeof(data))
Expand Down

30 changes: 30 additions & 0 deletions
tests/test_standalone_interactive_communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
 interface_send,
 interface_receive,
 SocketInterface,
 ExecutorlibSocketError,
)
from executorlib.standalone.serialize import cloudpickle_register
from executorlib.standalone.interactive.spawner import MpiExecSpawner
Expand DownExpand Up@@ -114,6 +115,35 @@ def test_interface_serial_with_debug(self):
 )
 interface.shutdown(wait=True)

 def test_interface_serial_with_stopped_process(self):
 cloudpickle_register(ind=1)
 task_dict = {"fn": calc, "args": (), "kwargs": {"i": 2}}
 interface = SocketInterface(
 spawner=MpiExecSpawner(cwd=None, cores=1, openmpi_oversubscribe=False),
 log_obj_size=True,
 )
 interface.bootup(
 command_lst=[
 sys.executable,
 os.path.abspath(
 os.path.join(
 __file__,
 "..",
 "..",
 "executorlib",
 "backend",
 "interactive_serial.py",
 )
 ),
 "--zmqport",
 str(interface.bind_to_random_port()),
 ]
 )
 interface.send_dict(input_dict=task_dict)
 interface._spawner._process.terminate()
 with self.assertRaises(ExecutorlibSocketError):
 interface.receive_dict()


Comment on lines

+118
 to 
+146

@coderabbitaicoderabbitaiBotAug 30, 2025

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Verification agent
🧩 Analysis chain
Likely inverted liveness check in receive_dict; test may mask the bug.
receive_dict() currently raises when the spawner is alive and may loop forever when it exited with a non-zero code (common after terminate()). This can cause this test to hang instead of raising, or to raise for the wrong reason.
  1. Proposed fix (outside this file) in executorlib/standalone/interactive/communication.py:
 def receive_dict(self) -> dict:
"""
Receive a dictionary from a connected client process.
"""
- response_lst: list[tuple[Any, int]] = []- while len(response_lst) == 0:- response_lst = self._poller.poll(self._time_out_ms)- if not self._spawner.poll():- raise ExecutorlibSocketError()+ response_lst: list[tuple[Any, int]] = []+ while True:+ # If the spawner exited (any return code), fail fast.+ if self._spawner.poll() is not None:+ raise ExecutorlibSocketError("Spawner terminated while waiting for response")+ response_lst = self._poller.poll(self._time_out_ms)+ if response_lst:+ break
data = self._socket.recv(zmq.NOBLOCK)
  1. Verify the current logic and ensure the fix is applied:
🏁 Script executed:
#!/bin/bashset -euo pipefail
# Show the receive_dict implementation and nearby lines
rg -n -C3 $'def\\s+receive_dict\\b' executorlib/standalone/interactive/communication.py
# Detect the inverted check; this should return no matches after the fix
rg -n $'if\\s+not\\s+self\\._spawner\\.poll\\(\\)\\s*:' executorlib/standalone/interactive/communication.py ||true
Length of output: 506
Fix inverted liveness check in receive_dict

executorlib/standalone/interactive/communication.py:69 still uses
ifnotself._spawner.poll():
raiseExecutorlibSocketError()
This is inverted – it hangs when the spawner exits. Change to:
ifself._spawner.poll() isnotNone:
raiseExecutorlibSocketError("Spawner terminated while waiting for response")
to fail fast on termination.
🤖 Prompt for AI Agents
In executorlib/standalone/interactive/communication.py around line 69, the
liveness check is inverted: currently it raises when poll() is falsy and thus
hangs when the spawner exits; change the condition to check for termination
using if self._spawner.poll() is not None and raise ExecutorlibSocketError with
a clear message like "Spawner terminated while waiting for response" so the code
fails fast when the spawner has exited.


class TestZMQ(unittest.TestCase):
 def test_interface_receive(self):
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Communication: Use zmq.Poller() rather than waiting infinitely#789

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jan-janssen
merged 4 commits into

from

Aug 30, 2025
 Merged


Communication: Use zmq.Poller() rather than waiting infinitely
#789
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
2b6055c

Communication: Use zmq.Poller() rather than waiting infinitely


jan-janssen Aug 29, 2025
453d133

Format black


pyiron-runner Aug 29, 2025
6f205ee

fix spelling


jan-janssen Aug 29, 2025
5df5362

Add test


jan-janssen Aug 30, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


26 changes: 21 additions & 5 deletions
executorlib/standalone/interactive/communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,35 +1,46 @@
import logging
import sys
from socket import gethostname
from typing import Optional
from typing import Any, Optional

import cloudpickle
import zmq


class ExecutorlibSocketError(RuntimeError):
 pass


class SocketInterface:
 """
 The SocketInterface is an abstraction layer on top of the zero message queue.

 Args:
 spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
 log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
 time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
 """

 def __init__(self, spawner=None, log_obj_size=False):
 def __init__(
 self, spawner=None, log_obj_size: bool = False, time_out_ms: int = 1000
 ):
 """
 Initialize the SocketInterface.

 Args:
 spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
 log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
 time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
 """
 self._context = zmq.Context()
 self._socket = self._context.socket(zmq.PAIR)
 self._poller = zmq.Poller()
 self._poller.register(self._socket, zmq.POLLIN)
 self._process = None
 self._time_out_ms = time_out_ms
 self._logger: Optional[logging.Logger] = None
 if log_obj_size:
 self._logger = logging.getLogger("executorlib")
 else:
 self._logger = None
 self._spawner = spawner

 def send_dict(self, input_dict: dict):
Expand All@@ -52,7 +63,12 @@ def receive_dict(self) -> dict:
 Returns:
 dict: dictionary with response received from the connected client
 """
 data = self._socket.recv()
 response_lst: list[tuple[Any, int]] = []
 while len(response_lst) == 0:
 response_lst = self._poller.poll(self._time_out_ms)
 if not self._spawner.poll():
 raise ExecutorlibSocketError()
 data = self._socket.recv(zmq.NOBLOCK)
 if self._logger is not None:
 self._logger.warning(
 "Received dictionary of size: " + str(sys.getsizeof(data))
Expand Down

30 changes: 30 additions & 0 deletions
tests/test_standalone_interactive_communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
 interface_send,
 interface_receive,
 SocketInterface,
 ExecutorlibSocketError,
)
from executorlib.standalone.serialize import cloudpickle_register
from executorlib.standalone.interactive.spawner import MpiExecSpawner
Expand DownExpand Up@@ -114,6 +115,35 @@ def test_interface_serial_with_debug(self):
 )
 interface.shutdown(wait=True)

 def test_interface_serial_with_stopped_process(self):
 cloudpickle_register(ind=1)
 task_dict = {"fn": calc, "args": (), "kwargs": {"i": 2}}
 interface = SocketInterface(
 spawner=MpiExecSpawner(cwd=None, cores=1, openmpi_oversubscribe=False),
 log_obj_size=True,
 )
 interface.bootup(
 command_lst=[
 sys.executable,
 os.path.abspath(
 os.path.join(
 __file__,
 "..",
 "..",
 "executorlib",
 "backend",
 "interactive_serial.py",
 )
 ),
 "--zmqport",
 str(interface.bind_to_random_port()),
 ]
 )
 interface.send_dict(input_dict=task_dict)
 interface._spawner._process.terminate()
 with self.assertRaises(ExecutorlibSocketError):
 interface.receive_dict()


Comment on lines

+118
 to 
+146

@coderabbitaicoderabbitaiBotAug 30, 2025

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Verification agent
🧩 Analysis chain
Likely inverted liveness check in receive_dict; test may mask the bug.
receive_dict() currently raises when the spawner is alive and may loop forever when it exited with a non-zero code (common after terminate()). This can cause this test to hang instead of raising, or to raise for the wrong reason.
  1. Proposed fix (outside this file) in executorlib/standalone/interactive/communication.py:
 def receive_dict(self) -> dict:
"""
Receive a dictionary from a connected client process.
"""
- response_lst: list[tuple[Any, int]] = []- while len(response_lst) == 0:- response_lst = self._poller.poll(self._time_out_ms)- if not self._spawner.poll():- raise ExecutorlibSocketError()+ response_lst: list[tuple[Any, int]] = []+ while True:+ # If the spawner exited (any return code), fail fast.+ if self._spawner.poll() is not None:+ raise ExecutorlibSocketError("Spawner terminated while waiting for response")+ response_lst = self._poller.poll(self._time_out_ms)+ if response_lst:+ break
data = self._socket.recv(zmq.NOBLOCK)
  1. Verify the current logic and ensure the fix is applied:
🏁 Script executed:
#!/bin/bashset -euo pipefail
# Show the receive_dict implementation and nearby lines
rg -n -C3 $'def\\s+receive_dict\\b' executorlib/standalone/interactive/communication.py
# Detect the inverted check; this should return no matches after the fix
rg -n $'if\\s+not\\s+self\\._spawner\\.poll\\(\\)\\s*:' executorlib/standalone/interactive/communication.py ||true
Length of output: 506
Fix inverted liveness check in receive_dict

executorlib/standalone/interactive/communication.py:69 still uses
ifnotself._spawner.poll():
raiseExecutorlibSocketError()
This is inverted – it hangs when the spawner exits. Change to:
ifself._spawner.poll() isnotNone:
raiseExecutorlibSocketError("Spawner terminated while waiting for response")
to fail fast on termination.
🤖 Prompt for AI Agents
In executorlib/standalone/interactive/communication.py around line 69, the
liveness check is inverted: currently it raises when poll() is falsy and thus
hangs when the spawner exits; change the condition to check for termination
using if self._spawner.poll() is not None and raise ExecutorlibSocketError with
a clear message like "Spawner terminated while waiting for response" so the code
fails fast when the spawner has exited.


class TestZMQ(unittest.TestCase):
 def test_interface_receive(self):
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Communication: Use zmq.Poller() rather than waiting infinitely#789

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jan-janssen
merged 4 commits into

from

Aug 30, 2025
 Merged


Communication: Use zmq.Poller() rather than waiting infinitely
#789
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
2b6055c

Communication: Use zmq.Poller() rather than waiting infinitely


jan-janssen Aug 29, 2025
453d133

Format black


pyiron-runner Aug 29, 2025
6f205ee

fix spelling


jan-janssen Aug 29, 2025
5df5362

Add test


jan-janssen Aug 30, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


26 changes: 21 additions & 5 deletions
executorlib/standalone/interactive/communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,35 +1,46 @@
import logging
import sys
from socket import gethostname
from typing import Optional
from typing import Any, Optional

import cloudpickle
import zmq


class ExecutorlibSocketError(RuntimeError):
 pass


class SocketInterface:
 """
 The SocketInterface is an abstraction layer on top of the zero message queue.

 Args:
 spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
 log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
 time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
 """

 def __init__(self, spawner=None, log_obj_size=False):
 def __init__(
 self, spawner=None, log_obj_size: bool = False, time_out_ms: int = 1000
 ):
 """
 Initialize the SocketInterface.

 Args:
 spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
 log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
 time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
 """
 self._context = zmq.Context()
 self._socket = self._context.socket(zmq.PAIR)
 self._poller = zmq.Poller()
 self._poller.register(self._socket, zmq.POLLIN)
 self._process = None
 self._time_out_ms = time_out_ms
 self._logger: Optional[logging.Logger] = None
 if log_obj_size:
 self._logger = logging.getLogger("executorlib")
 else:
 self._logger = None
 self._spawner = spawner

 def send_dict(self, input_dict: dict):
Expand All@@ -52,7 +63,12 @@ def receive_dict(self) -> dict:
 Returns:
 dict: dictionary with response received from the connected client
 """
 data = self._socket.recv()
 response_lst: list[tuple[Any, int]] = []
 while len(response_lst) == 0:
 response_lst = self._poller.poll(self._time_out_ms)
 if not self._spawner.poll():
 raise ExecutorlibSocketError()
 data = self._socket.recv(zmq.NOBLOCK)
 if self._logger is not None:
 self._logger.warning(
 "Received dictionary of size: " + str(sys.getsizeof(data))
Expand Down

30 changes: 30 additions & 0 deletions
tests/test_standalone_interactive_communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
 interface_send,
 interface_receive,
 SocketInterface,
 ExecutorlibSocketError,
)
from executorlib.standalone.serialize import cloudpickle_register
from executorlib.standalone.interactive.spawner import MpiExecSpawner
Expand DownExpand Up@@ -114,6 +115,35 @@ def test_interface_serial_with_debug(self):
 )
 interface.shutdown(wait=True)

 def test_interface_serial_with_stopped_process(self):
 cloudpickle_register(ind=1)
 task_dict = {"fn": calc, "args": (), "kwargs": {"i": 2}}
 interface = SocketInterface(
 spawner=MpiExecSpawner(cwd=None, cores=1, openmpi_oversubscribe=False),
 log_obj_size=True,
 )
 interface.bootup(
 command_lst=[
 sys.executable,
 os.path.abspath(
 os.path.join(
 __file__,
 "..",
 "..",
 "executorlib",
 "backend",
 "interactive_serial.py",
 )
 ),
 "--zmqport",
 str(interface.bind_to_random_port()),
 ]
 )
 interface.send_dict(input_dict=task_dict)
 interface._spawner._process.terminate()
 with self.assertRaises(ExecutorlibSocketError):
 interface.receive_dict()


Comment on lines

+118
 to 
+146

@coderabbitaicoderabbitaiBotAug 30, 2025

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Verification agent
🧩 Analysis chain
Likely inverted liveness check in receive_dict; test may mask the bug.
receive_dict() currently raises when the spawner is alive and may loop forever when it exited with a non-zero code (common after terminate()). This can cause this test to hang instead of raising, or to raise for the wrong reason.
  1. Proposed fix (outside this file) in executorlib/standalone/interactive/communication.py:
 def receive_dict(self) -> dict:
"""
Receive a dictionary from a connected client process.
"""
- response_lst: list[tuple[Any, int]] = []- while len(response_lst) == 0:- response_lst = self._poller.poll(self._time_out_ms)- if not self._spawner.poll():- raise ExecutorlibSocketError()+ response_lst: list[tuple[Any, int]] = []+ while True:+ # If the spawner exited (any return code), fail fast.+ if self._spawner.poll() is not None:+ raise ExecutorlibSocketError("Spawner terminated while waiting for response")+ response_lst = self._poller.poll(self._time_out_ms)+ if response_lst:+ break
data = self._socket.recv(zmq.NOBLOCK)
  1. Verify the current logic and ensure the fix is applied:
🏁 Script executed:
#!/bin/bashset -euo pipefail
# Show the receive_dict implementation and nearby lines
rg -n -C3 $'def\\s+receive_dict\\b' executorlib/standalone/interactive/communication.py
# Detect the inverted check; this should return no matches after the fix
rg -n $'if\\s+not\\s+self\\._spawner\\.poll\\(\\)\\s*:' executorlib/standalone/interactive/communication.py ||true
Length of output: 506
Fix inverted liveness check in receive_dict

executorlib/standalone/interactive/communication.py:69 still uses
ifnotself._spawner.poll():
raiseExecutorlibSocketError()
This is inverted – it hangs when the spawner exits. Change to:
ifself._spawner.poll() isnotNone:
raiseExecutorlibSocketError("Spawner terminated while waiting for response")
to fail fast on termination.
🤖 Prompt for AI Agents
In executorlib/standalone/interactive/communication.py around line 69, the
liveness check is inverted: currently it raises when poll() is falsy and thus
hangs when the spawner exits; change the condition to check for termination
using if self._spawner.poll() is not None and raise ExecutorlibSocketError with
a clear message like "Spawner terminated while waiting for response" so the code
fails fast when the spawner has exited.


class TestZMQ(unittest.TestCase):
 def test_interface_receive(self):
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Communication: Use zmq.Poller() rather than waiting infinitely#789

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jan-janssen
merged 4 commits into

from

Aug 30, 2025
 Merged


Communication: Use zmq.Poller() rather than waiting infinitely
#789
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
2b6055c

Communication: Use zmq.Poller() rather than waiting infinitely


jan-janssen Aug 29, 2025
453d133

Format black


pyiron-runner Aug 29, 2025
6f205ee

fix spelling


jan-janssen Aug 29, 2025
5df5362

Add test


jan-janssen Aug 30, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


26 changes: 21 additions & 5 deletions
executorlib/standalone/interactive/communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,35 +1,46 @@
import logging
import sys
from socket import gethostname
from typing import Optional
from typing import Any, Optional

import cloudpickle
import zmq


class ExecutorlibSocketError(RuntimeError):
 pass


class SocketInterface:
 """
 The SocketInterface is an abstraction layer on top of the zero message queue.

 Args:
 spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
 log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
 time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
 """

 def __init__(self, spawner=None, log_obj_size=False):
 def __init__(
 self, spawner=None, log_obj_size: bool = False, time_out_ms: int = 1000
 ):
 """
 Initialize the SocketInterface.

 Args:
 spawner (executorlib.shared.spawner.BaseSpawner): Interface for starting the parallel process
 log_obj_size (boolean): Enable debug mode which reports the size of the communicated objects.
 time_out_ms (int): Time out for waiting for a message on socket in milliseconds.
 """
 self._context = zmq.Context()
 self._socket = self._context.socket(zmq.PAIR)
 self._poller = zmq.Poller()
 self._poller.register(self._socket, zmq.POLLIN)
 self._process = None
 self._time_out_ms = time_out_ms
 self._logger: Optional[logging.Logger] = None
 if log_obj_size:
 self._logger = logging.getLogger("executorlib")
 else:
 self._logger = None
 self._spawner = spawner

 def send_dict(self, input_dict: dict):
Expand All@@ -52,7 +63,12 @@ def receive_dict(self) -> dict:
 Returns:
 dict: dictionary with response received from the connected client
 """
 data = self._socket.recv()
 response_lst: list[tuple[Any, int]] = []
 while len(response_lst) == 0:
 response_lst = self._poller.poll(self._time_out_ms)
 if not self._spawner.poll():
 raise ExecutorlibSocketError()
 data = self._socket.recv(zmq.NOBLOCK)
 if self._logger is not None:
 self._logger.warning(
 "Received dictionary of size: " + str(sys.getsizeof(data))
Expand Down

30 changes: 30 additions & 0 deletions
tests/test_standalone_interactive_communication.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
 interface_send,
 interface_receive,
 SocketInterface,
 ExecutorlibSocketError,
)
from executorlib.standalone.serialize import cloudpickle_register
from executorlib.standalone.interactive.spawner import MpiExecSpawner
Expand DownExpand Up@@ -114,6 +115,35 @@ def test_interface_serial_with_debug(self):
 )
 interface.shutdown(wait=True)

 def test_interface_serial_with_stopped_process(self):
 cloudpickle_register(ind=1)
 task_dict = {"fn": calc, "args": (), "kwargs": {"i": 2}}
 interface = SocketInterface(
 spawner=MpiExecSpawner(cwd=None, cores=1, openmpi_oversubscribe=False),
 log_obj_size=True,
 )
 interface.bootup(
 command_lst=[
 sys.executable,
 os.path.abspath(
 os.path.join(
 __file__,
 "..",
 "..",
 "executorlib",
 "backend",
 "interactive_serial.py",
 )
 ),
 "--zmqport",
 str(interface.bind_to_random_port()),
 ]
 )
 interface.send_dict(input_dict=task_dict)
 interface._spawner._process.terminate()
 with self.assertRaises(ExecutorlibSocketError):
 interface.receive_dict()


Comment on lines

+118
 to 
+146

@coderabbitaicoderabbitaiBotAug 30, 2025

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Verification agent
🧩 Analysis chain
Likely inverted liveness check in receive_dict; test may mask the bug.
receive_dict() currently raises when the spawner is alive and may loop forever when it exited with a non-zero code (common after terminate()). This can cause this test to hang instead of raising, or to raise for the wrong reason.
  1. Proposed fix (outside this file) in executorlib/standalone/interactive/communication.py:
 def receive_dict(self) -> dict:
"""
Receive a dictionary from a connected client process.
"""
- response_lst: list[tuple[Any, int]] = []- while len(response_lst) == 0:- response_lst = self._poller.poll(self._time_out_ms)- if not self._spawner.poll():- raise ExecutorlibSocketError()+ response_lst: list[tuple[Any, int]] = []+ while True:+ # If the spawner exited (any return code), fail fast.+ if self._spawner.poll() is not None:+ raise ExecutorlibSocketError("Spawner terminated while waiting for response")+ response_lst = self._poller.poll(self._time_out_ms)+ if response_lst:+ break
data = self._socket.recv(zmq.NOBLOCK)
  1. Verify the current logic and ensure the fix is applied:
🏁 Script executed:
#!/bin/bashset -euo pipefail
# Show the receive_dict implementation and nearby lines
rg -n -C3 $'def\\s+receive_dict\\b' executorlib/standalone/interactive/communication.py
# Detect the inverted check; this should return no matches after the fix
rg -n $'if\\s+not\\s+self\\._spawner\\.poll\\(\\)\\s*:' executorlib/standalone/interactive/communication.py ||true
Length of output: 506
Fix inverted liveness check in receive_dict

executorlib/standalone/interactive/communication.py:69 still uses
ifnotself._spawner.poll():
raiseExecutorlibSocketError()
This is inverted – it hangs when the spawner exits. Change to:
ifself._spawner.poll() isnotNone:
raiseExecutorlibSocketError("Spawner terminated while waiting for response")
to fail fast on termination.
🤖 Prompt for AI Agents
In executorlib/standalone/interactive/communication.py around line 69, the
liveness check is inverted: currently it raises when poll() is falsy and thus
hangs when the spawner exits; change the condition to check for termination
using if self._spawner.poll() is not None and raise ExecutorlibSocketError with
a clear message like "Spawner terminated while waiting for response" so the code
fails fast when the spawner has exited.


class TestZMQ(unittest.TestCase):
 def test_interface_receive(self):
Expand Down
Loading