Skip to content

fix loky/concurrency issue - #1042

Merged
PGijsbers merged 1 commit into
developfrom
fix_log_handling
Mar 30, 2021
Merged

fix loky/concurrency issue#1042
PGijsbers merged 1 commit into
developfrom
fix_log_handling

Conversation

@mfeurer

Copy link
Copy Markdown
Collaborator

Should fix issue of kind which are there because the directory which should contain the log files does not exist yet...


self = <joblib.executor.MemmappingExecutor object at 0x7f41d3ea8f10>
fn = <joblib._parallel_backends.SafeFunction object at 0x7f41c093f250>
args = (), kwargs = {}
def submit(self, fn, *args, **kwargs):
with self._flags.shutdown_lock:
if self._flags.broken is not None:
> raise self._flags.broken
E joblib.externals.loky.process_executor.BrokenProcessPool: A task has failed to un-serialize. Please ensure that the arguments of the function are all picklable.
/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py:1102: BrokenProcessPool
------------------------------ Captured log call -------------------------------
ERROR concurrent.futures:_base.py:627 exception calling callback for <Future at 0x7f41c0a318d0 state=finished raised BrokenProcessPool>
joblib.externals.loky.process_executor._RemoteTraceback: """
Traceback (most recent call last):
File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 404, in _process_worker
call_item = call_queue.get(block=True, timeout=timeout)
File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/multiprocessing/queues.py", line 113, in get
return _ForkingPickler.loads(res)
File "/home/runner/work/openml-python/openml-python/openml/__init__.py", line 20, in <module>
from . import _api_calls
File "/home/runner/work/openml-python/openml-python/openml/_api_calls.py", line 15, in <module>
from . import config
File "/home/runner/work/openml-python/openml-python/openml/config.py", line 341, in <module>
_setup()
File "/home/runner/work/openml-python/openml-python/openml/config.py", line 235, in _setup
_create_log_handlers()
File "/home/runner/work/openml-python/openml-python/openml/config.py", line 41, in _create_log_handlers
log_path, maxBytes=one_mb, backupCount=1, delay=True
File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/logging/handlers.py", line 148, in __init__
BaseRotatingHandler.__init__(self, filename, mode, encoding, delay)
File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/logging/handlers.py", line 55, in __init__
logging.FileHandler.__init__(self, filename, mode, encoding, delay)
File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/logging/__init__.py", line 1077, in __init__
self.baseFilename = os.path.abspath(filename)
File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/posixpath.py", line 383, in abspath
cwd = os.getcwd()
FileNotFoundError: [Errno 2] No such file or directory
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/site-packages/joblib/externals/loky/_base.py", line 625, in _invoke_callbacks
callback(self)
File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/site-packages/joblib/parallel.py", line 359, in __call__
self.parallel.dispatch_next()
File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/site-packages/joblib/parallel.py", line 792, in dispatch_next
if not self.dispatch_one_batch(self._original_iterator):
File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/site-packages/joblib/parallel.py", line 859, in dispatch_one_batch
self._dispatch(tasks)
File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/site-packages/joblib/parallel.py", line 777, in _dispatch
job = self._backend.apply_async(batch, callback=cb)
File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 531, in apply_async
future = self._workers.submit(SafeFunction(func))
File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/site-packages/joblib/externals/loky/reusable_executor.py", line 178, in submit
fn, *args, **kwargs)
File "/opt/hostedtoolcache/Python/3.7.10/x64/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 1102, in submit
raise self._flags.broken
joblib.externals.loky.process_executor.BrokenProcessPool: A task has failed to un-serialize. Please ensure that the arguments of the function are all picklable.

@mfeurer
mfeurer requested a review from PGijsbersMarch 29, 2021 13:51
@mfeurer

Copy link
Copy Markdown
CollaboratorAuthor

Failures are only due to the test server being flaky and the unit test output fail not being there.

@PGijsbers
PGijsbers merged commit 5511fa0 into developMar 30, 2021
@PGijsbers
PGijsbers deleted the fix_log_handling branch March 30, 2021 08:12
github-actionsBot pushed a commit to vishalbelsare/openml-python that referenced this pull request Apr 4, 2021
PGijsbers pushed a commit to Mirkazemi/openml-python that referenced this pull request Feb 23, 2023
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

2 participants

@mfeurer@PGijsbers