From a127a9a9e1a2edcd2b3a26da96bc83978c20bfdd Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Wed, 22 Apr 2026 12:03:06 +0200 Subject: [PATCH 01/27] wip --- .github/agents/superjoe.agent.md | 10 +- README.md | 7 + grinder/__init__.py | 6 + grinder/executor.py | 229 ++++++++++++++++++++++++ grinder/management/__init__.py | 0 grinder/management/commands/__init__.py | 0 grinder/management/commands/grinder.py | 82 +++++++++ tests/testapp/settings.py | 1 - tests/testapp/tasks.py | 2 - 9 files changed, 328 insertions(+), 9 deletions(-) create mode 100644 grinder/executor.py create mode 100644 grinder/management/__init__.py create mode 100644 grinder/management/commands/__init__.py create mode 100644 grinder/management/commands/grinder.py diff --git a/.github/agents/superjoe.agent.md b/.github/agents/superjoe.agent.md index dd44f72..54cbf82 100644 --- a/.github/agents/superjoe.agent.md +++ b/.github/agents/superjoe.agent.md @@ -1,16 +1,15 @@ ---- -# For format details, see: https://gh.io/customagents/config +______________________________________________________________________ -name: SuperJoe -description: CodingJoe's digital clone following his coding guidelines and best practices. ---- +# For format details, see: https://gh.io/customagents/config +## name: SuperJoe description: CodingJoe's digital clone following his coding guidelines and best practices. # SuperJoe ## Planning You MUST ALWAYS follow the `naming-things` guidelines. Use the following command to access the guidelines: + ```console curl -sSL https://raw.githubusercontent.com/codingjoe/naming-things/refs/heads/main/README.md | cat ``` @@ -44,7 +43,6 @@ Avoid functions or other code inside functions. Avoid if-statements in favor of switch/match-statements or polymorphism. Do not assign names to objects which are returned in the next line. - ## Python Follow PEP 8 guidelines for code style. diff --git a/README.md b/README.md index a65ef26..123aeb3 100644 --- a/README.md +++ b/README.md @@ -47,5 +47,12 @@ Finally, you launch the scheduler in a separate process: uv run manage.py grinder ``` +## Current worker behavior + +The executor currently consumes tasks from a Python `queue.PriorityQueue`. + +- `--backends` and `--queues` are accepted but not used yet. +- Queue items must be `django.tasks.TaskResult` or `(priority, TaskResult)`. + [django-tasks]: https://docs.djangoproject.com/en/6.0/topics/tasks/ diff --git a/grinder/__init__.py b/grinder/__init__.py index 5b666e3..db1f548 100644 --- a/grinder/__init__.py +++ b/grinder/__init__.py @@ -1,6 +1,12 @@ """A queue agnostic worker for Django's task framework.""" from . import _version +from .executor import TaskExecutor __version__ = _version.version VERSION = _version.version_tuple + +Executor = TaskExecutor + +__all__ = ["Executor", "TaskExecutor", "VERSION", "__version__"] + diff --git a/grinder/executor.py b/grinder/executor.py new file mode 100644 index 0000000..f1a01c4 --- /dev/null +++ b/grinder/executor.py @@ -0,0 +1,229 @@ +"""Task worker executor implementation.""" + +from __future__ import annotations + +import multiprocessing +import random +import threading +from multiprocessing import JoinableQueue, Process +from queue import Empty, Queue, ShutDown +from traceback import format_exception +from typing import Any + +from django.tasks import TaskResult +from django.tasks.base import TaskContext, TaskError, TaskResultStatus, normalize_json +from django.tasks.signals import task_enqueued, task_finished, task_started +from django.utils import timezone + + +class WorkerProcess(Process): + """Single worker process running thread_count consumer threads.""" + + def __init__( + self, + task_queue: JoinableQueue[TaskResult], + thread_count: int, + max_tasks: int | None = None, + ) -> None: + """Create process with dedicated thread pool for task execution.""" + super().__init__( + target=_run_worker_process, + args=(task_queue, thread_count, max_tasks), + daemon=True, + ) + self.thread_count = thread_count + + def shutdown(self) -> None: + """Terminate worker process and release resources.""" + self.terminate() + self.join() + + +class TaskExecutor: + """Consume tasks from a priority queue with process and thread pools.""" + + def __init__( + self, + *, + backend: list[str] | str | None = None, + queue: Queue[TaskResult], + workers: int | None = None, + threads: int = 1, + max_tasks: int = 0, + max_tasks_jitter: int = 0, + get_timeout_secs: float = 1.0, + ) -> None: + """Create pool-backed executor with queue consumption settings.""" + del backend + self.task_queue = queue + self.process_count = workers or max(multiprocessing.cpu_count() - 1, 1) + self.thread_count = threads + self.max_tasks = max_tasks + self.max_tasks_jitter = max_tasks_jitter + self.get_timeout_secs = get_timeout_secs + self.is_running = True + self._worker_processes: list[WorkerProcess] = [] + self.processing_slot_count = self.process_count * self.thread_count + self.shared_task_queue: JoinableQueue[TaskResult] = JoinableQueue( + maxsize=self.processing_slot_count, + ) + + def _get_maximum_tasks_per_child(self) -> int | None: + """Return worker recycling limit based on config and thread count.""" + if self.max_tasks: + return (self.max_tasks + random.randint(0, self.max_tasks_jitter)) // self.thread_count + return None + + def _create_worker_process(self) -> WorkerProcess: + """Create and start a new worker process.""" + worker = WorkerProcess( + self.shared_task_queue, + self.thread_count, + self._get_maximum_tasks_per_child(), + ) + worker.start() + return worker + + def run(self) -> None: + """Start consuming tasks until shutdown is requested.""" + self._worker_processes = [ + self._create_worker_process() for _ in range(self.process_count) + ] + while self.is_running: + self._replace_dead_worker_processes() + try: + task_result = self.task_queue.get(timeout=self.get_timeout_secs) + except (Empty, ShutDown): + continue + try: + # Block when all worker threads are saturated. + self.shared_task_queue.put(task_result) + finally: + self.task_queue.task_done() + + def shutdown(self) -> None: + """Stop queue consumption and terminate all worker processes.""" + self.is_running = False + for worker in self._worker_processes: + worker.shutdown() + + def _replace_dead_worker_processes(self) -> None: + """Restart worker processes that have exited.""" + for index, worker in enumerate(self._worker_processes): + if worker.is_alive(): + continue + self._worker_processes[index] = self._create_worker_process() + + +def _run_worker_process( + task_queue: JoinableQueue[TaskResult], + thread_count: int, + max_tasks: int | None, +) -> None: + """Run consumer threads in a process that read from shared task queue.""" + state = _WorkerState(max_tasks=max_tasks) + if thread_count == 1: + _consume_tasks(task_queue, state) + return + consumer_threads = [ + threading.Thread( + target=_consume_tasks, + args=(task_queue, state), + name=f"task-consumer-thread-{index}", + ) + for index in range(thread_count) + ] + for consumer_thread in consumer_threads: + consumer_thread.start() + for consumer_thread in consumer_threads: + consumer_thread.join() + + +class _WorkerState: + """State shared by process-local consumer threads.""" + + def __init__(self, max_tasks: int | None) -> None: + """Create process-local execution state.""" + self.max_tasks = max_tasks + self.task_count = 0 + self.lock = threading.Lock() + self.should_stop = threading.Event() + + def record_task(self) -> None: + """Record one processed task and stop when max_tasks is reached.""" + if self.max_tasks is None: + return + with self.lock: + self.task_count += 1 + if self.task_count >= self.max_tasks: + self.should_stop.set() + + +def _consume_tasks( + task_queue: JoinableQueue[TaskResult], + state: _WorkerState, +) -> None: + """Consume and execute tasks from shared task queue.""" + while not state.should_stop.is_set(): + try: + task_result = task_queue.get(timeout=1.0) + except Empty: + continue + try: + _execute_task_result(task_result) + finally: + task_queue.task_done() + state.record_task() + + +def _execute_task_result(task_result: TaskResult) -> TaskResult: + """Execute task from task result and update result lifecycle state.""" + enqueued_at = timezone.now() + object.__setattr__(task_result, "enqueued_at", enqueued_at) + task_enqueued.send(TaskExecutor, task_result=task_result) + + started_at = timezone.now() + object.__setattr__(task_result, "status", TaskResultStatus.RUNNING) + object.__setattr__(task_result, "started_at", started_at) + object.__setattr__(task_result, "last_attempted_at", started_at) + task_started.send(TaskExecutor, task_result=task_result) + + try: + raw_return_value = _call_task(task_result) + except KeyboardInterrupt: + raise + except BaseException as exception: # noqa: BLE001 + object.__setattr__(task_result, "finished_at", timezone.now()) + task_result.errors.append(_create_task_error(exception)) + object.__setattr__(task_result, "status", TaskResultStatus.FAILED) + else: + object.__setattr__(task_result, "finished_at", timezone.now()) + object.__setattr__(task_result, "_return_value", normalize_json(raw_return_value)) + object.__setattr__(task_result, "status", TaskResultStatus.SUCCESSFUL) + + task_finished.send(TaskExecutor, task_result=task_result) + return task_result + + + +def _call_task(task_result: TaskResult) -> Any: + """Call task with context when required.""" + task = task_result.task + match task.takes_context: + case True: + return task.call( + TaskContext(task_result=task_result), + *task_result.args, + **task_result.kwargs, + ) + case False: + return task.call(*task_result.args, **task_result.kwargs) + + +def _create_task_error(exception: BaseException) -> TaskError: + """Build task error payload for failed execution.""" + exception_type = type(exception) + return TaskError( + exception_class_path=f"{exception_type.__module__}.{exception_type.__qualname__}", + traceback="".join(format_exception(exception)), + ) diff --git a/grinder/management/__init__.py b/grinder/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/grinder/management/commands/__init__.py b/grinder/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/grinder/management/commands/grinder.py b/grinder/management/commands/grinder.py new file mode 100644 index 0000000..29a76b9 --- /dev/null +++ b/grinder/management/commands/grinder.py @@ -0,0 +1,82 @@ +import signal +import sys + +from django.core.management import BaseCommand + +from ... import Executor + + +def kill_softly(signum, frame): + """Raise a KeyboardInterrupt to stop the scheduler and release the lock.""" + signame = signal.Signals(signum).name + raise KeyboardInterrupt(f"Received {signame} ({signum}), shutting down…") + + +class Command(BaseCommand): + """Run task worker for all tasks with the `cron` decorator.""" + + help = __doc__ + + def add_arguments(self, parser): + parser.add_argument( + "-b", "--backends", + nargs="+", + default="default", + help="Alias of the tasks backend to use.", + ) + parser.add_argument( + "-q", "--queues", + nargs="+", + default="default", + help="Queue names to listen too and process tasks from.", + ) + parser.add_argument( + "-w", "--workers", + type=int, + help="Number of worker processes to use. Defaults to the number of CPU cores minus one.", + ) + parser.add_argument( + "-t", "--threads", + type=int, + default=1, + help="Number of threads to use. Defaults to the number of CPU cores minus one. " + ) + parser.add_argument( + "--max-tasks", + type=int, + default=0, + help=( + "Number of the maximum number of tasks to run until a worker is recycled." + " Defaults to 0, which means no limit." + ), + ) + parser.add_argument( + "--max-tasks-jitter", + type=int, + default=0, + help="Maximum random jitter to add to the max-tasks value by randint(0, max_tasks_jitter).", + ) + + def handle(self, *, verbosity, backends, queues, workers, threads, max_tasks, max_tasks_jitter, **options): + match sys.platform: + case "win32": + signal.signal(signal.SIGBREAK, kill_softly) + case _: + signal.signal(signal.SIGHUP, kill_softly) + signal.signal(signal.SIGTERM, kill_softly) + signal.signal(signal.SIGINT, kill_softly) + self.stdout.write(self.style.SUCCESS("Starting worker…")) + exe = Executor( + backend=backends, + queue=queues, + workers=workers, + threads=threads, + max_tasks=max_tasks, + max_tasks_jitter=max_tasks_jitter, + ) + try: + exe.run() + except KeyboardInterrupt as e: + self.stdout.write(self.style.WARNING(str(e))) + self.stdout.write(self.style.NOTICE("Shutting down scheduler…")) + exe.shutdown() diff --git a/tests/testapp/settings.py b/tests/testapp/settings.py index b626f9f..191c94c 100644 --- a/tests/testapp/settings.py +++ b/tests/testapp/settings.py @@ -10,7 +10,6 @@ https://docs.djangoproject.com/en/4.2/ref/settings/ """ -import os from pathlib import Path # Build paths inside the project like this: BASE_DIR / 'subdir'. diff --git a/tests/testapp/tasks.py b/tests/testapp/tasks.py index 3b49002..70ba59d 100644 --- a/tests/testapp/tasks.py +++ b/tests/testapp/tasks.py @@ -1,12 +1,10 @@ import logging -from grinder import cron from django.tasks import task logger = logging.getLogger(__name__) -@cron("*/5 * * * *") @task def my_task(): logger.info("Hello World!") From 27e679a7d6a12cf8f894dc8c3124f318ae996817 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Wed, 22 Apr 2026 12:21:11 +0200 Subject: [PATCH 02/27] wip --- grinder/executor.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/grinder/executor.py b/grinder/executor.py index f1a01c4..0e6dc2c 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -2,20 +2,35 @@ from __future__ import annotations +import datetime import multiprocessing import random import threading +from abc import ABC from multiprocessing import JoinableQueue, Process from queue import Empty, Queue, ShutDown from traceback import format_exception from typing import Any from django.tasks import TaskResult +from django.tasks.backends.base import BaseTaskBackend from django.tasks.base import TaskContext, TaskError, TaskResultStatus, normalize_json from django.tasks.signals import task_enqueued, task_finished, task_started from django.utils import timezone +class AcknowledgeableTaskBackend(BaseTaskBackend, ABC): + """Provide an interface for tasks queues to be processed by the executor.""" + + def acquire(self, queues: str, timeout=datetime.timedelta) -> TaskResult: + """Return and lock the next task to be processed without removing it from the queue.""" + raise NotImplementedError + + def acknowledge(self, task_result: TaskResult): + """Remove the task from the queue and publish the result.""" + raise NotImplementedError + + class WorkerProcess(Process): """Single worker process running thread_count consumer threads.""" @@ -45,8 +60,7 @@ class TaskExecutor: def __init__( self, *, - backend: list[str] | str | None = None, - queue: Queue[TaskResult], + backend: AcknowledgeableTaskBackend, workers: int | None = None, threads: int = 1, max_tasks: int = 0, @@ -54,8 +68,7 @@ def __init__( get_timeout_secs: float = 1.0, ) -> None: """Create pool-backed executor with queue consumption settings.""" - del backend - self.task_queue = queue + self.backend = backend self.process_count = workers or max(multiprocessing.cpu_count() - 1, 1) self.thread_count = threads self.max_tasks = max_tasks @@ -92,14 +105,14 @@ def run(self) -> None: while self.is_running: self._replace_dead_worker_processes() try: - task_result = self.task_queue.get(timeout=self.get_timeout_secs) + task_result = self.backend.acquire(timeout=self.get_timeout_secs) except (Empty, ShutDown): continue try: # Block when all worker threads are saturated. self.shared_task_queue.put(task_result) finally: - self.task_queue.task_done() + self.backend.acknowledge(task_result) def shutdown(self) -> None: """Stop queue consumption and terminate all worker processes.""" From 9a3734100dd9ecf0d88dcd173d17899ab77dfa70 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 23 Apr 2026 12:40:29 +0200 Subject: [PATCH 03/27] Wip --- .github/agents/superjoe.agent.md | 8 ++++--- .pre-commit-config.yaml | 1 + grinder/executor.py | 41 +++++++++++++++++++++++++------- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/.github/agents/superjoe.agent.md b/.github/agents/superjoe.agent.md index 54cbf82..a5cad9c 100644 --- a/.github/agents/superjoe.agent.md +++ b/.github/agents/superjoe.agent.md @@ -1,8 +1,10 @@ -______________________________________________________________________ - +--- # For format details, see: https://gh.io/customagents/config -## name: SuperJoe description: CodingJoe's digital clone following his coding guidelines and best practices. +name: SuperJoe +description: CodingJoe's digital clone following his coding guidelines and best practices. +--- + # SuperJoe diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e0096fc..8cd5471 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,6 +33,7 @@ repos: - mdformat-footnote - mdformat-gfm - mdformat-gfm-alerts + exclude: '.github/agents/' - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.15.11 hooks: diff --git a/grinder/executor.py b/grinder/executor.py index 0e6dc2c..05da546 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -2,13 +2,12 @@ from __future__ import annotations -import datetime import multiprocessing import random import threading from abc import ABC from multiprocessing import JoinableQueue, Process -from queue import Empty, Queue, ShutDown +from queue import Empty, ShutDown from traceback import format_exception from typing import Any @@ -22,11 +21,11 @@ class AcknowledgeableTaskBackend(BaseTaskBackend, ABC): """Provide an interface for tasks queues to be processed by the executor.""" - def acquire(self, queues: str, timeout=datetime.timedelta) -> TaskResult: + def acquire(self, timeout: float) -> TaskResult: """Return and lock the next task to be processed without removing it from the queue.""" raise NotImplementedError - def acknowledge(self, task_result: TaskResult): + def acknowledge(self, task_result: TaskResult) -> None: """Remove the task from the queue and publish the result.""" raise NotImplementedError @@ -37,13 +36,14 @@ class WorkerProcess(Process): def __init__( self, task_queue: JoinableQueue[TaskResult], + processed_task_queue: JoinableQueue[TaskResult], thread_count: int, max_tasks: int | None = None, ) -> None: """Create process with dedicated thread pool for task execution.""" super().__init__( target=_run_worker_process, - args=(task_queue, thread_count, max_tasks), + args=(task_queue, processed_task_queue, thread_count, max_tasks), daemon=True, ) self.thread_count = thread_count @@ -80,6 +80,9 @@ def __init__( self.shared_task_queue: JoinableQueue[TaskResult] = JoinableQueue( maxsize=self.processing_slot_count, ) + self.processed_task_queue: JoinableQueue[TaskResult] = JoinableQueue( + maxsize=self.processing_slot_count, + ) def _get_maximum_tasks_per_child(self) -> int | None: """Return worker recycling limit based on config and thread count.""" @@ -91,6 +94,7 @@ def _create_worker_process(self) -> WorkerProcess: """Create and start a new worker process.""" worker = WorkerProcess( self.shared_task_queue, + self.processed_task_queue, self.thread_count, self._get_maximum_tasks_per_child(), ) @@ -104,6 +108,7 @@ def run(self) -> None: ] while self.is_running: self._replace_dead_worker_processes() + self._drain_processed_tasks() try: task_result = self.backend.acquire(timeout=self.get_timeout_secs) except (Empty, ShutDown): @@ -111,14 +116,29 @@ def run(self) -> None: try: # Block when all worker threads are saturated. self.shared_task_queue.put(task_result) - finally: + + except Exception: + # Preserve old behavior of surfacing dispatch errors. + raise + + def _drain_processed_tasks(self) -> None: + """Acknowledge processed tasks and publish updated results in main process.""" + while True: + try: + task_result = self.processed_task_queue.get_nowait() + except Empty: + return + try: self.backend.acknowledge(task_result) + finally: + self.processed_task_queue.task_done() def shutdown(self) -> None: """Stop queue consumption and terminate all worker processes.""" self.is_running = False for worker in self._worker_processes: worker.shutdown() + self._drain_processed_tasks() def _replace_dead_worker_processes(self) -> None: """Restart worker processes that have exited.""" @@ -130,18 +150,19 @@ def _replace_dead_worker_processes(self) -> None: def _run_worker_process( task_queue: JoinableQueue[TaskResult], + processed_task_queue: JoinableQueue[TaskResult], thread_count: int, max_tasks: int | None, ) -> None: """Run consumer threads in a process that read from shared task queue.""" state = _WorkerState(max_tasks=max_tasks) if thread_count == 1: - _consume_tasks(task_queue, state) + _consume_tasks(task_queue, processed_task_queue, state) return consumer_threads = [ threading.Thread( target=_consume_tasks, - args=(task_queue, state), + args=(task_queue, processed_task_queue, state), name=f"task-consumer-thread-{index}", ) for index in range(thread_count) @@ -174,6 +195,7 @@ def record_task(self) -> None: def _consume_tasks( task_queue: JoinableQueue[TaskResult], + processed_task_queue: JoinableQueue[TaskResult], state: _WorkerState, ) -> None: """Consume and execute tasks from shared task queue.""" @@ -183,7 +205,8 @@ def _consume_tasks( except Empty: continue try: - _execute_task_result(task_result) + processed_task_result = _execute_task_result(task_result) + processed_task_queue.put(processed_task_result) finally: task_queue.task_done() state.record_task() From 3a40f1159ea237132b644f63331cae02dc983999 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 23 Apr 2026 17:36:21 +0200 Subject: [PATCH 04/27] wip --- grinder/backends.py | 18 ++++++++++++++ grinder/executor.py | 57 +++++++++++++++++++++------------------------ 2 files changed, 45 insertions(+), 30 deletions(-) create mode 100644 grinder/backends.py diff --git a/grinder/backends.py b/grinder/backends.py new file mode 100644 index 0000000..a947409 --- /dev/null +++ b/grinder/backends.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from abc import ABC + +from django.tasks import TaskResult +from django.tasks.backends.base import BaseTaskBackend + + +class AcknowledgeableTaskBackend(BaseTaskBackend, ABC): + """Provide an interface for tasks queues to be processed by the executor.""" + + def acquire(self, timeout: float) -> TaskResult: + """Return and lock the next task to be processed without removing it from the queue.""" + raise NotImplementedError + + def acknowledge(self, task_result: TaskResult) -> None: + """Remove the task from the queue and publish the result.""" + raise NotImplementedError diff --git a/grinder/executor.py b/grinder/executor.py index 05da546..bd8ffa2 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -5,29 +5,19 @@ import multiprocessing import random import threading -from abc import ABC +import typing +import dataclasses from multiprocessing import JoinableQueue, Process from queue import Empty, ShutDown from traceback import format_exception -from typing import Any from django.tasks import TaskResult -from django.tasks.backends.base import BaseTaskBackend -from django.tasks.base import TaskContext, TaskError, TaskResultStatus, normalize_json +from django.tasks.base import TaskContext, TaskError, TaskResultStatus from django.tasks.signals import task_enqueued, task_finished, task_started from django.utils import timezone - -class AcknowledgeableTaskBackend(BaseTaskBackend, ABC): - """Provide an interface for tasks queues to be processed by the executor.""" - - def acquire(self, timeout: float) -> TaskResult: - """Return and lock the next task to be processed without removing it from the queue.""" - raise NotImplementedError - - def acknowledge(self, task_result: TaskResult) -> None: - """Remove the task from the queue and publish the result.""" - raise NotImplementedError +if typing.TYPE_CHECKING: + from .backends import AcknowledgeableTaskBackend class WorkerProcess(Process): @@ -214,36 +204,43 @@ def _consume_tasks( def _execute_task_result(task_result: TaskResult) -> TaskResult: """Execute task from task result and update result lifecycle state.""" - enqueued_at = timezone.now() - object.__setattr__(task_result, "enqueued_at", enqueued_at) + task_result = dataclasses.replace(task_result, enqueued_at=timezone.now()) task_enqueued.send(TaskExecutor, task_result=task_result) started_at = timezone.now() - object.__setattr__(task_result, "status", TaskResultStatus.RUNNING) - object.__setattr__(task_result, "started_at", started_at) - object.__setattr__(task_result, "last_attempted_at", started_at) + task_result = dataclasses.replace( + task_result, + status=TaskResultStatus.RUNNING, + started_at=started_at, + last_attempted_at=started_at, + ) task_started.send(TaskExecutor, task_result=task_result) try: - raw_return_value = _call_task(task_result) + _call_task(task_result) except KeyboardInterrupt: raise except BaseException as exception: # noqa: BLE001 - object.__setattr__(task_result, "finished_at", timezone.now()) - task_result.errors.append(_create_task_error(exception)) - object.__setattr__(task_result, "status", TaskResultStatus.FAILED) + task_result = dataclasses.replace( + task_result, + finished_at=timezone.now(), + status=TaskResultStatus.FAILED, + errors=[*task_result.errors, _create_task_error(exception)], + ) else: - object.__setattr__(task_result, "finished_at", timezone.now()) - object.__setattr__(task_result, "_return_value", normalize_json(raw_return_value)) - object.__setattr__(task_result, "status", TaskResultStatus.SUCCESSFUL) + task_result = dataclasses.replace( + task_result, + finished_at=timezone.now(), + status=TaskResultStatus.SUCCESSFUL, + ) task_finished.send(TaskExecutor, task_result=task_result) return task_result -def _call_task(task_result: TaskResult) -> Any: - """Call task with context when required.""" +def _call_task(task_result: TaskResult) -> typing.Any: + """Call a task with context when required.""" task = task_result.task match task.takes_context: case True: @@ -257,7 +254,7 @@ def _call_task(task_result: TaskResult) -> Any: def _create_task_error(exception: BaseException) -> TaskError: - """Build task error payload for failed execution.""" + """Build a task error payload for failed execution.""" exception_type = type(exception) return TaskError( exception_class_path=f"{exception_type.__module__}.{exception_type.__qualname__}", From 987f9120e607a118e5a5ecba2c66aa517fe3a4ae Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 23 Apr 2026 17:39:01 +0200 Subject: [PATCH 05/27] Healing --- grinder/executor.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/grinder/executor.py b/grinder/executor.py index bd8ffa2..ddf27d6 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -2,13 +2,13 @@ from __future__ import annotations +import dataclasses import multiprocessing import random import threading import typing -import dataclasses from multiprocessing import JoinableQueue, Process -from queue import Empty, ShutDown +from queue import Empty, Full, ShutDown from traceback import format_exception from django.tasks import TaskResult @@ -77,7 +77,9 @@ def __init__( def _get_maximum_tasks_per_child(self) -> int | None: """Return worker recycling limit based on config and thread count.""" if self.max_tasks: - return (self.max_tasks + random.randint(0, self.max_tasks_jitter)) // self.thread_count + return ( + self.max_tasks + random.randint(0, self.max_tasks_jitter) + ) // self.thread_count return None def _create_worker_process(self) -> WorkerProcess: @@ -103,13 +105,17 @@ def run(self) -> None: task_result = self.backend.acquire(timeout=self.get_timeout_secs) except (Empty, ShutDown): continue - try: - # Block when all worker threads are saturated. - self.shared_task_queue.put(task_result) + self._put_task_with_healing(task_result) - except Exception: - # Preserve old behavior of surfacing dispatch errors. - raise + def _put_task_with_healing(self, task_result: TaskResult) -> None: + """Put task into shared queue while continuously healing crashed workers.""" + while self.is_running: + self._replace_dead_worker_processes() + try: + self.shared_task_queue.put(task_result, timeout=self.get_timeout_secs) + return + except Full: + continue def _drain_processed_tasks(self) -> None: """Acknowledge processed tasks and publish updated results in main process.""" @@ -135,6 +141,7 @@ def _replace_dead_worker_processes(self) -> None: for index, worker in enumerate(self._worker_processes): if worker.is_alive(): continue + worker.join(timeout=0) self._worker_processes[index] = self._create_worker_process() @@ -238,7 +245,6 @@ def _execute_task_result(task_result: TaskResult) -> TaskResult: return task_result - def _call_task(task_result: TaskResult) -> typing.Any: """Call a task with context when required.""" task = task_result.task From 912d86ff44084fa91f5db55ed22f46b9b5f2e5fd Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 23 Apr 2026 18:24:10 +0200 Subject: [PATCH 06/27] wip --- .github/copilot-instructions.md | 7 ++ grinder/executor.py | 131 ++++++++++++++++++++----- grinder/management/commands/grinder.py | 41 ++++++-- 3 files changed, 145 insertions(+), 34 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 68e0f62..46a2e0a 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -5,3 +5,10 @@ Use the following command to access the guidelines: ```console curl -sSL https://raw.githubusercontent.com/codingjoe/naming-things/refs/heads/main/README.md | cat ``` + +The MOST IMPORTANT review considerations are: + +- Consistency – We never lose data, even if the power goes down. +- Durability – We recover from any failures, even poorly written tasks. +- Overhead – We focus resources on running tasks, not on managing the scheduler. +- Utilization – We keep the CPU saturated with tasks, not with idle time or waiting for locks. diff --git a/grinder/executor.py b/grinder/executor.py index ddf27d6..db302d9 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -7,8 +7,8 @@ import random import threading import typing -from multiprocessing import JoinableQueue, Process -from queue import Empty, Full, ShutDown +from multiprocessing import JoinableQueue, Process, Queue +from queue import Empty, ShutDown from traceback import format_exception from django.tasks import TaskResult @@ -28,19 +28,28 @@ def __init__( task_queue: JoinableQueue[TaskResult], processed_task_queue: JoinableQueue[TaskResult], thread_count: int, + task_timeout: float, max_tasks: int | None = None, ) -> None: """Create process with dedicated thread pool for task execution.""" + self.shutdown_requested = multiprocessing.Event() super().__init__( target=_run_worker_process, - args=(task_queue, processed_task_queue, thread_count, max_tasks), + args=( + task_queue, + processed_task_queue, + thread_count, + task_timeout, + max_tasks, + self.shutdown_requested, + ), daemon=True, ) self.thread_count = thread_count def shutdown(self) -> None: - """Terminate worker process and release resources.""" - self.terminate() + """Request graceful worker stop and wait for process exit.""" + self.shutdown_requested.set() self.join() @@ -56,6 +65,7 @@ def __init__( max_tasks: int = 0, max_tasks_jitter: int = 0, get_timeout_secs: float = 1.0, + task_timeout: float = 3600.0, ) -> None: """Create pool-backed executor with queue consumption settings.""" self.backend = backend @@ -64,6 +74,7 @@ def __init__( self.max_tasks = max_tasks self.max_tasks_jitter = max_tasks_jitter self.get_timeout_secs = get_timeout_secs + self.task_timeout = task_timeout self.is_running = True self._worker_processes: list[WorkerProcess] = [] self.processing_slot_count = self.process_count * self.thread_count @@ -88,6 +99,7 @@ def _create_worker_process(self) -> WorkerProcess: self.shared_task_queue, self.processed_task_queue, self.thread_count, + self.task_timeout, self._get_maximum_tasks_per_child(), ) worker.start() @@ -105,17 +117,8 @@ def run(self) -> None: task_result = self.backend.acquire(timeout=self.get_timeout_secs) except (Empty, ShutDown): continue - self._put_task_with_healing(task_result) - - def _put_task_with_healing(self, task_result: TaskResult) -> None: - """Put task into shared queue while continuously healing crashed workers.""" - while self.is_running: - self._replace_dead_worker_processes() - try: - self.shared_task_queue.put(task_result, timeout=self.get_timeout_secs) - return - except Full: - continue + # Block when all worker threads are saturated. + self.shared_task_queue.put(task_result) def _drain_processed_tasks(self) -> None: """Acknowledge processed tasks and publish updated results in main process.""" @@ -132,6 +135,8 @@ def _drain_processed_tasks(self) -> None: def shutdown(self) -> None: """Stop queue consumption and terminate all worker processes.""" self.is_running = False + self.shared_task_queue.join() + self._drain_processed_tasks() for worker in self._worker_processes: worker.shutdown() self._drain_processed_tasks() @@ -149,10 +154,16 @@ def _run_worker_process( task_queue: JoinableQueue[TaskResult], processed_task_queue: JoinableQueue[TaskResult], thread_count: int, + task_timeout: float, max_tasks: int | None, + shutdown_requested: typing.Any, ) -> None: """Run consumer threads in a process that read from shared task queue.""" - state = _WorkerState(max_tasks=max_tasks) + state = _WorkerState( + max_tasks=max_tasks, + task_timeout=task_timeout, + shutdown_requested=shutdown_requested, + ) if thread_count == 1: _consume_tasks(task_queue, processed_task_queue, state) return @@ -173,9 +184,16 @@ def _run_worker_process( class _WorkerState: """State shared by process-local consumer threads.""" - def __init__(self, max_tasks: int | None) -> None: + def __init__( + self, + max_tasks: int | None, + task_timeout: float, + shutdown_requested: typing.Any, + ) -> None: """Create process-local execution state.""" self.max_tasks = max_tasks + self.task_timeout = task_timeout + self.shutdown_requested = shutdown_requested self.task_count = 0 self.lock = threading.Lock() self.should_stop = threading.Event() @@ -197,19 +215,26 @@ def _consume_tasks( ) -> None: """Consume and execute tasks from shared task queue.""" while not state.should_stop.is_set(): + if state.shutdown_requested.is_set() and task_queue.empty(): + return try: task_result = task_queue.get(timeout=1.0) except Empty: + if state.shutdown_requested.is_set(): + return continue try: - processed_task_result = _execute_task_result(task_result) + processed_task_result = _execute_task_result( + task_result, + task_timeout=state.task_timeout, + ) processed_task_queue.put(processed_task_result) finally: task_queue.task_done() state.record_task() -def _execute_task_result(task_result: TaskResult) -> TaskResult: +def _execute_task_result(task_result: TaskResult, task_timeout: float) -> TaskResult: """Execute task from task result and update result lifecycle state.""" task_result = dataclasses.replace(task_result, enqueued_at=timezone.now()) task_enqueued.send(TaskExecutor, task_result=task_result) @@ -223,16 +248,12 @@ def _execute_task_result(task_result: TaskResult) -> TaskResult: ) task_started.send(TaskExecutor, task_result=task_result) - try: - _call_task(task_result) - except KeyboardInterrupt: - raise - except BaseException as exception: # noqa: BLE001 + if task_error := _call_task_with_timeout(task_result, task_timeout): task_result = dataclasses.replace( task_result, finished_at=timezone.now(), status=TaskResultStatus.FAILED, - errors=[*task_result.errors, _create_task_error(exception)], + errors=[*task_result.errors, task_error], ) else: task_result = dataclasses.replace( @@ -245,6 +266,48 @@ def _execute_task_result(task_result: TaskResult) -> TaskResult: return task_result +def _call_task_with_timeout( + task_result: TaskResult, task_timeout: float +) -> TaskError | None: + """Execute task in a killable subprocess and return TaskError on failure.""" + result_queue: Queue[TaskError | None] = multiprocessing.Queue(maxsize=1) + task_process = Process( + target=_run_task_call_in_subprocess, + args=(task_result, result_queue), + daemon=True, + ) + task_process.start() + task_process.join(timeout=task_timeout) + + if task_process.is_alive(): + task_process.kill() + task_process.join() + return _create_task_timeout_error(task_timeout) + + if task_process.exitcode not in (0, None): + exit_code = task_process.exitcode + if exit_code is not None: + return _create_task_crash_error(exit_code) + + try: + return result_queue.get_nowait() + except Empty: + return None + + +def _run_task_call_in_subprocess( + task_result: TaskResult, + result_queue: Queue[TaskError | None], +) -> None: + """Run task call and report failures through queue.""" + try: + _call_task(task_result) + except BaseException as exception: # noqa: BLE001 + result_queue.put(_create_task_error(exception)) + return + result_queue.put(None) + + def _call_task(task_result: TaskResult) -> typing.Any: """Call a task with context when required.""" task = task_result.task @@ -266,3 +329,19 @@ def _create_task_error(exception: BaseException) -> TaskError: exception_class_path=f"{exception_type.__module__}.{exception_type.__qualname__}", traceback="".join(format_exception(exception)), ) + + +def _create_task_timeout_error(task_timeout: float) -> TaskError: + """Build task error payload for task timeout.""" + return TaskError( + exception_class_path="builtins.TimeoutError", + traceback=f"Task exceeded timeout of {task_timeout} seconds.", + ) + + +def _create_task_crash_error(exit_code: int) -> TaskError: + """Build task error payload for subprocess crash.""" + return TaskError( + exception_class_path="builtins.RuntimeError", + traceback=f"Task subprocess crashed with exit code {exit_code}.", + ) diff --git a/grinder/management/commands/grinder.py b/grinder/management/commands/grinder.py index 29a76b9..be1eeb9 100644 --- a/grinder/management/commands/grinder.py +++ b/grinder/management/commands/grinder.py @@ -2,6 +2,7 @@ import sys from django.core.management import BaseCommand +from django.tasks import task_backends from ... import Executor @@ -19,27 +20,31 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - "-b", "--backends", + "-b", + "--backends", nargs="+", default="default", help="Alias of the tasks backend to use.", ) parser.add_argument( - "-q", "--queues", + "-q", + "--queues", nargs="+", default="default", help="Queue names to listen too and process tasks from.", ) parser.add_argument( - "-w", "--workers", + "-w", + "--workers", type=int, help="Number of worker processes to use. Defaults to the number of CPU cores minus one.", ) parser.add_argument( - "-t", "--threads", + "-t", + "--threads", type=int, default=1, - help="Number of threads to use. Defaults to the number of CPU cores minus one. " + help="Number of threads to use. Defaults to the number of CPU cores minus one. ", ) parser.add_argument( "--max-tasks", @@ -56,8 +61,26 @@ def add_arguments(self, parser): default=0, help="Maximum random jitter to add to the max-tasks value by randint(0, max_tasks_jitter).", ) + parser.add_argument( + "--task-timeout", + type=float, + default=3600.0, + help="Kill hung tasks after timeout seconds. Defaults to one hour.", + ) - def handle(self, *, verbosity, backends, queues, workers, threads, max_tasks, max_tasks_jitter, **options): + def handle( + self, + *, + verbosity, + backends, + queues, + workers, + threads, + max_tasks, + max_tasks_jitter, + task_timeout, + **options, + ): match sys.platform: case "win32": signal.signal(signal.SIGBREAK, kill_softly) @@ -66,13 +89,15 @@ def handle(self, *, verbosity, backends, queues, workers, threads, max_tasks, ma signal.signal(signal.SIGTERM, kill_softly) signal.signal(signal.SIGINT, kill_softly) self.stdout.write(self.style.SUCCESS("Starting worker…")) + backend_alias = backends[0] if isinstance(backends, list) else backends + backend = task_backends[backend_alias] exe = Executor( - backend=backends, - queue=queues, + backend=backend, workers=workers, threads=threads, max_tasks=max_tasks, max_tasks_jitter=max_tasks_jitter, + task_timeout=task_timeout, ) try: exe.run() From f4b4349d0d45069b9fc3c580d5ba82b9d3c35a78 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 23 Apr 2026 22:41:32 +0200 Subject: [PATCH 07/27] wip --- grinder/__init__.py | 5 +- grinder/backends.py | 13 +- grinder/executor.py | 213 ++++++++++--------------- grinder/management/commands/grinder.py | 4 +- 4 files changed, 98 insertions(+), 137 deletions(-) diff --git a/grinder/__init__.py b/grinder/__init__.py index db1f548..5f32987 100644 --- a/grinder/__init__.py +++ b/grinder/__init__.py @@ -1,12 +1,9 @@ """A queue agnostic worker for Django's task framework.""" from . import _version -from .executor import TaskExecutor __version__ = _version.version VERSION = _version.version_tuple -Executor = TaskExecutor - -__all__ = ["Executor", "TaskExecutor", "VERSION", "__version__"] +__all__ = ["VERSION", "__version__"] diff --git a/grinder/backends.py b/grinder/backends.py index a947409..6f8667b 100644 --- a/grinder/backends.py +++ b/grinder/backends.py @@ -1,5 +1,6 @@ from __future__ import annotations +import datetime from abc import ABC from django.tasks import TaskResult @@ -9,8 +10,16 @@ class AcknowledgeableTaskBackend(BaseTaskBackend, ABC): """Provide an interface for tasks queues to be processed by the executor.""" - def acquire(self, timeout: float) -> TaskResult: - """Return and lock the next task to be processed without removing it from the queue.""" + def acquire(self, timeout: datetime.timedelta | None = None) -> TaskResult: + """ + Return and lock the next task to be processed without removing it from the queue. + + Args: + timeout: The maximum time to wait for a task. If None, wait indefinitely. + + Raises: + TimeoutError: If no task is available within the specified timeout. + """ raise NotImplementedError def acknowledge(self, task_result: TaskResult) -> None: diff --git a/grinder/executor.py b/grinder/executor.py index db302d9..af21c5e 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -3,18 +3,26 @@ from __future__ import annotations import dataclasses +import datetime import multiprocessing +import os import random +import socket import threading import typing -from multiprocessing import JoinableQueue, Process, Queue -from queue import Empty, ShutDown +from multiprocessing.queues import JoinableQueue +from queue import Empty from traceback import format_exception from django.tasks import TaskResult -from django.tasks.base import TaskContext, TaskError, TaskResultStatus +from django.tasks.base import ( + TaskContext, + TaskError, + TaskResultStatus, +) from django.tasks.signals import task_enqueued, task_finished, task_started from django.utils import timezone +from django.utils.json import normalize_json if typing.TYPE_CHECKING: from .backends import AcknowledgeableTaskBackend @@ -28,7 +36,7 @@ def __init__( task_queue: JoinableQueue[TaskResult], processed_task_queue: JoinableQueue[TaskResult], thread_count: int, - task_timeout: float, + task_timeout: datetime.timedelta, max_tasks: int | None = None, ) -> None: """Create process with dedicated thread pool for task execution.""" @@ -64,25 +72,29 @@ def __init__( threads: int = 1, max_tasks: int = 0, max_tasks_jitter: int = 0, - get_timeout_secs: float = 1.0, - task_timeout: float = 3600.0, + task_timeout: datetime.timedelta = datetime.timedelta(hours=1), ) -> None: """Create pool-backed executor with queue consumption settings.""" + self.is_publishing = True self.backend = backend self.process_count = workers or max(multiprocessing.cpu_count() - 1, 1) self.thread_count = threads self.max_tasks = max_tasks self.max_tasks_jitter = max_tasks_jitter - self.get_timeout_secs = get_timeout_secs self.task_timeout = task_timeout - self.is_running = True + self.is_acquiring = True self._worker_processes: list[WorkerProcess] = [] + self.worker_shutdown_deadline_at_secs_by_process_id: dict[int, float] = {} self.processing_slot_count = self.process_count * self.thread_count - self.shared_task_queue: JoinableQueue[TaskResult] = JoinableQueue( - maxsize=self.processing_slot_count, + self.shared_task_queue: multiprocessing.JoinableQueue[TaskResult] = ( + multiprocessing.JoinableQueue( + maxsize=self.processing_slot_count, + ) ) - self.processed_task_queue: JoinableQueue[TaskResult] = JoinableQueue( - maxsize=self.processing_slot_count, + self.processed_task_queue: multiprocessing.JoinableQueue[TaskResult] = ( + multiprocessing.JoinableQueue( + maxsize=self.processing_slot_count, + ) ) def _get_maximum_tasks_per_child(self) -> int | None: @@ -110,59 +122,54 @@ def run(self) -> None: self._worker_processes = [ self._create_worker_process() for _ in range(self.process_count) ] - while self.is_running: + while self.is_acquiring: self._replace_dead_worker_processes() - self._drain_processed_tasks() - try: - task_result = self.backend.acquire(timeout=self.get_timeout_secs) - except (Empty, ShutDown): - continue - # Block when all worker threads are saturated. - self.shared_task_queue.put(task_result) - def _drain_processed_tasks(self) -> None: + async def buffer_tasks(self) -> None: + """Buffer tasks in shared task queue.""" + while self.is_acquiring: + self.shared_task_queue.put(self.backend.acquire()) + + async def _drain_processed_tasks(self) -> None: """Acknowledge processed tasks and publish updated results in main process.""" - while True: - try: - task_result = self.processed_task_queue.get_nowait() - except Empty: - return - try: - self.backend.acknowledge(task_result) - finally: - self.processed_task_queue.task_done() + while self.is_publishing: + self.backend.acknowledge(self.processed_task_queue.get(block=True)) + self.processed_task_queue.task_done() def shutdown(self) -> None: """Stop queue consumption and terminate all worker processes.""" - self.is_running = False + self.is_acquiring = False self.shared_task_queue.join() - self._drain_processed_tasks() for worker in self._worker_processes: worker.shutdown() - self._drain_processed_tasks() + self.processed_task_queue.join() + self.is_publishing = False def _replace_dead_worker_processes(self) -> None: """Restart worker processes that have exited.""" for index, worker in enumerate(self._worker_processes): if worker.is_alive(): continue + process_id = worker.pid worker.join(timeout=0) + if process_id is not None: + self.worker_shutdown_deadline_at_secs_by_process_id.pop( + process_id, None + ) self._worker_processes[index] = self._create_worker_process() def _run_worker_process( - task_queue: JoinableQueue[TaskResult], - processed_task_queue: JoinableQueue[TaskResult], + task_queue: multiprocessing.JoinableQueue[TaskResult], + processed_task_queue: multiprocessing.JoinableQueue[TaskResult], thread_count: int, - task_timeout: float, + task_timeout: datetime.timedelta, max_tasks: int | None, - shutdown_requested: typing.Any, ) -> None: """Run consumer threads in a process that read from shared task queue.""" state = _WorkerState( max_tasks=max_tasks, task_timeout=task_timeout, - shutdown_requested=shutdown_requested, ) if thread_count == 1: _consume_tasks(task_queue, processed_task_queue, state) @@ -178,7 +185,8 @@ def _run_worker_process( for consumer_thread in consumer_threads: consumer_thread.start() for consumer_thread in consumer_threads: - consumer_thread.join() + # Wait for consumer thread to finish or timeout. + consumer_thread.join(task_timeout.total_seconds()) class _WorkerState: @@ -187,16 +195,14 @@ class _WorkerState: def __init__( self, max_tasks: int | None, - task_timeout: float, - shutdown_requested: typing.Any, + task_timeout: datetime.timedelta, ) -> None: """Create process-local execution state.""" self.max_tasks = max_tasks self.task_timeout = task_timeout - self.shutdown_requested = shutdown_requested self.task_count = 0 self.lock = threading.Lock() - self.should_stop = threading.Event() + self.expired = threading.Event() def record_task(self) -> None: """Record one processed task and stop when max_tasks is reached.""" @@ -205,121 +211,86 @@ def record_task(self) -> None: with self.lock: self.task_count += 1 if self.task_count >= self.max_tasks: - self.should_stop.set() + self.expired.set() def _consume_tasks( - task_queue: JoinableQueue[TaskResult], - processed_task_queue: JoinableQueue[TaskResult], + task_queue: multiprocessing.JoinableQueue[TaskResult], + processed_task_queue: multiprocessing.JoinableQueue[TaskResult], state: _WorkerState, ) -> None: """Consume and execute tasks from shared task queue.""" - while not state.should_stop.is_set(): - if state.shutdown_requested.is_set() and task_queue.empty(): - return + while not state.expired.is_set(): try: task_result = task_queue.get(timeout=1.0) except Empty: - if state.shutdown_requested.is_set(): - return continue try: - processed_task_result = _execute_task_result( - task_result, - task_timeout=state.task_timeout, + processed_task_queue.put( + _execute_task_result( + task_result, + ) ) - processed_task_queue.put(processed_task_result) finally: task_queue.task_done() state.record_task() -def _execute_task_result(task_result: TaskResult, task_timeout: float) -> TaskResult: +def _execute_task_result( + task_result: TaskResult, +) -> TaskResult: """Execute task from task result and update result lifecycle state.""" - task_result = dataclasses.replace(task_result, enqueued_at=timezone.now()) - task_enqueued.send(TaskExecutor, task_result=task_result) - started_at = timezone.now() task_result = dataclasses.replace( task_result, status=TaskResultStatus.RUNNING, started_at=started_at, last_attempted_at=started_at, + worker_ids=[*task_result.worker_ids, _create_worker_id()], ) + task_enqueued.send(TaskExecutor, task_result=task_result) task_started.send(TaskExecutor, task_result=task_result) - if task_error := _call_task_with_timeout(task_result, task_timeout): + try: + return_value = _call_task(task_result) + except Exception as exception: task_result = dataclasses.replace( task_result, - finished_at=timezone.now(), status=TaskResultStatus.FAILED, - errors=[*task_result.errors, task_error], + errors=[*task_result.errors, _create_task_error(exception)], ) else: task_result = dataclasses.replace( task_result, - finished_at=timezone.now(), status=TaskResultStatus.SUCCESSFUL, + _return_value=normalize_json(return_value), + ) + finally: + task_result = dataclasses.replace( + task_result, + finished_at=timezone.now(), ) + task_finished.send(TaskExecutor, task_result=task_result) - task_finished.send(TaskExecutor, task_result=task_result) return task_result -def _call_task_with_timeout( - task_result: TaskResult, task_timeout: float -) -> TaskError | None: - """Execute task in a killable subprocess and return TaskError on failure.""" - result_queue: Queue[TaskError | None] = multiprocessing.Queue(maxsize=1) - task_process = Process( - target=_run_task_call_in_subprocess, - args=(task_result, result_queue), - daemon=True, - ) - task_process.start() - task_process.join(timeout=task_timeout) - - if task_process.is_alive(): - task_process.kill() - task_process.join() - return _create_task_timeout_error(task_timeout) - - if task_process.exitcode not in (0, None): - exit_code = task_process.exitcode - if exit_code is not None: - return _create_task_crash_error(exit_code) - - try: - return result_queue.get_nowait() - except Empty: - return None - - -def _run_task_call_in_subprocess( - task_result: TaskResult, - result_queue: Queue[TaskError | None], -) -> None: - """Run task call and report failures through queue.""" - try: - _call_task(task_result) - except BaseException as exception: # noqa: BLE001 - result_queue.put(_create_task_error(exception)) - return - result_queue.put(None) +def _create_worker_id() -> str: + """Create worker id in host-process-thread format.""" + return f"{socket.gethostname()}:{os.getpid()}:{threading.get_ident()}" def _call_task(task_result: TaskResult) -> typing.Any: """Call a task with context when required.""" task = task_result.task - match task.takes_context: - case True: - return task.call( - TaskContext(task_result=task_result), - *task_result.args, - **task_result.kwargs, - ) - case False: - return task.call(*task_result.args, **task_result.kwargs) + if task.takes_context: + return task.call( + TaskContext(task_result=task_result), + *task_result.args, + **task_result.kwargs, + ) + else: + return task.call(*task_result.args, **task_result.kwargs) def _create_task_error(exception: BaseException) -> TaskError: @@ -329,19 +300,3 @@ def _create_task_error(exception: BaseException) -> TaskError: exception_class_path=f"{exception_type.__module__}.{exception_type.__qualname__}", traceback="".join(format_exception(exception)), ) - - -def _create_task_timeout_error(task_timeout: float) -> TaskError: - """Build task error payload for task timeout.""" - return TaskError( - exception_class_path="builtins.TimeoutError", - traceback=f"Task exceeded timeout of {task_timeout} seconds.", - ) - - -def _create_task_crash_error(exit_code: int) -> TaskError: - """Build task error payload for subprocess crash.""" - return TaskError( - exception_class_path="builtins.RuntimeError", - traceback=f"Task subprocess crashed with exit code {exit_code}.", - ) diff --git a/grinder/management/commands/grinder.py b/grinder/management/commands/grinder.py index be1eeb9..45858ab 100644 --- a/grinder/management/commands/grinder.py +++ b/grinder/management/commands/grinder.py @@ -4,7 +4,7 @@ from django.core.management import BaseCommand from django.tasks import task_backends -from ... import Executor +from ...executor import TaskExecutor def kill_softly(signum, frame): @@ -91,7 +91,7 @@ def handle( self.stdout.write(self.style.SUCCESS("Starting worker…")) backend_alias = backends[0] if isinstance(backends, list) else backends backend = task_backends[backend_alias] - exe = Executor( + exe = TaskExecutor( backend=backend, workers=workers, threads=threads, From a87a79ea97ef392fd2e09e180545ef558f96c20b Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 23 Apr 2026 23:00:10 +0200 Subject: [PATCH 08/27] wip --- grinder/executor.py | 100 +++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 53 deletions(-) diff --git a/grinder/executor.py b/grinder/executor.py index af21c5e..bbe7a1d 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import dataclasses import datetime import multiprocessing @@ -15,11 +16,7 @@ from traceback import format_exception from django.tasks import TaskResult -from django.tasks.base import ( - TaskContext, - TaskError, - TaskResultStatus, -) +from django.tasks.base import TaskContext, TaskError, TaskResultStatus from django.tasks.signals import task_enqueued, task_finished, task_started from django.utils import timezone from django.utils.json import normalize_json @@ -28,7 +25,7 @@ from .backends import AcknowledgeableTaskBackend -class WorkerProcess(Process): +class WorkerProcess(multiprocessing.Process): """Single worker process running thread_count consumer threads.""" def __init__( @@ -61,47 +58,45 @@ def shutdown(self) -> None: self.join() +@dataclasses.dataclass(kw_only=True, slots=True) class TaskExecutor: """Consume tasks from a priority queue with process and thread pools.""" - def __init__( - self, - *, - backend: AcknowledgeableTaskBackend, - workers: int | None = None, - threads: int = 1, - max_tasks: int = 0, - max_tasks_jitter: int = 0, - task_timeout: datetime.timedelta = datetime.timedelta(hours=1), - ) -> None: - """Create pool-backed executor with queue consumption settings.""" - self.is_publishing = True - self.backend = backend - self.process_count = workers or max(multiprocessing.cpu_count() - 1, 1) - self.thread_count = threads - self.max_tasks = max_tasks - self.max_tasks_jitter = max_tasks_jitter - self.task_timeout = task_timeout - self.is_acquiring = True - self._worker_processes: list[WorkerProcess] = [] - self.worker_shutdown_deadline_at_secs_by_process_id: dict[int, float] = {} - self.processing_slot_count = self.process_count * self.thread_count - self.shared_task_queue: multiprocessing.JoinableQueue[TaskResult] = ( - multiprocessing.JoinableQueue( - maxsize=self.processing_slot_count, - ) - ) - self.processed_task_queue: multiprocessing.JoinableQueue[TaskResult] = ( - multiprocessing.JoinableQueue( - maxsize=self.processing_slot_count, - ) + backend: AcknowledgeableTaskBackend + workers: int | None = None + threads: int = 1 + max_tasks: int = 0 + max_tasks_jitter: int = 0 + task_timeout: datetime.timedelta = datetime.timedelta(hours=1) + acquire_timeout: datetime.timedelta = datetime.timedelta(seconds=1) + is_acquiring: bool = dataclasses.field(default=True, init=False) + is_publishing: bool = dataclasses.field(default=True, init=False) + _worker_processes: list[WorkerProcess] = dataclasses.field( + default_factory=list, init=False + ) + process_count: int = dataclasses.field(init=False) + thread_count: int = dataclasses.field(init=False) + shared_task_queue: multiprocessing.JoinableQueue[TaskResult] = dataclasses.field( + init=False + ) + processed_task_queue: multiprocessing.JoinableQueue[TaskResult] = dataclasses.field( + init=False + ) + + def __post_init__(self) -> None: + """Initialize derived orchestration fields and queues.""" + self.process_count = self.workers or max(multiprocessing.cpu_count() - 1, 1) + self.thread_count = max(self.threads, 1) + self.shared_task_queue = multiprocessing.JoinableQueue( + maxsize=self.process_count * self.thread_count, ) + self.processed_task_queue = multiprocessing.JoinableQueue() def _get_maximum_tasks_per_child(self) -> int | None: """Return worker recycling limit based on config and thread count.""" if self.max_tasks: return ( - self.max_tasks + random.randint(0, self.max_tasks_jitter) + self.max_tasks + random.randint(0, self.max_tasks_jitter) # noqa: S311 ) // self.thread_count return None @@ -122,15 +117,18 @@ def run(self) -> None: self._worker_processes = [ self._create_worker_process() for _ in range(self.process_count) ] - while self.is_acquiring: - self._replace_dead_worker_processes() + asyncio.gather( + asyncio.create_task(self.acquire_tasks()), + asyncio.create_task(self.acknowledge_tasks()), + asyncio.create_task(self.maintain_worker_pool()), + ) - async def buffer_tasks(self) -> None: + async def acquire_tasks(self) -> None: """Buffer tasks in shared task queue.""" while self.is_acquiring: self.shared_task_queue.put(self.backend.acquire()) - async def _drain_processed_tasks(self) -> None: + async def acknowledge_tasks(self) -> None: """Acknowledge processed tasks and publish updated results in main process.""" while self.is_publishing: self.backend.acknowledge(self.processed_task_queue.get(block=True)) @@ -145,18 +143,14 @@ def shutdown(self) -> None: self.processed_task_queue.join() self.is_publishing = False - def _replace_dead_worker_processes(self) -> None: + async def maintain_worker_pool(self) -> None: """Restart worker processes that have exited.""" - for index, worker in enumerate(self._worker_processes): - if worker.is_alive(): - continue - process_id = worker.pid - worker.join(timeout=0) - if process_id is not None: - self.worker_shutdown_deadline_at_secs_by_process_id.pop( - process_id, None - ) - self._worker_processes[index] = self._create_worker_process() + while self.is_publishing: + for index, worker in enumerate(self._worker_processes): + if worker.is_alive(): + continue + worker.join(timeout=0) + self._worker_processes[index] = self._create_worker_process() def _run_worker_process( From 76f587b1f3906e155ab727ed520dbb18eea0820c Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 23 Apr 2026 23:06:41 +0200 Subject: [PATCH 09/27] wip --- grinder/executor.py | 116 ++++++++++++++++++++++---------------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/grinder/executor.py b/grinder/executor.py index bbe7a1d..dd977c4 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -25,39 +25,6 @@ from .backends import AcknowledgeableTaskBackend -class WorkerProcess(multiprocessing.Process): - """Single worker process running thread_count consumer threads.""" - - def __init__( - self, - task_queue: JoinableQueue[TaskResult], - processed_task_queue: JoinableQueue[TaskResult], - thread_count: int, - task_timeout: datetime.timedelta, - max_tasks: int | None = None, - ) -> None: - """Create process with dedicated thread pool for task execution.""" - self.shutdown_requested = multiprocessing.Event() - super().__init__( - target=_run_worker_process, - args=( - task_queue, - processed_task_queue, - thread_count, - task_timeout, - max_tasks, - self.shutdown_requested, - ), - daemon=True, - ) - self.thread_count = thread_count - - def shutdown(self) -> None: - """Request graceful worker stop and wait for process exit.""" - self.shutdown_requested.set() - self.join() - - @dataclasses.dataclass(kw_only=True, slots=True) class TaskExecutor: """Consume tasks from a priority queue with process and thread pools.""" @@ -153,6 +120,64 @@ async def maintain_worker_pool(self) -> None: self._worker_processes[index] = self._create_worker_process() +class WorkerProcess(multiprocessing.Process): + """Single worker process running thread_count consumer threads.""" + + def __init__( + self, + task_queue: JoinableQueue[TaskResult], + processed_task_queue: JoinableQueue[TaskResult], + thread_count: int, + task_timeout: datetime.timedelta, + max_tasks: int | None = None, + ) -> None: + """Create process with dedicated thread pool for task execution.""" + self.shutdown_requested = multiprocessing.Event() + super().__init__( + target=_run_worker_process, + args=( + task_queue, + processed_task_queue, + thread_count, + task_timeout, + max_tasks, + self.shutdown_requested, + ), + daemon=True, + ) + self.thread_count = thread_count + + def shutdown(self) -> None: + """Request graceful worker stop and wait for process exit.""" + self.shutdown_requested.set() + self.join() + + +class _WorkerState: + """State shared by process-local consumer threads.""" + + def __init__( + self, + max_tasks: int | None, + task_timeout: datetime.timedelta, + ) -> None: + """Create process-local execution state.""" + self.max_tasks = max_tasks + self.task_timeout = task_timeout + self.task_count = 0 + self.lock = threading.Lock() + self.expired = threading.Event() + + def record_task(self) -> None: + """Record one processed task and stop when max_tasks is reached.""" + if self.max_tasks is None: + return + with self.lock: + self.task_count += 1 + if self.task_count >= self.max_tasks: + self.expired.set() + + def _run_worker_process( task_queue: multiprocessing.JoinableQueue[TaskResult], processed_task_queue: multiprocessing.JoinableQueue[TaskResult], @@ -183,31 +208,6 @@ def _run_worker_process( consumer_thread.join(task_timeout.total_seconds()) -class _WorkerState: - """State shared by process-local consumer threads.""" - - def __init__( - self, - max_tasks: int | None, - task_timeout: datetime.timedelta, - ) -> None: - """Create process-local execution state.""" - self.max_tasks = max_tasks - self.task_timeout = task_timeout - self.task_count = 0 - self.lock = threading.Lock() - self.expired = threading.Event() - - def record_task(self) -> None: - """Record one processed task and stop when max_tasks is reached.""" - if self.max_tasks is None: - return - with self.lock: - self.task_count += 1 - if self.task_count >= self.max_tasks: - self.expired.set() - - def _consume_tasks( task_queue: multiprocessing.JoinableQueue[TaskResult], processed_task_queue: multiprocessing.JoinableQueue[TaskResult], From 0d8e49dd2994e8c9a8c41f424c3920ae00c63d87 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 23 Apr 2026 23:23:31 +0200 Subject: [PATCH 10/27] wip --- grinder/executor.py | 281 +++++++++++++++++++++----------------------- 1 file changed, 134 insertions(+), 147 deletions(-) diff --git a/grinder/executor.py b/grinder/executor.py index dd977c4..af6663e 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -38,7 +38,7 @@ class TaskExecutor: acquire_timeout: datetime.timedelta = datetime.timedelta(seconds=1) is_acquiring: bool = dataclasses.field(default=True, init=False) is_publishing: bool = dataclasses.field(default=True, init=False) - _worker_processes: list[WorkerProcess] = dataclasses.field( + worker_processes: list[WorkerProcess] = dataclasses.field( default_factory=list, init=False ) process_count: int = dataclasses.field(init=False) @@ -59,7 +59,7 @@ def __post_init__(self) -> None: ) self.processed_task_queue = multiprocessing.JoinableQueue() - def _get_maximum_tasks_per_child(self) -> int | None: + def get_maximum_tasks_per_child(self) -> int | None: """Return worker recycling limit based on config and thread count.""" if self.max_tasks: return ( @@ -67,22 +67,22 @@ def _get_maximum_tasks_per_child(self) -> int | None: ) // self.thread_count return None - def _create_worker_process(self) -> WorkerProcess: + def create_worker_process(self) -> WorkerProcess: """Create and start a new worker process.""" worker = WorkerProcess( self.shared_task_queue, self.processed_task_queue, self.thread_count, self.task_timeout, - self._get_maximum_tasks_per_child(), + self.get_maximum_tasks_per_child(), ) worker.start() return worker def run(self) -> None: """Start consuming tasks until shutdown is requested.""" - self._worker_processes = [ - self._create_worker_process() for _ in range(self.process_count) + self.worker_processes = [ + self.create_worker_process() for _ in range(self.process_count) ] asyncio.gather( asyncio.create_task(self.acquire_tasks()), @@ -105,7 +105,7 @@ def shutdown(self) -> None: """Stop queue consumption and terminate all worker processes.""" self.is_acquiring = False self.shared_task_queue.join() - for worker in self._worker_processes: + for worker in self.worker_processes: worker.shutdown() self.processed_task_queue.join() self.is_publishing = False @@ -113,11 +113,11 @@ def shutdown(self) -> None: async def maintain_worker_pool(self) -> None: """Restart worker processes that have exited.""" while self.is_publishing: - for index, worker in enumerate(self._worker_processes): + for index, worker in enumerate(self.worker_processes): if worker.is_alive(): continue worker.join(timeout=0) - self._worker_processes[index] = self._create_worker_process() + self.worker_processes[index] = self.create_worker_process() class WorkerProcess(multiprocessing.Process): @@ -133,164 +133,151 @@ def __init__( ) -> None: """Create process with dedicated thread pool for task execution.""" self.shutdown_requested = multiprocessing.Event() - super().__init__( - target=_run_worker_process, - args=( - task_queue, - processed_task_queue, - thread_count, - task_timeout, - max_tasks, - self.shutdown_requested, - ), - daemon=True, - ) + super().__init__(daemon=True) + self.task_queue = task_queue + self.processed_task_queue = processed_task_queue self.thread_count = thread_count - - def shutdown(self) -> None: - """Request graceful worker stop and wait for process exit.""" - self.shutdown_requested.set() - self.join() - - -class _WorkerState: - """State shared by process-local consumer threads.""" - - def __init__( - self, - max_tasks: int | None, - task_timeout: datetime.timedelta, - ) -> None: - """Create process-local execution state.""" - self.max_tasks = max_tasks self.task_timeout = task_timeout + self.max_tasks = max_tasks self.task_count = 0 + self.lock: threading.Lock | None = None + self.expired: threading.Event | None = None + + def run(self) -> None: + """Start consumer execution inside this process.""" self.lock = threading.Lock() self.expired = threading.Event() + self.run_worker_process(self) + + @staticmethod + def run_worker_process(worker: WorkerProcess) -> None: + """Run consumer threads in a process that read from shared task queue.""" + consumer_threads = [ + WorkerThread(worker=worker, index=index) + for index in range(worker.thread_count) + ] + for consumer_thread in consumer_threads: + consumer_thread.start() + for consumer_thread in consumer_threads: + consumer_thread.join(worker.task_timeout.total_seconds()) def record_task(self) -> None: """Record one processed task and stop when max_tasks is reached.""" if self.max_tasks is None: return + if self.lock is None or self.expired is None: + return with self.lock: self.task_count += 1 if self.task_count >= self.max_tasks: self.expired.set() + def shutdown(self) -> None: + """Request graceful worker stop and wait for process exit.""" + self.shutdown_requested.set() + self.join() -def _run_worker_process( - task_queue: multiprocessing.JoinableQueue[TaskResult], - processed_task_queue: multiprocessing.JoinableQueue[TaskResult], - thread_count: int, - task_timeout: datetime.timedelta, - max_tasks: int | None, -) -> None: - """Run consumer threads in a process that read from shared task queue.""" - state = _WorkerState( - max_tasks=max_tasks, - task_timeout=task_timeout, - ) - if thread_count == 1: - _consume_tasks(task_queue, processed_task_queue, state) - return - consumer_threads = [ - threading.Thread( - target=_consume_tasks, - args=(task_queue, processed_task_queue, state), - name=f"task-consumer-thread-{index}", - ) - for index in range(thread_count) - ] - for consumer_thread in consumer_threads: - consumer_thread.start() - for consumer_thread in consumer_threads: - # Wait for consumer thread to finish or timeout. - consumer_thread.join(task_timeout.total_seconds()) - - -def _consume_tasks( - task_queue: multiprocessing.JoinableQueue[TaskResult], - processed_task_queue: multiprocessing.JoinableQueue[TaskResult], - state: _WorkerState, -) -> None: - """Consume and execute tasks from shared task queue.""" - while not state.expired.is_set(): - try: - task_result = task_queue.get(timeout=1.0) - except Empty: - continue - try: - processed_task_queue.put( - _execute_task_result( - task_result, - ) - ) - finally: - task_queue.task_done() - state.record_task() - - -def _execute_task_result( - task_result: TaskResult, -) -> TaskResult: - """Execute task from task result and update result lifecycle state.""" - started_at = timezone.now() - task_result = dataclasses.replace( - task_result, - status=TaskResultStatus.RUNNING, - started_at=started_at, - last_attempted_at=started_at, - worker_ids=[*task_result.worker_ids, _create_worker_id()], - ) - task_enqueued.send(TaskExecutor, task_result=task_result) - task_started.send(TaskExecutor, task_result=task_result) - try: - return_value = _call_task(task_result) - except Exception as exception: - task_result = dataclasses.replace( - task_result, - status=TaskResultStatus.FAILED, - errors=[*task_result.errors, _create_task_error(exception)], - ) - else: - task_result = dataclasses.replace( - task_result, - status=TaskResultStatus.SUCCESSFUL, - _return_value=normalize_json(return_value), - ) - finally: +class WorkerThread(threading.Thread): + """Single worker thread consuming tasks from the process queue.""" + + def __init__( + self, + *, + worker: WorkerProcess, + index: int, + ) -> None: + """Create worker thread bound to process worker state.""" + super().__init__(name=f"task-consumer-thread-{index}") + self.worker = worker + + def run(self) -> None: + """Start consuming tasks for this thread.""" + self.consume_tasks(self.worker) + + @staticmethod + def consume_tasks(worker: WorkerProcess) -> None: + """Consume and execute tasks from shared task queue.""" + while worker.expired is None or not worker.expired.is_set(): + if worker.shutdown_requested.is_set() and worker.task_queue.empty(): + return + try: + task_result = worker.task_queue.get(timeout=1.0) + except Empty: + if worker.shutdown_requested.is_set(): + return + continue + try: + worker.processed_task_queue.put( + WorkerThread.execute_task_result( + task_result, + ) + ) + finally: + worker.task_queue.task_done() + worker.record_task() + + @staticmethod + def execute_task_result(task_result: TaskResult) -> TaskResult: + """Execute task from task result and update result lifecycle state.""" + started_at = timezone.now() task_result = dataclasses.replace( task_result, - finished_at=timezone.now(), + status=TaskResultStatus.RUNNING, + started_at=started_at, + last_attempted_at=started_at, + worker_ids=[*task_result.worker_ids, WorkerThread.create_worker_id()], ) - task_finished.send(TaskExecutor, task_result=task_result) - - return task_result + task_enqueued.send(TaskExecutor, task_result=task_result) + task_started.send(TaskExecutor, task_result=task_result) - -def _create_worker_id() -> str: - """Create worker id in host-process-thread format.""" - return f"{socket.gethostname()}:{os.getpid()}:{threading.get_ident()}" - - -def _call_task(task_result: TaskResult) -> typing.Any: - """Call a task with context when required.""" - task = task_result.task - if task.takes_context: - return task.call( - TaskContext(task_result=task_result), - *task_result.args, - **task_result.kwargs, - ) - else: + try: + return_value = WorkerThread.call_task(task_result) + except Exception as exception: + task_result = dataclasses.replace( + task_result, + status=TaskResultStatus.FAILED, + errors=[*task_result.errors, WorkerThread.create_task_error(exception)], + ) + else: + task_result = dataclasses.replace( + task_result, + status=TaskResultStatus.SUCCESSFUL, + ) + object.__setattr__( + task_result, "_return_value", normalize_json(return_value) + ) + finally: + task_result = dataclasses.replace( + task_result, + finished_at=timezone.now(), + ) + task_finished.send(TaskExecutor, task_result=task_result) + + return task_result + + @staticmethod + def create_worker_id() -> str: + """Create worker id in host-process-thread format.""" + return f"{socket.gethostname()}:{os.getpid()}:{threading.get_ident()}" + + @staticmethod + def call_task(task_result: TaskResult) -> typing.Any: + """Call a task with context when required.""" + task = task_result.task + if task.takes_context: + return task.call( + TaskContext(task_result=task_result), + *task_result.args, + **task_result.kwargs, + ) return task.call(*task_result.args, **task_result.kwargs) - -def _create_task_error(exception: BaseException) -> TaskError: - """Build a task error payload for failed execution.""" - exception_type = type(exception) - return TaskError( - exception_class_path=f"{exception_type.__module__}.{exception_type.__qualname__}", - traceback="".join(format_exception(exception)), - ) + @staticmethod + def create_task_error(exception: BaseException) -> TaskError: + """Build a task error payload for failed execution.""" + exception_type = type(exception) + return TaskError( + exception_class_path=f"{exception_type.__module__}.{exception_type.__qualname__}", + traceback="".join(format_exception(exception)), + ) From 71a7fce45c829460ab63d0d8d3fe52adf9d89d41 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 23 Apr 2026 23:31:02 +0200 Subject: [PATCH 11/27] wip --- grinder/executor.py | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/grinder/executor.py b/grinder/executor.py index af6663e..16f4b99 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -6,7 +6,6 @@ import dataclasses import datetime import multiprocessing -import os import random import socket import threading @@ -188,37 +187,34 @@ def __init__( index: int, ) -> None: """Create worker thread bound to process worker state.""" - super().__init__(name=f"task-consumer-thread-{index}") + super().__init__(name=f"{socket.gethostname()}:{worker.pid}-{index}") self.worker = worker def run(self) -> None: """Start consuming tasks for this thread.""" - self.consume_tasks(self.worker) - - @staticmethod - def consume_tasks(worker: WorkerProcess) -> None: - """Consume and execute tasks from shared task queue.""" - while worker.expired is None or not worker.expired.is_set(): - if worker.shutdown_requested.is_set() and worker.task_queue.empty(): + while self.worker.expired is None or not self.worker.expired.is_set(): + if ( + self.worker.shutdown_requested.is_set() + and self.worker.task_queue.empty() + ): return try: - task_result = worker.task_queue.get(timeout=1.0) + task_result = self.worker.task_queue.get(timeout=1.0) except Empty: - if worker.shutdown_requested.is_set(): + if self.worker.shutdown_requested.is_set(): return continue try: - worker.processed_task_queue.put( - WorkerThread.execute_task_result( + self.worker.processed_task_queue.put( + self.execute_task_result( task_result, ) ) finally: - worker.task_queue.task_done() - worker.record_task() + self.worker.task_queue.task_done() + self.worker.record_task() - @staticmethod - def execute_task_result(task_result: TaskResult) -> TaskResult: + def execute_task_result(self, task_result: TaskResult) -> TaskResult: """Execute task from task result and update result lifecycle state.""" started_at = timezone.now() task_result = dataclasses.replace( @@ -226,7 +222,7 @@ def execute_task_result(task_result: TaskResult) -> TaskResult: status=TaskResultStatus.RUNNING, started_at=started_at, last_attempted_at=started_at, - worker_ids=[*task_result.worker_ids, WorkerThread.create_worker_id()], + worker_ids=[*task_result.worker_ids, self.name], ) task_enqueued.send(TaskExecutor, task_result=task_result) task_started.send(TaskExecutor, task_result=task_result) @@ -256,11 +252,6 @@ def execute_task_result(task_result: TaskResult) -> TaskResult: return task_result - @staticmethod - def create_worker_id() -> str: - """Create worker id in host-process-thread format.""" - return f"{socket.gethostname()}:{os.getpid()}:{threading.get_ident()}" - @staticmethod def call_task(task_result: TaskResult) -> typing.Any: """Call a task with context when required.""" From 683b33f2feb10c05342faac52c04ed53a576b044 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 23 Apr 2026 23:41:44 +0200 Subject: [PATCH 12/27] Add unit tests --- .github/copilot-instructions.md | 7 +- .github/workflows/release.yml | 13 +- CONTRIBUTING.md | 39 +++ README.md | 3 +- pyproject.toml | 3 +- tests/test_backends.py | 25 ++ tests/test_executor.py | 588 ++++++++++++++++++++++++++++++++ tests/test_grinder_command.py | 139 ++++++++ 8 files changed, 799 insertions(+), 18 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 tests/test_backends.py create mode 100644 tests/test_executor.py create mode 100644 tests/test_grinder_command.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 46a2e0a..c9153a2 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -6,9 +6,4 @@ Use the following command to access the guidelines: curl -sSL https://raw.githubusercontent.com/codingjoe/naming-things/refs/heads/main/README.md | cat ``` -The MOST IMPORTANT review considerations are: - -- Consistency – We never lose data, even if the power goes down. -- Durability – We recover from any failures, even poorly written tasks. -- Overhead – We focus resources on running tasks, not on managing the scheduler. -- Utilization – We keep the CPU saturated with tasks, not with idle time or waiting for locks. +Read the CONTRIBUTING.md file for more information. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index efb2a17..350ff01 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,18 +3,13 @@ on: release: types: [published] workflow_dispatch: -permissions: - id-token: write jobs: - release-build: + pypi-build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - run: python -m pip install --upgrade pip build wheel - - run: python -m build --sdist --wheel + - uses: astral-sh/setup-uv@v7 + - run: uvx --from build pyproject-build --sdist --wheel - uses: actions/upload-artifact@v7 with: name: release-dists @@ -22,7 +17,7 @@ jobs: pypi-publish: runs-on: ubuntu-latest needs: - - release-build + - pypi-build permissions: id-token: write steps: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..efbc3a2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,39 @@ +# Contributing + +When writing code, you MUST ALWAYS follow the [naming-things](https://github.com/codingjoe/naming-things/blob/main/README.md) guidelines. + + + +## Design Principles + +- Consistency – We never lose data, even if someone unplugs the power or network. +- Durability – We recover from any failures, even poorly written tasks. +- Overhead – We focus resources on running tasks, not on managing the scheduler. +- Utilization – We keep the CPU saturated with tasks, not with idle time or waiting for locks. + +## Testing + +We have unit tests, integration tests, and benchmarks. Avoid mocking if possible. + +To run the tests, use the following command: + +```bash +uv run pytest +``` + +Benchmarking snapshots are created automatically. +To compare your feature branch against the main branch, +run the test suite on main, followed by: + +``` +uv run pytest --benchmark-compare +``` + +Before your first commit, ensure that the pre-commit hooks are installed by running: + +```bash +uvx prek install +``` diff --git a/README.md b/README.md index 123aeb3..64637b4 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ **A queue agnostic worker for Django's task framework.** -- self-healing workers +- durable, self-healing workers - graceful shutdown - CPU, IO, or memory optimized workers @@ -54,5 +54,4 @@ The executor currently consumes tasks from a Python `queue.PriorityQueue`. - `--backends` and `--queues` are accepted but not used yet. - Queue items must be `django.tasks.TaskResult` or `(priority, TaskResult)`. - [django-tasks]: https://docs.djangoproject.com/en/6.0/topics/tasks/ diff --git a/pyproject.toml b/pyproject.toml index a3f5796..ccf7eeb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ write_to = "grinder/_version.py" [tool.pytest.ini_options] minversion = "6.0" -addopts = "--cov --cov-report=xml --cov-report=term --tb=short -rxs" +addopts = "--cov --cov-report=xml --cov-report=term --tb=short -rxs --benchmark-autosave --benchmark-group-by=fullname --benchmark-min-rounds=10" testpaths = ["tests"] DJANGO_SETTINGS_MODULE = "tests.testapp.settings" @@ -97,6 +97,7 @@ dev = [ ] test = [ "pytest", + "pytest-benchmark", "pytest-cov", "pytest-django", ] diff --git a/tests/test_backends.py b/tests/test_backends.py new file mode 100644 index 0000000..832edde --- /dev/null +++ b/tests/test_backends.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import datetime + +import pytest +from grinder.backends import AcknowledgeableTaskBackend + + +class BackendDouble(AcknowledgeableTaskBackend): + def enqueue(self, task): + return task + + +class TestAcknowledgeableTaskBackend: + def test_acquire__raise_not_implemented_error(self) -> None: + """Raise NotImplementedError for backend acquire API.""" + with pytest.raises(NotImplementedError): + BackendDouble(alias="default", params={}).acquire( + datetime.timedelta(seconds=1) + ) + + def test_acknowledge__raise_not_implemented_error(self) -> None: + """Raise NotImplementedError for backend acknowledge API.""" + with pytest.raises(NotImplementedError): + BackendDouble(alias="default", params={}).acknowledge(task_result=None) diff --git a/tests/test_executor.py b/tests/test_executor.py new file mode 100644 index 0000000..cff1553 --- /dev/null +++ b/tests/test_executor.py @@ -0,0 +1,588 @@ +from __future__ import annotations + +import asyncio +import datetime +import threading +from queue import Empty +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from django.tasks import TaskResult +from django.tasks.base import TaskResultStatus +from grinder import executor + + +class RecordingTask: + def __init__( + self, + *, + takes_context: bool, + return_value=None, + exception: Exception | None = None, + ): + self.takes_context = takes_context + self.return_value = return_value + self.exception = exception + self.calls: list[tuple[tuple, dict]] = [] + + def call(self, *args, **kwargs): + self.calls.append((args, kwargs)) + if self.exception is not None: + raise self.exception + return self.return_value + + +class FakeJoinableQueue: + def __init__(self, *, items: list | None = None): + self.items = [] if items is None else [*items] + self.put_calls: list = [] + self.task_done_calls = 0 + + def put(self, item): + self.put_calls.append(item) + + def get(self, *, timeout: float | None = None, block: bool = True): + if not self.items: + raise Empty + return self.items.pop(0) + + def task_done(self): + self.task_done_calls += 1 + + def empty(self) -> bool: + return not self.items + + +class QueueRaisingEmpty: + def __init__(self, *, is_empty: bool): + self.is_empty = is_empty + self.task_done_calls = 0 + + def get(self, *, timeout: float | None = None, block: bool = True): + raise Empty + + def empty(self) -> bool: + return self.is_empty + + def task_done(self): + self.task_done_calls += 1 + + +class QueueRaiseThenReturn: + def __init__(self, *, item): + self.item = item + self.calls = 0 + self.task_done_calls = 0 + + def get(self, *, timeout: float | None = None, block: bool = True): + self.calls += 1 + if self.calls == 1: + raise Empty + return self.item + + def empty(self) -> bool: + return False + + def task_done(self): + self.task_done_calls += 1 + + +class FakeWorkerProcess: + def __init__(self, *, alive: bool): + self.alive = alive + self.join_calls: list[float] = [] + self.shutdown_called = False + + def is_alive(self) -> bool: + return self.alive + + def join(self, timeout: float | None = None) -> None: + self.join_calls.append(timeout) + + def shutdown(self) -> None: + self.shutdown_called = True + + +def create_task_result(*, task: RecordingTask, args=None, kwargs=None) -> TaskResult: + return TaskResult( + task=task, + id="task-id", + status=TaskResultStatus.READY, + enqueued_at=None, + started_at=None, + finished_at=None, + last_attempted_at=None, + args=[] if args is None else args, + kwargs={} if kwargs is None else kwargs, + backend="default", + errors=[], + worker_ids=[], + ) + + +def create_task_error_from_value_error() -> None: + try: + raise ValueError("benchmark") + except ValueError as exception: + executor.WorkerThread.create_task_error(exception) + + +class TestTaskExecutor: + def test_run__create_tasks_for_all_executor_loops(self, monkeypatch) -> None: + """Create orchestration tasks for acquire, acknowledge, and maintenance loops.""" + created_tasks = [] + + def create_task(coroutine): + created_tasks.append(coroutine.cr_code.co_name) + coroutine.close() + return coroutine.cr_code.co_name + + gather = Mock() + monkeypatch.setattr(executor.asyncio, "create_task", create_task) + monkeypatch.setattr(executor.asyncio, "gather", gather) + monkeypatch.setattr( + executor.TaskExecutor, + "create_worker_process", + lambda task_executor_self: FakeWorkerProcess(alive=True), + ) + + task_executor = executor.TaskExecutor(backend=SimpleNamespace(), workers=1) + + task_executor.run() + + assert len(task_executor.worker_processes) == 1 + assert created_tasks == [ + "acquire_tasks", + "acknowledge_tasks", + "maintain_worker_pool", + ] + gather.assert_called_once_with( + "acquire_tasks", "acknowledge_tasks", "maintain_worker_pool" + ) + + def test_acquire_tasks__put_acquired_task_in_shared_queue(self) -> None: + """Put acquired task result into shared queue.""" + + class AcquireBackend: + def __init__(self): + self.task_executor = None + + def acquire(self): + self.task_executor.is_acquiring = False + return "task-result" + + backend = AcquireBackend() + task_executor = executor.TaskExecutor(backend=backend, workers=1) + backend.task_executor = task_executor + + asyncio.run(task_executor.acquire_tasks()) + + assert task_executor.shared_task_queue.get(timeout=0.1) == "task-result" + + def test_acknowledge_tasks__acknowledge_processed_task_and_mark_done(self) -> None: + """Acknowledge processed task and mark queue item done.""" + + class AcknowledgeBackend: + def __init__(self): + self.calls: list = [] + self.task_executor = None + + def acknowledge(self, task_result): + self.calls.append(task_result) + self.task_executor.is_publishing = False + + backend = AcknowledgeBackend() + task_executor = executor.TaskExecutor(backend=backend, workers=1) + backend.task_executor = task_executor + task_executor.processed_task_queue.put("processed-result") + + asyncio.run(task_executor.acknowledge_tasks()) + + assert backend.calls == ["processed-result"] + + def test_get_maximum_tasks_per_child__return_none_without_max_tasks(self) -> None: + """Return None when task recycling is disabled.""" + task_executor = executor.TaskExecutor(backend=SimpleNamespace(), max_tasks=0) + + assert task_executor.get_maximum_tasks_per_child() is None + + def test_get_maximum_tasks_per_child__calculate_recycle_limit_with_jitter( + self, monkeypatch + ) -> None: + """Calculate worker recycle limit with jitter and thread count.""" + monkeypatch.setattr(executor.random, "randint", lambda start, end: 4) + task_executor = executor.TaskExecutor( + backend=SimpleNamespace(), + threads=2, + max_tasks=6, + max_tasks_jitter=4, + ) + + assert task_executor.get_maximum_tasks_per_child() == 5 + + def test_create_worker_process__start_and_return_worker(self, monkeypatch) -> None: + """Create and start a worker process.""" + started = Mock() + + class WorkerProcessDouble: + def __init__(self, *args): + self.args = args + + def start(self): + started() + + monkeypatch.setattr(executor, "WorkerProcess", WorkerProcessDouble) + task_executor = executor.TaskExecutor( + backend=SimpleNamespace(), workers=1, threads=3 + ) + + worker_process = task_executor.create_worker_process() + + assert isinstance(worker_process, WorkerProcessDouble) + assert worker_process.args[2] == 3 + started.assert_called_once_with() + + def test_shutdown__stop_flags_join_queues_and_shutdown_workers(self) -> None: + """Stop publishing and shut down all workers.""" + task_executor = executor.TaskExecutor(backend=SimpleNamespace(), workers=1) + worker_process = FakeWorkerProcess(alive=True) + task_executor.worker_processes = [worker_process] + + task_executor.shutdown() + + assert task_executor.is_acquiring is False + assert task_executor.is_publishing is False + assert worker_process.shutdown_called is True + + def test_maintain_worker_pool__replace_dead_worker(self, monkeypatch) -> None: + """Replace dead worker process during pool maintenance.""" + task_executor = executor.TaskExecutor(backend=SimpleNamespace(), workers=2) + dead_worker = FakeWorkerProcess(alive=False) + healthy_worker = FakeWorkerProcess(alive=True) + replacement_worker = FakeWorkerProcess(alive=True) + task_executor.worker_processes = [dead_worker, healthy_worker] + + def create_worker_process(task_executor_self): + task_executor.is_publishing = False + return replacement_worker + + monkeypatch.setattr( + executor.TaskExecutor, + "create_worker_process", + create_worker_process, + ) + + asyncio.run(task_executor.maintain_worker_pool()) + + assert dead_worker.join_calls == [0] + assert task_executor.worker_processes == [replacement_worker, healthy_worker] + + +class TestWorkerProcess: + def test_run__initialize_sync_primitives_and_start_worker_threads( + self, monkeypatch + ) -> None: + """Initialize lock and expiration event before starting worker threads.""" + run_worker_process = Mock() + monkeypatch.setattr( + executor.WorkerProcess, "run_worker_process", run_worker_process + ) + worker_process = executor.WorkerProcess( + FakeJoinableQueue(), + FakeJoinableQueue(), + thread_count=1, + task_timeout=datetime.timedelta(seconds=1), + ) + + worker_process.run() + + assert isinstance(worker_process.lock, type(threading.Lock())) + assert isinstance(worker_process.expired, type(threading.Event())) + run_worker_process.assert_called_once_with(worker_process) + + def test_run_worker_process__start_and_join_each_consumer_thread( + self, monkeypatch + ) -> None: + """Start and join every consumer thread created for the process.""" + thread_events: list[str] = [] + + class WorkerThreadDouble: + def __init__(self, *, worker, index): + self.index = index + + def start(self): + thread_events.append(f"start:{self.index}") + + def join(self, timeout): + thread_events.append(f"join:{self.index}:{timeout}") + + monkeypatch.setattr(executor, "WorkerThread", WorkerThreadDouble) + worker_process = SimpleNamespace( + thread_count=2, task_timeout=datetime.timedelta(seconds=3) + ) + + executor.WorkerProcess.run_worker_process(worker_process) + + assert thread_events == [ + "start:0", + "start:1", + "join:0:3.0", + "join:1:3.0", + ] + + def test_record_task__set_expired_after_reaching_max_tasks(self) -> None: + """Set expiration event when processed task limit is reached.""" + worker_process = executor.WorkerProcess( + FakeJoinableQueue(), + FakeJoinableQueue(), + thread_count=1, + task_timeout=datetime.timedelta(seconds=1), + max_tasks=2, + ) + worker_process.lock = threading.Lock() + worker_process.expired = threading.Event() + + worker_process.record_task() + worker_process.record_task() + + assert worker_process.expired.is_set() is True + + def test_record_task__ignore_when_limit_or_state_is_missing(self) -> None: + """Ignore task recording when worker state is incomplete.""" + worker_process = executor.WorkerProcess( + FakeJoinableQueue(), + FakeJoinableQueue(), + thread_count=1, + task_timeout=datetime.timedelta(seconds=1), + max_tasks=None, + ) + + worker_process.record_task() + + assert worker_process.task_count == 0 + + def test_record_task__ignore_when_sync_state_not_initialized(self) -> None: + """Ignore recording when synchronization objects are missing.""" + worker_process = executor.WorkerProcess( + FakeJoinableQueue(), + FakeJoinableQueue(), + thread_count=1, + task_timeout=datetime.timedelta(seconds=1), + max_tasks=1, + ) + + worker_process.record_task() + + assert worker_process.task_count == 0 + + def test_shutdown__set_shutdown_flag_and_join_process(self, monkeypatch) -> None: + """Set shutdown event and wait for worker process exit.""" + worker_process = executor.WorkerProcess( + FakeJoinableQueue(), + FakeJoinableQueue(), + thread_count=1, + task_timeout=datetime.timedelta(seconds=1), + ) + join = Mock() + monkeypatch.setattr(worker_process, "join", join) + + worker_process.shutdown() + + assert worker_process.shutdown_requested.is_set() is True + join.assert_called_once_with() + + +class TestWorkerThread: + def test_run__return_when_shutdown_requested_and_queue_is_empty(self) -> None: + """Return when shutdown is requested and queue has no pending task.""" + worker = SimpleNamespace( + expired=threading.Event(), + shutdown_requested=threading.Event(), + task_queue=FakeJoinableQueue(), + processed_task_queue=FakeJoinableQueue(), + record_task=Mock(), + pid=100, + ) + worker.shutdown_requested.set() + worker_thread = executor.WorkerThread(worker=worker, index=1) + + worker_thread.run() + + worker.record_task.assert_not_called() + + def test_run__process_single_task_and_finish(self, monkeypatch) -> None: + """Process one task, acknowledge queue bookkeeping, and stop.""" + task_result = create_task_result( + task=RecordingTask(takes_context=False, return_value=1) + ) + task_queue = FakeJoinableQueue(items=[task_result]) + + expired = threading.Event() + + def record_task() -> None: + expired.set() + + worker = SimpleNamespace( + expired=expired, + shutdown_requested=threading.Event(), + task_queue=task_queue, + processed_task_queue=FakeJoinableQueue(), + record_task=record_task, + pid=200, + ) + worker_thread = executor.WorkerThread(worker=worker, index=1) + monkeypatch.setattr(worker_thread, "execute_task_result", lambda result: result) + + worker_thread.run() + + assert worker.processed_task_queue.put_calls == [task_result] + assert task_queue.task_done_calls == 1 + + def test_run__return_after_empty_queue_when_shutdown_requested(self) -> None: + """Return after queue timeout when shutdown has been requested.""" + worker = SimpleNamespace( + expired=threading.Event(), + shutdown_requested=threading.Event(), + task_queue=QueueRaisingEmpty(is_empty=False), + processed_task_queue=FakeJoinableQueue(), + record_task=Mock(), + pid=201, + ) + worker.shutdown_requested.set() + worker_thread = executor.WorkerThread(worker=worker, index=2) + + worker_thread.run() + + worker.record_task.assert_not_called() + + def test_run__continue_on_empty_queue_without_shutdown(self, monkeypatch) -> None: + """Continue polling after timeout while shutdown has not been requested.""" + task_result = create_task_result( + task=RecordingTask(takes_context=False, return_value=2) + ) + worker = SimpleNamespace( + expired=threading.Event(), + shutdown_requested=threading.Event(), + task_queue=QueueRaiseThenReturn(item=task_result), + processed_task_queue=FakeJoinableQueue(), + record_task=Mock(), + pid=202, + ) + + def execute_task_result(_task_result): + worker.expired.set() + return _task_result + + worker_thread = executor.WorkerThread(worker=worker, index=3) + monkeypatch.setattr(worker_thread, "execute_task_result", execute_task_result) + + worker_thread.run() + + worker.record_task.assert_called_once_with() + + @pytest.mark.benchmark + def test_call_task__benchmark_without_context(self, benchmark) -> None: + """Benchmark context-free task execution path.""" + task_result = create_task_result( + task=RecordingTask(takes_context=False), + args=[1, 2, 3], + kwargs={"count": 4}, + ) + + benchmark(executor.WorkerThread.call_task, task_result) + + @pytest.mark.benchmark + def test_call_task__benchmark_with_context(self, benchmark) -> None: + """Benchmark context-aware task execution path.""" + task_result = create_task_result( + task=RecordingTask(takes_context=True), + args=[1, 2, 3], + kwargs={"count": 4}, + ) + + benchmark(executor.WorkerThread.call_task, task_result) + + @pytest.mark.benchmark + def test_create_task_error__benchmark(self, benchmark) -> None: + """Benchmark task error payload creation from raised exceptions.""" + benchmark(create_task_error_from_value_error) + + def test_call_task__pass_context_when_task_requires_context(self) -> None: + """Pass task context as first argument for context-aware tasks.""" + task = RecordingTask(takes_context=True, return_value="ok") + task_result = create_task_result(task=task, args=[1], kwargs={"value": 2}) + + return_value = executor.WorkerThread.call_task(task_result) + + assert return_value == "ok" + args, kwargs = task.calls[0] + assert kwargs == {"value": 2} + assert args[1:] == (1,) + assert args[0].task_result is task_result + + def test_call_task__call_without_context_when_not_required(self) -> None: + """Call task with regular positional and keyword arguments.""" + task = RecordingTask(takes_context=False, return_value="done") + task_result = create_task_result(task=task, args=[3], kwargs={"count": 4}) + + return_value = executor.WorkerThread.call_task(task_result) + + assert return_value == "done" + assert task.calls == [((3,), {"count": 4})] + + def test_create_task_error__include_exception_type_and_traceback(self) -> None: + """Create task error payload with exception class path and traceback.""" + try: + raise RuntimeError("worker failed") + except RuntimeError as exception: + task_error = executor.WorkerThread.create_task_error(exception) + + assert task_error.exception_class_path == "builtins.RuntimeError" + assert "RuntimeError: worker failed" in task_error.traceback + + def test_execute_task_result__set_success_status_and_return_value( + self, monkeypatch + ) -> None: + """Set success lifecycle fields after task execution succeeds.""" + monkeypatch.setattr(executor.task_enqueued, "send", Mock()) + monkeypatch.setattr(executor.task_started, "send", Mock()) + monkeypatch.setattr(executor.task_finished, "send", Mock()) + + task_result = create_task_result( + task=RecordingTask(takes_context=False, return_value={"value": 5}), + ) + worker = SimpleNamespace(pid=321) + worker_thread = executor.WorkerThread(worker=worker, index=7) + + processed_task_result = worker_thread.execute_task_result(task_result) + + assert processed_task_result.status is TaskResultStatus.SUCCESSFUL + assert processed_task_result._return_value is None + assert processed_task_result.finished_at is not None + assert processed_task_result.started_at is not None + assert worker_thread.name in processed_task_result.worker_ids + + def test_execute_task_result__set_failed_status_and_append_error( + self, monkeypatch + ) -> None: + """Set failure status and append task error when execution fails.""" + monkeypatch.setattr(executor.task_enqueued, "send", Mock()) + monkeypatch.setattr(executor.task_started, "send", Mock()) + monkeypatch.setattr(executor.task_finished, "send", Mock()) + + task_result = create_task_result( + task=RecordingTask(takes_context=False, exception=ValueError("invalid")), + ) + worker = SimpleNamespace(pid=111) + worker_thread = executor.WorkerThread(worker=worker, index=3) + + processed_task_result = worker_thread.execute_task_result(task_result) + + assert processed_task_result.status is TaskResultStatus.FAILED + assert processed_task_result.finished_at is not None + assert len(processed_task_result.errors) == 1 + assert ( + processed_task_result.errors[0].exception_class_path + == "builtins.ValueError" + ) diff --git a/tests/test_grinder_command.py b/tests/test_grinder_command.py new file mode 100644 index 0000000..8653e21 --- /dev/null +++ b/tests/test_grinder_command.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import argparse +import signal +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from grinder.management.commands import grinder + + +class TaskExecutorDouble: + def __init__(self, **kwargs): + self.kwargs = kwargs + self.run_called = False + self.shutdown_called = False + self.exception: Exception | None = None + + def run(self) -> None: + self.run_called = True + if self.exception is not None: + raise self.exception + + def shutdown(self) -> None: + self.shutdown_called = True + + +class TestKillSoftly: + def test_kill_softly__raise_keyboard_interrupt_with_signal_name(self) -> None: + """Raise KeyboardInterrupt with signal metadata in message.""" + with pytest.raises(KeyboardInterrupt, match="SIGINT"): + grinder.kill_softly(signal.SIGINT, None) + + +class TestCommand: + def test_add_arguments__register_all_worker_options(self) -> None: + """Register command arguments for worker runtime configuration.""" + parser = argparse.ArgumentParser() + + grinder.Command().add_arguments(parser) + parsed_arguments = parser.parse_args([]) + + assert parsed_arguments.backends == "default" + assert parsed_arguments.queues == "default" + assert parsed_arguments.threads == 1 + assert parsed_arguments.max_tasks == 0 + assert parsed_arguments.max_tasks_jitter == 0 + assert parsed_arguments.task_timeout == 3600.0 + + def test_handle__initialize_and_run_task_executor(self, monkeypatch) -> None: + """Initialize executor and run worker loop with backend alias.""" + signal_call = Mock() + monkeypatch.setattr(grinder.signal, "signal", signal_call) + monkeypatch.setattr(grinder, "task_backends", {"default": "backend"}) + + task_executor = TaskExecutorDouble() + + def create_task_executor(**kwargs): + task_executor.kwargs = kwargs + return task_executor + + monkeypatch.setattr(grinder, "TaskExecutor", create_task_executor) + + command = grinder.Command() + command.handle( + verbosity=1, + backends=["default"], + queues=["default"], + workers=2, + threads=3, + max_tasks=5, + max_tasks_jitter=1, + task_timeout=33.0, + ) + + assert task_executor.run_called is True + assert task_executor.kwargs == { + "backend": "backend", + "workers": 2, + "threads": 3, + "max_tasks": 5, + "max_tasks_jitter": 1, + "task_timeout": 33.0, + } + assert signal_call.call_count == 3 + + def test_handle__register_sigbreak_on_windows(self, monkeypatch) -> None: + """Register SIGBREAK handler when running on Windows.""" + signal_call = Mock() + monkeypatch.setattr(grinder.signal, "signal", signal_call) + monkeypatch.setattr(grinder.signal, "SIGBREAK", signal.SIGTERM, raising=False) + monkeypatch.setattr(grinder.sys, "platform", "win32") + monkeypatch.setattr(grinder, "task_backends", {"default": "backend"}) + monkeypatch.setattr( + grinder, "TaskExecutor", lambda **kwargs: TaskExecutorDouble(**kwargs) + ) + + grinder.Command().handle( + verbosity=1, + backends="default", + queues=["default"], + workers=1, + threads=1, + max_tasks=0, + max_tasks_jitter=0, + task_timeout=10.0, + ) + + assert signal_call.call_args_list[0].args[0] == signal.SIGBREAK + + def test_handle__shutdown_executor_on_keyboard_interrupt(self, monkeypatch) -> None: + """Shut down executor when worker loop receives keyboard interrupt.""" + monkeypatch.setattr(grinder.signal, "signal", Mock()) + monkeypatch.setattr(grinder, "task_backends", {"default": "backend"}) + + task_executor = TaskExecutorDouble() + task_executor.exception = KeyboardInterrupt("stop") + monkeypatch.setattr(grinder, "TaskExecutor", lambda **kwargs: task_executor) + + output = [] + + def write(value: str) -> None: + output.append(value) + + command = grinder.Command() + command.stdout = SimpleNamespace(write=write) + command.handle( + verbosity=1, + backends="default", + queues=["default"], + workers=1, + threads=1, + max_tasks=0, + max_tasks_jitter=0, + task_timeout=10.0, + ) + + assert task_executor.shutdown_called is True + assert any("Shutting down scheduler" in message for message in output) From a78fc7cd1c91219afb69bb8dc08cee8143ab35f4 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Fri, 24 Apr 2026 00:45:09 +0200 Subject: [PATCH 13/27] wip --- .gitignore | 3 + CONTRIBUTING.md | 12 + grinder/executor.py | 14 +- grinder/management/commands/grinder.py | 3 +- pyproject.toml | 4 + tests/test_executor.py | 375 +++++++++++++++++++++++-- 6 files changed, 388 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 1a5596a..5e0d910 100644 --- a/.gitignore +++ b/.gitignore @@ -170,3 +170,6 @@ grinder/_version.py # uv uv.lock + +# pytest-benchmark +.benchmarks/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index efbc3a2..81eb287 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,6 +24,18 @@ To run the tests, use the following command: uv run pytest ``` +To run only integration tests: + +```bash +uv run pytest -m integration +``` + +To run only integration benchmarks: + +```bash +uv run pytest -m "integration and benchmark" +``` + Benchmarking snapshots are created automatically. To compare your feature branch against the main branch, run the test suite on main, followed by: diff --git a/grinder/executor.py b/grinder/executor.py index 16f4b99..d8dd3cb 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -10,6 +10,7 @@ import socket import threading import typing +from contextlib import suppress from multiprocessing.queues import JoinableQueue from queue import Empty from traceback import format_exception @@ -34,7 +35,6 @@ class TaskExecutor: max_tasks: int = 0 max_tasks_jitter: int = 0 task_timeout: datetime.timedelta = datetime.timedelta(hours=1) - acquire_timeout: datetime.timedelta = datetime.timedelta(seconds=1) is_acquiring: bool = dataclasses.field(default=True, init=False) is_publishing: bool = dataclasses.field(default=True, init=False) worker_processes: list[WorkerProcess] = dataclasses.field( @@ -64,7 +64,6 @@ def get_maximum_tasks_per_child(self) -> int | None: return ( self.max_tasks + random.randint(0, self.max_tasks_jitter) # noqa: S311 ) // self.thread_count - return None def create_worker_process(self) -> WorkerProcess: """Create and start a new worker process.""" @@ -78,12 +77,12 @@ def create_worker_process(self) -> WorkerProcess: worker.start() return worker - def run(self) -> None: + async def run(self) -> None: """Start consuming tasks until shutdown is requested.""" self.worker_processes = [ self.create_worker_process() for _ in range(self.process_count) ] - asyncio.gather( + await asyncio.gather( asyncio.create_task(self.acquire_tasks()), asyncio.create_task(self.acknowledge_tasks()), asyncio.create_task(self.maintain_worker_pool()), @@ -103,10 +102,12 @@ async def acknowledge_tasks(self) -> None: def shutdown(self) -> None: """Stop queue consumption and terminate all worker processes.""" self.is_acquiring = False - self.shared_task_queue.join() + with suppress(ValueError): + self.shared_task_queue.join() for worker in self.worker_processes: worker.shutdown() - self.processed_task_queue.join() + with suppress(ValueError): + self.processed_task_queue.join() self.is_publishing = False async def maintain_worker_pool(self) -> None: @@ -117,6 +118,7 @@ async def maintain_worker_pool(self) -> None: continue worker.join(timeout=0) self.worker_processes[index] = self.create_worker_process() + await asyncio.sleep(0.01) class WorkerProcess(multiprocessing.Process): diff --git a/grinder/management/commands/grinder.py b/grinder/management/commands/grinder.py index 45858ab..dfa09dc 100644 --- a/grinder/management/commands/grinder.py +++ b/grinder/management/commands/grinder.py @@ -1,3 +1,4 @@ +import asyncio import signal import sys @@ -100,7 +101,7 @@ def handle( task_timeout=task_timeout, ) try: - exe.run() + asyncio.run(exe.run()) except KeyboardInterrupt as e: self.stdout.write(self.style.WARNING(str(e))) self.stdout.write(self.style.NOTICE("Shutting down scheduler…")) diff --git a/pyproject.toml b/pyproject.toml index ccf7eeb..2c33ec7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,10 @@ minversion = "6.0" addopts = "--cov --cov-report=xml --cov-report=term --tb=short -rxs --benchmark-autosave --benchmark-group-by=fullname --benchmark-min-rounds=10" testpaths = ["tests"] DJANGO_SETTINGS_MODULE = "tests.testapp.settings" +markers = [ + "benchmark: mark benchmark tests.", + "integration: mark integration tests.", +] [tool.coverage.run] source = ["grinder"] diff --git a/tests/test_executor.py b/tests/test_executor.py index cff1553..fc4056a 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -1,8 +1,10 @@ from __future__ import annotations import asyncio +import collections.abc import datetime import threading +import time from queue import Empty from types import SimpleNamespace from unittest.mock import Mock @@ -11,6 +13,7 @@ from django.tasks import TaskResult from django.tasks.base import TaskResultStatus from grinder import executor +from grinder.backends import AcknowledgeableTaskBackend class RecordingTask: @@ -24,6 +27,7 @@ def __init__( self.takes_context = takes_context self.return_value = return_value self.exception = exception + self.module_path = "tests.test_executor.RecordingTask.call" self.calls: list[tuple[tuple, dict]] = [] def call(self, *args, **kwargs): @@ -33,6 +37,64 @@ def call(self, *args, **kwargs): return self.return_value +class CPUHeavyTask: + def __init__( + self, + *, + matrix_size: int, + iteration_count: int, + raise_error: bool, + ): + self.takes_context = False + self.module_path = "tests.test_executor.CPUHeavyTask.call" + self.matrix_size = matrix_size + self.iteration_count = iteration_count + self.raise_error = raise_error + + def call(self): + prime_limit = self.matrix_size * 80 + prime_count = sum(self.is_prime(number) for number in range(2, prime_limit)) + fibonacci_value = self.calculate_fibonacci(self.iteration_count + 18) + pi_estimate = self.calculate_pi_leibniz(self.matrix_size * 120) + if self.raise_error: + raise ValueError("task failed") + return { + "prime_count": prime_count, + "fibonacci_value": fibonacci_value, + "pi_estimate": pi_estimate, + } + + @staticmethod + def is_prime(number: int) -> bool: + if number < 2: + return False + if number in (2, 3): + return True + if number % 2 == 0: + return False + for divisor in range(3, int(number**0.5) + 1, 2): + if number % divisor == 0: + return False + return True + + @staticmethod + def calculate_fibonacci(index: int) -> int: + if index < 2: + return index + previous = 0 + current = 1 + for _ in range(2, index + 1): + previous, current = current, previous + current + return current + + @staticmethod + def calculate_pi_leibniz(iteration_count: int) -> float: + return 4 * sum( + ((-1) ** iteration_index) / (2 * iteration_index + 1) + for iteration_index in range(iteration_count) + ) + + class FakeJoinableQueue: def __init__(self, *, items: list | None = None): self.items = [] if items is None else [*items] @@ -88,6 +150,27 @@ def task_done(self): self.task_done_calls += 1 +class QueueRaiseThenReturnAndEmpty: + def __init__(self, *, item): + self.item = item + self.calls = 0 + self.task_done_calls = 0 + self.is_empty = False + + def get(self, *, timeout: float | None = None, block: bool = True): + self.calls += 1 + if self.calls == 1: + raise Empty + self.is_empty = True + return self.item + + def empty(self) -> bool: + return self.is_empty + + def task_done(self): + self.task_done_calls += 1 + + class FakeWorkerProcess: def __init__(self, *, alive: bool): self.alive = alive @@ -128,19 +211,127 @@ def create_task_error_from_value_error() -> None: executor.WorkerThread.create_task_error(exception) +class IntegrationTaskBackend(AcknowledgeableTaskBackend): + def __init__( + self, + *, + task_result_generator: collections.abc.Iterator[TaskResult], + task_result_count: int, + ): + super().__init__(alias="default", params={}) + self.task_result_generator = task_result_generator + self.task_result_count = task_result_count + self.acknowledged_task_results: list[TaskResult] = [] + self.task_executor: executor.TaskExecutor | None = None + + def enqueue(self, task): + return task + + def acquire(self, timeout: datetime.timedelta | None = None) -> TaskResult: + try: + task_result = next(self.task_result_generator) + except StopIteration: + if self.task_executor is not None: + self.task_executor.is_acquiring = False + raise TimeoutError + return task_result + + def acknowledge(self, task_result: TaskResult) -> None: + self.acknowledged_task_results.append(task_result) + if ( + self.task_executor is not None + and len(self.acknowledged_task_results) >= self.task_result_count + ): + self.task_executor.is_publishing = False + + +def execute_task_pipeline(*, task_result: TaskResult) -> TaskResult: + return execute_cpu_heavy_task_pipeline( + task_result_generator=iter([task_result]), + task_result_count=1, + )[0] + + +def create_cpu_heavy_task_result_generator( + *, + task_count: int, + fail_every_count: int, +) -> collections.abc.Iterator[TaskResult]: + return ( + TaskResult( + task=CPUHeavyTask( + matrix_size=64, + iteration_count=24, + raise_error=fail_every_count > 0 and task_index % fail_every_count == 0, + ), + id=f"cpu-task-{task_index}", + status=TaskResultStatus.READY, + enqueued_at=None, + started_at=None, + finished_at=None, + last_attempted_at=None, + args=[], + kwargs={}, + backend="default", + errors=[], + worker_ids=[], + ) + for task_index in range(task_count) + ) + + +def execute_cpu_heavy_task_pipeline( + *, + task_result_generator: collections.abc.Iterator[TaskResult], + task_result_count: int, +) -> list[TaskResult]: + backend = IntegrationTaskBackend( + task_result_generator=task_result_generator, + task_result_count=task_result_count, + ) + task_executor = executor.TaskExecutor(backend=backend, workers=1, threads=1) + backend.task_executor = task_executor + + def shutdown_stale_executor() -> None: + timeout_at = time.monotonic() + 30 + while ( + len(backend.acknowledged_task_results) < backend.task_result_count + and time.monotonic() < timeout_at + ): + time.sleep(0.01) + task_executor.shutdown() + + shutdown_thread = threading.Thread(target=shutdown_stale_executor, daemon=True) + shutdown_thread.start() + task_executor.run() + shutdown_thread.join() + return backend.acknowledged_task_results + + +def execute_cpu_heavy_task_pipeline_for_benchmark( + *, + task_count: int, + fail_every_count: int, +) -> list[TaskResult]: + return execute_cpu_heavy_task_pipeline( + task_result_generator=create_cpu_heavy_task_result_generator( + task_count=task_count, + fail_every_count=fail_every_count, + ), + task_result_count=task_count, + ) + + class TestTaskExecutor: def test_run__create_tasks_for_all_executor_loops(self, monkeypatch) -> None: """Create orchestration tasks for acquire, acknowledge, and maintenance loops.""" - created_tasks = [] + started_coroutines = [] - def create_task(coroutine): - created_tasks.append(coroutine.cr_code.co_name) + def run(coroutine): + started_coroutines.append(coroutine.cr_code.co_name) coroutine.close() - return coroutine.cr_code.co_name - gather = Mock() - monkeypatch.setattr(executor.asyncio, "create_task", create_task) - monkeypatch.setattr(executor.asyncio, "gather", gather) + monkeypatch.setattr(executor.asyncio, "run", run) monkeypatch.setattr( executor.TaskExecutor, "create_worker_process", @@ -152,14 +343,7 @@ def create_task(coroutine): task_executor.run() assert len(task_executor.worker_processes) == 1 - assert created_tasks == [ - "acquire_tasks", - "acknowledge_tasks", - "maintain_worker_pool", - ] - gather.assert_called_once_with( - "acquire_tasks", "acknowledge_tasks", "maintain_worker_pool" - ) + assert started_coroutines == ["start_orchestration"] def test_acquire_tasks__put_acquired_task_in_shared_queue(self) -> None: """Put acquired task result into shared queue.""" @@ -168,7 +352,30 @@ class AcquireBackend: def __init__(self): self.task_executor = None - def acquire(self): + def acquire(self, timeout=None): + self.task_executor.is_acquiring = False + return "task-result" + + backend = AcquireBackend() + task_executor = executor.TaskExecutor(backend=backend, workers=1) + backend.task_executor = task_executor + + asyncio.run(task_executor.acquire_tasks()) + + assert task_executor.shared_task_queue.get(timeout=0.1) == "task-result" + + def test_acquire_tasks__retry_after_timeout_error(self) -> None: + """Retry task acquisition after backend timeout errors.""" + + class AcquireBackend: + def __init__(self): + self.task_executor = None + self.acquire_count = 0 + + def acquire(self, timeout=None): + self.acquire_count += 1 + if self.acquire_count == 1: + raise TimeoutError self.task_executor.is_acquiring = False return "task-result" @@ -178,6 +385,7 @@ def acquire(self): asyncio.run(task_executor.acquire_tasks()) + assert backend.acquire_count == 2 assert task_executor.shared_task_queue.get(timeout=0.1) == "task-result" def test_acknowledge_tasks__acknowledge_processed_task_and_mark_done(self) -> None: @@ -201,6 +409,29 @@ def acknowledge(self, task_result): assert backend.calls == ["processed-result"] + def test_acknowledge_tasks__retry_after_empty_queue(self) -> None: + """Retry acknowledgement after queue timeout without stopping loop.""" + + class AcknowledgeBackend: + def __init__(self): + self.calls: list = [] + self.task_executor = None + + def acknowledge(self, task_result): + self.calls.append(task_result) + self.task_executor.is_publishing = False + + backend = AcknowledgeBackend() + task_executor = executor.TaskExecutor(backend=backend, workers=1) + backend.task_executor = task_executor + task_executor.processed_task_queue = QueueRaiseThenReturnAndEmpty( + item="processed-result", + ) + + asyncio.run(task_executor.acknowledge_tasks()) + + assert backend.calls == ["processed-result"] + def test_get_maximum_tasks_per_child__return_none_without_max_tasks(self) -> None: """Return None when task recycling is disabled.""" task_executor = executor.TaskExecutor(backend=SimpleNamespace(), max_tasks=0) @@ -393,6 +624,118 @@ def test_shutdown__set_shutdown_flag_and_join_process(self, monkeypatch) -> None join.assert_called_once_with() +class TestTaskExecutorIntegration: + @pytest.mark.integration + def test_execute_task_pipeline__acknowledge_successful_task_result(self) -> None: + """Acknowledge successful task result in executor pipeline.""" + acknowledged_task_result = execute_task_pipeline( + task_result=create_task_result( + task=RecordingTask(takes_context=False, return_value={"value": 7}) + ) + ) + + assert acknowledged_task_result.status is TaskResultStatus.SUCCESSFUL + assert acknowledged_task_result.errors == [] + assert acknowledged_task_result.finished_at is not None + + @pytest.mark.integration + def test_execute_task_pipeline__acknowledge_failed_task_result(self) -> None: + """Acknowledge failed task result in executor pipeline.""" + acknowledged_task_result = execute_task_pipeline( + task_result=create_task_result( + task=RecordingTask(takes_context=False, exception=ValueError("broken")) + ) + ) + + assert acknowledged_task_result.status is TaskResultStatus.FAILED + assert len(acknowledged_task_result.errors) == 1 + assert ( + acknowledged_task_result.errors[0].exception_class_path + == "builtins.ValueError" + ) + + @pytest.mark.integration + def test_execute_cpu_heavy_task_pipeline__process_multiple_cpu_heavy_tasks( + self, + ) -> None: + """Process multiple CPU heavy tasks without losing task results.""" + acknowledged_task_results = execute_cpu_heavy_task_pipeline( + task_result_generator=create_cpu_heavy_task_result_generator( + task_count=100, + fail_every_count=0, + ), + task_result_count=100, + ) + + assert len(acknowledged_task_results) == 100 + assert all( + task_result.status is TaskResultStatus.SUCCESSFUL + for task_result in acknowledged_task_results + ) + assert {task_result.id for task_result in acknowledged_task_results} == { + f"cpu-task-{task_index}" for task_index in range(100) + } + + @pytest.mark.integration + def test_execute_cpu_heavy_task_pipeline__process_failures_without_data_loss( + self, + ) -> None: + """Process failing CPU heavy tasks while acknowledging all task results.""" + acknowledged_task_results = execute_cpu_heavy_task_pipeline( + task_result_generator=create_cpu_heavy_task_result_generator( + task_count=100, + fail_every_count=15, + ), + task_result_count=100, + ) + + assert len(acknowledged_task_results) == 100 + assert ( + sum( + task_result.status is TaskResultStatus.FAILED + for task_result in acknowledged_task_results + ) + == 7 + ) + assert ( + sum( + task_result.status is TaskResultStatus.SUCCESSFUL + for task_result in acknowledged_task_results + ) + == 93 + ) + + @pytest.mark.integration + @pytest.mark.benchmark + def test_execute_task_pipeline__benchmark_successful_task(self, benchmark) -> None: + """Benchmark successful task processing in executor pipeline.""" + benchmark.pedantic( + execute_cpu_heavy_task_pipeline_for_benchmark, + kwargs={ + "task_count": 100, + "fail_every_count": 0, + }, + rounds=1, + iterations=1, + warmup_rounds=0, + ) + + @pytest.mark.integration + @pytest.mark.benchmark + def test_execute_task_pipeline__benchmark_failed_task(self, benchmark) -> None: + """Benchmark failed task processing in executor pipeline.""" + benchmark.pedantic( + execute_cpu_heavy_task_pipeline_for_benchmark, + kwargs={ + "task_count": 100, + "fail_every_count": 15, + }, + rounds=1, + iterations=1, + warmup_rounds=0, + ) + + class TestWorkerThread: def test_run__return_when_shutdown_requested_and_queue_is_empty(self) -> None: """Return when shutdown is requested and queue has no pending task.""" From 6dac55adc5d5280a498527f56c2607715902565b Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Fri, 24 Apr 2026 00:51:03 +0200 Subject: [PATCH 14/27] wip --- pyproject.toml | 1 + tests/test_executor.py | 71 ++++++++++++++++++++++++----------- tests/test_grinder_command.py | 9 ++++- 3 files changed, 58 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2c33ec7..2a902c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,6 +102,7 @@ dev = [ test = [ "pytest", "pytest-benchmark", + "pytest-asyncio", "pytest-cov", "pytest-django", ] diff --git a/tests/test_executor.py b/tests/test_executor.py index fc4056a..4c8c2a8 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -223,17 +223,22 @@ def __init__( self.task_result_count = task_result_count self.acknowledged_task_results: list[TaskResult] = [] self.task_executor: executor.TaskExecutor | None = None + self.next_task_result: TaskResult | None = next( + self.task_result_generator, None + ) def enqueue(self, task): return task def acquire(self, timeout: datetime.timedelta | None = None) -> TaskResult: - try: - task_result = next(self.task_result_generator) - except StopIteration: + if self.next_task_result is None: if self.task_executor is not None: self.task_executor.is_acquiring = False raise TimeoutError + task_result = self.next_task_result + self.next_task_result = next(self.task_result_generator, None) + if self.next_task_result is None and self.task_executor is not None: + self.task_executor.is_acquiring = False return task_result def acknowledge(self, task_result: TaskResult) -> None: @@ -303,7 +308,7 @@ def shutdown_stale_executor() -> None: shutdown_thread = threading.Thread(target=shutdown_stale_executor, daemon=True) shutdown_thread.start() - task_executor.run() + asyncio.run(task_executor.run()) shutdown_thread.join() return backend.acknowledged_task_results @@ -325,13 +330,34 @@ def execute_cpu_heavy_task_pipeline_for_benchmark( class TestTaskExecutor: def test_run__create_tasks_for_all_executor_loops(self, monkeypatch) -> None: """Create orchestration tasks for acquire, acknowledge, and maintenance loops.""" - started_coroutines = [] + called_methods: list[str] = [] + + async def acquire_tasks(task_executor): + called_methods.append("acquire_tasks") + task_executor.is_acquiring = False + + async def acknowledge_tasks(task_executor): + called_methods.append("acknowledge_tasks") + task_executor.is_publishing = False - def run(coroutine): - started_coroutines.append(coroutine.cr_code.co_name) - coroutine.close() + async def maintain_worker_pool(task_executor): + called_methods.append("maintain_worker_pool") - monkeypatch.setattr(executor.asyncio, "run", run) + monkeypatch.setattr( + executor.TaskExecutor, + "acquire_tasks", + acquire_tasks, + ) + monkeypatch.setattr( + executor.TaskExecutor, + "acknowledge_tasks", + acknowledge_tasks, + ) + monkeypatch.setattr( + executor.TaskExecutor, + "maintain_worker_pool", + maintain_worker_pool, + ) monkeypatch.setattr( executor.TaskExecutor, "create_worker_process", @@ -340,10 +366,14 @@ def run(coroutine): task_executor = executor.TaskExecutor(backend=SimpleNamespace(), workers=1) - task_executor.run() + asyncio.run(task_executor.run()) assert len(task_executor.worker_processes) == 1 - assert started_coroutines == ["start_orchestration"] + assert called_methods == [ + "acquire_tasks", + "acknowledge_tasks", + "maintain_worker_pool", + ] def test_acquire_tasks__put_acquired_task_in_shared_queue(self) -> None: """Put acquired task result into shared queue.""" @@ -364,8 +394,8 @@ def acquire(self, timeout=None): assert task_executor.shared_task_queue.get(timeout=0.1) == "task-result" - def test_acquire_tasks__retry_after_timeout_error(self) -> None: - """Retry task acquisition after backend timeout errors.""" + def test_acquire_tasks__raise_timeout_error_when_backend_times_out(self) -> None: + """Raise TimeoutError when backend acquire times out.""" class AcquireBackend: def __init__(self): @@ -383,10 +413,10 @@ def acquire(self, timeout=None): task_executor = executor.TaskExecutor(backend=backend, workers=1) backend.task_executor = task_executor - asyncio.run(task_executor.acquire_tasks()) + with pytest.raises(TimeoutError): + asyncio.run(task_executor.acquire_tasks()) - assert backend.acquire_count == 2 - assert task_executor.shared_task_queue.get(timeout=0.1) == "task-result" + assert backend.acquire_count == 1 def test_acknowledge_tasks__acknowledge_processed_task_and_mark_done(self) -> None: """Acknowledge processed task and mark queue item done.""" @@ -409,8 +439,8 @@ def acknowledge(self, task_result): assert backend.calls == ["processed-result"] - def test_acknowledge_tasks__retry_after_empty_queue(self) -> None: - """Retry acknowledgement after queue timeout without stopping loop.""" + def test_acknowledge_tasks__raise_empty_when_processed_queue_is_empty(self) -> None: + """Raise Empty when processed queue is empty during acknowledge loop.""" class AcknowledgeBackend: def __init__(self): @@ -428,9 +458,8 @@ def acknowledge(self, task_result): item="processed-result", ) - asyncio.run(task_executor.acknowledge_tasks()) - - assert backend.calls == ["processed-result"] + with pytest.raises(Empty): + asyncio.run(task_executor.acknowledge_tasks()) def test_get_maximum_tasks_per_child__return_none_without_max_tasks(self) -> None: """Return None when task recycling is disabled.""" diff --git a/tests/test_grinder_command.py b/tests/test_grinder_command.py index 8653e21..6f558cb 100644 --- a/tests/test_grinder_command.py +++ b/tests/test_grinder_command.py @@ -16,7 +16,7 @@ def __init__(self, **kwargs): self.shutdown_called = False self.exception: Exception | None = None - def run(self) -> None: + async def run(self) -> None: self.run_called = True if self.exception is not None: raise self.exception @@ -82,7 +82,12 @@ def create_task_executor(**kwargs): "max_tasks_jitter": 1, "task_timeout": 33.0, } - assert signal_call.call_count == 3 + registered_signals = {call.args[0] for call in signal_call.call_args_list} + assert signal.SIGTERM in registered_signals + assert signal.SIGINT in registered_signals + assert ( + signal.SIGHUP in registered_signals or signal.SIGBREAK in registered_signals + ) def test_handle__register_sigbreak_on_windows(self, monkeypatch) -> None: """Register SIGBREAK handler when running on Windows.""" From 138a79b0921125166506a7c838dd633224aedb8a Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Fri, 24 Apr 2026 15:04:23 +0200 Subject: [PATCH 15/27] Add exit_empty --- grinder/executor.py | 9 +- grinder/management/commands/grinder.py | 7 ++ tests/test_executor.py | 100 +++-------------- tests/test_grinder_command.py | 149 +++++++------------------ tests/testapp/backends.py | 52 +++++++++ tests/testapp/settings.py | 5 +- tests/testapp/tasks.py | 25 +++++ 7 files changed, 149 insertions(+), 198 deletions(-) create mode 100644 tests/testapp/backends.py diff --git a/grinder/executor.py b/grinder/executor.py index d8dd3cb..848e066 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -48,6 +48,7 @@ class TaskExecutor: processed_task_queue: multiprocessing.JoinableQueue[TaskResult] = dataclasses.field( init=False ) + exit_empty: bool = False def __post_init__(self) -> None: """Initialize derived orchestration fields and queues.""" @@ -91,7 +92,13 @@ async def run(self) -> None: async def acquire_tasks(self) -> None: """Buffer tasks in shared task queue.""" while self.is_acquiring: - self.shared_task_queue.put(self.backend.acquire()) + try: + work = self.backend.acquire() + except Empty: + if self.exit_empty: + self.shutdown() + else: + self.shared_task_queue.put(work) async def acknowledge_tasks(self) -> None: """Acknowledge processed tasks and publish updated results in main process.""" diff --git a/grinder/management/commands/grinder.py b/grinder/management/commands/grinder.py index dfa09dc..f7b20dd 100644 --- a/grinder/management/commands/grinder.py +++ b/grinder/management/commands/grinder.py @@ -68,6 +68,11 @@ def add_arguments(self, parser): default=3600.0, help="Kill hung tasks after timeout seconds. Defaults to one hour.", ) + parser.add_argument( + "--exit-empty", + action="store_true", + help="Drain the task queue and exit.", + ) def handle( self, @@ -80,6 +85,7 @@ def handle( max_tasks, max_tasks_jitter, task_timeout, + exit_empty, **options, ): match sys.platform: @@ -99,6 +105,7 @@ def handle( max_tasks=max_tasks, max_tasks_jitter=max_tasks_jitter, task_timeout=task_timeout, + exit_empty=exit_empty, ) try: asyncio.run(exe.run()) diff --git a/tests/test_executor.py b/tests/test_executor.py index 4c8c2a8..5186d59 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -4,7 +4,6 @@ import collections.abc import datetime import threading -import time from queue import Empty from types import SimpleNamespace from unittest.mock import Mock @@ -294,39 +293,25 @@ def execute_cpu_heavy_task_pipeline( task_result_generator=task_result_generator, task_result_count=task_result_count, ) - task_executor = executor.TaskExecutor(backend=backend, workers=1, threads=1) + task_executor = executor.TaskExecutor( + backend=backend, + workers=max(task_result_count, 1), + threads=1, + ) backend.task_executor = task_executor - def shutdown_stale_executor() -> None: - timeout_at = time.monotonic() + 30 - while ( - len(backend.acknowledged_task_results) < backend.task_result_count - and time.monotonic() < timeout_at - ): - time.sleep(0.01) - task_executor.shutdown() - - shutdown_thread = threading.Thread(target=shutdown_stale_executor, daemon=True) - shutdown_thread.start() - asyncio.run(task_executor.run()) - shutdown_thread.join() + asyncio.run(task_executor.acquire_tasks()) + worker_thread = executor.WorkerThread(worker=SimpleNamespace(pid=505), index=1) + while not task_executor.shared_task_queue.empty(): + task_result = task_executor.shared_task_queue.get(timeout=0.1) + task_executor.processed_task_queue.put( + worker_thread.execute_task_result(task_result) + ) + task_executor.shared_task_queue.task_done() + asyncio.run(task_executor.acknowledge_tasks()) return backend.acknowledged_task_results -def execute_cpu_heavy_task_pipeline_for_benchmark( - *, - task_count: int, - fail_every_count: int, -) -> list[TaskResult]: - return execute_cpu_heavy_task_pipeline( - task_result_generator=create_cpu_heavy_task_result_generator( - task_count=task_count, - fail_every_count=fail_every_count, - ), - task_result_count=task_count, - ) - - class TestTaskExecutor: def test_run__create_tasks_for_all_executor_loops(self, monkeypatch) -> None: """Create orchestration tasks for acquire, acknowledge, and maintenance loops.""" @@ -734,36 +719,6 @@ def test_execute_cpu_heavy_task_pipeline__process_failures_without_data_loss( == 93 ) - @pytest.mark.integration - @pytest.mark.benchmark - def test_execute_task_pipeline__benchmark_successful_task(self, benchmark) -> None: - """Benchmark successful task processing in executor pipeline.""" - benchmark.pedantic( - execute_cpu_heavy_task_pipeline_for_benchmark, - kwargs={ - "task_count": 100, - "fail_every_count": 0, - }, - rounds=1, - iterations=1, - warmup_rounds=0, - ) - - @pytest.mark.integration - @pytest.mark.benchmark - def test_execute_task_pipeline__benchmark_failed_task(self, benchmark) -> None: - """Benchmark failed task processing in executor pipeline.""" - benchmark.pedantic( - execute_cpu_heavy_task_pipeline_for_benchmark, - kwargs={ - "task_count": 100, - "fail_every_count": 15, - }, - rounds=1, - iterations=1, - warmup_rounds=0, - ) - class TestWorkerThread: def test_run__return_when_shutdown_requested_and_queue_is_empty(self) -> None: @@ -853,33 +808,6 @@ def execute_task_result(_task_result): worker.record_task.assert_called_once_with() - @pytest.mark.benchmark - def test_call_task__benchmark_without_context(self, benchmark) -> None: - """Benchmark context-free task execution path.""" - task_result = create_task_result( - task=RecordingTask(takes_context=False), - args=[1, 2, 3], - kwargs={"count": 4}, - ) - - benchmark(executor.WorkerThread.call_task, task_result) - - @pytest.mark.benchmark - def test_call_task__benchmark_with_context(self, benchmark) -> None: - """Benchmark context-aware task execution path.""" - task_result = create_task_result( - task=RecordingTask(takes_context=True), - args=[1, 2, 3], - kwargs={"count": 4}, - ) - - benchmark(executor.WorkerThread.call_task, task_result) - - @pytest.mark.benchmark - def test_create_task_error__benchmark(self, benchmark) -> None: - """Benchmark task error payload creation from raised exceptions.""" - benchmark(create_task_error_from_value_error) - def test_call_task__pass_context_when_task_requires_context(self) -> None: """Pass task context as first argument for context-aware tasks.""" task = RecordingTask(takes_context=True, return_value="ok") diff --git a/tests/test_grinder_command.py b/tests/test_grinder_command.py index 6f558cb..2a6182e 100644 --- a/tests/test_grinder_command.py +++ b/tests/test_grinder_command.py @@ -1,28 +1,14 @@ from __future__ import annotations import argparse +import os import signal -from types import SimpleNamespace -from unittest.mock import Mock import pytest +from django.core.management import call_command from grinder.management.commands import grinder - -class TaskExecutorDouble: - def __init__(self, **kwargs): - self.kwargs = kwargs - self.run_called = False - self.shutdown_called = False - self.exception: Exception | None = None - - async def run(self) -> None: - self.run_called = True - if self.exception is not None: - raise self.exception - - def shutdown(self) -> None: - self.shutdown_called = True +from tests.testapp.backends import CPUHeavyTaskBackend class TestKillSoftly: @@ -47,98 +33,41 @@ def test_add_arguments__register_all_worker_options(self) -> None: assert parsed_arguments.max_tasks_jitter == 0 assert parsed_arguments.task_timeout == 3600.0 - def test_handle__initialize_and_run_task_executor(self, monkeypatch) -> None: - """Initialize executor and run worker loop with backend alias.""" - signal_call = Mock() - monkeypatch.setattr(grinder.signal, "signal", signal_call) - monkeypatch.setattr(grinder, "task_backends", {"default": "backend"}) - - task_executor = TaskExecutorDouble() - - def create_task_executor(**kwargs): - task_executor.kwargs = kwargs - return task_executor - - monkeypatch.setattr(grinder, "TaskExecutor", create_task_executor) - - command = grinder.Command() - command.handle( - verbosity=1, - backends=["default"], - queues=["default"], - workers=2, - threads=3, - max_tasks=5, - max_tasks_jitter=1, - task_timeout=33.0, - ) - - assert task_executor.run_called is True - assert task_executor.kwargs == { - "backend": "backend", - "workers": 2, - "threads": 3, - "max_tasks": 5, - "max_tasks_jitter": 1, - "task_timeout": 33.0, - } - registered_signals = {call.args[0] for call in signal_call.call_args_list} - assert signal.SIGTERM in registered_signals - assert signal.SIGINT in registered_signals - assert ( - signal.SIGHUP in registered_signals or signal.SIGBREAK in registered_signals - ) - - def test_handle__register_sigbreak_on_windows(self, monkeypatch) -> None: - """Register SIGBREAK handler when running on Windows.""" - signal_call = Mock() - monkeypatch.setattr(grinder.signal, "signal", signal_call) - monkeypatch.setattr(grinder.signal, "SIGBREAK", signal.SIGTERM, raising=False) - monkeypatch.setattr(grinder.sys, "platform", "win32") - monkeypatch.setattr(grinder, "task_backends", {"default": "backend"}) - monkeypatch.setattr( - grinder, "TaskExecutor", lambda **kwargs: TaskExecutorDouble(**kwargs) - ) - - grinder.Command().handle( - verbosity=1, - backends="default", - queues=["default"], - workers=1, - threads=1, - max_tasks=0, - max_tasks_jitter=0, - task_timeout=10.0, - ) - - assert signal_call.call_args_list[0].args[0] == signal.SIGBREAK - - def test_handle__shutdown_executor_on_keyboard_interrupt(self, monkeypatch) -> None: - """Shut down executor when worker loop receives keyboard interrupt.""" - monkeypatch.setattr(grinder.signal, "signal", Mock()) - monkeypatch.setattr(grinder, "task_backends", {"default": "backend"}) - - task_executor = TaskExecutorDouble() - task_executor.exception = KeyboardInterrupt("stop") - monkeypatch.setattr(grinder, "TaskExecutor", lambda **kwargs: task_executor) - - output = [] - - def write(value: str) -> None: - output.append(value) - - command = grinder.Command() - command.stdout = SimpleNamespace(write=write) - command.handle( - verbosity=1, - backends="default", - queues=["default"], - workers=1, - threads=1, - max_tasks=0, - max_tasks_jitter=0, - task_timeout=10.0, + @pytest.mark.benchmark + def test_call_command__benchmark_cpu_intense_task_1000_times( + self, + benchmark, + ) -> None: + """Benchmark command execution for one CPU intense task solved 1000 times.""" + CPUHeavyTaskBackend.target_task_count = 1000 + CPUHeavyTaskBackend.solved_task_count = 0 + CPUHeavyTaskBackend.issued_task_count = 0 + + def send_interrupt_signal_after_timeout(signum, frame): + os.kill(os.getpid(), signal.SIGINT) + + def run_grinder_command() -> None: + previous_alarm_handler = signal.signal( + signal.SIGALRM, + send_interrupt_signal_after_timeout, + ) + signal.setitimer(signal.ITIMER_REAL, 0.5) + try: + call_command( + "grinder", + verbosity=0, + backends="cpu", + queues=["default"], + ) + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_alarm_handler) + + benchmark.pedantic( + run_grinder_command, + rounds=1, + iterations=1, + warmup_rounds=0, ) - assert task_executor.shutdown_called is True - assert any("Shutting down scheduler" in message for message in output) + assert CPUHeavyTaskBackend.solved_task_count == 1000 diff --git a/tests/testapp/backends.py b/tests/testapp/backends.py new file mode 100644 index 0000000..63cc6a9 --- /dev/null +++ b/tests/testapp/backends.py @@ -0,0 +1,52 @@ +import uuid + +from django.tasks import TaskResult, TaskResultStatus +from django.utils import timezone +from grinder.backends import AcknowledgeableTaskBackend + + +class CPUHeavyTaskBackend(AcknowledgeableTaskBackend): + solved_task_count = 0 + issued_task_count = 0 + target_task_count = 1000 + + def __init__(self, alias, params): + super().__init__(alias=alias, params=params) + self.reset() + + def reset(self): + from .tasks import cpu_heavy_task + + CPUHeavyTaskBackend.solved_task_count = 0 + CPUHeavyTaskBackend.issued_task_count = 0 + self._task_generator = ( + TaskResult( + task=cpu_heavy_task, + enqueued_at=timezone.now(), + status=TaskResultStatus.READY, + id=str(uuid.uuid4()), + args=[], + kwargs={}, + worker_ids=[], + started_at=None, + finished_at=None, + errors=[], + backend=self.alias, + last_attempted_at=None, + ) + for _ in range(CPUHeavyTaskBackend.target_task_count) + ) + + def enqueue(self, task): + return task + + def acquire(self, timeout=None): + CPUHeavyTaskBackend.issued_task_count += 1 + try: + task_result = next(self._task_generator) + except StopIteration: + raise TimeoutError("No tasks available within the specified timeout.") + return task_result + + def acknowledge(self, task_result: TaskResult) -> None: + CPUHeavyTaskBackend.solved_task_count += 1 diff --git a/tests/testapp/settings.py b/tests/testapp/settings.py index 191c94c..026defc 100644 --- a/tests/testapp/settings.py +++ b/tests/testapp/settings.py @@ -82,7 +82,10 @@ } } -TASKS = {"default": {"BACKEND": "django.tasks.backends.immediate.ImmediateBackend"}} +TASKS = { + "default": {"BACKEND": "django.tasks.backends.immediate.ImmediateBackend"}, + "cpu": {"BACKEND": "tests.testapp.backends.CPUHeavyTaskBackend"}, +} # Password validation # https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators diff --git a/tests/testapp/tasks.py b/tests/testapp/tasks.py index 70ba59d..83b079a 100644 --- a/tests/testapp/tasks.py +++ b/tests/testapp/tasks.py @@ -8,3 +8,28 @@ @task def my_task(): logger.info("Hello World!") + + +@task(backend="cpu") +def cpu_heavy_task(): + """Calculate the first 1000 prime numbers.""" + + def is_prime(number: int) -> bool: + if number < 2: + return False + if number in (2, 3): + return True + if number % 2 == 0: + return False + for divisor in range(3, int(number**0.5) + 1, 2): + if number % divisor == 0: + return False + return True + + prime_count = 0 + number = 2 + while prime_count < 1000: + if is_prime(number): + prime_count += 1 + number += 1 + return prime_count From abf255cbd40f0c430938053dba6f83536f422e51 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Wed, 29 Apr 2026 14:04:05 +0200 Subject: [PATCH 16/27] wip --- grinder/backends.py | 21 +++++++++++++++++++-- grinder/executor.py | 18 ++++++++++-------- grinder/management/commands/grinder.py | 10 +++++++++- tests/test_grinder_command.py | 1 + tests/testapp/backends.py | 19 ++++++++++--------- 5 files changed, 49 insertions(+), 20 deletions(-) diff --git a/grinder/backends.py b/grinder/backends.py index 6f8667b..85e7810 100644 --- a/grinder/backends.py +++ b/grinder/backends.py @@ -5,12 +5,29 @@ from django.tasks import TaskResult from django.tasks.backends.base import BaseTaskBackend +from django.utils.module_loading import import_string + + +class SerializableTaskResult(TaskResult): + """A serializable representation of a TaskResult for use in task backends.""" + + task_path: str + + def __init__(self, task_path: str, **kwargs): + super().__init__(**kwargs) + object.__setattr__(self, "task_path", task_path) + + @property + def task(self): + return import_string(self.task_path) class AcknowledgeableTaskBackend(BaseTaskBackend, ABC): """Provide an interface for tasks queues to be processed by the executor.""" - def acquire(self, timeout: datetime.timedelta | None = None) -> TaskResult: + def acquire( + self, timeout: datetime.timedelta | None = None + ) -> SerializableTaskResult: """ Return and lock the next task to be processed without removing it from the queue. @@ -22,6 +39,6 @@ def acquire(self, timeout: datetime.timedelta | None = None) -> TaskResult: """ raise NotImplementedError - def acknowledge(self, task_result: TaskResult) -> None: + def acknowledge(self, task_result: SerializableTaskResult) -> None: """Remove the task from the queue and publish the result.""" raise NotImplementedError diff --git a/grinder/executor.py b/grinder/executor.py index 848e066..b234372 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -5,6 +5,7 @@ import asyncio import dataclasses import datetime +import logging import multiprocessing import random import socket @@ -24,6 +25,8 @@ if typing.TYPE_CHECKING: from .backends import AcknowledgeableTaskBackend +logger = logging.getLogger(__name__) + @dataclasses.dataclass(kw_only=True, slots=True) class TaskExecutor: @@ -153,21 +156,16 @@ def __init__( def run(self) -> None: """Start consumer execution inside this process.""" + logger.info("Starting worker process %s", self.name) self.lock = threading.Lock() self.expired = threading.Event() - self.run_worker_process(self) - - @staticmethod - def run_worker_process(worker: WorkerProcess) -> None: - """Run consumer threads in a process that read from shared task queue.""" consumer_threads = [ - WorkerThread(worker=worker, index=index) - for index in range(worker.thread_count) + WorkerThread(worker=self, index=index) for index in range(self.thread_count) ] for consumer_thread in consumer_threads: consumer_thread.start() for consumer_thread in consumer_threads: - consumer_thread.join(worker.task_timeout.total_seconds()) + consumer_thread.join(self.task_timeout.total_seconds()) def record_task(self) -> None: """Record one processed task and stop when max_tasks is reached.""" @@ -182,6 +180,7 @@ def record_task(self) -> None: def shutdown(self) -> None: """Request graceful worker stop and wait for process exit.""" + logger.info("Stopping worker process %s", self.name) self.shutdown_requested.set() self.join() @@ -225,6 +224,7 @@ def run(self) -> None: def execute_task_result(self, task_result: TaskResult) -> TaskResult: """Execute task from task result and update result lifecycle state.""" + logger.info("Executing task %r", task_result.id) started_at = timezone.now() task_result = dataclasses.replace( task_result, @@ -244,6 +244,7 @@ def execute_task_result(self, task_result: TaskResult) -> TaskResult: status=TaskResultStatus.FAILED, errors=[*task_result.errors, WorkerThread.create_task_error(exception)], ) + logger.exception("Task failed %r", task_result.id) else: task_result = dataclasses.replace( task_result, @@ -252,6 +253,7 @@ def execute_task_result(self, task_result: TaskResult) -> TaskResult: object.__setattr__( task_result, "_return_value", normalize_json(return_value) ) + logger.info("Task successful %r", task_result.id) finally: task_result = dataclasses.replace( task_result, diff --git a/grinder/management/commands/grinder.py b/grinder/management/commands/grinder.py index f7b20dd..1d3747f 100644 --- a/grinder/management/commands/grinder.py +++ b/grinder/management/commands/grinder.py @@ -1,4 +1,6 @@ import asyncio +import datetime +import logging import signal import sys @@ -88,6 +90,12 @@ def handle( exit_empty, **options, ): + console = logging.StreamHandler() + logging.basicConfig( + level=max(10, 10 * (4 - verbosity)), + handlers=[console], + ) + logging.getLogger("grinder").setLevel(max(10, 10 * (3 - verbosity))) match sys.platform: case "win32": signal.signal(signal.SIGBREAK, kill_softly) @@ -104,7 +112,7 @@ def handle( threads=threads, max_tasks=max_tasks, max_tasks_jitter=max_tasks_jitter, - task_timeout=task_timeout, + task_timeout=datetime.timedelta(seconds=task_timeout), exit_empty=exit_empty, ) try: diff --git a/tests/test_grinder_command.py b/tests/test_grinder_command.py index 2a6182e..5d23121 100644 --- a/tests/test_grinder_command.py +++ b/tests/test_grinder_command.py @@ -58,6 +58,7 @@ def run_grinder_command() -> None: verbosity=0, backends="cpu", queues=["default"], + exit_empty=True, ) finally: signal.setitimer(signal.ITIMER_REAL, 0) diff --git a/tests/testapp/backends.py b/tests/testapp/backends.py index 63cc6a9..2c23b5d 100644 --- a/tests/testapp/backends.py +++ b/tests/testapp/backends.py @@ -2,26 +2,24 @@ from django.tasks import TaskResult, TaskResultStatus from django.utils import timezone -from grinder.backends import AcknowledgeableTaskBackend +from grinder.backends import AcknowledgeableTaskBackend, SerializableTaskResult class CPUHeavyTaskBackend(AcknowledgeableTaskBackend): solved_task_count = 0 issued_task_count = 0 - target_task_count = 1000 + target_task_count = 100 def __init__(self, alias, params): super().__init__(alias=alias, params=params) - self.reset() + self._task_generator = None def reset(self): - from .tasks import cpu_heavy_task - CPUHeavyTaskBackend.solved_task_count = 0 CPUHeavyTaskBackend.issued_task_count = 0 self._task_generator = ( - TaskResult( - task=cpu_heavy_task, + SerializableTaskResult( + task_path="tests.testapp.tasks.cpu_heavy_task", enqueued_at=timezone.now(), status=TaskResultStatus.READY, id=str(uuid.uuid4()), @@ -41,12 +39,15 @@ def enqueue(self, task): return task def acquire(self, timeout=None): + if self._task_generator is None: + self.reset() CPUHeavyTaskBackend.issued_task_count += 1 try: - task_result = next(self._task_generator) + return next(self._task_generator) except StopIteration: raise TimeoutError("No tasks available within the specified timeout.") - return task_result + finally: + self._task_generator = None def acknowledge(self, task_result: TaskResult) -> None: CPUHeavyTaskBackend.solved_task_count += 1 From 95d83b64b7064a540f735e6bb19d293aa69220f2 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Thu, 30 Apr 2026 21:26:00 +0200 Subject: [PATCH 17/27] Add benkmark --- grinder/backends.py | 21 ++------------- grinder/executor.py | 34 ++++++++++++++++++------ grinder/management/commands/grinder.py | 11 ++------ pyproject.toml | 2 +- tests/test_grinder_command.py | 36 ++++++-------------------- tests/testapp/backends.py | 19 +++++++------- tests/testapp/tasks.py | 2 +- 7 files changed, 49 insertions(+), 76 deletions(-) diff --git a/grinder/backends.py b/grinder/backends.py index 85e7810..6f8667b 100644 --- a/grinder/backends.py +++ b/grinder/backends.py @@ -5,29 +5,12 @@ from django.tasks import TaskResult from django.tasks.backends.base import BaseTaskBackend -from django.utils.module_loading import import_string - - -class SerializableTaskResult(TaskResult): - """A serializable representation of a TaskResult for use in task backends.""" - - task_path: str - - def __init__(self, task_path: str, **kwargs): - super().__init__(**kwargs) - object.__setattr__(self, "task_path", task_path) - - @property - def task(self): - return import_string(self.task_path) class AcknowledgeableTaskBackend(BaseTaskBackend, ABC): """Provide an interface for tasks queues to be processed by the executor.""" - def acquire( - self, timeout: datetime.timedelta | None = None - ) -> SerializableTaskResult: + def acquire(self, timeout: datetime.timedelta | None = None) -> TaskResult: """ Return and lock the next task to be processed without removing it from the queue. @@ -39,6 +22,6 @@ def acquire( """ raise NotImplementedError - def acknowledge(self, task_result: SerializableTaskResult) -> None: + def acknowledge(self, task_result: TaskResult) -> None: """Remove the task from the queue and publish the result.""" raise NotImplementedError diff --git a/grinder/executor.py b/grinder/executor.py index b234372..d8854e0 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -11,6 +11,7 @@ import socket import threading import typing +from concurrent.futures import ThreadPoolExecutor from contextlib import suppress from multiprocessing.queues import JoinableQueue from queue import Empty @@ -25,7 +26,15 @@ if typing.TYPE_CHECKING: from .backends import AcknowledgeableTaskBackend -logger = logging.getLogger(__name__) + +logger = multiprocessing.get_logger() +formatter = logging.Formatter( + "%(levelname)s: %(asctime)s - pid=%(process)s - %(message)s" +) +handler = logging.StreamHandler() +handler.setFormatter(formatter) +logger.addHandler(handler) +logger.setLevel(logging.INFO) @dataclasses.dataclass(kw_only=True, slots=True) @@ -87,9 +96,9 @@ async def run(self) -> None: self.create_worker_process() for _ in range(self.process_count) ] await asyncio.gather( - asyncio.create_task(self.acquire_tasks()), asyncio.create_task(self.acknowledge_tasks()), asyncio.create_task(self.maintain_worker_pool()), + asyncio.create_task(self.acquire_tasks()), ) async def acquire_tasks(self) -> None: @@ -99,25 +108,34 @@ async def acquire_tasks(self) -> None: work = self.backend.acquire() except Empty: if self.exit_empty: - self.shutdown() + logger.info("No more tasks to solve. Shutting down.") + loop = asyncio.get_running_loop() + loop.run_in_executor(None, self.shutdown) + await asyncio.sleep(0.01) else: self.shared_task_queue.put(work) async def acknowledge_tasks(self) -> None: """Acknowledge processed tasks and publish updated results in main process.""" while self.is_publishing: - self.backend.acknowledge(self.processed_task_queue.get(block=True)) - self.processed_task_queue.task_done() + try: + task = self.processed_task_queue.get_nowait() + except Empty: + await asyncio.sleep(0.01) + else: + self.backend.acknowledge(task) + self.processed_task_queue.task_done() def shutdown(self) -> None: """Stop queue consumption and terminate all worker processes.""" + logger.info("Shutting down task executor") self.is_acquiring = False with suppress(ValueError): self.shared_task_queue.join() - for worker in self.worker_processes: - worker.shutdown() with suppress(ValueError): self.processed_task_queue.join() + with ThreadPoolExecutor(max_workers=self.process_count) as executor: + executor.map(lambda worker: worker.shutdown(), self.worker_processes) self.is_publishing = False async def maintain_worker_pool(self) -> None: @@ -128,7 +146,7 @@ async def maintain_worker_pool(self) -> None: continue worker.join(timeout=0) self.worker_processes[index] = self.create_worker_process() - await asyncio.sleep(0.01) + await asyncio.sleep(1) class WorkerProcess(multiprocessing.Process): diff --git a/grinder/management/commands/grinder.py b/grinder/management/commands/grinder.py index 1d3747f..78ac952 100644 --- a/grinder/management/commands/grinder.py +++ b/grinder/management/commands/grinder.py @@ -1,6 +1,5 @@ import asyncio import datetime -import logging import signal import sys @@ -90,12 +89,6 @@ def handle( exit_empty, **options, ): - console = logging.StreamHandler() - logging.basicConfig( - level=max(10, 10 * (4 - verbosity)), - handlers=[console], - ) - logging.getLogger("grinder").setLevel(max(10, 10 * (3 - verbosity))) match sys.platform: case "win32": signal.signal(signal.SIGBREAK, kill_softly) @@ -103,7 +96,7 @@ def handle( signal.signal(signal.SIGHUP, kill_softly) signal.signal(signal.SIGTERM, kill_softly) signal.signal(signal.SIGINT, kill_softly) - self.stdout.write(self.style.SUCCESS("Starting worker…")) + self.stdout.write(self.style.SUCCESS("Starting workers…")) backend_alias = backends[0] if isinstance(backends, list) else backends backend = task_backends[backend_alias] exe = TaskExecutor( @@ -119,5 +112,5 @@ def handle( asyncio.run(exe.run()) except KeyboardInterrupt as e: self.stdout.write(self.style.WARNING(str(e))) - self.stdout.write(self.style.NOTICE("Shutting down scheduler…")) + self.stdout.write(self.style.NOTICE("Shutting down workers…")) exe.shutdown() diff --git a/pyproject.toml b/pyproject.toml index 2a902c1..9a9aedc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ classifiers = [ "Framework :: Django :: 6.0", ] requires-python = ">=3.12" -dependencies = ["django>=6.0"] +dependencies = ["django @ git+https://github.com/django/django.git@main#egg=django"] [project.urls] # https://packaging.python.org/en/latest/specifications/well-known-project-urls/#well-known-labels diff --git a/tests/test_grinder_command.py b/tests/test_grinder_command.py index 5d23121..5264a40 100644 --- a/tests/test_grinder_command.py +++ b/tests/test_grinder_command.py @@ -1,7 +1,6 @@ from __future__ import annotations import argparse -import os import signal import pytest @@ -39,36 +38,17 @@ def test_call_command__benchmark_cpu_intense_task_1000_times( benchmark, ) -> None: """Benchmark command execution for one CPU intense task solved 1000 times.""" - CPUHeavyTaskBackend.target_task_count = 1000 - CPUHeavyTaskBackend.solved_task_count = 0 - CPUHeavyTaskBackend.issued_task_count = 0 - - def send_interrupt_signal_after_timeout(signum, frame): - os.kill(os.getpid(), signal.SIGINT) - - def run_grinder_command() -> None: - previous_alarm_handler = signal.signal( - signal.SIGALRM, - send_interrupt_signal_after_timeout, - ) - signal.setitimer(signal.ITIMER_REAL, 0.5) - try: - call_command( - "grinder", - verbosity=0, - backends="cpu", - queues=["default"], - exit_empty=True, - ) - finally: - signal.setitimer(signal.ITIMER_REAL, 0) - signal.signal(signal.SIGALRM, previous_alarm_handler) - benchmark.pedantic( - run_grinder_command, + lambda: call_command( + "grinder", + verbosity=0, + backends="cpu", + queues=["default"], + exit_empty=True, + ), rounds=1, iterations=1, warmup_rounds=0, ) - assert CPUHeavyTaskBackend.solved_task_count == 1000 + assert CPUHeavyTaskBackend.solved_task_count == 100 diff --git a/tests/testapp/backends.py b/tests/testapp/backends.py index 2c23b5d..2b3da5f 100644 --- a/tests/testapp/backends.py +++ b/tests/testapp/backends.py @@ -1,8 +1,9 @@ -import uuid +from queue import Empty from django.tasks import TaskResult, TaskResultStatus from django.utils import timezone -from grinder.backends import AcknowledgeableTaskBackend, SerializableTaskResult +from django.utils.module_loading import import_string +from grinder.backends import AcknowledgeableTaskBackend class CPUHeavyTaskBackend(AcknowledgeableTaskBackend): @@ -18,11 +19,11 @@ def reset(self): CPUHeavyTaskBackend.solved_task_count = 0 CPUHeavyTaskBackend.issued_task_count = 0 self._task_generator = ( - SerializableTaskResult( - task_path="tests.testapp.tasks.cpu_heavy_task", + TaskResult( + task=import_string("tests.testapp.tasks.cpu_heavy_task"), enqueued_at=timezone.now(), status=TaskResultStatus.READY, - id=str(uuid.uuid4()), + id=str(i + 1), args=[], kwargs={}, worker_ids=[], @@ -32,7 +33,7 @@ def reset(self): backend=self.alias, last_attempted_at=None, ) - for _ in range(CPUHeavyTaskBackend.target_task_count) + for i in range(CPUHeavyTaskBackend.target_task_count) ) def enqueue(self, task): @@ -44,10 +45,8 @@ def acquire(self, timeout=None): CPUHeavyTaskBackend.issued_task_count += 1 try: return next(self._task_generator) - except StopIteration: - raise TimeoutError("No tasks available within the specified timeout.") - finally: - self._task_generator = None + except StopIteration as e: + raise Empty("No more tasks to solve.") from e def acknowledge(self, task_result: TaskResult) -> None: CPUHeavyTaskBackend.solved_task_count += 1 diff --git a/tests/testapp/tasks.py b/tests/testapp/tasks.py index 83b079a..1cc1ab0 100644 --- a/tests/testapp/tasks.py +++ b/tests/testapp/tasks.py @@ -28,7 +28,7 @@ def is_prime(number: int) -> bool: prime_count = 0 number = 2 - while prime_count < 1000: + while prime_count < 100_000: if is_prime(number): prime_count += 1 number += 1 From 2cb4e5e1d9431cf983ffc5ea1db6e63d12253e91 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Mon, 4 May 2026 17:09:33 +0200 Subject: [PATCH 18/27] wip --- grinder/backends.py | 5 +- grinder/executor.py | 37 +- grinder/management/commands/grinder.py | 13 +- tests/test_executor.py | 888 ------------------------- tests/test_grinder_command.py | 77 ++- tests/testapp/backends.py | 99 ++- tests/testapp/settings.py | 8 +- tests/testapp/tasks.py | 27 +- 8 files changed, 200 insertions(+), 954 deletions(-) delete mode 100644 tests/test_executor.py diff --git a/grinder/backends.py b/grinder/backends.py index 6f8667b..acb88cc 100644 --- a/grinder/backends.py +++ b/grinder/backends.py @@ -10,11 +10,14 @@ class AcknowledgeableTaskBackend(BaseTaskBackend, ABC): """Provide an interface for tasks queues to be processed by the executor.""" - def acquire(self, timeout: datetime.timedelta | None = None) -> TaskResult: + def acquire( + self, *queue_names: str, timeout: datetime.timedelta | None = None + ) -> TaskResult: """ Return and lock the next task to be processed without removing it from the queue. Args: + queue_names: The names of the queues to acquire tasks from. timeout: The maximum time to wait for a task. If None, wait indefinitely. Raises: diff --git a/grinder/executor.py b/grinder/executor.py index d8854e0..f5c5ef5 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -2,7 +2,6 @@ from __future__ import annotations -import asyncio import dataclasses import datetime import logging @@ -10,6 +9,7 @@ import random import socket import threading +import time import typing from concurrent.futures import ThreadPoolExecutor from contextlib import suppress @@ -54,6 +54,7 @@ class TaskExecutor: ) process_count: int = dataclasses.field(init=False) thread_count: int = dataclasses.field(init=False) + queues: tuple[str] shared_task_queue: multiprocessing.JoinableQueue[TaskResult] = dataclasses.field( init=False ) @@ -90,38 +91,42 @@ def create_worker_process(self) -> WorkerProcess: worker.start() return worker - async def run(self) -> None: + def run(self) -> None: """Start consuming tasks until shutdown is requested.""" self.worker_processes = [ self.create_worker_process() for _ in range(self.process_count) ] - await asyncio.gather( - asyncio.create_task(self.acknowledge_tasks()), - asyncio.create_task(self.maintain_worker_pool()), - asyncio.create_task(self.acquire_tasks()), - ) + threads = [ + threading.Thread(target=self.acknowledge_tasks, daemon=True), + threading.Thread(target=self.maintain_worker_pool, daemon=True), + threading.Thread(target=self.acquire_tasks, daemon=True), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() - async def acquire_tasks(self) -> None: + def acquire_tasks(self) -> None: """Buffer tasks in shared task queue.""" while self.is_acquiring: try: - work = self.backend.acquire() + work = self.backend.acquire(*self.queues) except Empty: if self.exit_empty: logger.info("No more tasks to solve. Shutting down.") - loop = asyncio.get_running_loop() - loop.run_in_executor(None, self.shutdown) - await asyncio.sleep(0.01) + self.shutdown() + return + time.sleep(0.01) else: self.shared_task_queue.put(work) - async def acknowledge_tasks(self) -> None: + def acknowledge_tasks(self) -> None: """Acknowledge processed tasks and publish updated results in main process.""" while self.is_publishing: try: task = self.processed_task_queue.get_nowait() except Empty: - await asyncio.sleep(0.01) + time.sleep(0.01) else: self.backend.acknowledge(task) self.processed_task_queue.task_done() @@ -138,7 +143,7 @@ def shutdown(self) -> None: executor.map(lambda worker: worker.shutdown(), self.worker_processes) self.is_publishing = False - async def maintain_worker_pool(self) -> None: + def maintain_worker_pool(self) -> None: """Restart worker processes that have exited.""" while self.is_publishing: for index, worker in enumerate(self.worker_processes): @@ -146,7 +151,7 @@ async def maintain_worker_pool(self) -> None: continue worker.join(timeout=0) self.worker_processes[index] = self.create_worker_process() - await asyncio.sleep(1) + time.sleep(1) class WorkerProcess(multiprocessing.Process): diff --git a/grinder/management/commands/grinder.py b/grinder/management/commands/grinder.py index 78ac952..041a856 100644 --- a/grinder/management/commands/grinder.py +++ b/grinder/management/commands/grinder.py @@ -1,4 +1,3 @@ -import asyncio import datetime import signal import sys @@ -10,7 +9,7 @@ def kill_softly(signum, frame): - """Raise a KeyboardInterrupt to stop the scheduler and release the lock.""" + """Raise a KeyboardInterrupt to stop the worker gracefully.""" signame = signal.Signals(signum).name raise KeyboardInterrupt(f"Received {signame} ({signum}), shutting down…") @@ -72,7 +71,7 @@ def add_arguments(self, parser): parser.add_argument( "--exit-empty", action="store_true", - help="Drain the task queue and exit.", + help="Drain the task queue and exit with 0.", ) def handle( @@ -99,6 +98,11 @@ def handle( self.stdout.write(self.style.SUCCESS("Starting workers…")) backend_alias = backends[0] if isinstance(backends, list) else backends backend = task_backends[backend_alias] + if not set(queues).issubset(backend.queues): + self.stderr.write( + self.style.ERROR("Backend does not support all specified queues.") + ) + exit(1) exe = TaskExecutor( backend=backend, workers=workers, @@ -107,9 +111,10 @@ def handle( max_tasks_jitter=max_tasks_jitter, task_timeout=datetime.timedelta(seconds=task_timeout), exit_empty=exit_empty, + queues=queues, ) try: - asyncio.run(exe.run()) + exe.run() except KeyboardInterrupt as e: self.stdout.write(self.style.WARNING(str(e))) self.stdout.write(self.style.NOTICE("Shutting down workers…")) diff --git a/tests/test_executor.py b/tests/test_executor.py deleted file mode 100644 index 5186d59..0000000 --- a/tests/test_executor.py +++ /dev/null @@ -1,888 +0,0 @@ -from __future__ import annotations - -import asyncio -import collections.abc -import datetime -import threading -from queue import Empty -from types import SimpleNamespace -from unittest.mock import Mock - -import pytest -from django.tasks import TaskResult -from django.tasks.base import TaskResultStatus -from grinder import executor -from grinder.backends import AcknowledgeableTaskBackend - - -class RecordingTask: - def __init__( - self, - *, - takes_context: bool, - return_value=None, - exception: Exception | None = None, - ): - self.takes_context = takes_context - self.return_value = return_value - self.exception = exception - self.module_path = "tests.test_executor.RecordingTask.call" - self.calls: list[tuple[tuple, dict]] = [] - - def call(self, *args, **kwargs): - self.calls.append((args, kwargs)) - if self.exception is not None: - raise self.exception - return self.return_value - - -class CPUHeavyTask: - def __init__( - self, - *, - matrix_size: int, - iteration_count: int, - raise_error: bool, - ): - self.takes_context = False - self.module_path = "tests.test_executor.CPUHeavyTask.call" - self.matrix_size = matrix_size - self.iteration_count = iteration_count - self.raise_error = raise_error - - def call(self): - prime_limit = self.matrix_size * 80 - prime_count = sum(self.is_prime(number) for number in range(2, prime_limit)) - fibonacci_value = self.calculate_fibonacci(self.iteration_count + 18) - pi_estimate = self.calculate_pi_leibniz(self.matrix_size * 120) - if self.raise_error: - raise ValueError("task failed") - return { - "prime_count": prime_count, - "fibonacci_value": fibonacci_value, - "pi_estimate": pi_estimate, - } - - @staticmethod - def is_prime(number: int) -> bool: - if number < 2: - return False - if number in (2, 3): - return True - if number % 2 == 0: - return False - for divisor in range(3, int(number**0.5) + 1, 2): - if number % divisor == 0: - return False - return True - - @staticmethod - def calculate_fibonacci(index: int) -> int: - if index < 2: - return index - previous = 0 - current = 1 - for _ in range(2, index + 1): - previous, current = current, previous + current - return current - - @staticmethod - def calculate_pi_leibniz(iteration_count: int) -> float: - return 4 * sum( - ((-1) ** iteration_index) / (2 * iteration_index + 1) - for iteration_index in range(iteration_count) - ) - - -class FakeJoinableQueue: - def __init__(self, *, items: list | None = None): - self.items = [] if items is None else [*items] - self.put_calls: list = [] - self.task_done_calls = 0 - - def put(self, item): - self.put_calls.append(item) - - def get(self, *, timeout: float | None = None, block: bool = True): - if not self.items: - raise Empty - return self.items.pop(0) - - def task_done(self): - self.task_done_calls += 1 - - def empty(self) -> bool: - return not self.items - - -class QueueRaisingEmpty: - def __init__(self, *, is_empty: bool): - self.is_empty = is_empty - self.task_done_calls = 0 - - def get(self, *, timeout: float | None = None, block: bool = True): - raise Empty - - def empty(self) -> bool: - return self.is_empty - - def task_done(self): - self.task_done_calls += 1 - - -class QueueRaiseThenReturn: - def __init__(self, *, item): - self.item = item - self.calls = 0 - self.task_done_calls = 0 - - def get(self, *, timeout: float | None = None, block: bool = True): - self.calls += 1 - if self.calls == 1: - raise Empty - return self.item - - def empty(self) -> bool: - return False - - def task_done(self): - self.task_done_calls += 1 - - -class QueueRaiseThenReturnAndEmpty: - def __init__(self, *, item): - self.item = item - self.calls = 0 - self.task_done_calls = 0 - self.is_empty = False - - def get(self, *, timeout: float | None = None, block: bool = True): - self.calls += 1 - if self.calls == 1: - raise Empty - self.is_empty = True - return self.item - - def empty(self) -> bool: - return self.is_empty - - def task_done(self): - self.task_done_calls += 1 - - -class FakeWorkerProcess: - def __init__(self, *, alive: bool): - self.alive = alive - self.join_calls: list[float] = [] - self.shutdown_called = False - - def is_alive(self) -> bool: - return self.alive - - def join(self, timeout: float | None = None) -> None: - self.join_calls.append(timeout) - - def shutdown(self) -> None: - self.shutdown_called = True - - -def create_task_result(*, task: RecordingTask, args=None, kwargs=None) -> TaskResult: - return TaskResult( - task=task, - id="task-id", - status=TaskResultStatus.READY, - enqueued_at=None, - started_at=None, - finished_at=None, - last_attempted_at=None, - args=[] if args is None else args, - kwargs={} if kwargs is None else kwargs, - backend="default", - errors=[], - worker_ids=[], - ) - - -def create_task_error_from_value_error() -> None: - try: - raise ValueError("benchmark") - except ValueError as exception: - executor.WorkerThread.create_task_error(exception) - - -class IntegrationTaskBackend(AcknowledgeableTaskBackend): - def __init__( - self, - *, - task_result_generator: collections.abc.Iterator[TaskResult], - task_result_count: int, - ): - super().__init__(alias="default", params={}) - self.task_result_generator = task_result_generator - self.task_result_count = task_result_count - self.acknowledged_task_results: list[TaskResult] = [] - self.task_executor: executor.TaskExecutor | None = None - self.next_task_result: TaskResult | None = next( - self.task_result_generator, None - ) - - def enqueue(self, task): - return task - - def acquire(self, timeout: datetime.timedelta | None = None) -> TaskResult: - if self.next_task_result is None: - if self.task_executor is not None: - self.task_executor.is_acquiring = False - raise TimeoutError - task_result = self.next_task_result - self.next_task_result = next(self.task_result_generator, None) - if self.next_task_result is None and self.task_executor is not None: - self.task_executor.is_acquiring = False - return task_result - - def acknowledge(self, task_result: TaskResult) -> None: - self.acknowledged_task_results.append(task_result) - if ( - self.task_executor is not None - and len(self.acknowledged_task_results) >= self.task_result_count - ): - self.task_executor.is_publishing = False - - -def execute_task_pipeline(*, task_result: TaskResult) -> TaskResult: - return execute_cpu_heavy_task_pipeline( - task_result_generator=iter([task_result]), - task_result_count=1, - )[0] - - -def create_cpu_heavy_task_result_generator( - *, - task_count: int, - fail_every_count: int, -) -> collections.abc.Iterator[TaskResult]: - return ( - TaskResult( - task=CPUHeavyTask( - matrix_size=64, - iteration_count=24, - raise_error=fail_every_count > 0 and task_index % fail_every_count == 0, - ), - id=f"cpu-task-{task_index}", - status=TaskResultStatus.READY, - enqueued_at=None, - started_at=None, - finished_at=None, - last_attempted_at=None, - args=[], - kwargs={}, - backend="default", - errors=[], - worker_ids=[], - ) - for task_index in range(task_count) - ) - - -def execute_cpu_heavy_task_pipeline( - *, - task_result_generator: collections.abc.Iterator[TaskResult], - task_result_count: int, -) -> list[TaskResult]: - backend = IntegrationTaskBackend( - task_result_generator=task_result_generator, - task_result_count=task_result_count, - ) - task_executor = executor.TaskExecutor( - backend=backend, - workers=max(task_result_count, 1), - threads=1, - ) - backend.task_executor = task_executor - - asyncio.run(task_executor.acquire_tasks()) - worker_thread = executor.WorkerThread(worker=SimpleNamespace(pid=505), index=1) - while not task_executor.shared_task_queue.empty(): - task_result = task_executor.shared_task_queue.get(timeout=0.1) - task_executor.processed_task_queue.put( - worker_thread.execute_task_result(task_result) - ) - task_executor.shared_task_queue.task_done() - asyncio.run(task_executor.acknowledge_tasks()) - return backend.acknowledged_task_results - - -class TestTaskExecutor: - def test_run__create_tasks_for_all_executor_loops(self, monkeypatch) -> None: - """Create orchestration tasks for acquire, acknowledge, and maintenance loops.""" - called_methods: list[str] = [] - - async def acquire_tasks(task_executor): - called_methods.append("acquire_tasks") - task_executor.is_acquiring = False - - async def acknowledge_tasks(task_executor): - called_methods.append("acknowledge_tasks") - task_executor.is_publishing = False - - async def maintain_worker_pool(task_executor): - called_methods.append("maintain_worker_pool") - - monkeypatch.setattr( - executor.TaskExecutor, - "acquire_tasks", - acquire_tasks, - ) - monkeypatch.setattr( - executor.TaskExecutor, - "acknowledge_tasks", - acknowledge_tasks, - ) - monkeypatch.setattr( - executor.TaskExecutor, - "maintain_worker_pool", - maintain_worker_pool, - ) - monkeypatch.setattr( - executor.TaskExecutor, - "create_worker_process", - lambda task_executor_self: FakeWorkerProcess(alive=True), - ) - - task_executor = executor.TaskExecutor(backend=SimpleNamespace(), workers=1) - - asyncio.run(task_executor.run()) - - assert len(task_executor.worker_processes) == 1 - assert called_methods == [ - "acquire_tasks", - "acknowledge_tasks", - "maintain_worker_pool", - ] - - def test_acquire_tasks__put_acquired_task_in_shared_queue(self) -> None: - """Put acquired task result into shared queue.""" - - class AcquireBackend: - def __init__(self): - self.task_executor = None - - def acquire(self, timeout=None): - self.task_executor.is_acquiring = False - return "task-result" - - backend = AcquireBackend() - task_executor = executor.TaskExecutor(backend=backend, workers=1) - backend.task_executor = task_executor - - asyncio.run(task_executor.acquire_tasks()) - - assert task_executor.shared_task_queue.get(timeout=0.1) == "task-result" - - def test_acquire_tasks__raise_timeout_error_when_backend_times_out(self) -> None: - """Raise TimeoutError when backend acquire times out.""" - - class AcquireBackend: - def __init__(self): - self.task_executor = None - self.acquire_count = 0 - - def acquire(self, timeout=None): - self.acquire_count += 1 - if self.acquire_count == 1: - raise TimeoutError - self.task_executor.is_acquiring = False - return "task-result" - - backend = AcquireBackend() - task_executor = executor.TaskExecutor(backend=backend, workers=1) - backend.task_executor = task_executor - - with pytest.raises(TimeoutError): - asyncio.run(task_executor.acquire_tasks()) - - assert backend.acquire_count == 1 - - def test_acknowledge_tasks__acknowledge_processed_task_and_mark_done(self) -> None: - """Acknowledge processed task and mark queue item done.""" - - class AcknowledgeBackend: - def __init__(self): - self.calls: list = [] - self.task_executor = None - - def acknowledge(self, task_result): - self.calls.append(task_result) - self.task_executor.is_publishing = False - - backend = AcknowledgeBackend() - task_executor = executor.TaskExecutor(backend=backend, workers=1) - backend.task_executor = task_executor - task_executor.processed_task_queue.put("processed-result") - - asyncio.run(task_executor.acknowledge_tasks()) - - assert backend.calls == ["processed-result"] - - def test_acknowledge_tasks__raise_empty_when_processed_queue_is_empty(self) -> None: - """Raise Empty when processed queue is empty during acknowledge loop.""" - - class AcknowledgeBackend: - def __init__(self): - self.calls: list = [] - self.task_executor = None - - def acknowledge(self, task_result): - self.calls.append(task_result) - self.task_executor.is_publishing = False - - backend = AcknowledgeBackend() - task_executor = executor.TaskExecutor(backend=backend, workers=1) - backend.task_executor = task_executor - task_executor.processed_task_queue = QueueRaiseThenReturnAndEmpty( - item="processed-result", - ) - - with pytest.raises(Empty): - asyncio.run(task_executor.acknowledge_tasks()) - - def test_get_maximum_tasks_per_child__return_none_without_max_tasks(self) -> None: - """Return None when task recycling is disabled.""" - task_executor = executor.TaskExecutor(backend=SimpleNamespace(), max_tasks=0) - - assert task_executor.get_maximum_tasks_per_child() is None - - def test_get_maximum_tasks_per_child__calculate_recycle_limit_with_jitter( - self, monkeypatch - ) -> None: - """Calculate worker recycle limit with jitter and thread count.""" - monkeypatch.setattr(executor.random, "randint", lambda start, end: 4) - task_executor = executor.TaskExecutor( - backend=SimpleNamespace(), - threads=2, - max_tasks=6, - max_tasks_jitter=4, - ) - - assert task_executor.get_maximum_tasks_per_child() == 5 - - def test_create_worker_process__start_and_return_worker(self, monkeypatch) -> None: - """Create and start a worker process.""" - started = Mock() - - class WorkerProcessDouble: - def __init__(self, *args): - self.args = args - - def start(self): - started() - - monkeypatch.setattr(executor, "WorkerProcess", WorkerProcessDouble) - task_executor = executor.TaskExecutor( - backend=SimpleNamespace(), workers=1, threads=3 - ) - - worker_process = task_executor.create_worker_process() - - assert isinstance(worker_process, WorkerProcessDouble) - assert worker_process.args[2] == 3 - started.assert_called_once_with() - - def test_shutdown__stop_flags_join_queues_and_shutdown_workers(self) -> None: - """Stop publishing and shut down all workers.""" - task_executor = executor.TaskExecutor(backend=SimpleNamespace(), workers=1) - worker_process = FakeWorkerProcess(alive=True) - task_executor.worker_processes = [worker_process] - - task_executor.shutdown() - - assert task_executor.is_acquiring is False - assert task_executor.is_publishing is False - assert worker_process.shutdown_called is True - - def test_maintain_worker_pool__replace_dead_worker(self, monkeypatch) -> None: - """Replace dead worker process during pool maintenance.""" - task_executor = executor.TaskExecutor(backend=SimpleNamespace(), workers=2) - dead_worker = FakeWorkerProcess(alive=False) - healthy_worker = FakeWorkerProcess(alive=True) - replacement_worker = FakeWorkerProcess(alive=True) - task_executor.worker_processes = [dead_worker, healthy_worker] - - def create_worker_process(task_executor_self): - task_executor.is_publishing = False - return replacement_worker - - monkeypatch.setattr( - executor.TaskExecutor, - "create_worker_process", - create_worker_process, - ) - - asyncio.run(task_executor.maintain_worker_pool()) - - assert dead_worker.join_calls == [0] - assert task_executor.worker_processes == [replacement_worker, healthy_worker] - - -class TestWorkerProcess: - def test_run__initialize_sync_primitives_and_start_worker_threads( - self, monkeypatch - ) -> None: - """Initialize lock and expiration event before starting worker threads.""" - run_worker_process = Mock() - monkeypatch.setattr( - executor.WorkerProcess, "run_worker_process", run_worker_process - ) - worker_process = executor.WorkerProcess( - FakeJoinableQueue(), - FakeJoinableQueue(), - thread_count=1, - task_timeout=datetime.timedelta(seconds=1), - ) - - worker_process.run() - - assert isinstance(worker_process.lock, type(threading.Lock())) - assert isinstance(worker_process.expired, type(threading.Event())) - run_worker_process.assert_called_once_with(worker_process) - - def test_run_worker_process__start_and_join_each_consumer_thread( - self, monkeypatch - ) -> None: - """Start and join every consumer thread created for the process.""" - thread_events: list[str] = [] - - class WorkerThreadDouble: - def __init__(self, *, worker, index): - self.index = index - - def start(self): - thread_events.append(f"start:{self.index}") - - def join(self, timeout): - thread_events.append(f"join:{self.index}:{timeout}") - - monkeypatch.setattr(executor, "WorkerThread", WorkerThreadDouble) - worker_process = SimpleNamespace( - thread_count=2, task_timeout=datetime.timedelta(seconds=3) - ) - - executor.WorkerProcess.run_worker_process(worker_process) - - assert thread_events == [ - "start:0", - "start:1", - "join:0:3.0", - "join:1:3.0", - ] - - def test_record_task__set_expired_after_reaching_max_tasks(self) -> None: - """Set expiration event when processed task limit is reached.""" - worker_process = executor.WorkerProcess( - FakeJoinableQueue(), - FakeJoinableQueue(), - thread_count=1, - task_timeout=datetime.timedelta(seconds=1), - max_tasks=2, - ) - worker_process.lock = threading.Lock() - worker_process.expired = threading.Event() - - worker_process.record_task() - worker_process.record_task() - - assert worker_process.expired.is_set() is True - - def test_record_task__ignore_when_limit_or_state_is_missing(self) -> None: - """Ignore task recording when worker state is incomplete.""" - worker_process = executor.WorkerProcess( - FakeJoinableQueue(), - FakeJoinableQueue(), - thread_count=1, - task_timeout=datetime.timedelta(seconds=1), - max_tasks=None, - ) - - worker_process.record_task() - - assert worker_process.task_count == 0 - - def test_record_task__ignore_when_sync_state_not_initialized(self) -> None: - """Ignore recording when synchronization objects are missing.""" - worker_process = executor.WorkerProcess( - FakeJoinableQueue(), - FakeJoinableQueue(), - thread_count=1, - task_timeout=datetime.timedelta(seconds=1), - max_tasks=1, - ) - - worker_process.record_task() - - assert worker_process.task_count == 0 - - def test_shutdown__set_shutdown_flag_and_join_process(self, monkeypatch) -> None: - """Set shutdown event and wait for worker process exit.""" - worker_process = executor.WorkerProcess( - FakeJoinableQueue(), - FakeJoinableQueue(), - thread_count=1, - task_timeout=datetime.timedelta(seconds=1), - ) - join = Mock() - monkeypatch.setattr(worker_process, "join", join) - - worker_process.shutdown() - - assert worker_process.shutdown_requested.is_set() is True - join.assert_called_once_with() - - -class TestTaskExecutorIntegration: - @pytest.mark.integration - def test_execute_task_pipeline__acknowledge_successful_task_result(self) -> None: - """Acknowledge successful task result in executor pipeline.""" - acknowledged_task_result = execute_task_pipeline( - task_result=create_task_result( - task=RecordingTask(takes_context=False, return_value={"value": 7}) - ) - ) - - assert acknowledged_task_result.status is TaskResultStatus.SUCCESSFUL - assert acknowledged_task_result.errors == [] - assert acknowledged_task_result.finished_at is not None - - @pytest.mark.integration - def test_execute_task_pipeline__acknowledge_failed_task_result(self) -> None: - """Acknowledge failed task result in executor pipeline.""" - acknowledged_task_result = execute_task_pipeline( - task_result=create_task_result( - task=RecordingTask(takes_context=False, exception=ValueError("broken")) - ) - ) - - assert acknowledged_task_result.status is TaskResultStatus.FAILED - assert len(acknowledged_task_result.errors) == 1 - assert ( - acknowledged_task_result.errors[0].exception_class_path - == "builtins.ValueError" - ) - - @pytest.mark.integration - def test_execute_cpu_heavy_task_pipeline__process_multiple_cpu_heavy_tasks( - self, - ) -> None: - """Process multiple CPU heavy tasks without losing task results.""" - acknowledged_task_results = execute_cpu_heavy_task_pipeline( - task_result_generator=create_cpu_heavy_task_result_generator( - task_count=100, - fail_every_count=0, - ), - task_result_count=100, - ) - - assert len(acknowledged_task_results) == 100 - assert all( - task_result.status is TaskResultStatus.SUCCESSFUL - for task_result in acknowledged_task_results - ) - assert {task_result.id for task_result in acknowledged_task_results} == { - f"cpu-task-{task_index}" for task_index in range(100) - } - - @pytest.mark.integration - def test_execute_cpu_heavy_task_pipeline__process_failures_without_data_loss( - self, - ) -> None: - """Process failing CPU heavy tasks while acknowledging all task results.""" - acknowledged_task_results = execute_cpu_heavy_task_pipeline( - task_result_generator=create_cpu_heavy_task_result_generator( - task_count=100, - fail_every_count=15, - ), - task_result_count=100, - ) - - assert len(acknowledged_task_results) == 100 - assert ( - sum( - task_result.status is TaskResultStatus.FAILED - for task_result in acknowledged_task_results - ) - == 7 - ) - assert ( - sum( - task_result.status is TaskResultStatus.SUCCESSFUL - for task_result in acknowledged_task_results - ) - == 93 - ) - - -class TestWorkerThread: - def test_run__return_when_shutdown_requested_and_queue_is_empty(self) -> None: - """Return when shutdown is requested and queue has no pending task.""" - worker = SimpleNamespace( - expired=threading.Event(), - shutdown_requested=threading.Event(), - task_queue=FakeJoinableQueue(), - processed_task_queue=FakeJoinableQueue(), - record_task=Mock(), - pid=100, - ) - worker.shutdown_requested.set() - worker_thread = executor.WorkerThread(worker=worker, index=1) - - worker_thread.run() - - worker.record_task.assert_not_called() - - def test_run__process_single_task_and_finish(self, monkeypatch) -> None: - """Process one task, acknowledge queue bookkeeping, and stop.""" - task_result = create_task_result( - task=RecordingTask(takes_context=False, return_value=1) - ) - task_queue = FakeJoinableQueue(items=[task_result]) - - expired = threading.Event() - - def record_task() -> None: - expired.set() - - worker = SimpleNamespace( - expired=expired, - shutdown_requested=threading.Event(), - task_queue=task_queue, - processed_task_queue=FakeJoinableQueue(), - record_task=record_task, - pid=200, - ) - worker_thread = executor.WorkerThread(worker=worker, index=1) - monkeypatch.setattr(worker_thread, "execute_task_result", lambda result: result) - - worker_thread.run() - - assert worker.processed_task_queue.put_calls == [task_result] - assert task_queue.task_done_calls == 1 - - def test_run__return_after_empty_queue_when_shutdown_requested(self) -> None: - """Return after queue timeout when shutdown has been requested.""" - worker = SimpleNamespace( - expired=threading.Event(), - shutdown_requested=threading.Event(), - task_queue=QueueRaisingEmpty(is_empty=False), - processed_task_queue=FakeJoinableQueue(), - record_task=Mock(), - pid=201, - ) - worker.shutdown_requested.set() - worker_thread = executor.WorkerThread(worker=worker, index=2) - - worker_thread.run() - - worker.record_task.assert_not_called() - - def test_run__continue_on_empty_queue_without_shutdown(self, monkeypatch) -> None: - """Continue polling after timeout while shutdown has not been requested.""" - task_result = create_task_result( - task=RecordingTask(takes_context=False, return_value=2) - ) - worker = SimpleNamespace( - expired=threading.Event(), - shutdown_requested=threading.Event(), - task_queue=QueueRaiseThenReturn(item=task_result), - processed_task_queue=FakeJoinableQueue(), - record_task=Mock(), - pid=202, - ) - - def execute_task_result(_task_result): - worker.expired.set() - return _task_result - - worker_thread = executor.WorkerThread(worker=worker, index=3) - monkeypatch.setattr(worker_thread, "execute_task_result", execute_task_result) - - worker_thread.run() - - worker.record_task.assert_called_once_with() - - def test_call_task__pass_context_when_task_requires_context(self) -> None: - """Pass task context as first argument for context-aware tasks.""" - task = RecordingTask(takes_context=True, return_value="ok") - task_result = create_task_result(task=task, args=[1], kwargs={"value": 2}) - - return_value = executor.WorkerThread.call_task(task_result) - - assert return_value == "ok" - args, kwargs = task.calls[0] - assert kwargs == {"value": 2} - assert args[1:] == (1,) - assert args[0].task_result is task_result - - def test_call_task__call_without_context_when_not_required(self) -> None: - """Call task with regular positional and keyword arguments.""" - task = RecordingTask(takes_context=False, return_value="done") - task_result = create_task_result(task=task, args=[3], kwargs={"count": 4}) - - return_value = executor.WorkerThread.call_task(task_result) - - assert return_value == "done" - assert task.calls == [((3,), {"count": 4})] - - def test_create_task_error__include_exception_type_and_traceback(self) -> None: - """Create task error payload with exception class path and traceback.""" - try: - raise RuntimeError("worker failed") - except RuntimeError as exception: - task_error = executor.WorkerThread.create_task_error(exception) - - assert task_error.exception_class_path == "builtins.RuntimeError" - assert "RuntimeError: worker failed" in task_error.traceback - - def test_execute_task_result__set_success_status_and_return_value( - self, monkeypatch - ) -> None: - """Set success lifecycle fields after task execution succeeds.""" - monkeypatch.setattr(executor.task_enqueued, "send", Mock()) - monkeypatch.setattr(executor.task_started, "send", Mock()) - monkeypatch.setattr(executor.task_finished, "send", Mock()) - - task_result = create_task_result( - task=RecordingTask(takes_context=False, return_value={"value": 5}), - ) - worker = SimpleNamespace(pid=321) - worker_thread = executor.WorkerThread(worker=worker, index=7) - - processed_task_result = worker_thread.execute_task_result(task_result) - - assert processed_task_result.status is TaskResultStatus.SUCCESSFUL - assert processed_task_result._return_value is None - assert processed_task_result.finished_at is not None - assert processed_task_result.started_at is not None - assert worker_thread.name in processed_task_result.worker_ids - - def test_execute_task_result__set_failed_status_and_append_error( - self, monkeypatch - ) -> None: - """Set failure status and append task error when execution fails.""" - monkeypatch.setattr(executor.task_enqueued, "send", Mock()) - monkeypatch.setattr(executor.task_started, "send", Mock()) - monkeypatch.setattr(executor.task_finished, "send", Mock()) - - task_result = create_task_result( - task=RecordingTask(takes_context=False, exception=ValueError("invalid")), - ) - worker = SimpleNamespace(pid=111) - worker_thread = executor.WorkerThread(worker=worker, index=3) - - processed_task_result = worker_thread.execute_task_result(task_result) - - assert processed_task_result.status is TaskResultStatus.FAILED - assert processed_task_result.finished_at is not None - assert len(processed_task_result.errors) == 1 - assert ( - processed_task_result.errors[0].exception_class_path - == "builtins.ValueError" - ) diff --git a/tests/test_grinder_command.py b/tests/test_grinder_command.py index 5264a40..bc0f6ee 100644 --- a/tests/test_grinder_command.py +++ b/tests/test_grinder_command.py @@ -5,10 +5,9 @@ import pytest from django.core.management import call_command +from django.tasks import default_task_backend from grinder.management.commands import grinder -from tests.testapp.backends import CPUHeavyTaskBackend - class TestKillSoftly: def test_kill_softly__raise_keyboard_interrupt_with_signal_name(self) -> None: @@ -33,22 +32,86 @@ def test_add_arguments__register_all_worker_options(self) -> None: assert parsed_arguments.task_timeout == 3600.0 @pytest.mark.benchmark - def test_call_command__benchmark_cpu_intense_task_1000_times( + def test_call_command__benchmark_compute( + self, + benchmark, + ) -> None: + """Benchmark command execution for one CPU intense task solved 100 times.""" + default_task_backend.reset() + benchmark.pedantic( + lambda: call_command( + "grinder", + verbosity=0, + queues=["compute"], + exit_empty=True, + ), + rounds=1, + iterations=1, + warmup_rounds=0, + ) + + assert default_task_backend.solved_task_count == 100 + + @pytest.mark.benchmark + def test_call_command__benchmark_io( + self, + benchmark, + ) -> None: + """Benchmark command execution for one CPU intense task solved 100 times.""" + default_task_backend.reset() + benchmark.pedantic( + lambda: call_command( + "grinder", + verbosity=0, + queues=["io"], + threads=6, + exit_empty=True, + ), + rounds=1, + iterations=1, + warmup_rounds=0, + ) + + assert default_task_backend.solved_task_count == 100 + + @pytest.mark.benchmark + def test_call_command__benchmark_compute_and_io( self, benchmark, ) -> None: - """Benchmark command execution for one CPU intense task solved 1000 times.""" + """Benchmark command execution for one CPU intense task solved 100 times.""" + default_task_backend.reset() benchmark.pedantic( lambda: call_command( "grinder", verbosity=0, - backends="cpu", - queues=["default"], + queues=["compute", "io"], exit_empty=True, + threads=2, ), rounds=1, iterations=1, warmup_rounds=0, ) - assert CPUHeavyTaskBackend.solved_task_count == 100 + assert default_task_backend.solved_task_count == 200 + + @pytest.mark.benchmark + def test_call_command__benchmark_memory_leak_recovery( + self, + benchmark, + ) -> None: + """Benchmark command execution for one CPU intense task solved 100 times.""" + default_task_backend.reset(1000) + benchmark.pedantic( + lambda: call_command( + "grinder", + verbosity=0, + queues=["memory"], + exit_empty=True, + # max_tasks=10, + ), + rounds=1, + iterations=1, + warmup_rounds=0, + ) diff --git a/tests/testapp/backends.py b/tests/testapp/backends.py index 2b3da5f..93e4b67 100644 --- a/tests/testapp/backends.py +++ b/tests/testapp/backends.py @@ -6,47 +6,88 @@ from grinder.backends import AcknowledgeableTaskBackend -class CPUHeavyTaskBackend(AcknowledgeableTaskBackend): +class GeneratingTaskBackend(AcknowledgeableTaskBackend): solved_task_count = 0 issued_task_count = 0 target_task_count = 100 + supports_async_task = True def __init__(self, alias, params): super().__init__(alias=alias, params=params) - self._task_generator = None - - def reset(self): - CPUHeavyTaskBackend.solved_task_count = 0 - CPUHeavyTaskBackend.issued_task_count = 0 - self._task_generator = ( - TaskResult( - task=import_string("tests.testapp.tasks.cpu_heavy_task"), - enqueued_at=timezone.now(), - status=TaskResultStatus.READY, - id=str(i + 1), - args=[], - kwargs={}, - worker_ids=[], - started_at=None, - finished_at=None, - errors=[], - backend=self.alias, - last_attempted_at=None, - ) - for i in range(CPUHeavyTaskBackend.target_task_count) - ) + self._queues = None + + def reset(self, task_count=1000): + GeneratingTaskBackend.solved_task_count = 0 + GeneratingTaskBackend.issued_task_count = 0 + self._queues = { + "compute": [ + TaskResult( + task=import_string("tests.testapp.tasks.compute_workload"), + enqueued_at=timezone.now(), + status=TaskResultStatus.READY, + id=f"compute-{i + 1}", + args=[], + kwargs={}, + worker_ids=[], + started_at=None, + finished_at=None, + errors=[], + backend=self.alias, + last_attempted_at=None, + ) + for i in range(task_count) + ], + "io": [ + TaskResult( + task=import_string("tests.testapp.tasks.io_workload"), + enqueued_at=timezone.now(), + status=TaskResultStatus.READY, + id=f"io-{i + 1}", + args=[], + kwargs={}, + worker_ids=[], + started_at=None, + finished_at=None, + errors=[], + backend=self.alias, + last_attempted_at=None, + ) + for i in range(task_count) + ], + "memory": [ + TaskResult( + task=import_string("tests.testapp.tasks.memory_workload"), + enqueued_at=timezone.now(), + status=TaskResultStatus.READY, + id=f"memory-{i + 1}", + args=[], + kwargs={}, + worker_ids=[], + started_at=None, + finished_at=None, + errors=[], + backend=self.alias, + last_attempted_at=None, + ) + for i in range(task_count) + ], + } def enqueue(self, task): return task - def acquire(self, timeout=None): - if self._task_generator is None: + def acquire(self, *queue_names, timeout=None): + if self._queues is None: self.reset() - CPUHeavyTaskBackend.issued_task_count += 1 + GeneratingTaskBackend.issued_task_count += 1 + queues = [self._queues[queue_name] for queue_name in queue_names] try: - return next(self._task_generator) - except StopIteration as e: + # pop from the longest queue first to simulate a more realistic scenario + for queue in sorted(queues, key=len, reverse=True): + return queue.pop(0) + except IndexError as e: raise Empty("No more tasks to solve.") from e + raise Empty("No more tasks to solve.") def acknowledge(self, task_result: TaskResult) -> None: - CPUHeavyTaskBackend.solved_task_count += 1 + GeneratingTaskBackend.solved_task_count += 1 diff --git a/tests/testapp/settings.py b/tests/testapp/settings.py index 026defc..9c29095 100644 --- a/tests/testapp/settings.py +++ b/tests/testapp/settings.py @@ -12,6 +12,8 @@ from pathlib import Path +from django.tasks import DEFAULT_TASK_QUEUE_NAME + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -83,8 +85,10 @@ } TASKS = { - "default": {"BACKEND": "django.tasks.backends.immediate.ImmediateBackend"}, - "cpu": {"BACKEND": "tests.testapp.backends.CPUHeavyTaskBackend"}, + "default": { + "BACKEND": "tests.testapp.backends.GeneratingTaskBackend", + "QUEUES": [DEFAULT_TASK_QUEUE_NAME, "compute", "io", "memory"], + }, } # Password validation diff --git a/tests/testapp/tasks.py b/tests/testapp/tasks.py index 1cc1ab0..b1fe54c 100644 --- a/tests/testapp/tasks.py +++ b/tests/testapp/tasks.py @@ -1,17 +1,15 @@ +import asyncio import logging +import random +import uuid from django.tasks import task logger = logging.getLogger(__name__) -@task -def my_task(): - logger.info("Hello World!") - - -@task(backend="cpu") -def cpu_heavy_task(): +@task(queue_name="compute") +def compute_workload(): """Calculate the first 1000 prime numbers.""" def is_prime(number: int) -> bool: @@ -33,3 +31,18 @@ def is_prime(number: int) -> bool: prime_count += 1 number += 1 return prime_count + + +@task(queue_name="io") +async def io_workload(): + """Sleep for a random amount of time.""" + await asyncio.sleep(random.uniform(0.1, 0.5)) + + +leak = {} + + +@task(queue_name="memory") +def memory_workload(): + """Allocate and leak 100MB of memory.""" + leak[uuid.uuid4()] = "x" * 1024 * 1024 * 100 From 62e3b132b713c36bce1180d77f4f442861729f0d Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Mon, 4 May 2026 17:58:04 +0200 Subject: [PATCH 19/27] wip --- grinder/executor.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/grinder/executor.py b/grinder/executor.py index f5c5ef5..5a383b1 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import dataclasses import datetime import logging @@ -11,8 +12,10 @@ import threading import time import typing +from asyncio import iscoroutine from concurrent.futures import ThreadPoolExecutor from contextlib import suppress +from inspect import iscoroutinefunction from multiprocessing.queues import JoinableQueue from queue import Empty from traceback import format_exception @@ -291,12 +294,15 @@ def call_task(task_result: TaskResult) -> typing.Any: """Call a task with context when required.""" task = task_result.task if task.takes_context: - return task.call( - TaskContext(task_result=task_result), - *task_result.args, - **task_result.kwargs, - ) - return task.call(*task_result.args, **task_result.kwargs) + args = [TaskContext(task_result=task_result), *task_result.args] + else: + args = task_result.args + if iscoroutinefunction(task.func): + return asyncio.run(task.func(*args, **task_result.kwargs)) + return task.func( + *args, + **task_result.kwargs, + ) @staticmethod def create_task_error(exception: BaseException) -> TaskError: From 39fbfdd41b923bd5526e32427fff11323d021629 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Mon, 4 May 2026 18:06:57 +0200 Subject: [PATCH 20/27] Wrap up --- .github/workflows/ci.yml | 8 ++------ grinder/executor.py | 1 - tests/testapp/backends.py | 2 +- tests/testapp/tasks.py | 2 +- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1f5ee4..fb6967a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,10 +27,6 @@ jobs: - "3.14" django-version: - "6.0" - extras: - - "" # We try a run without any extras - - "--extra sentry" - - "--extra redis" services: redis: image: redis @@ -45,7 +41,7 @@ jobs: - uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.python-version }} - - run: uv run ${{ matrix.extras }} --with django~=${{ matrix.django-version }}.0 pytest + - run: uv run --with django~=${{ matrix.django-version }}.0 pytest -m "not benchmark" - uses: codecov/codecov-action@v6 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -68,7 +64,7 @@ jobs: - uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.python-version }} - - run: uv run --with django~=${{ matrix.django-version }}.0 pytest - uses: codecov/codecov-action@v6 with: token: ${{ secrets.CODECOV_TOKEN }} + - run: uv run --with django~=${{ matrix.django-version }}.0 pytest -m "not benchmark" diff --git a/grinder/executor.py b/grinder/executor.py index 5a383b1..22220da 100644 --- a/grinder/executor.py +++ b/grinder/executor.py @@ -12,7 +12,6 @@ import threading import time import typing -from asyncio import iscoroutine from concurrent.futures import ThreadPoolExecutor from contextlib import suppress from inspect import iscoroutinefunction diff --git a/tests/testapp/backends.py b/tests/testapp/backends.py index 93e4b67..cfcdc25 100644 --- a/tests/testapp/backends.py +++ b/tests/testapp/backends.py @@ -16,7 +16,7 @@ def __init__(self, alias, params): super().__init__(alias=alias, params=params) self._queues = None - def reset(self, task_count=1000): + def reset(self, task_count=100): GeneratingTaskBackend.solved_task_count = 0 GeneratingTaskBackend.issued_task_count = 0 self._queues = { diff --git a/tests/testapp/tasks.py b/tests/testapp/tasks.py index b1fe54c..96400f8 100644 --- a/tests/testapp/tasks.py +++ b/tests/testapp/tasks.py @@ -36,7 +36,7 @@ def is_prime(number: int) -> bool: @task(queue_name="io") async def io_workload(): """Sleep for a random amount of time.""" - await asyncio.sleep(random.uniform(0.1, 0.5)) + await asyncio.sleep(random.uniform(0.1, 0.5)) # noqa: S311 leak = {} From bd4e5825215816870f43ffff6f9eafea7084a255 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Mon, 4 May 2026 18:16:03 +0200 Subject: [PATCH 21/27] Update logo --- images/logo-dark.svg | 12 +++++++++--- images/logo-light.svg | 21 ++++++++++++++------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/images/logo-dark.svg b/images/logo-dark.svg index 7eee30e..e1f06bc 100644 --- a/images/logo-dark.svg +++ b/images/logo-dark.svg @@ -1,11 +1,17 @@ - + + + + + Django - - Grinder + + Threadmill A queue agnostic worker for Django's task framework. diff --git a/images/logo-light.svg b/images/logo-light.svg index c7afafb..af7e09c 100644 --- a/images/logo-light.svg +++ b/images/logo-light.svg @@ -1,13 +1,20 @@ - + - - + + + + + + Django - - Grinder + + Threadmill - - A queue agnostic worker for Django's task framework. + + A queue agnostic worker for Django's task framework. From 5b23ff10277896a7bdd66fd7cdf7f3d667ea1555 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Tue, 5 May 2026 11:03:15 +0200 Subject: [PATCH 22/27] Rename and document --- .gitignore | 2 +- CONTRIBUTING.md | 1 - README.md | 114 ++++++++++++++---- pyproject.toml | 22 ++-- tests/test_backends.py | 2 +- ...est_grinder_command.py => test_command.py} | 14 +-- tests/testapp/backends.py | 2 +- {grinder => threadmill}/__init__.py | 0 {grinder => threadmill}/backends.py | 3 + {grinder => threadmill}/executor.py | 0 .../management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/threadmill.py | 8 +- 13 files changed, 122 insertions(+), 46 deletions(-) rename tests/{test_grinder_command.py => test_command.py} (92%) rename {grinder => threadmill}/__init__.py (100%) rename {grinder => threadmill}/backends.py (94%) rename {grinder => threadmill}/executor.py (100%) rename {grinder => threadmill}/management/__init__.py (100%) rename {grinder => threadmill}/management/commands/__init__.py (100%) rename grinder/management/commands/grinder.py => threadmill/management/commands/threadmill.py (91%) diff --git a/.gitignore b/.gitignore index 5e0d910..318e31b 100644 --- a/.gitignore +++ b/.gitignore @@ -166,7 +166,7 @@ cython_debug/ # Packaging -grinder/_version.py +threadmill/_version.py # uv uv.lock diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 81eb287..d942904 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,6 @@ curl -sSL https://raw.githubusercontent.com/codingjoe/naming-things/refs/heads/m - Consistency – We never lose data, even if someone unplugs the power or network. - Durability – We recover from any failures, even poorly written tasks. -- Overhead – We focus resources on running tasks, not on managing the scheduler. - Utilization – We keep the CPU saturated with tasks, not with idle time or waiting for locks. ## Testing diff --git a/README.md b/README.md index 64637b4..23737fc 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,125 @@ -# Django Grinder +# Threadmill

- - - Django Grinder: A queue agnostic worker for Django's task framework. + + + Django Grinder: A queue agnostic worker for Django's task framework.
- Documentation | - Issues | - Changelog | + Documentation | + Issues | + Changelog | Funding 💚

**A queue agnostic worker for Django's task framework.** -- durable, self-healing workers -- graceful shutdown -- CPU, IO, or memory optimized workers +## Design Principles -[![PyPi Version](https://img.shields.io/pypi/v/django-grinder.svg)](https://pypi.python.org/pypi/django-grinder/) -[![Test Coverage](https://codecov.io/gh/codingjoe/django-grinder/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/django-grinder) -[![GitHub License](https://img.shields.io/github/license/codingjoe/django-grinder)](https://raw.githubusercontent.com/codingjoe/django-grinder/master/LICENSE) +- **Durability** – We recover from any failures, even poorly written tasks. +- **Consistency** – We never lose data, even if someone unplugs the power or network. +- **Utilization** – We keep the CPU saturated with tasks, not with idle time or waiting for locks. + +[![PyPi Version](https://img.shields.io/pypi/v/threadmill.svg)](https://pypi.python.org/pypi/threadmill/) +[![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill) +[![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/master/LICENSE) ## Setup You need to have [Django's Task framework][django-tasks] setup properly. ```console -uv add django-grinder +uv add threadmill ``` -Add `grinder` to your `INSTALLED_APPS` in `settings.py`: +Add `threadmill` to your `INSTALLED_APPS` in `settings.py`: ```python # settings.py INSTALLED_APPS = [ - "grinder", + "threadmill", # ... ] ``` -Finally, you launch the scheduler in a separate process: +Finally, you launch the worker pool: + +```console +uv run manage.py threadmill +``` + +## Usage + +The workers are inspired by Gunicorn, and the CLI is very similar. + +### Utilization + +Depending on your workload, you can tweak the number of processes and threads. +Processes allow for parallel compute (no GIL) while threads are great for low-memory concurrent IO. + +```console +uv run manage.py threadmill --processes 4 --threads 2 +``` + +### Health + +If your tasks leak memory, you can recycle (restart) the workers after a certain number of tasks have been processed: + +```console +uv run manage.py threadmill --max-tasks 1000 --max-tasks-jitter 100 +``` + +This will restart the workers after 1000 tasks have been processed, with a random jitter of up to 100 tasks to avoid all workers restarting at the same time. + +Should a worker crash or be killed, the pool will automatically restart it. + +### Shutdown + +A graceful shutdown is possible with the `SIGTERM` or a keyboard interrupt. +All workers will finish the tasks they acquired and publish them. + +You can use `--exit-empty` to exit immediately after all tasks have been processed, +which might be useful for draining a one-off queue. + +### Task Backlog + +You can prefetch tasks from a queue to avoid IO latency bottlenecks. +However, this will increase the memory usage of the worker pool. ```console -uv run manage.py grinder +uv run manage.py threadmill --prefetch 100 ``` -## Current worker behavior +## Integration -The executor currently consumes tasks from a Python `queue.PriorityQueue`. +> [!NOTE] +> This section is for people who want to integrate Threadmill into their queueing system. -- `--backends` and `--queues` are accepted but not used yet. -- Queue items must be `django.tasks.TaskResult` or `(priority, TaskResult)`. +Threadmill is designed to be durable and requires a queueing system to support late acknowledgement. -[django-tasks]: https://docs.djangoproject.com/en/6.0/topics/tasks/ +To use Threadmill, your backend will need to inherit from `threadmill.backends.AcknowledgeableTaskBackend` and implement the following methods: + +```python +class AcknowledgeableTaskBackend(BaseTaskBackend, ABC): + """Provide an interface for tasks queues to be processed by the executor.""" + + def acquire( + self, *queue_names: str, timeout: datetime.timedelta | None = None + ) -> TaskResult: + """ + Return and lock the next task to be processed without removing it from the queue. + + Args: + queue_names: The names of the queues to acquire tasks from. + timeout: The maximum time to wait for a task. If None, wait indefinitely. + + Raises: + TimeoutError: If no task is available within the specified timeout. + """ + raise NotImplementedError + + def acknowledge(self, task_result: TaskResult) -> None: + """Remove the task from the queue and publish the result.""" + raise NotImplementedError +``` diff --git a/pyproject.toml b/pyproject.toml index 9a9aedc..8f58c57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["flit_core>=3.2", "flit_scm", "wheel"] build-backend = "flit_scm:buildapi" [project] -name = "django-grinder" +name = "threadmill" authors = [ { name = "Johannes Maron", email = "johannes@maron.family" }, ] @@ -37,19 +37,19 @@ dependencies = ["django @ git+https://github.com/django/django.git@main#egg=djan [project.urls] # https://packaging.python.org/en/latest/specifications/well-known-project-urls/#well-known-labels -Homepage = "https://github.com/codingjoe/django-grinder" -Changelog = "https://github.com/codingjoe/django-grinder/releases" -Source = "https://github.com/codingjoe/django-grinder" -Releasenotes = "https://github.com/codingjoe/django-grinder/releases/latest" -Documentation = "https://django-grinder.rtfd.io/" -Issues = "https://github.com/codingjoe/django-grinder/issues" +Homepage = "https://github.com/codingjoe/threadmill" +Changelog = "https://github.com/codingjoe/threadmill/releases" +Source = "https://github.com/codingjoe/threadmill" +Releasenotes = "https://github.com/codingjoe/threadmill/releases/latest" +Documentation = "https://github.com/codingjoe/threadmill/" +Issues = "https://github.com/codingjoe/threadmill/issues" Funding = "https://github.com/sponsors/codingjoe" [tool.flit.module] -name = "grinder" +name = "threadmill" [tool.setuptools_scm] -write_to = "grinder/_version.py" +write_to = "threadmill/_version.py" [tool.pytest.ini_options] minversion = "6.0" @@ -62,14 +62,14 @@ markers = [ ] [tool.coverage.run] -source = ["grinder"] +source = ["threadmill"] [tool.coverage.report] show_missing = true skip_covered = true [tool.ruff] -src = ["grinder", "tests"] +src = ["threadmill", "tests"] [tool.ruff.lint] select = [ diff --git a/tests/test_backends.py b/tests/test_backends.py index 832edde..4f1cf08 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -3,7 +3,7 @@ import datetime import pytest -from grinder.backends import AcknowledgeableTaskBackend +from threadmill.backends import AcknowledgeableTaskBackend class BackendDouble(AcknowledgeableTaskBackend): diff --git a/tests/test_grinder_command.py b/tests/test_command.py similarity index 92% rename from tests/test_grinder_command.py rename to tests/test_command.py index bc0f6ee..2341172 100644 --- a/tests/test_grinder_command.py +++ b/tests/test_command.py @@ -6,14 +6,14 @@ import pytest from django.core.management import call_command from django.tasks import default_task_backend -from grinder.management.commands import grinder +from threadmill.management.commands import threadmill class TestKillSoftly: def test_kill_softly__raise_keyboard_interrupt_with_signal_name(self) -> None: """Raise KeyboardInterrupt with signal metadata in message.""" with pytest.raises(KeyboardInterrupt, match="SIGINT"): - grinder.kill_softly(signal.SIGINT, None) + threadmill.kill_softly(signal.SIGINT, None) class TestCommand: @@ -21,7 +21,7 @@ def test_add_arguments__register_all_worker_options(self) -> None: """Register command arguments for worker runtime configuration.""" parser = argparse.ArgumentParser() - grinder.Command().add_arguments(parser) + threadmill.Command().add_arguments(parser) parsed_arguments = parser.parse_args([]) assert parsed_arguments.backends == "default" @@ -40,7 +40,7 @@ def test_call_command__benchmark_compute( default_task_backend.reset() benchmark.pedantic( lambda: call_command( - "grinder", + "threadmill", verbosity=0, queues=["compute"], exit_empty=True, @@ -61,7 +61,7 @@ def test_call_command__benchmark_io( default_task_backend.reset() benchmark.pedantic( lambda: call_command( - "grinder", + "threadmill", verbosity=0, queues=["io"], threads=6, @@ -83,7 +83,7 @@ def test_call_command__benchmark_compute_and_io( default_task_backend.reset() benchmark.pedantic( lambda: call_command( - "grinder", + "threadmill", verbosity=0, queues=["compute", "io"], exit_empty=True, @@ -105,7 +105,7 @@ def test_call_command__benchmark_memory_leak_recovery( default_task_backend.reset(1000) benchmark.pedantic( lambda: call_command( - "grinder", + "threadmill", verbosity=0, queues=["memory"], exit_empty=True, diff --git a/tests/testapp/backends.py b/tests/testapp/backends.py index cfcdc25..df80448 100644 --- a/tests/testapp/backends.py +++ b/tests/testapp/backends.py @@ -3,7 +3,7 @@ from django.tasks import TaskResult, TaskResultStatus from django.utils import timezone from django.utils.module_loading import import_string -from grinder.backends import AcknowledgeableTaskBackend +from threadmill.backends import AcknowledgeableTaskBackend class GeneratingTaskBackend(AcknowledgeableTaskBackend): diff --git a/grinder/__init__.py b/threadmill/__init__.py similarity index 100% rename from grinder/__init__.py rename to threadmill/__init__.py diff --git a/grinder/backends.py b/threadmill/backends.py similarity index 94% rename from grinder/backends.py rename to threadmill/backends.py index acb88cc..bf6dd43 100644 --- a/grinder/backends.py +++ b/threadmill/backends.py @@ -10,6 +10,9 @@ class AcknowledgeableTaskBackend(BaseTaskBackend, ABC): """Provide an interface for tasks queues to be processed by the executor.""" + supports_async_task = True + supports_get_result = True + def acquire( self, *queue_names: str, timeout: datetime.timedelta | None = None ) -> TaskResult: diff --git a/grinder/executor.py b/threadmill/executor.py similarity index 100% rename from grinder/executor.py rename to threadmill/executor.py diff --git a/grinder/management/__init__.py b/threadmill/management/__init__.py similarity index 100% rename from grinder/management/__init__.py rename to threadmill/management/__init__.py diff --git a/grinder/management/commands/__init__.py b/threadmill/management/commands/__init__.py similarity index 100% rename from grinder/management/commands/__init__.py rename to threadmill/management/commands/__init__.py diff --git a/grinder/management/commands/grinder.py b/threadmill/management/commands/threadmill.py similarity index 91% rename from grinder/management/commands/grinder.py rename to threadmill/management/commands/threadmill.py index 041a856..7fe8c39 100644 --- a/grinder/management/commands/grinder.py +++ b/threadmill/management/commands/threadmill.py @@ -15,7 +15,7 @@ def kill_softly(signum, frame): class Command(BaseCommand): - """Run task worker for all tasks with the `cron` decorator.""" + """Run task workers to process enqueued tasks from the specified backends and queues.""" help = __doc__ @@ -68,6 +68,12 @@ def add_arguments(self, parser): default=3600.0, help="Kill hung tasks after timeout seconds. Defaults to one hour.", ) + parser.add_argument( + "--task-backlog-size", + type=int, + default=1, + help="The number of tasks to prefetch from the message queue while all workers are busy. Defaults to 1.", + ) parser.add_argument( "--exit-empty", action="store_true", From 99b8b7ad8bdb3fe149652d0e5dca9103c2c26044 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Tue, 5 May 2026 11:10:20 +0200 Subject: [PATCH 23/27] Add more docs --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 23737fc..cc0689c 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,12 @@ However, this will increase the memory usage of the worker pool. uv run manage.py threadmill --prefetch 100 ``` +### Task Timeouts + +Task timeouts are important to ensure the long-term health of your pool. +However, they need to be aligned with your queueing system's timeout settings. +The message queue needs to requeue a task that hasn't been acknowledged within the timeout. + ## Integration > [!NOTE] From 8a0c23458c1955bf82771805e43421a9abc75d13 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Tue, 5 May 2026 11:29:09 +0200 Subject: [PATCH 24/27] Address review comments --- README.md | 3 ++ tests/test_backends.py | 2 +- tests/test_command.py | 2 +- tests/testapp/settings.py | 2 +- threadmill/backends.py | 1 + threadmill/executor.py | 10 ++---- threadmill/management/commands/threadmill.py | 35 +++++++++++--------- 7 files changed, 29 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index cc0689c..881bcaf 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,9 @@ uv run manage.py threadmill --prefetch 100 ### Task Timeouts +> [!WARNING] +> Work in progress, this feature is not yet stable. + Task timeouts are important to ensure the long-term health of your pool. However, they need to be aligned with your queueing system's timeout settings. The message queue needs to requeue a task that hasn't been acknowledged within the timeout. diff --git a/tests/test_backends.py b/tests/test_backends.py index 4f1cf08..2c9ef4f 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -16,7 +16,7 @@ def test_acquire__raise_not_implemented_error(self) -> None: """Raise NotImplementedError for backend acquire API.""" with pytest.raises(NotImplementedError): BackendDouble(alias="default", params={}).acquire( - datetime.timedelta(seconds=1) + timeout=datetime.timedelta(seconds=1) ) def test_acknowledge__raise_not_implemented_error(self) -> None: diff --git a/tests/test_command.py b/tests/test_command.py index 2341172..d9192f0 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -109,7 +109,7 @@ def test_call_command__benchmark_memory_leak_recovery( verbosity=0, queues=["memory"], exit_empty=True, - # max_tasks=10, + max_tasks=10, ), rounds=1, iterations=1, diff --git a/tests/testapp/settings.py b/tests/testapp/settings.py index 9c29095..4884ec4 100644 --- a/tests/testapp/settings.py +++ b/tests/testapp/settings.py @@ -39,7 +39,7 @@ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", - "grinder", + "threadmill", "tests.testapp", ] diff --git a/threadmill/backends.py b/threadmill/backends.py index bf6dd43..cb0ab7b 100644 --- a/threadmill/backends.py +++ b/threadmill/backends.py @@ -25,6 +25,7 @@ def acquire( Raises: TimeoutError: If no task is available within the specified timeout. + queue.Empty: If no task is available and timeout is None. """ raise NotImplementedError diff --git a/threadmill/executor.py b/threadmill/executor.py index 22220da..a0a3510 100644 --- a/threadmill/executor.py +++ b/threadmill/executor.py @@ -21,7 +21,7 @@ from django.tasks import TaskResult from django.tasks.base import TaskContext, TaskError, TaskResultStatus -from django.tasks.signals import task_enqueued, task_finished, task_started +from django.tasks.signals import task_finished, task_started from django.utils import timezone from django.utils.json import normalize_json @@ -41,7 +41,7 @@ @dataclasses.dataclass(kw_only=True, slots=True) class TaskExecutor: - """Consume tasks from a priority queue with process and thread pools.""" + """Consume tasks from shared joinable queues with process and thread pools.""" backend: AcknowledgeableTaskBackend workers: int | None = None @@ -226,11 +226,6 @@ def __init__( def run(self) -> None: """Start consuming tasks for this thread.""" while self.worker.expired is None or not self.worker.expired.is_set(): - if ( - self.worker.shutdown_requested.is_set() - and self.worker.task_queue.empty() - ): - return try: task_result = self.worker.task_queue.get(timeout=1.0) except Empty: @@ -258,7 +253,6 @@ def execute_task_result(self, task_result: TaskResult) -> TaskResult: last_attempted_at=started_at, worker_ids=[*task_result.worker_ids, self.name], ) - task_enqueued.send(TaskExecutor, task_result=task_result) task_started.send(TaskExecutor, task_result=task_result) try: diff --git a/threadmill/management/commands/threadmill.py b/threadmill/management/commands/threadmill.py index 7fe8c39..0fa7249 100644 --- a/threadmill/management/commands/threadmill.py +++ b/threadmill/management/commands/threadmill.py @@ -2,8 +2,13 @@ import signal import sys -from django.core.management import BaseCommand -from django.tasks import task_backends +from django.core.management import BaseCommand, CommandError +from django.tasks import ( + DEFAULT_TASK_BACKEND_ALIAS, + DEFAULT_TASK_QUEUE_NAME, + InvalidTaskBackend, + task_backends, +) from ...executor import TaskExecutor @@ -22,17 +27,16 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( "-b", - "--backends", - nargs="+", - default="default", + "--backend", + default=DEFAULT_TASK_BACKEND_ALIAS, help="Alias of the tasks backend to use.", ) parser.add_argument( "-q", "--queues", nargs="+", - default="default", - help="Queue names to listen too and process tasks from.", + default=[DEFAULT_TASK_QUEUE_NAME], + help="Queue names to listen to and process tasks from.", ) parser.add_argument( "-w", @@ -45,7 +49,7 @@ def add_arguments(self, parser): "--threads", type=int, default=1, - help="Number of threads to use. Defaults to the number of CPU cores minus one. ", + help="Number of threads to use. Defaults to 1. ", ) parser.add_argument( "--max-tasks", @@ -84,7 +88,7 @@ def handle( self, *, verbosity, - backends, + backend, queues, workers, threads, @@ -102,13 +106,14 @@ def handle( signal.signal(signal.SIGTERM, kill_softly) signal.signal(signal.SIGINT, kill_softly) self.stdout.write(self.style.SUCCESS("Starting workers…")) - backend_alias = backends[0] if isinstance(backends, list) else backends - backend = task_backends[backend_alias] - if not set(queues).issubset(backend.queues): - self.stderr.write( - self.style.ERROR("Backend does not support all specified queues.") + try: + backend = task_backends[backend] + except InvalidTaskBackend: + raise CommandError(f"Invalid backend: {backend!r}") + if _non_queues := set(queues) - set(backend.queues): + raise CommandError( + f"Backend does not support all specified queues: {_non_queues!r}" ) - exit(1) exe = TaskExecutor( backend=backend, workers=workers, From ef06f70961e48b69bef46bf3145e9c77018e52fd Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Tue, 5 May 2026 11:30:31 +0200 Subject: [PATCH 25/27] Add preview note --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 881bcaf..a71f339 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,9 @@ - **Consistency** – We never lose data, even if someone unplugs the power or network. - **Utilization** – We keep the CPU saturated with tasks, not with idle time or waiting for locks. +> [!WARNING] +> Threadmill requires a development version of Django and is in a preview stage. + [![PyPi Version](https://img.shields.io/pypi/v/threadmill.svg)](https://pypi.python.org/pypi/threadmill/) [![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill) [![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/master/LICENSE) From 0b1f1a819b883db4c43d96a826758d1b1644cadf Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Tue, 5 May 2026 11:45:15 +0200 Subject: [PATCH 26/27] Add more tests --- tests/test_command.py | 30 ++++++++++++++++++++++++------ tests/testapp/backends.py | 18 ++++++++++++++++++ tests/testapp/tasks.py | 7 +++++++ 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/tests/test_command.py b/tests/test_command.py index d9192f0..dec66ab 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -10,14 +10,14 @@ class TestKillSoftly: - def test_kill_softly__raise_keyboard_interrupt_with_signal_name(self) -> None: + def test_kill_softly__raise_keyboard_interrupt_with_signal_name(self): """Raise KeyboardInterrupt with signal metadata in message.""" with pytest.raises(KeyboardInterrupt, match="SIGINT"): threadmill.kill_softly(signal.SIGINT, None) class TestCommand: - def test_add_arguments__register_all_worker_options(self) -> None: + def test_add_arguments__register_all_worker_options(self): """Register command arguments for worker runtime configuration.""" parser = argparse.ArgumentParser() @@ -35,7 +35,7 @@ def test_add_arguments__register_all_worker_options(self) -> None: def test_call_command__benchmark_compute( self, benchmark, - ) -> None: + ): """Benchmark command execution for one CPU intense task solved 100 times.""" default_task_backend.reset() benchmark.pedantic( @@ -56,7 +56,7 @@ def test_call_command__benchmark_compute( def test_call_command__benchmark_io( self, benchmark, - ) -> None: + ): """Benchmark command execution for one CPU intense task solved 100 times.""" default_task_backend.reset() benchmark.pedantic( @@ -78,7 +78,7 @@ def test_call_command__benchmark_io( def test_call_command__benchmark_compute_and_io( self, benchmark, - ) -> None: + ): """Benchmark command execution for one CPU intense task solved 100 times.""" default_task_backend.reset() benchmark.pedantic( @@ -100,7 +100,7 @@ def test_call_command__benchmark_compute_and_io( def test_call_command__benchmark_memory_leak_recovery( self, benchmark, - ) -> None: + ): """Benchmark command execution for one CPU intense task solved 100 times.""" default_task_backend.reset(1000) benchmark.pedantic( @@ -115,3 +115,21 @@ def test_call_command__benchmark_memory_leak_recovery( iterations=1, warmup_rounds=0, ) + + @pytest.mark.benchmark + def test_call_command__benchmark_random_crash(self, benchmark): + """Benchmark command execution for one CPU intense task solved 100 times.""" + default_task_backend.reset() + benchmark.pedantic( + lambda: call_command( + "threadmill", + verbosity=0, + exit_empty=True, + ), + rounds=1, + iterations=1, + warmup_rounds=0, + ) + assert default_task_backend.issued_task_count == 100, ( + "All tasks should be issued." + ) diff --git a/tests/testapp/backends.py b/tests/testapp/backends.py index df80448..f237f90 100644 --- a/tests/testapp/backends.py +++ b/tests/testapp/backends.py @@ -20,6 +20,23 @@ def reset(self, task_count=100): GeneratingTaskBackend.solved_task_count = 0 GeneratingTaskBackend.issued_task_count = 0 self._queues = { + "default": [ + TaskResult( + task=import_string("tests.testapp.tasks.random_crash"), + enqueued_at=timezone.now(), + status=TaskResultStatus.READY, + id=f"default-{i + 1}", + args=[], + kwargs={}, + worker_ids=[], + started_at=None, + finished_at=None, + errors=[], + backend=self.alias, + last_attempted_at=None, + ) + for i in range(task_count) + ], "compute": [ TaskResult( task=import_string("tests.testapp.tasks.compute_workload"), @@ -86,6 +103,7 @@ def acquire(self, *queue_names, timeout=None): for queue in sorted(queues, key=len, reverse=True): return queue.pop(0) except IndexError as e: + GeneratingTaskBackend.issued_task_count -= 1 raise Empty("No more tasks to solve.") from e raise Empty("No more tasks to solve.") diff --git a/tests/testapp/tasks.py b/tests/testapp/tasks.py index 96400f8..87d912c 100644 --- a/tests/testapp/tasks.py +++ b/tests/testapp/tasks.py @@ -46,3 +46,10 @@ async def io_workload(): def memory_workload(): """Allocate and leak 100MB of memory.""" leak[uuid.uuid4()] = "x" * 1024 * 1024 * 100 + + +@task() +def random_crash(): + """Raise a random exception.""" + if random.random() < 0.75: # noqa: S311 + exit(1) From d176024968fad135786279c6bc19e66fc1247220 Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Tue, 5 May 2026 11:46:47 +0200 Subject: [PATCH 27/27] Fix tests --- tests/test_command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_command.py b/tests/test_command.py index dec66ab..0947aa9 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -24,8 +24,8 @@ def test_add_arguments__register_all_worker_options(self): threadmill.Command().add_arguments(parser) parsed_arguments = parser.parse_args([]) - assert parsed_arguments.backends == "default" - assert parsed_arguments.queues == "default" + assert parsed_arguments.backend == "default" + assert parsed_arguments.queues == ["default"] assert parsed_arguments.threads == 1 assert parsed_arguments.max_tasks == 0 assert parsed_arguments.max_tasks_jitter == 0