From e7bc71943500660f981579b5e5f57a5451e053b7 Mon Sep 17 00:00:00 2001 From: "waldemort-auto[bot]" <223556219+Copilot@users.noreply.github.com> Date: Thu, 13 Aug 2026 19:11:47 +0000 Subject: [PATCH 1/3] Build abi3 wheels so the SDK installs on Python 3.9+ (fixes #6) Published wheels were tagged cp312 only, so every other interpreter fell back to the sdist. That path requires a Rust toolchain and OpenSSL dev headers, and on Python 3.14 it cannot succeed at all because PyO3 0.23 supports at most 3.13. Declared metadata (requires-python >=3.9) invited that attempt. Build the extension against CPython's stable ABI via pyo3/abi3-py39, so one cp39-abi3 wheel per platform serves 3.9 and every later version. Verified: a wheel built on CPython 3.12 installs and runs the repository smoke orchestration to completion on 3.9.25, 3.12.13, and 3.14.7. The published 0.1.27 release resolves to "no usable wheels" on 3.9 and 3.14. The companion .github/workflows/publish.yml change (widen both smoke matrices to 3.9/3.12/3.13/3.14 and fail the build on any non-abi3 wheel) is attached to the pull request instead of committed here: this App has no GitHub Actions `workflows` permission, so the push was rejected. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 16 ++++++++++++++++ Cargo.toml | 2 +- README.md | 4 ++++ pyproject.toml | 1 + 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8064a7..44a0ffa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **Wheels now install on Python 3.9 and later, not just 3.12.** The published + wheels were tagged `cp312` only, so every other interpreter fell back to the + sdist, which requires a Rust toolchain and OpenSSL headers and fails outright + on Python 3.14 (PyO3 0.23 supports at most 3.13). The extension is now built + against CPython's stable ABI (`pyo3/abi3-py39`), producing one `cp39-abi3` + wheel per platform that works on 3.9 through current and future releases. + +### Changed + +- **Packaging CI now smoke-tests Python 3.9, 3.12, 3.13, and 3.14** on every + supported OS, and fails the build if any produced wheel is not `abi3`-tagged. + The previous matrix tested only 3.12, so the version gap was invisible to CI. +- Added the Python 3.14 classifier. + ## [0.1.27] - 2026-07-29 ### Changed diff --git a/Cargo.toml b/Cargo.toml index aebaff5..f11ad6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ crate-type = ["cdylib"] [dependencies] duroxide = { version = "0.1.30", features = ["sqlite"] } duroxide-pg = "0.1.34" -pyo3 = { version = "0.23", features = ["extension-module"] } +pyo3 = { version = "0.23", features = ["extension-module", "abi3-py39"] } async-trait = "0.1" tokio = { version = "1", features = ["full"] } serde = { version = "1", features = ["derive"] } diff --git a/README.md b/README.md index 676c663..caddebf 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ pip install duroxide Prebuilt wheels are published for macOS arm64/x64, Linux manylinux x86_64 and aarch64, and Windows x86_64. +Wheels are built against CPython's stable ABI (`abi3`, minimum 3.9), so a single +wheel per platform works on Python 3.9 and every later version, including +releases published after this SDK. + ## Quick Start ```python diff --git a/pyproject.toml b/pyproject.toml index ec2812c..521ebcb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] [project.optional-dependencies] From d8c57415d33db01a61a54a51aa1d18e12069bc2e Mon Sep 17 00:00:00 2001 From: Pino de Candia Date: Fri, 14 Aug 2026 21:32:46 +0000 Subject: [PATCH 2/3] CI: smoke-test 3.9/3.12/3.13/3.14 and require abi3-tagged wheels --- .github/workflows/publish.yml | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 14d55b3..0292143 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -128,7 +128,9 @@ jobs: shell: bash - os: windows-latest shell: pwsh - python: ["3.12"] + # abi3 wheels must install on every supported minor, not just the + # one CI happens to build with (see issue 6). + python: ["3.9", "3.12", "3.13", "3.14"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -144,6 +146,24 @@ jobs: - name: List wheels shell: bash run: ls -lh wheels/ + - name: Assert wheels are abi3-tagged + shell: bash + run: | + # A cp3XX-cp3XX wheel silently restricts the SDK to a single Python + # minor (issue 6). Every built wheel must carry the stable-ABI tag. + shopt -s nullglob + bad=() + for w in wheels/*.whl; do + case "$(basename "$w")" in + *-abi3-*) ;; + *) bad+=("$w") ;; + esac + done + if [ ${#bad[@]} -gt 0 ]; then + printf 'non-abi3 wheel: %s\n' "${bad[@]}" + exit 1 + fi + echo "all wheels abi3-tagged" - name: Run smoke (local wheel) if: matrix.shell == 'bash' env: @@ -201,7 +221,9 @@ jobs: shell: bash - os: windows-latest shell: pwsh - python: ["3.12"] + # abi3 wheels must install on every supported minor, not just the + # one CI happens to build with (see issue 6). + python: ["3.9", "3.12", "3.13", "3.14"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 From 024027902b5ddfcaf4b5cdc85789f53c5468d72d Mon Sep 17 00:00:00 2001 From: Pino de Candia Date: Fri, 14 Aug 2026 22:21:31 +0000 Subject: [PATCH 3/3] Build native wheels for CPython 3.9-3.14 --- .agents/skills/pyo3-interop/SKILL.md | 38 +++--- .github/copilot-instructions.md | 14 +- .github/workflows/publish.yml | 186 ++++++++++++++------------- CHANGELOG.md | 14 +- Cargo.toml | 2 +- README.md | 6 +- ci/smoke/run-local.ps1 | 2 +- ci/smoke/run-local.sh | 2 +- ci/smoke/run-registry.ps1 | 2 +- ci/smoke/run-registry.sh | 2 +- docs/architecture.md | 34 ++--- src/client.rs | 48 +++---- src/handlers.rs | 10 +- src/lib.rs | 2 +- src/pg_provider.rs | 2 +- src/runtime.rs | 6 +- src/types.rs | 26 ++-- 17 files changed, 202 insertions(+), 194 deletions(-) diff --git a/.agents/skills/pyo3-interop/SKILL.md b/.agents/skills/pyo3-interop/SKILL.md index 7ed788f..fea35bf 100644 --- a/.agents/skills/pyo3-interop/SKILL.md +++ b/.agents/skills/pyo3-interop/SKILL.md @@ -1,6 +1,6 @@ --- name: pyo3-interop -description: Rust↔Python interop architecture in duroxide-python. Use when modifying the PyO3 bridge, adding ScheduledTask types, fixing GIL deadlocks, changing tracing delegation, or debugging block_in_place / with_gil behavior. +description: Rust↔Python interop architecture in duroxide-python. Use when modifying the PyO3 bridge, adding ScheduledTask types, fixing GIL deadlocks, changing tracing delegation, or debugging block_in_place / attach behavior. --- # PyO3 Interop Architecture @@ -17,7 +17,7 @@ duroxide-python bridges Rust's duroxide runtime to Python via PyO3/maturin. The | `src/types.rs` | `ScheduledTask` enum — the protocol between Python and Rust | | `src/lib.rs` | PyO3 module entry point, `#[pyfunction]` trace functions | | `src/runtime.rs` | `PyRuntime` — wraps `duroxide::Runtime`, global tokio runtime | -| `src/client.rs` | `PyClient` — wraps `duroxide::Client`, all methods with `py.allow_threads()` | +| `src/client.rs` | `PyClient` — wraps `duroxide::Client`, all methods with `py.detach()` | | `src/provider.rs` | `PySqliteProvider` | | `src/pg_provider.rs` | `PyPostgresProvider` | | `python/duroxide/__init__.py` | Python wrapper: SqliteProvider, PostgresProvider, Client, Runtime, decorators | @@ -30,7 +30,7 @@ This is the most important difference between duroxide-python and duroxide-node. ### The Problem -PyO3 holds the GIL when Python calls into Rust `#[pymethods]`. If that method calls `TOKIO_RT.block_on()`, it blocks the thread while holding the GIL. Meanwhile, orchestration handlers running on tokio threads need the GIL via `Python::with_gil()` — **deadlock**. +PyO3 holds the GIL when Python calls into Rust `#[pymethods]`. If that method calls `TOKIO_RT.block_on()`, it blocks the thread while holding the GIL. Meanwhile, orchestration handlers running on tokio threads need the GIL via `Python::attach()` — **deadlock**. ``` Thread A (Python → Rust): @@ -40,16 +40,16 @@ Thread A (Python → Rust): Thread B (Tokio → Python): orchestration handler invoked - → block_in_place + Python::with_gil() ← BLOCKS, waiting for GIL + → block_in_place + Python::attach() ← BLOCKS, waiting for GIL ``` ### The Fix -EVERY method that calls `block_on` must use `py.allow_threads()` to release the GIL before blocking: +EVERY method that calls `block_on` must use `py.detach()` to release the GIL before blocking: ```rust fn wait_for_orchestration(&self, py: Python<'_>, id: String, timeout: u64) -> PyResult<...> { - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { self.client.wait_for_orchestration(&id, timeout).await .map_err(|e| format!("{e}")) @@ -63,14 +63,14 @@ This pattern is applied to ALL 20+ methods in `client.rs` and `runtime.rs`. ### Error Handling Across the Boundary -`PyErr` is not `Send`, so you can't return `PyResult` from inside `allow_threads`. Pattern: -1. Inside `allow_threads`: map errors to `String` via `.map_err(|e| format!("{e}"))` -2. Outside `allow_threads`: map `String` to `PyErr` via `.map_err(PyRuntimeError::new_err)` +`PyErr` is not `Send`, so you can't return `PyResult` from inside `detach`. Pattern: +1. Inside `detach`: map errors to `String` via `.map_err(|e| format!("{e}"))` +2. Outside `detach`: map `String` to `PyErr` via `.map_err(PyRuntimeError::new_err)` ### Rules for ANY New Method 1. **Add `py: Python<'_>` parameter** to the method signature -2. **Wrap `TOKIO_RT.block_on()` in `py.allow_threads(|| { ... })`** +2. **Wrap `TOKIO_RT.block_on()` in `py.detach(|| { ... })`** 3. **Map errors to `String` inside, to `PyErr` outside** 4. **Never hold the GIL while blocking on tokio** @@ -78,12 +78,12 @@ This pattern is applied to ALL 20+ methods in `client.rs` and `runtime.rs`. The replay engine calls `poll_once()` on the handler future. If the future isn't ready in one poll, it's **dropped**. -**Solution: `block_in_place` + `with_gil`** +**Solution: `block_in_place` + `attach`** ```rust fn call_create_blocking(&self, payload: String) -> Result { tokio::task::block_in_place(|| { - Python::with_gil(|py| { + Python::attach(|py| { let result = self.create_fn.call1(py, (payload,))?; // parse result... }) @@ -99,7 +99,7 @@ Rust (tokio thread) Python (GIL) 1. invoke(ctx, input) ├─ Store ctx in ORCHESTRATION_CTXS[instance_id] ├─ call_create_blocking(payload) ──────► create_generator(payload) - │ (block_in_place + with_gil) ├─ Create OrchestrationContext + │ (block_in_place + attach) ├─ Create OrchestrationContext │ ├─ Create generator: fn(ctx, input) │ ├─ gen.send(None) → first yield │ └─ Return {"status": "yielded", "task": ...} @@ -107,7 +107,7 @@ Rust (tokio thread) Python (GIL) ├─ Loop: │ ├─ execute_task(ctx, task) // Real DurableFuture or replay │ ├─ call_next_blocking(result) ──────► next_step(result) - │ │ (block_in_place + with_gil) ├─ gen.send(value) or gen.throw(exc) + │ │ (block_in_place + attach) ├─ gen.send(value) or gen.throw(exc) │ │ └─ Return next task or completion │ │◄────────────────────────────────────┘ │ └─ If completed/error: break @@ -116,7 +116,7 @@ Rust (tokio thread) Python (GIL) ## Activity Interop (Synchronous GIL Call) -Activities in duroxide-python are **synchronous** functions (unlike duroxide-node's async activities). They run on tokio threads via `block_in_place` + `with_gil`: +Activities in duroxide-python are **synchronous** functions (unlike duroxide-node's async activities). They run on tokio threads via `block_in_place` + `attach`: ``` Rust Python @@ -124,7 +124,7 @@ Rust Python invoke(ctx, input) ├─ Generate unique token (act-0, act-1, ...) ├─ Store ctx in ACTIVITY_CTXS[token] - ├─ block_in_place + with_gil ─────────────► wrapped_fn(payload) + ├─ block_in_place + attach ───────────────► wrapped_fn(payload) │ ├─ Parse ctx, create ActivityContext │ ├─ Call user's function (synchronous) │ └─ Return JSON result @@ -209,7 +209,7 @@ static TOKIO_RT: LazyLock = LazyLock::new(|| { }); ``` -All async operations go through `TOKIO_RT.block_on()` (with GIL released via `py.allow_threads()`). No pyo3-async-runtimes needed. +All async operations go through `TOKIO_RT.block_on()` (with GIL released via `py.detach()`). No pyo3-async-runtimes needed. ## Provider Polymorphism @@ -248,8 +248,8 @@ def _parse_status(raw): | Pitfall | What Happens | Fix | |---------|-------------|-----| -| Missing `py.allow_threads()` around `block_on` | GIL deadlock — process hangs forever | Wrap ALL `TOKIO_RT.block_on()` calls | -| Returning `PyErr` from inside `allow_threads` | Compile error — `PyErr` is not `Send` | Map to `String` inside, `PyErr` outside | +| Missing `py.detach()` around `block_on` | GIL deadlock — process hangs forever | Wrap ALL `TOKIO_RT.block_on()` calls | +| Returning `PyErr` from inside `detach` | Compile error — `PyErr` is not `Send` | Map to `String` inside, `PyErr` outside | | Thread-local for cross-thread context | Lookup returns `None` — traces silently fail | Use global `HashMap` | | Mutating PyO3 `#[pyclass]` fields from Python | `TypeError` or silently ignored | Use Python wrapper objects | | `cargo build` instead of `maturin develop` | Python imports stale `.so` — changes don't take effect | Always use `maturin develop` | diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 46667d6..2bc4271 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -28,7 +28,7 @@ Python (generators) ←→ PyO3 bridge ←→ duroxide (Rust core) ``` - **Orchestrations**: Python generators that `yield` scheduling commands (dicts) to Rust -- **Activities**: Regular Python functions called by Rust via `block_in_place` + `Python::with_gil()` +- **Activities**: Regular Python functions called by Rust via `block_in_place` + `Python::attach()` - **Providers**: PostgreSQL (`duroxide-pg`) or SQLite — configured at startup - **Tracing**: Delegates to Rust `tracing` — controlled by `RUST_LOG` env var @@ -42,7 +42,7 @@ Python (generators) ←→ PyO3 bridge ←→ duroxide (Rust core) | `types.rs` | `ScheduledTask` enum (Python→Rust protocol) | | `handlers.rs` | Core interop: orchestration loop, activity handler, execute_task, select/join | | `runtime.rs` | `PyRuntime` — global tokio runtime, start/shutdown | -| `client.rs` | `PyClient` — all client methods with `py.allow_threads()` | +| `client.rs` | `PyClient` — all client methods with `py.detach()` | | `provider.rs` | `PySqliteProvider` | | `pg_provider.rs` | `PyPostgresProvider` | @@ -91,7 +91,7 @@ RUST_LOG=info pytest -s # see orchestration/activity traces ```rust fn my_method(&self, py: Python<'_>, ...) -> PyResult<...> { - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { ... }) .map_err(|e| format!("{e}")) }) @@ -99,7 +99,7 @@ fn my_method(&self, py: Python<'_>, ...) -> PyResult<...> { } ``` -Without `py.allow_threads()`, Python holds the GIL while blocking on tokio. Orchestration handlers on tokio threads need the GIL → **deadlock**. See `pyo3-interop` skill for full details. +Without `py.detach()`, Python holds the GIL while blocking on tokio. Orchestration handlers on tokio threads need the GIL → **deadlock**. See `pyo3-interop` skill for full details. ## Interop Model @@ -113,7 +113,7 @@ Python generators yield scheduling commands as plain dicts. The Rust handler loo ### Activities: Synchronous Call -Rust calls Python activity functions synchronously via `block_in_place` + `Python::with_gil()`. Activities are regular `def` functions (not generators, not async). +Rust calls Python activity functions synchronously via `block_in_place` + `Python::attach()`. Activities are regular `def` functions (not generators, not async). ### Tracing: Global Context Maps @@ -149,12 +149,12 @@ pub enum ScheduledTask { ## Key Patterns -### Error Handling Across py.allow_threads +### Error Handling Across py.detach `PyErr` is not `Send`. Map errors inside, convert outside: ```rust -py.allow_threads(|| { +py.detach(|| { TOKIO_RT.block_on(async { ... }).map_err(|e| format!("{e}")) }) .map_err(PyRuntimeError::new_err) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0292143..56a6c2a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,46 +15,68 @@ permissions: jobs: # ── macOS (ARM64 + x86_64) ───────────────────────────────── macos: - runs-on: macos-latest + runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - target: [aarch64-apple-darwin, x86_64-apple-darwin] + platform: [macos-arm64, macos-x64] + python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + include: + - platform: macos-arm64 + os: macos-14 + architecture: arm64 + target: aarch64-apple-darwin + - platform: macos-x64 + os: macos-15-intel + architecture: x64 + target: x86_64-apple-darwin steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: ${{ matrix.python }} + architecture: ${{ matrix.architecture }} - name: Build wheel uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} - args: --release --out dist + args: --release --out dist --interpreter python sccache: "true" + - name: Smoke wheel + env: + WHEEL_DIR: ${{ github.workspace }}/dist + SMOKE_SCRIPT: ${{ github.workspace }}/ci/smoke/smoke.py + run: bash ci/smoke/run-local.sh - uses: actions/upload-artifact@v4 with: - name: wheels-macos-${{ matrix.target }} + name: wheels-${{ matrix.platform }}-py${{ matrix.python }} path: dist/*.whl # ── Linux (x86_64 + aarch64, manylinux) ─────────────────── linux: strategy: + fail-fast: false matrix: + platform: [linux-x64, linux-arm64] + python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] include: - - os: ubuntu-latest + - platform: linux-x64 + os: ubuntu-latest target: x86_64-unknown-linux-gnu - - os: ubuntu-24.04-arm + - platform: linux-arm64 + os: ubuntu-24.04-arm target: aarch64-unknown-linux-gnu runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: ${{ matrix.python }} - name: Build wheel uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} - args: --release --out dist --interpreter python3.12 + args: --release --out dist --interpreter python${{ matrix.python }} manylinux: "2_28" before-script-linux: | # Install OpenSSL dev headers for native-tls (replaces ring/rustls) @@ -66,33 +88,47 @@ jobs: apt-get update && apt-get install -y libssl-dev pkg-config fi # maturin_2_28 images expose pythons at /opt/python/*/bin but not on PATH - ln -sf /opt/python/cp312-cp312/bin/python3 /usr/local/bin/python3 || true + pyver="${{ matrix.python }}" + pytag="cp${pyver//./}-cp${pyver//./}" + ln -sf "/opt/python/$pytag/bin/python3" "/usr/local/bin/python$pyver" sccache: "true" + - name: Smoke wheel + env: + WHEEL_DIR: ${{ github.workspace }}/dist + SMOKE_SCRIPT: ${{ github.workspace }}/ci/smoke/smoke.py + run: bash ci/smoke/run-local.sh - uses: actions/upload-artifact@v4 with: - name: wheels-linux-${{ matrix.target }} + name: wheels-${{ matrix.platform }}-py${{ matrix.python }} path: dist/*.whl # ── Windows (x86_64) ───────────────────────────────────────── windows: runs-on: windows-latest strategy: + fail-fast: false matrix: - target: [x86_64-pc-windows-msvc] + python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: ${{ matrix.python }} - name: Build wheel uses: PyO3/maturin-action@v1 with: - target: ${{ matrix.target }} - args: --release --out dist + target: x86_64-pc-windows-msvc + args: --release --out dist --interpreter python sccache: "true" + - name: Smoke wheel + shell: pwsh + env: + WHEEL_DIR: ${{ github.workspace }}\dist + SMOKE_SCRIPT: ${{ github.workspace }}\ci\smoke\smoke.py + run: pwsh ci/smoke/run-local.ps1 - uses: actions/upload-artifact@v4 with: - name: wheels-windows-${{ matrix.target }} + name: wheels-windows-x64-py${{ matrix.python }} path: dist/*.whl # ── Source distribution ───────────────────────────────────── @@ -110,80 +146,53 @@ jobs: name: wheels-sdist path: dist/*.tar.gz - # ── Pre-publish smoke: install wheel into fresh venv on real OSes ── - smoke-local: - name: Smoke (local) ${{ matrix.os }} py${{ matrix.python }} + # ── Verify all native wheel assets exist before smoke testing ── + verify-wheels: + name: Verify native wheel set needs: [macos, linux, windows] - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-latest - shell: bash - - os: ubuntu-24.04-arm - shell: bash - - os: macos-14 # arm64 - shell: bash - - os: macos-13 # x64 - shell: bash - - os: windows-latest - shell: pwsh - # abi3 wheels must install on every supported minor, not just the - # one CI happens to build with (see issue 6). - python: ["3.9", "3.12", "3.13", "3.14"] - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python }} - - name: Download wheels - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v4 with: pattern: wheels-* merge-multiple: true path: wheels - - name: List wheels - shell: bash - run: ls -lh wheels/ - - name: Assert wheels are abi3-tagged + - name: Require every native wheel target shell: bash run: | - # A cp3XX-cp3XX wheel silently restricts the SDK to a single Python - # minor (issue 6). Every built wheel must carry the stable-ABI tag. shopt -s nullglob - bad=() - for w in wheels/*.whl; do - case "$(basename "$w")" in - *-abi3-*) ;; - *) bad+=("$w") ;; - esac - done - if [ ${#bad[@]} -gt 0 ]; then - printf 'non-abi3 wheel: %s\n' "${bad[@]}" + wheels=(wheels/*.whl) + if [ ${#wheels[@]} -ne 30 ]; then + printf 'expected 30 wheels, found %d\n' "${#wheels[@]}" + printf '%s\n' "${wheels[@]}" exit 1 fi - echo "all wheels abi3-tagged" - - name: Run smoke (local wheel) - if: matrix.shell == 'bash' - env: - WHEEL_DIR: ${{ github.workspace }}/wheels - SMOKE_SCRIPT: ${{ github.workspace }}/ci/smoke/smoke.py - run: bash ci/smoke/run-local.sh - - name: Run smoke (local wheel, Windows) - if: matrix.shell == 'pwsh' - shell: pwsh - env: - WHEEL_DIR: ${{ github.workspace }}/wheels - SMOKE_SCRIPT: ${{ github.workspace }}/ci/smoke/smoke.py - run: pwsh ci/smoke/run-local.ps1 + + require_one() { + local label="$1" + shift + local matches=("$@") + if [ ${#matches[@]} -ne 1 ]; then + printf 'expected one %s wheel, found %d\n' "$label" "${#matches[@]}" + printf '%s\n' "${matches[@]}" + exit 1 + fi + } + + for tag in cp39 cp310 cp311 cp312 cp313 cp314; do + require_one "$tag macOS arm64" wheels/*-"$tag"-"$tag"-macosx_*_arm64.whl + require_one "$tag macOS x64" wheels/*-"$tag"-"$tag"-macosx_*_x86_64.whl + require_one "$tag manylinux arm64" wheels/*-"$tag"-"$tag"-manylinux_2_28_aarch64.whl + require_one "$tag manylinux x64" wheels/*-"$tag"-"$tag"-manylinux_2_28_x86_64.whl + require_one "$tag Windows x64" wheels/*-"$tag"-"$tag"-win_amd64.whl + done # ── Publish to PyPI ───────────────────────────────────────── publish: name: Publish to PyPI runs-on: ubuntu-latest if: github.event_name == 'release' - needs: [macos, linux, windows, sdist, smoke-local] + needs: [sdist, verify-wheels] environment: name: pypi url: https://pypi.org/p/duroxide @@ -204,26 +213,25 @@ jobs: # ── Post-publish smoke: install from real PyPI on real OSes ── smoke-registry: - name: Smoke (registry) ${{ matrix.os }} py${{ matrix.python }} + name: Smoke (registry) ${{ matrix.platform }} py${{ matrix.python }} if: github.event_name == 'release' needs: [publish] strategy: fail-fast: false matrix: + platform: [linux-x64, linux-arm64, macos-arm64, macos-x64, windows-x64] + python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] include: - - os: ubuntu-latest - shell: bash - - os: ubuntu-24.04-arm - shell: bash - - os: macos-14 - shell: bash - - os: macos-13 - shell: bash - - os: windows-latest - shell: pwsh - # abi3 wheels must install on every supported minor, not just the - # one CI happens to build with (see issue 6). - python: ["3.9", "3.12", "3.13", "3.14"] + - platform: linux-x64 + os: ubuntu-latest + - platform: linux-arm64 + os: ubuntu-24.04-arm + - platform: macos-arm64 + os: macos-14 + - platform: macos-x64 + os: macos-15-intel + - platform: windows-x64 + os: windows-latest runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -238,13 +246,13 @@ jobs: V="${V#v}" echo "version=$V" >> "$GITHUB_OUTPUT" - name: Run smoke (registry install) - if: matrix.shell == 'bash' + if: runner.os != 'Windows' env: DUROXIDE_VERSION: ${{ steps.ver.outputs.version }} SMOKE_SCRIPT: ${{ github.workspace }}/ci/smoke/smoke.py run: bash ci/smoke/run-registry.sh - name: Run smoke (registry install, Windows) - if: matrix.shell == 'pwsh' + if: runner.os == 'Windows' shell: pwsh env: DUROXIDE_VERSION: ${{ steps.ver.outputs.version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a0ffa..ab02b7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,18 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **Wheels now install on Python 3.9 and later, not just 3.12.** The published +- **Wheels now install on Python 3.9 through 3.14, not just 3.12.** The published wheels were tagged `cp312` only, so every other interpreter fell back to the sdist, which requires a Rust toolchain and OpenSSL headers and fails outright - on Python 3.14 (PyO3 0.23 supports at most 3.13). The extension is now built - against CPython's stable ABI (`pyo3/abi3-py39`), producing one `cp39-abi3` - wheel per platform that works on 3.9 through current and future releases. + on Python 3.14 (PyO3 0.23 supports at most 3.13). Releases now include a native + wheel for each supported CPython minor and platform. ### Changed -- **Packaging CI now smoke-tests Python 3.9, 3.12, 3.13, and 3.14** on every - supported OS, and fails the build if any produced wheel is not `abi3`-tagged. - The previous matrix tested only 3.12, so the version gap was invisible to CI. +- **Packaging CI now builds and smoke-tests Python 3.9 through 3.14** on every + supported OS, and fails if the expected native wheel set is incomplete. The + previous matrix tested only 3.12, so the version gap was invisible to CI. +- Updated PyO3 from 0.23 to 0.29.2 for Python 3.14 support. - Added the Python 3.14 classifier. ## [0.1.27] - 2026-07-29 diff --git a/Cargo.toml b/Cargo.toml index f11ad6c..1e58bc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ crate-type = ["cdylib"] [dependencies] duroxide = { version = "0.1.30", features = ["sqlite"] } duroxide-pg = "0.1.34" -pyo3 = { version = "0.23", features = ["extension-module", "abi3-py39"] } +pyo3 = { version = "0.29.2", features = ["extension-module"] } async-trait = "0.1" tokio = { version = "1", features = ["full"] } serde = { version = "1", features = ["derive"] } diff --git a/README.md b/README.md index caddebf..bbc8a06 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,9 @@ pip install duroxide Prebuilt wheels are published for macOS arm64/x64, Linux manylinux x86_64 and aarch64, and Windows x86_64. -Wheels are built against CPython's stable ABI (`abi3`, minimum 3.9), so a single -wheel per platform works on Python 3.9 and every later version, including -releases published after this SDK. +Wheels are published for each supported CPython minor version from 3.9 through +3.14 on every supported platform. This lets the extension use CPython's full API +and version-specific optimizations without requiring a Rust toolchain to install. ## Quick Start diff --git a/ci/smoke/run-local.ps1 b/ci/smoke/run-local.ps1 index fc7d058..1b49f0f 100644 --- a/ci/smoke/run-local.ps1 +++ b/ci/smoke/run-local.ps1 @@ -20,7 +20,7 @@ try { & .\.venv\Scripts\Activate.ps1 python -m pip install --upgrade pip | Out-Null - pip install --no-index --find-links "$env:WHEEL_DIR" duroxide + pip install --only-binary :all: --no-index --find-links "$env:WHEEL_DIR" duroxide if ($LASTEXITCODE -ne 0) { throw "pip install failed ($LASTEXITCODE)" } Copy-Item $env:SMOKE_SCRIPT ./smoke.py diff --git a/ci/smoke/run-local.sh b/ci/smoke/run-local.sh index f4daac5..71afffe 100755 --- a/ci/smoke/run-local.sh +++ b/ci/smoke/run-local.sh @@ -26,7 +26,7 @@ python -m pip install --upgrade pip >/dev/null # pip picks the matching wheel for the current interpreter/platform tags. # --no-index + --find-links ensures we install from LOCAL wheels only — no PyPI fallback. -pip install --no-index --find-links "$WHEEL_DIR" duroxide +pip install --only-binary :all: --no-index --find-links "$WHEEL_DIR" duroxide cp "$SMOKE_SCRIPT" ./smoke.py python ./smoke.py diff --git a/ci/smoke/run-registry.ps1 b/ci/smoke/run-registry.ps1 index 2885e20..bf2521f 100644 --- a/ci/smoke/run-registry.ps1 +++ b/ci/smoke/run-registry.ps1 @@ -21,7 +21,7 @@ try { $attempts = 6 for ($i = 1; $i -le $attempts; $i++) { - pip install --pre "duroxide==$env:DUROXIDE_VERSION" + pip install --only-binary :all: --pre "duroxide==$env:DUROXIDE_VERSION" if ($LASTEXITCODE -eq 0) { break } if ($i -eq $attempts) { throw "pip install failed after $attempts attempts" } $sleep = $i * 10 diff --git a/ci/smoke/run-registry.sh b/ci/smoke/run-registry.sh index d4ac975..065a69d 100755 --- a/ci/smoke/run-registry.sh +++ b/ci/smoke/run-registry.sh @@ -26,7 +26,7 @@ python -m pip install --upgrade pip >/dev/null attempts=6 for i in $(seq 1 "$attempts"); do # --pre allows prereleases (0.1.20rc0); harmless for stable. - if pip install --pre "duroxide==$DUROXIDE_VERSION"; then + if pip install --only-binary :all: --pre "duroxide==$DUROXIDE_VERSION"; then break fi if [ "$i" -eq "$attempts" ]; then diff --git a/docs/architecture.md b/docs/architecture.md index 3b645cc..3468358 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -84,7 +84,7 @@ The generator yields a plain descriptor dict (e.g., `{"type": "activity", "name" ### Why Activities Are Synchronous -Unlike duroxide-node where activities are async JS functions called via ThreadsafeFunction, duroxide-python activities are regular synchronous functions called via `block_in_place` + `Python::with_gil()`. Activities run on tokio threads that acquire the GIL to call into Python. +Unlike duroxide-node where activities are async JS functions called via ThreadsafeFunction, duroxide-python activities are regular synchronous functions called via `block_in_place` + `Python::attach()`. Activities run on tokio threads that acquire the GIL to call into Python. Users can use `asyncio.run()` internally if they need async I/O, or use synchronous libraries like `requests`. @@ -94,7 +94,7 @@ This is the most critical difference between duroxide-python and duroxide-node. ### The Problem -PyO3 holds the GIL when Python calls into Rust `#[pymethods]`. If that method calls `TOKIO_RT.block_on()`, it blocks the thread while holding the GIL. Meanwhile, orchestration handlers running on tokio threads need the GIL via `Python::with_gil()` — **deadlock**. +PyO3 holds the GIL when Python calls into Rust `#[pymethods]`. If that method calls `TOKIO_RT.block_on()`, it blocks the thread while holding the GIL. Meanwhile, orchestration handlers running on tokio threads need the GIL via `Python::attach()` — **deadlock**. ``` Thread A (Python → Rust): @@ -104,16 +104,16 @@ Thread A (Python → Rust): Thread B (Tokio → Python): orchestration handler invoked - → block_in_place + Python::with_gil() ← BLOCKS, waiting for GIL + → block_in_place + Python::attach() ← BLOCKS, waiting for GIL ``` ### The Fix -EVERY method that calls `block_on` must use `py.allow_threads()` to release the GIL before blocking: +EVERY method that calls `block_on` must use `py.detach()` to release the GIL before blocking: ```rust fn wait_for_orchestration(&self, py: Python<'_>, id: String, timeout: u64) -> PyResult<...> { - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { self.client.wait_for_orchestration(&id, timeout).await .map_err(|e| format!("{e}")) @@ -127,7 +127,7 @@ This pattern is applied to ALL 20+ methods in `client.rs` and `runtime.rs`. ### Error Handling Across the Boundary -`PyErr` is not `Send`, so you can't return `PyResult` from inside `allow_threads`. Pattern: map errors to `String` inside `allow_threads`, then `.map_err(PyRuntimeError::new_err)` outside. +`PyErr` is not `Send`, so you can't return `PyResult` from inside `detach`. Pattern: map errors to `String` inside `detach`, then `.map_err(PyRuntimeError::new_err)` outside. ## Orchestration Handler Loop @@ -141,7 +141,7 @@ The core of the interop is in `src/handlers.rs`. Here's the sequence for a singl ├─ Store ctx in ORCHESTRATION_CTXS map │ ├─ call_create_blocking(payload) ──────────────────► create_generator(payload) - │ (block_in_place + with_gil) │ + │ (block_in_place + attach) │ │ ├─ Create OrchestrationContext │ ├─ Create generator: fn(ctx, input) │ ├─ gen.send(None) → first yield @@ -167,10 +167,10 @@ The core of the interop is in `src/handlers.rs`. Here's the sequence for a singl ### The block_in_place Fix -The replay engine's `poll_once()` drops the handler future after a single poll. We use `tokio::task::block_in_place()` + `Python::with_gil()` to synchronously call Python generator functions from tokio threads. This works because: +The replay engine's `poll_once()` drops the handler future after a single poll. We use `tokio::task::block_in_place()` + `Python::attach()` to synchronously call Python generator functions from tokio threads. This works because: - `block_in_place` tells tokio this thread is doing blocking work -- `with_gil()` acquires the GIL only when needed -- The GIL is released by `py.allow_threads()` in the client/runtime methods +- `attach()` acquires the GIL only when needed +- The GIL is released by `py.detach()` in the client/runtime methods ### Activity Handler @@ -184,7 +184,7 @@ invoke(ctx, input) ├─ Store ctx in ACTIVITY_CTXS map (token-keyed) ├─ Serialize ctx + input as payload │ - ├─ block_in_place + with_gil ─────────────────► wrapped_fn(payload) + ├─ block_in_place + attach ───────────────────► wrapped_fn(payload) │ │ │ ├─ Parse ctx, create ActivityContext │ ├─ Call user's function @@ -252,7 +252,7 @@ Internally, `PyRuntime` stores `Arc` so all provider operations ar ## Global Tokio Runtime -A single `static TOKIO_RT: LazyLock` is used for all async operations. All `TOKIO_RT.block_on()` calls release the GIL first via `py.allow_threads()`. No pyo3-async-runtimes needed — this keeps the design simple. +A single `static TOKIO_RT: LazyLock` is used for all async operations. All `TOKIO_RT.block_on()` calls release the GIL first via `py.detach()`. No pyo3-async-runtimes needed — this keeps the design simple. ## Crate Version Alignment @@ -285,7 +285,7 @@ Orchestration functions must be regular generators (`def` with `yield`), not asy ### Activities Are Synchronous -Activities are called via `block_in_place` + `with_gil()` on tokio threads. They are regular synchronous Python functions. For async I/O, use `asyncio.run()` inside the activity body. +Activities are called via `block_in_place` + `attach()` on tokio threads. They are regular synchronous Python functions. For async I/O, use `asyncio.run()` inside the activity body. ### select/race Supports 2 Tasks @@ -334,7 +334,7 @@ Python Rust (PyO3) Provider ────── ─────────── ───────────── client.wait_for_status_change(id, last_version, poll_ms, timeout_ms) │ - └─► py.allow_threads(|| { + └─► py.detach(|| { TOKIO_RT.block_on(async { loop { status = provider.get_status(id) @@ -369,13 +369,13 @@ ctx.set_kv_value("status", "ready") client.get_kv_value(id, "status") │ - └─► py.allow_threads(|| TOKIO_RT.block_on(client.get_kv_value(id, "status"))) + └─► py.detach(|| TOKIO_RT.block_on(client.get_kv_value(id, "status"))) │ └─► provider.get_kv_value(id, "status") client.wait_for_kv_value(id, "status", timeout_ms) │ - └─► py.allow_threads(|| TOKIO_RT.block_on(client.wait_for_kv_value(...))) + └─► py.detach(|| TOKIO_RT.block_on(client.wait_for_kv_value(...))) │ └─► repeated provider.get_kv_value(...) polling until key exists or timeout ``` @@ -393,7 +393,7 @@ Python Rust (PyO3) Provider ────── ─────────── ───────────── client.enqueue_event(id, "inbox", data) │ - └─► py.allow_threads(|| { + └─► py.detach(|| { TOKIO_RT.block_on(async { client.enqueue_event(id, "inbox", data).await }) diff --git a/src/client.rs b/src/client.rs index 90910fc..c11e3fe 100644 --- a/src/client.rs +++ b/src/client.rs @@ -124,7 +124,7 @@ impl PyClient { input: String, ) -> PyResult<()> { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { client .start_orchestration(&instance_id, &orchestration_name, input) @@ -145,7 +145,7 @@ impl PyClient { version: String, ) -> PyResult<()> { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { client .start_orchestration_versioned( @@ -164,7 +164,7 @@ impl PyClient { /// Get the current status of an orchestration instance. fn get_status(&self, py: Python<'_>, instance_id: String) -> PyResult { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let status = client .get_orchestration_status(&instance_id) @@ -185,7 +185,7 @@ impl PyClient { ) -> PyResult { let client = self.inner.clone(); let timeout = Duration::from_millis(timeout_ms as u64); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let status = client .wait_for_orchestration(&instance_id, timeout) @@ -206,7 +206,7 @@ impl PyClient { reason: Option, ) -> PyResult<()> { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { client .cancel_instance(&instance_id, reason.unwrap_or_default()) @@ -226,7 +226,7 @@ impl PyClient { data: String, ) -> PyResult<()> { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { client .raise_event(&instance_id, &event_name, data) @@ -246,7 +246,7 @@ impl PyClient { data: String, ) -> PyResult<()> { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { client .enqueue_event(&instance_id, &queue_name, data) @@ -269,7 +269,7 @@ impl PyClient { let client = self.inner.clone(); let poll_interval = Duration::from_millis(poll_interval_ms); let timeout = Duration::from_millis(timeout_ms); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let status = client .wait_for_status_change(&instance_id, last_seen_version, poll_interval, timeout) @@ -289,7 +289,7 @@ impl PyClient { key: String, ) -> PyResult> { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { client .get_kv_value(&instance_id, &key) @@ -310,7 +310,7 @@ impl PyClient { ) -> PyResult { let client = self.inner.clone(); let timeout = Duration::from_millis(timeout_ms); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { client .wait_for_kv_value(&instance_id, &key, timeout) @@ -324,7 +324,7 @@ impl PyClient { /// Get system metrics (if provider supports management). fn get_system_metrics(&self, py: Python<'_>) -> PyResult { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let metrics = client .get_system_metrics() @@ -350,7 +350,7 @@ impl PyClient { instance_id: String, ) -> PyResult> { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let stats = client .get_orchestration_stats(&instance_id) @@ -371,7 +371,7 @@ impl PyClient { /// Get queue depths (if provider supports management). fn get_queue_depths(&self, py: Python<'_>) -> PyResult { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let depths = client .get_queue_depths() @@ -390,7 +390,7 @@ impl PyClient { /// List all orchestration instance IDs. fn list_all_instances(&self, py: Python<'_>) -> PyResult> { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { client .list_all_instances() @@ -404,7 +404,7 @@ impl PyClient { /// List orchestration instance IDs by status. fn list_instances_by_status(&self, py: Python<'_>, status: String) -> PyResult> { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { client .list_instances_by_status(&status) @@ -418,7 +418,7 @@ impl PyClient { /// Get detailed info about a specific instance. fn get_instance_info(&self, py: Python<'_>, instance_id: String) -> PyResult { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let info = client .get_instance_info(&instance_id) @@ -448,7 +448,7 @@ impl PyClient { execution_id: i64, ) -> PyResult { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let info = client .get_execution_info(&instance_id, execution_id as u64) @@ -470,7 +470,7 @@ impl PyClient { /// List execution IDs for an instance. fn list_executions(&self, py: Python<'_>, instance_id: String) -> PyResult> { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let ids = client .list_executions(&instance_id) @@ -490,7 +490,7 @@ impl PyClient { execution_id: i64, ) -> PyResult> { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let events = client .read_execution_history(&instance_id, execution_id as u64) @@ -522,7 +522,7 @@ impl PyClient { /// Get the full instance tree (root + all descendants). fn get_instance_tree(&self, py: Python<'_>, instance_id: String) -> PyResult { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let tree = client .get_instance_tree(&instance_id) @@ -547,7 +547,7 @@ impl PyClient { force: bool, ) -> PyResult { let client = self.inner.clone(); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let result = client .delete_instance(&instance_id, force) @@ -576,7 +576,7 @@ impl PyClient { completed_before: filter.completed_before.map(|v| v as u64), limit: filter.limit.map(|v| v as u32), }; - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let result = client .delete_instance_bulk(rust_filter) @@ -605,7 +605,7 @@ impl PyClient { keep_last: options.keep_last.map(|v| v as u32), completed_before: options.completed_before.map(|v| v as u64), }; - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let result = client .prune_executions(&instance_id, rust_options) @@ -638,7 +638,7 @@ impl PyClient { keep_last: options.keep_last.map(|v| v as u32), completed_before: options.completed_before.map(|v| v as u64), }; - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { let result = client .prune_executions_bulk(rust_filter, rust_options) diff --git a/src/handlers.rs b/src/handlers.rs index 7387edb..24d3e5f 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -100,7 +100,7 @@ impl Drop for OrchestrationInvokeGuard { }; // Call dispose synchronously via GIL - if let Err(e) = Python::with_gil(|py| -> PyResult<()> { + if let Err(e) = Python::attach(|py| -> PyResult<()> { self.dispose_fn.call1(py, (gen_id.to_string(),))?; Ok(()) }) { @@ -240,7 +240,7 @@ impl PyActivityHandler { // Call the Python callable synchronously via GIL (activity functions are regular def) let result: String = tokio::task::block_in_place(|| { - Python::with_gil(|py| { + Python::attach(|py| { let result = self .callback .call1(py, (payload,)) @@ -284,7 +284,7 @@ impl PyOrchestrationHandler { /// Call the Python create function synchronously using block_in_place + GIL. fn call_create_blocking(&self, payload: String) -> Result { tokio::task::block_in_place(|| { - Python::with_gil(|py| { + Python::attach(|py| { let result = self .create_fn .call1(py, (payload,)) @@ -312,7 +312,7 @@ impl PyOrchestrationHandler { .to_string(); tokio::task::block_in_place(|| { - Python::with_gil(|py| { + Python::attach(|py| { let result = self .next_fn .call1(py, (payload,)) @@ -886,7 +886,7 @@ impl duroxide::runtime::OrchestrationHandler for PyOrchestrationHandler { .insert(instance_id.clone(), ctx.clone()); let mut guard = OrchestrationInvokeGuard::new( instance_id.clone(), - Python::with_gil(|py| self.dispose_fn.clone_ref(py)), + Python::attach(|py| self.dispose_fn.clone_ref(py)), ); let ctx_info = serde_json::json!({ diff --git a/src/lib.rs b/src/lib.rs index 1514f8a..e071a26 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -173,7 +173,7 @@ fn init_tracing( Ok(()) } -#[pymodule] +#[pymodule(gil_used = true)] fn _duroxide(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(activity_trace_log, m)?)?; m.add_function(wrap_pyfunction!(orchestration_trace_log, m)?)?; diff --git a/src/pg_provider.rs b/src/pg_provider.rs index bc663c9..c440d75 100644 --- a/src/pg_provider.rs +++ b/src/pg_provider.rs @@ -11,7 +11,7 @@ use duroxide_pg::{PostgresProvider, ProviderConfig}; /// Python-visible options for Entra ID (Azure AD) authentication. /// /// All fields are optional; omitting a field uses the duroxide-pg default. -#[pyclass] +#[pyclass(from_py_object)] #[derive(Clone, Default)] pub struct PyPostgresEntraOptions { pub audience: Option, diff --git a/src/runtime.rs b/src/runtime.rs index 887c2e4..0168c91 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -20,7 +20,7 @@ pub(crate) static TOKIO_RT: std::sync::LazyLock = }); /// Runtime options configurable from Python. -#[pyclass(name = "RuntimeOptions", get_all)] +#[pyclass(name = "RuntimeOptions", get_all, from_py_object)] #[derive(Debug, Clone)] pub struct PyRuntimeOptions { /// Orchestration concurrency (default: 4) @@ -270,7 +270,7 @@ impl PyRuntime { // Release GIL before blocking — orchestration handlers need GIL access let provider = self.provider.clone(); - let rt = py.allow_threads(|| { + let rt = py.detach(|| { TOKIO_RT.block_on(async { runtime::Runtime::start_with_options( provider, @@ -314,7 +314,7 @@ impl PyRuntime { fn shutdown(&mut self, py: Python<'_>, timeout_ms: Option) -> PyResult<()> { if let Some(rt) = self.inner.take() { let timeout = timeout_ms.map(|ms| ms as u64); - py.allow_threads(|| { + py.detach(|| { TOKIO_RT.block_on(async { rt.shutdown(timeout).await; }); diff --git a/src/types.rs b/src/types.rs index 14651b6..ffaaee0 100644 --- a/src/types.rs +++ b/src/types.rs @@ -138,7 +138,7 @@ pub enum GeneratorStepResult { } /// Orchestration status returned to Python. -#[pyclass(name = "OrchestrationStatus", get_all, set_all)] +#[pyclass(name = "OrchestrationStatus", get_all, set_all, skip_from_py_object)] #[derive(Debug, Clone)] pub struct PyOrchestrationStatus { pub status: String, @@ -149,7 +149,7 @@ pub struct PyOrchestrationStatus { } /// System metrics returned to Python. -#[pyclass(name = "SystemMetrics", get_all)] +#[pyclass(name = "SystemMetrics", get_all, skip_from_py_object)] #[derive(Debug, Clone)] pub struct PySystemMetrics { pub total_instances: i64, @@ -161,7 +161,7 @@ pub struct PySystemMetrics { } /// Per-orchestration runtime stats returned to Python. -#[pyclass(name = "SystemStats", get_all)] +#[pyclass(name = "SystemStats", get_all, skip_from_py_object)] #[derive(Debug, Clone)] pub struct PySystemStats { pub history_event_count: i64, @@ -172,7 +172,7 @@ pub struct PySystemStats { } /// Queue depths returned to Python. -#[pyclass(name = "QueueDepths", get_all)] +#[pyclass(name = "QueueDepths", get_all, skip_from_py_object)] #[derive(Debug, Clone)] pub struct PyQueueDepths { pub orchestrator_queue: i64, @@ -181,7 +181,7 @@ pub struct PyQueueDepths { } /// Instance info returned to Python. -#[pyclass(name = "InstanceInfo", get_all, set_all)] +#[pyclass(name = "InstanceInfo", get_all, set_all, skip_from_py_object)] #[derive(Debug, Clone)] pub struct PyInstanceInfo { pub instance_id: String, @@ -196,7 +196,7 @@ pub struct PyInstanceInfo { } /// Execution info returned to Python. -#[pyclass(name = "ExecutionInfo", get_all, set_all)] +#[pyclass(name = "ExecutionInfo", get_all, set_all, skip_from_py_object)] #[derive(Debug, Clone)] pub struct PyExecutionInfo { pub execution_id: i64, @@ -208,7 +208,7 @@ pub struct PyExecutionInfo { } /// Instance tree returned to Python. -#[pyclass(name = "InstanceTree", get_all)] +#[pyclass(name = "InstanceTree", get_all, skip_from_py_object)] #[derive(Debug, Clone)] pub struct PyInstanceTree { pub root_id: String, @@ -217,7 +217,7 @@ pub struct PyInstanceTree { } /// Delete result returned to Python. -#[pyclass(name = "DeleteInstanceResult", get_all)] +#[pyclass(name = "DeleteInstanceResult", get_all, skip_from_py_object)] #[derive(Debug, Clone)] pub struct PyDeleteInstanceResult { pub instances_deleted: i64, @@ -227,7 +227,7 @@ pub struct PyDeleteInstanceResult { } /// Prune options from Python. -#[pyclass(name = "PruneOptions", get_all)] +#[pyclass(name = "PruneOptions", get_all, skip_from_py_object)] #[derive(Debug, Clone)] pub struct PyPruneOptions { pub keep_last: Option, @@ -247,7 +247,7 @@ impl PyPruneOptions { } /// Prune result returned to Python. -#[pyclass(name = "PruneResult", get_all)] +#[pyclass(name = "PruneResult", get_all, skip_from_py_object)] #[derive(Debug, Clone)] pub struct PyPruneResult { pub instances_processed: i64, @@ -256,7 +256,7 @@ pub struct PyPruneResult { } /// Instance filter from Python. -#[pyclass(name = "InstanceFilter", get_all)] +#[pyclass(name = "InstanceFilter", get_all, skip_from_py_object)] #[derive(Debug, Clone)] pub struct PyInstanceFilter { pub instance_ids: Option>, @@ -282,7 +282,7 @@ impl PyInstanceFilter { } /// Runtime metrics snapshot returned to Python. -#[pyclass(name = "MetricsSnapshot", get_all)] +#[pyclass(name = "MetricsSnapshot", get_all, skip_from_py_object)] #[derive(Debug, Clone)] pub struct PyMetricsSnapshot { pub orch_starts: u64, @@ -305,7 +305,7 @@ pub struct PyMetricsSnapshot { } /// A single history event returned to Python. -#[pyclass(name = "Event", get_all)] +#[pyclass(name = "Event", get_all, skip_from_py_object)] #[derive(Debug, Clone)] pub struct PyEvent { pub event_id: i64,