feat(decdn_node): expose operator-facing config knobs (#29)

ansible/molecule/default/converge.yml

@@ -27,6 +27,26 @@
  # it (verify.yml asserts owner/group/mode). Keep in sync with verify.yml.
  decdn_cache_origin_kind: "fs"
  decdn_cache_origin_path: "/var/lib/decdn/origin"
+ # Operator-facing tuning knobs (#29 + siblings): exercise the new emission +
+ # validation. Chosen to cover the forms a naive template would get wrong — an
+ # explicit 0 (must EMIT, not be dropped), bare-integer Bytes/Percent (never
+ # "2MiB"/"4.0"), a bool (rendered lowercase via `| lower`), and a string list.
+ # verify.yml asserts each renders with the exact value AND type. Keep in sync.
+ decdn_node_to_node_pull_through_enabled: true
+ decdn_gc_interval_sec: 0
+ decdn_pull_ahead_bytes: 2097152
+ decdn_pull_share_ratio_percent: 400
+ decdn_enable_0rtt: false
+ decdn_delivery_floor: 0
+ decdn_settlement_auto_threshold_micro_usdc: 50000000
+ # Two entries each so the {% for %} comma-join branch (loop.last) is exercised —
+ # a single-element list never renders the separator. verify.yml asserts order.
+ decdn_relay_urls:
+ - "https://relay1.example.invalid:443"
+ - "https://relay2.example.invalid:443"
+ decdn_pinned_hashes:
+ - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ - "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
  # Exercise baseline's named-sudo-users feature (see pre_tasks below). Throwaway
  # keys, generated for this test only — public keys, not credentials. Two users
  # on purpose: `alice.smith` has a '.' (so the sudoers.d filename-sanitize maps it

ansible/molecule/default/verify.yml

@@ -96,6 +96,39 @@
  if origin.get("kind") != "fs" or origin.get("path") != "/var/lib/decdn/origin":
  print("node.toml [cache.origin] mismatch (got):", origin, file=sys.stderr)
  sys.exit(1)
+ # Operator-facing tuning knobs (#29 + siblings), set in converge.yml. Assert
+ # exact value AND type: a dropped `0`, a stringified Bytes, or a bool rendered
+ # as an int/string are real regressions a loose `==` would miss (in Python
+ # `0 == False` and `1 == True`, so isinstance guards are load-bearing here).
+ def _is_int(x):
+ return isinstance(x, int) and not isinstance(x, bool)
+ cache, net, pay, bc = (d.get(s, {}) for s in ("cache", "network", "payment", "blockchain"))
+ knob_checks = [
+ ("cache.node_to_node_pull_through_enabled",
+ cache.get("node_to_node_pull_through_enabled") is True),
+ ("cache.gc_interval_sec (explicit 0 must emit)",
+ _is_int(cache.get("gc_interval_sec")) and cache.get("gc_interval_sec") == 0),
+ ("cache.pull_ahead_bytes (bare-int Bytes)",
+ _is_int(cache.get("pull_ahead_bytes")) and cache.get("pull_ahead_bytes") == 2097152),
+ ("cache.pull_share_ratio_percent (bare-int Percent)",
+ _is_int(cache.get("pull_share_ratio_percent")) and cache.get("pull_share_ratio_percent") == 400),
+ # Two-element lists in exact order — exercises the comma-join branch.
+ ("cache.pinned_hashes", cache.get("pinned_hashes") == ["a" * 64, "b" * 64]),
+ ("network.relay_urls",
+ net.get("relay_urls") == ["https://relay1.example.invalid:443",
+ "https://relay2.example.invalid:443"]),
+ ("network.enable_0rtt", net.get("enable_0rtt") is False),
+ ("payment.delivery_floor (explicit 0 must emit)",
+ _is_int(pay.get("delivery_floor")) and pay.get("delivery_floor") == 0),
+ ("blockchain.settlement_auto_threshold_micro_usdc",
+ _is_int(bc.get("settlement_auto_threshold_micro_usdc"))
+ and bc.get("settlement_auto_threshold_micro_usdc") == 50000000),
+ ]
+ knob_bad = [name for name, ok in knob_checks if not ok]
+ if knob_bad:
+ print("node.toml tuning-knob mismatch:", knob_bad, file=sys.stderr)
+ print(" cache=", cache, "\n network=", net, "\n payment=", pay, file=sys.stderr)
+ sys.exit(1)
 
  - name: Assert node.toml is valid TOML and renders the expected content
  ansible.builtin.command:

ansible/molecule/validation/converge.yml

@@ -0,0 +1,130 @@
+---
+# NEGATIVE-path scenario: proves the role's fail-loud tuning-knob validation actually
+# REJECTS bad values (AGENTS.md hard-rule 4), so a regression that weakens an assert
+# fails CI. Each case runs decdn_node with a VALID required config plus ONE bad
+# optional knob, wrapped in block/rescue. The role aborts at its validation asserts
+# (which run before any host mutation), so cases don't interfere and no service starts.
+#
+# The rescue records a case as "rejected" ONLY when the failing task is one of the
+# role's own "Validate optional ..." asserts (matched on ansible_failed_task.name).
+# That is what makes this a real regression guard: if a bad value slipped PAST
+# validation, the role would run on and fail later at a differently-named task (e.g.
+# the keystore gate) — which is NOT counted, so the final assert catches the miss.
+- name: Converge (expect validation failures)
+ hosts: all
+ become: true
+ vars:
+ stub_bin: "{{ lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') }}/molecule/default/files/decdn-node-stub"
+ # Valid baseline so execution reaches the OPTIONAL-knob asserts (required-var
+ # asserts all pass); each case below overrides exactly one optional knob.
+ decdn_node_install_method: manual
+ decdn_node_manual_bin_src: "{{ stub_bin }}"
+ decdn_cli_manual_bin_src: "{{ stub_bin }}"
+ decdn_rpc_url: "https://rpc.example.invalid/"
+ decdn_region: "US"
+ decdn_payment_channel_address: "0x1111111111111111111111111111111111111111"
+ decdn_capacity_bond_address: "0x2222222222222222222222222222222222222222"
+ decdn_slash_judge_address: "0x3333333333333333333333333333333333333333"
+ tasks:
+ - name: Initialize the rejected-by-validation tracker
+ ansible.builtin.set_fact:
+ decdn_rejected: []
+
+ # --- Case: cross-field (the regression guard for the effective-value fix) ------
+ # A raised pull_ahead with the leech cap left UNSET (resolves to 256 MiB): the
+ # daemon would reject floor>cap, so the role must too.
+ - name: "Case cross-field — pull_ahead above the default (unset) leech cap"
+ block:
+ - name: Run decdn_node with pull_ahead > default leech cap
+ ansible.builtin.include_role:
+ name: decdn_node
+ vars:
+ decdn_pull_ahead_bytes: 536870912
+ rescue:
+ - name: Record cross-field rejection (only if a validation assert failed)
+ ansible.builtin.set_fact:
+ decdn_rejected: "{{ decdn_rejected + ['cross-field'] }}"
+ when: ansible_failed_task.name is match('^Validate optional')
+
+ # --- Case: range (numeric but below the daemon's minimum) ---------------------
+ - name: "Case range — event_poll_interval_ms below 250"
+ block:
+ - name: Run decdn_node with event_poll below the daemon minimum
+ ansible.builtin.include_role:
+ name: decdn_node
+ vars:
+ decdn_event_poll_interval_ms: 100
+ rescue:
+ - name: Record range rejection (only if a validation assert failed)
+ ansible.builtin.set_fact:
+ decdn_rejected: "{{ decdn_rejected + ['range'] }}"
+ when: ansible_failed_task.name is match('^Validate optional')
+
+ # --- Case: shape (non-numeric string for an integer knob) ---------------------
+ - name: "Case shape — non-numeric integer knob"
+ block:
+ - name: Run decdn_node with a non-numeric integer knob
+ ansible.builtin.include_role:
+ name: decdn_node
+ vars:
+ decdn_gc_interval_sec: "12x"
+ rescue:
+ - name: Record shape rejection (only if a validation assert failed)
+ ansible.builtin.set_fact:
+ decdn_rejected: "{{ decdn_rejected + ['shape'] }}"
+ when: ansible_failed_task.name is match('^Validate optional')
+
+ # --- Case: bool (quoted non-boolean) ------------------------------------------
+ - name: "Case bool — quoted non-boolean"
+ block:
+ - name: Run decdn_node with a quoted non-boolean
+ ansible.builtin.include_role:
+ name: decdn_node
+ vars:
+ decdn_enable_0rtt: "yes"
+ rescue:
+ - name: Record bool rejection (only if a validation assert failed)
+ ansible.builtin.set_fact:
+ decdn_rejected: "{{ decdn_rejected + ['bool'] }}"
+ when: ansible_failed_task.name is match('^Validate optional')
+
+ # --- Case: list (uppercase pinned hash — upstream rejects uppercase) ----------
+ - name: "Case list — uppercase pinned hash"
+ block:
+ - name: Run decdn_node with an uppercase pinned hash
+ ansible.builtin.include_role:
+ name: decdn_node
+ vars:
+ decdn_pinned_hashes:
+ - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+ rescue:
+ - name: Record list rejection (only if a validation assert failed)
+ ansible.builtin.set_fact:
+ decdn_rejected: "{{ decdn_rejected + ['list'] }}"
+ when: ansible_failed_task.name is match('^Validate optional')
+
+ # --- Case: string (double-quote breaks the rendered TOML string) --------------
+ - name: "Case string — user_agent containing a double-quote"
+ block:
+ - name: Run decdn_node with a quote in user_agent
+ ansible.builtin.include_role:
+ name: decdn_node
+ vars:
+ decdn_cache_user_agent: 'bad"agent'
+ rescue:
+ - name: Record string rejection (only if a validation assert failed)
+ ansible.builtin.set_fact:
+ decdn_rejected: "{{ decdn_rejected + ['string'] }}"
+ when: ansible_failed_task.name is match('^Validate optional')
+
+ - name: Confirm every bad value was rejected by the role's own validation
+ ansible.builtin.assert:
+ that:
+ - decdn_rejected | sort == _decdn_expected | sort
+ fail_msg: >-
+ decdn_node did NOT reject every bad value at its validation asserts.
+ Expected {{ _decdn_expected | sort }}, got {{ decdn_rejected | sort }} —
+ a missing tag means that bad value slipped past validation (it would reach
+ the daemon and crash-loop it at config load).
+ vars:
+ _decdn_expected: ["cross-field", "range", "shape", "bool", "list", "string"]

ansible/molecule/validation/molecule.yml

@@ -0,0 +1,37 @@
+---
+# NEGATIVE-path scenario: proves the role's fail-loud tuning-knob validation
+# actually REJECTS bad values (AGENTS.md hard-rule 4). converge.yml runs the role
+# with an otherwise-valid config plus one bad knob per case and asserts each is
+# rejected — so a regression that weakens an assert fails this scenario. The role
+# aborts at its validation asserts (before any host mutation), so no service starts
+# here; there is no verify/idempotence step — the converge IS the assertion. Reuses
+# the default scenario's stub binary. `baseline` is not exercised (host-only — see
+# molecule/default/molecule.yml).
+dependency:
+ name: galaxy
+ options:
+ requirements-file: ../../requirements.yml
+driver:
+ name: docker
+platforms:
+ - name: decdn-node-validation
+ # Same digest-pinned image as the default scenario (repo convention). Re-resolve
+ # both together to bump.
+ image: geerlingguy/docker-debian12-ansible@sha256:4553092be2c00b1ffe580927b9ff03f3c3a0df32b7dd693a3eb02efb6c2b77b7
+ pre_build_image: true
+ command: /usr/lib/systemd/systemd
+ privileged: true
+ cgroupns_mode: host
+ volumes:
+ - /sys/fs/cgroup:/sys/fs/cgroup:rw
+provisioner:
+ name: ansible
+ env:
+ ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles"
+ ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections"
+scenario:
+ test_sequence:
+ - dependency
+ - create
+ - converge
+ - destroy

ansible/roles/decdn_node/README.md

@@ -118,6 +118,38 @@ Optional (omitted from `node.toml` unless set):
  defaults). A **timeout** only warns; it never fails the deploy — but a non-200 *answer*
  does. See [Readiness](#readiness).
 
+### Optional tuning knobs
+
+These expose daemon config fields that most operators never touch. Each defaults to
+`""` (or `[]`), meaning **omit the key and use the daemon's own default** — set one only
+to override. Unlike the `*_from_block` knobs above, an explicit `0`/`false` **is** emitted
+(`0` is meaningful — e.g. `decdn_gc_interval_sec: 0` disables the GC sweep). A malformed
+or out-of-range value fails loud at deploy time (the daemon would otherwise reject it at
+load and crash-loop). See `defaults/main.yml` for every knob's upstream default and unit.
+
+- **Node-to-node pull-through (#29)** — `decdn_node_to_node_pull_through_enabled` (bool)
+ gates all serve-time origin pull from upstream nodes; the tuning knobs
+ (`decdn_node_pull_probe_fanout`, `decdn_node_pull_timeout_sec`, `decdn_pull_ahead_bytes`,
+ `decdn_max_unrecouped_leech_bytes`, `decdn_pull_share_ratio_percent`,
+ `decdn_pull_through_require_authorized_origin`) only take effect when it is `true`.
+ `*_bytes` / `*_percent` are **bare integers** (bytes; percent at scale 100, `400` = 4.0×).
+- **Settlement / channels** — `decdn_redeem_threshold_micro_usdc`,
+ `decdn_buyer_deposit_micro_usdc`, `decdn_buyer_max_approve` (bool), and the opt-in
+ auto-close pair `decdn_settlement_auto_threshold_micro_usdc` /
+ `decdn_settlement_auto_by_voucher_nonce_span` (`""` disables — a configured `0` is
+ rejected upstream). All µUSDC.
+- **Delivery-rate clamps** — `decdn_delivery_floor` / `decdn_delivery_ceiling` (µUSDC;
+ floor ≤ ceiling) and `decdn_voucher_interval_mb` (`1..=1024`).
+- **Cache tuning** — `decdn_max_probe_holds`, `decdn_stake_lane_reserved_holds`,
+ `decdn_gc_interval_sec`, `decdn_pinned_hashes` (list of 64-char **lowercase-hex** BLAKE3
+ hashes), `decdn_cache_user_agent`.
+- **Blockchain watchers** — `decdn_rpc_watchdog_interval_sec` (`0` or `>= 10`),
+ `decdn_event_poll_interval_ms` (`>= 250`), `decdn_content_blacklist_poll_interval_sec` (`>= 1`).
+- **Network** — `decdn_relay_urls` (list; when non-empty the role emits it and omits the
+ singular `decdn_relay_url`), `decdn_enable_0rtt` (bool).
+- **Observability** — `decdn_otlp_endpoint` (OTLP span export; needs the node built
+ `--features otlp`), `decdn_region_accounting_interval_sec`.
+
 Source contract addresses / chain-id from the deCDN contract deployment for your target
 chain, or the relevant ADR — never guess. See `roles/decdn_node/defaults/main.yml` for the
 full knob list and defaults.

ansible/roles/decdn_node/defaults/main.yml

@@ -55,14 +55,56 @@ decdn_chain_id: 421614 # Arbitrum Sepolia (matches decdn-node's --chain
 
 # --- Network ------------------------------------------------------------------
 decdn_bind_port: 4433 # public QUIC (udp); opened in the baseline firewall
-decdn_relay_url: "" # optional iroh relay for NAT traversal
+decdn_relay_url: "" # optional iroh relay for NAT traversal (deprecated single alias)
+# iroh relay URLs; [] => n0 default relays. When this list is non-empty the template
+# emits it and omits decdn_relay_url (this list supersedes the singular alias).
+# e.g. ["https://relay1.example:443", "https://relay2.example:443"]
+decdn_relay_urls: []
+# The optional knobs below default to "" (or []) => "omit the key, use the daemon's
+# own default". Set a value only to override; an explicit 0/false IS emitted.
+decdn_enable_0rtt: "" # bool; daemon dflt true — QUIC 0-RTT for cdn/probe/v1 (ADR 015)
 
 # --- Economics ----------------------------------------------------------------
 decdn_rate_per_mb: 10 # USDC base units (6 decimals)
+# Delivery-rate clamps + voucher cadence (µUSDC; "" => daemon default).
+decdn_delivery_floor: "" # daemon dflt 0 — rate clamp floor before signing ProbeResponse
+decdn_delivery_ceiling: "" # daemon dflt 1_000_000_000_000 (protocol MAX) — rate clamp ceiling
+decdn_voucher_interval_mb: "" # daemon dflt 1; range 1..=1024 — voucher cadence (ADR 003)
+
+# --- Settlement / channels (all µUSDC; "" => daemon default) ------------------
+decdn_redeem_threshold_micro_usdc: "" # daemon dflt 1_000_000 (1 USDC) — accrued µUSDC to trigger withdraw
+decdn_buyer_deposit_micro_usdc: "" # daemon dflt 10_000_000 (10 USDC) — escrow on a buyer-channel open
+decdn_buyer_max_approve: "" # bool; daemon dflt true — one-time max USDC approval to PaymentChannel
+# Auto-closeChannel triggers (OR'd). "" => off (disabled upstream); a configured 0 is rejected.
+decdn_settlement_auto_threshold_micro_usdc: "" # un-redeemed µUSDC threshold
+decdn_settlement_auto_by_voucher_nonce_span: "" # nonce-span companion trigger
+
+# --- Blockchain watcher tuning ("" => daemon default) -------------------------
+decdn_rpc_watchdog_interval_sec: "" # daemon dflt 30; 0 disables, else >= 10 — RPC watchdog
+decdn_event_poll_interval_ms: "" # daemon dflt 7000; >= 250 — eth_getFilterChanges poll
+decdn_content_blacklist_poll_interval_sec: "" # daemon dflt 600; >= 1 — blacklist re-scope cadence
 
 # --- Cache --------------------------------------------------------------------
 decdn_cache_size_mb: 10240 # 10 GB
 decdn_max_blob_size_mb: 1024 # 1 GB
+# Cache tuning ("" / [] => omit + use the daemon default; an explicit 0 IS emitted —
+# 0 is meaningful here, e.g. gc_interval_sec = 0 disables the sweep and
+# max_unrecouped_leech_bytes = 0 turns the cap off).
+decdn_max_probe_holds: "" # daemon dflt 256 — eviction-exempt holds (ADR 005); 0 = has_blob answers false
+decdn_stake_lane_reserved_holds: "" # daemon dflt 0 — holds reserved for node-to-node probes (#757)
+decdn_gc_interval_sec: "" # daemon dflt 300 — iroh-blobs GC sweep; 0 = off
+decdn_pinned_hashes: [] # eviction-exempt pins; each a 64-char lowercase-hex BLAKE3 hash
+decdn_cache_user_agent: "" # daemon dflt decdn-node/<version> — HTTP origin User-Agent
+# Node-to-node pull-through (#29). The toggle gates all serve-time origin pull from
+# upstream nodes; the knobs below only take effect when it is true. *_bytes / *_percent
+# are BARE INTEGERS on the wire (bytes; percent at scale 100, so 400 = 4.0x).
+decdn_node_to_node_pull_through_enabled: "" # bool; daemon dflt false — enable pull from upstream nodes (#29)
+decdn_node_pull_probe_fanout: "" # daemon dflt 5 — providers probed per miss; 0 disables pull
+decdn_node_pull_timeout_sec: "" # daemon dflt 20 — per-upstream pull timeout
+decdn_pull_ahead_bytes: "" # daemon dflt 1048576 (1 MiB) — speculative pull window (ADR 037)
+decdn_max_unrecouped_leech_bytes: "" # daemon dflt 268435456 (256 MiB) — speculative-spend cap; 0 = off
+decdn_pull_share_ratio_percent: "" # daemon dflt 400 (=4.0x) — per-peer speculative ceiling
+decdn_pull_through_require_authorized_origin: "" # bool; daemon dflt false — gate pulls on the ADR 022 directory
 # Pull-through origin (#437): what the node fetches on a cache miss. Empty kind =>
 # the [cache.origin] table is omitted and misses fail NoOrigin — a serving node
 # needs an origin. Pick ONE kind and set that kind's fields; the others are ignored.
@@ -82,6 +124,9 @@ decdn_log_format: json
 decdn_metrics_port: 9090
 decdn_metrics_bind: "127.0.0.1" # keep loopback — Prometheus scrape is a follow-up
 decdn_admin_port: 9191
+# Optional ("" => omit + use the daemon default).
+decdn_otlp_endpoint: "" # OTLP span export; http(s)://; needs the node built --features otlp
+decdn_region_accounting_interval_sec: "" # daemon dflt 3600 — per-region bandwidth log; 0 = off
 
 # --- Readiness probe (advisory) -----------------------------------------------
 # After start, the role probes http://127.0.0.1:<metrics_port>/metrics as a