diff --git a/deployment/aliyun/host-rust-lob-recovery-queue.sh b/deployment/aliyun/host-rust-lob-recovery-queue.sh index 535333e7d..726f3eefb 100755 --- a/deployment/aliyun/host-rust-lob-recovery-queue.sh +++ b/deployment/aliyun/host-rust-lob-recovery-queue.sh @@ -233,6 +233,43 @@ has_incomplete_parts() { \) -print -quit) ]] } +has_uploaded_cleanup_markers() { + local spool=$1 + [[ -d $spool && ! -L $spool ]] || return 1 + [[ -n $(find "$spool" -type f \( \ + -name '*.uploaded-cleanup.json' -o \ + -name '*.uploaded-cleanup.json.tmp' \ + \) -print -quit) ]] +} + +has_undrained_complete_segments() { + # Manifest/data/success only. Leftover uploaded-cleanup markers are the + # production archiver's post-upload residue, not undrained segments. + # Isolating them would send drain through --upload-only, which deletes the + # marker without refreshing last_success_at, then fails triplet verify and + # leaves the job unresumable. + local spool=$1 + [[ -d $spool && ! -L $spool ]] || return 1 + [[ -n $(find "$spool" -type f \( \ + -name '*.manifest.json' -o \ + -name '*.jsonl.zst' -o \ + -name '*._SUCCESS' \ + \) -print -quit) ]] +} + +has_cleanup_marker_only_leftover() { + local spool=$1 + has_uploaded_cleanup_markers "$spool" || return 1 + has_incomplete_parts "$spool" && return 1 + has_undrained_complete_segments "$spool" && return 1 + return 0 +} + +needs_recovery_isolation() { + local spool=$1 + has_incomplete_parts "$spool" || has_undrained_complete_segments "$spool" +} + segment_artifacts() { local spool=$1 [[ -d $spool && ! -L $spool ]] || return 0 @@ -649,8 +686,8 @@ isolate_market() { fail "canonical spool is missing; refusing recovery fallback: $CANONICAL_SPOOL" fi secure_directory "$CANONICAL_SPOOL" "$hft_uid" "$hft_gid" - isolation_phase_begin incomplete-scan - if ! has_incomplete_parts "$CANONICAL_SPOOL"; then + isolation_phase_begin recovery-scan + if ! needs_recovery_isolation "$CANONICAL_SPOOL"; then isolation_phase_done exit 0 fi @@ -1285,6 +1322,9 @@ run_drain_job() { "$release_binary" --recover-parts-only \ || fail 'detached recovery of incomplete parts failed' fi + if has_cleanup_marker_only_leftover "$running_dir"; then + fail 'detached spool has leftover uploaded-cleanup markers without local segment artifacts; refusing fresh-upload verify so the marker remains inspectable' + fi # The uploader commits its own readback before removing local segments. An # interrupted controller may therefore have only metadata left. Re-running # an empty upload cannot refresh its timestamp and is not completion proof. diff --git a/deployment/aliyun/test-rust-lob-recovery-queue.sh b/deployment/aliyun/test-rust-lob-recovery-queue.sh index e8b87e0a0..862abb344 100755 --- a/deployment/aliyun/test-rust-lob-recovery-queue.sh +++ b/deployment/aliyun/test-rust-lob-recovery-queue.sh @@ -14,6 +14,17 @@ fi grep -Fq "active V2 controller is required" "$RECOVERY" grep -Fq 'monday.rust_lob_controller_release.v2' "$RECOVERY" grep -Fq 'monday.rust_lob_controller_release.v2' "$RECOVERY" +grep -Fq 'needs_recovery_isolation' "$RECOVERY" +grep -Fq 'has_undrained_complete_segments' "$RECOVERY" +grep -Fq 'has_cleanup_marker_only_leftover' "$RECOVERY" +if grep -Fq 'if ! has_incomplete_parts "$CANONICAL_SPOOL"' "$RECOVERY"; then + printf 'recovery isolate still skips complete undrained segments\n' >&2 + exit 1 +fi +if awk '/^has_undrained_complete_segments\(\)/,/^}/' "$RECOVERY" | grep -Fq -- "-name '*.uploaded-cleanup"; then + printf 'undrained-complete still treats cleanup markers as segments\n' >&2 + exit 1 +fi projection_contract='"$ACTIVE_CONTROLLER/deployment/binance-lob-archiver-production-$MARKET.env"' resolved_contract='secure_regular_file "$installed_env" 0' obsolete_contract='secure_regular_file "$ENV_FILE" 0' @@ -542,4 +553,154 @@ expect_rejected missing-remote-object fixture_drain unset FIXTURE_OSS_FAIL [[ $(jq -r .result "$attempt/result.json") == failed ]] [[ $(sha256sum "$EVIDENCE_ROOT/$RESUME_JOB_ID/result.json" | awk '{print $1}') == "$fixture_old_result_sha" ]] -printf 'Explicit recovery adoption, historical readback, mixed drain and interruption behavior passed\n' + +# Isolate/drain must cover rotated complete undrained segments, not only +# incomplete parts. Drain of a complete-only spool uses --upload-only. +empty_spool="$fixture/empty-isolate-spool" +complete_spool="$fixture/complete-isolate-spool" +part_spool="$fixture/part-isolate-spool" +mkdir -p "$empty_spool" "$complete_spool" "$part_spool" +printf '{}\n' >"$empty_spool/upload-status.json" +printf '{}\n' >"$empty_spool/health.json" +: >"$empty_spool/.binance-lob-archiver.lock" +if has_incomplete_parts "$empty_spool" || has_undrained_complete_segments "$empty_spool" \ + || needs_recovery_isolation "$empty_spool"; then + printf 'empty spool was treated as recovery work\n' >&2 + exit 1 +fi +printf 'sealed\n' >"$complete_spool/part-1.jsonl.zst" +printf '{}\n' >"$complete_spool/part-1.jsonl.zst.manifest.json" +printf 'ok\n' >"$complete_spool/part-1.jsonl.zst._SUCCESS" +has_undrained_complete_segments "$complete_spool" +needs_recovery_isolation "$complete_spool" +if has_incomplete_parts "$complete_spool"; then + printf 'complete segment was classified as an incomplete part\n' >&2 + exit 1 +fi +printf 'raw\n' >"$part_spool/part-2.jsonl.part" +has_incomplete_parts "$part_spool" +needs_recovery_isolation "$part_spool" +if has_undrained_complete_segments "$part_spool"; then + printf 'incomplete part was classified as a complete segment\n' >&2 + exit 1 +fi +cleanup_spool="$fixture/cleanup-isolate-spool" +mkdir -p "$cleanup_spool" +printf '%s\n' '{"schema":"monday.binance_lob.uploaded_cleanup.v1","data":"part-1.jsonl.zst","manifest":"part-1.jsonl.zst.manifest.json","success":"part-1.jsonl.zst._SUCCESS"}' \ + >"$cleanup_spool/part-1.jsonl.zst.manifest.json.uploaded-cleanup.json" +has_uploaded_cleanup_markers "$cleanup_spool" +has_cleanup_marker_only_leftover "$cleanup_spool" +if has_undrained_complete_segments "$cleanup_spool" || needs_recovery_isolation "$cleanup_spool"; then + printf 'cleanup-marker-only spool was treated as recovery work\n' >&2 + exit 1 +fi + +rm -f "$fixture/payload.calls" "$fixture/uploaded-files" +fixture_job 108 ready +printf 'already sealed\n' >"$fixture_job_dir/part-complete.jsonl.zst" +printf '{}\n' >"$fixture_job_dir/part-complete.jsonl.zst.manifest.json" +printf 'success\n' >"$fixture_job_dir/part-complete.jsonl.zst._SUCCESS" +fixture_resume >/dev/null +attempt=$(fixture_attempt) +fixture_drain >/dev/null +[[ -d $attempt/spool.done && $(jq -r .result "$attempt/result.json") == passed ]] +grep -Fq -- '--upload-only' "$fixture/payload.calls" +if grep -Fq -- '--recover-parts-only' "$fixture/payload.calls"; then + printf 'complete-only drain invoked recover-parts\n' >&2 + exit 1 +fi +grep -Fq 'part-complete.jsonl.zst' "$fixture/uploaded-files" + +rm -f "$fixture/payload.calls" "$fixture/uploaded-files" +fixture_job 109 ready +printf '%s\n' '{"schema":"monday.binance_lob.uploaded_cleanup.v1","data":"part-cleanup.jsonl.zst","manifest":"part-cleanup.jsonl.zst.manifest.json","success":"part-cleanup.jsonl.zst._SUCCESS"}' \ + >"$fixture_job_dir/part-cleanup.jsonl.zst.manifest.json.uploaded-cleanup.json" +fixture_resume >/dev/null +attempt=$(fixture_attempt) +expect_rejected leftover-cleanup-markers fixture_drain +grep -Fq 'leftover uploaded-cleanup markers' "$fixture/rejected.log" +[[ -d $QUEUE_MARKET_ROOT/$RESUME_JOB_ID.failed ]] +[[ -f $QUEUE_MARKET_ROOT/$RESUME_JOB_ID.failed/part-cleanup.jsonl.zst.manifest.json.uploaded-cleanup.json ]] +[[ $(jq -r .result "$attempt/result.json") == failed ]] +if [[ -e $fixture/payload.calls ]] && grep -Fq -- '--upload-only' "$fixture/payload.calls"; then + printf 'cleanup-marker-only drain entered fresh-upload verify\n' >&2 + exit 1 +fi +if [[ -e $fixture/payload.calls ]] && grep -Fq -- '--recover-parts-only' "$fixture/payload.calls"; then + printf 'cleanup-marker-only drain invoked recover-parts\n' >&2 + exit 1 +fi +fixture_drain >/dev/null +[[ -d $QUEUE_MARKET_ROOT/$RESUME_JOB_ID.failed ]] +[[ ! -d $QUEUE_MARKET_ROOT/$RESUME_JOB_ID.ready ]] + +saved_canonical=$CANONICAL_SPOOL +isolate_spool="$fixture/host/data/monday/spool/binance-lob/spot" +rm -rf -- "$isolate_spool" +mkdir -p "$isolate_spool" +printf '{}\n' >"$isolate_spool/upload-status.json" +: >"$isolate_spool/.binance-lob-archiver.lock" +chmod 0640 "$isolate_spool/.binance-lob-archiver.lock" "$isolate_spool/upload-status.json" +ready_before=$(find "$QUEUE_MARKET_ROOT" -mindepth 1 -maxdepth 1 -type d -name '*.ready' -print | wc -l) +fixture_isolate() { + ( + CURRENT_ACTION=isolate + # shellcheck disable=SC2317,SC2329 # Nested stub used by run_isolate in this subshell. + install() { + local mode=0750 dest src + while (($#)); do + case "$1" in + -d) shift ;; + -m) mode=$2; shift 2 ;; + -o|-g) shift 2 ;; + --) shift; break ;; + *) break ;; + esac + done + dest=${*: -1} + if (($# <= 1)); then + mkdir -p "$dest" + chmod "$mode" "$dest" 2>/dev/null || true + else + src=${*: -2:1} + cp -- "$src" "$dest" + chmod "$mode" "$dest" 2>/dev/null || true + fi + } + queue_lock + run_isolate + ) +} +if ! fixture_isolate >/dev/null; then + printf 'empty canonical spool isolate failed\n' >&2 + exit 1 +fi +ready_after=$(find "$QUEUE_MARKET_ROOT" -mindepth 1 -maxdepth 1 -type d -name '*.ready' -print | wc -l) +[[ $ready_before == "$ready_after" ]] \ + || { printf 'empty spool isolate queued a job\n' >&2; exit 1; } + +printf '%s\n' '{"schema":"monday.binance_lob.uploaded_cleanup.v1","data":"part-cleanup.jsonl.zst","manifest":"part-cleanup.jsonl.zst.manifest.json","success":"part-cleanup.jsonl.zst._SUCCESS"}' \ + >"$isolate_spool/part-cleanup.jsonl.zst.manifest.json.uploaded-cleanup.json" +if ! fixture_isolate >/dev/null; then + printf 'cleanup-marker-only isolate failed\n' >&2 + exit 1 +fi +ready_after_cleanup=$(find "$QUEUE_MARKET_ROOT" -mindepth 1 -maxdepth 1 -type d -name '*.ready' -print | wc -l) +[[ $ready_before == "$ready_after_cleanup" ]] \ + || { printf 'cleanup-marker-only isolate queued a job\n' >&2; exit 1; } +[[ -f $isolate_spool/part-cleanup.jsonl.zst.manifest.json.uploaded-cleanup.json ]] +rm -f -- "$isolate_spool/part-cleanup.jsonl.zst.manifest.json.uploaded-cleanup.json" + +printf 'sealed\n' >"$isolate_spool/part-pending.jsonl.zst" +printf '{}\n' >"$isolate_spool/part-pending.jsonl.zst.manifest.json" +printf 'ok\n' >"$isolate_spool/part-pending.jsonl.zst._SUCCESS" +fixture_isolate >/dev/null +ready_dir=$(find "$QUEUE_MARKET_ROOT" -mindepth 1 -maxdepth 1 -type d -name '*.ready' -print \ + | while IFS= read -r dir; do + [[ -f $dir/part-pending.jsonl.zst ]] && printf '%s\n' "$dir" + done | sed -n '1p') +[[ -n $ready_dir ]] || { printf 'complete undrained isolate did not queue a job\n' >&2; exit 1; } +[[ -f $ready_dir/part-pending.jsonl.zst && -f $ready_dir/part-pending.jsonl.zst.manifest.json ]] +[[ -f $isolate_spool/upload-status.json && ! -e $isolate_spool/part-pending.jsonl.zst ]] +CANONICAL_SPOOL=$saved_canonical +printf 'Explicit recovery adoption, historical readback, mixed drain and complete-segment isolation passed\n'