From fa3392f80917d241664016cb0ba96b4d97628772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ahmet=20So=C4=9Fuksu?= Date: Wed, 12 Aug 2026 18:00:08 +0300 Subject: [PATCH 1/3] raft: guard snapshot restore from deleting recently-seen instances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restore() previously did an unconditional set-difference delete: any instance present in the local backend but absent from the (best-effort, possibly stale) raft snapshot was forgotten via ForgetInstance, with no recency check. On a rolling restart with a leader change, this could wipe recently-discovered instances cluster-wide if the on-disk snapshot predated their discovery. Add ReadRecentlySeenInstanceKeyMap() and use it to skip forgetting any key whose last_seen is within UnseenInstanceForgetHours (reuses the existing config, no new setting). Restore's deletes become a strict subset of what ForgetLongUnseenInstances would already remove. Fixes #123 Signed-off-by: Ahmet Soğuksu --- go/inst/instance_dao.go | 23 +++++++++++++++++++++++ go/logic/snapshot_data.go | 22 ++++++++++++++++++---- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/go/inst/instance_dao.go b/go/inst/instance_dao.go index 43d7b73b..6933fd6d 100644 --- a/go/inst/instance_dao.go +++ b/go/inst/instance_dao.go @@ -2694,6 +2694,29 @@ func ReadAllInstanceKeys() ([]InstanceKey, error) { return res, log.Errore(err) } +// ReadRecentlySeenInstanceKeyMap returns the set of instance keys whose last_seen is +// within the given recency window (hours). Used to protect freshly-discovered instances +// from being purged during raft snapshot restore before they've propagated into a snapshot. +func ReadRecentlySeenInstanceKeyMap(recencyHours uint) (*InstanceKeyMap, error) { + keys := NewInstanceKeyMap() + query := ` + select + hostname, port + from + database_instance + where + last_seen > NOW() - interval ? hour` + err := db.QueryOrchestrator(query, sqlutils.Args(recencyHours), func(m sqlutils.RowMap) error { + instanceKey, merr := NewResolveInstanceKey(m.GetString("hostname"), m.GetInt("port")) + if merr != nil { + return log.Errore(merr) + } + keys.AddKey(*instanceKey) + return nil + }) + return keys, log.Errore(err) +} + // ReadAllInstanceKeysMasterKeys func ReadAllMinimalInstances() ([]MinimalInstance, error) { res := []MinimalInstance{} diff --git a/go/logic/snapshot_data.go b/go/logic/snapshot_data.go index bd6e9cd0..b1327b63 100644 --- a/go/logic/snapshot_data.go +++ b/go/logic/snapshot_data.go @@ -22,6 +22,7 @@ import ( "encoding/json" "io" + "github.com/proxysql/orchestrator/go/config" "github.com/proxysql/orchestrator/go/db" "github.com/proxysql/orchestrator/go/inst" @@ -152,13 +153,26 @@ func (s *SnapshotDataCreatorApplier) Restore(rc io.ReadCloser) error { } discardedKeys := 0 - // Forget instances that were not in snapshot + // Forget instances that were not in snapshot. + // Guard: only forget an instance absent from the snapshot if it is ALSO stale + // locally (not seen within UnseenInstanceForgetHours). A freshly discovered + // instance may legitimately exist in our local backend but not yet be captured + // in the (older) snapshot we are restoring; deleting it here races with discovery + // and can wipe recent discoveries cluster-wide on restart/leader-change. Genuine + // decommissions age out and are removed both here and by ForgetLongUnseenInstances(), + // and explicit forgets arrive via the replicated "forget" command. existingKeys, _ := inst.ReadAllInstanceKeys() + recentlySeenKeys, _ := inst.ReadRecentlySeenInstanceKeyMap(config.Config.UnseenInstanceForgetHours) for _, existingKey := range existingKeys { - if !snapshotInstanceKeyMap.HasKey(existingKey) { - _ = inst.ForgetInstance(&existingKey) - discardedKeys++ + if snapshotInstanceKeyMap.HasKey(existingKey) { + continue } + if recentlySeenKeys.HasKey(existingKey) { + log.Debugf("raft snapshot restore: retaining recently-seen instance %+v absent from snapshot", existingKey) + continue + } + _ = inst.ForgetInstance(&existingKey) + discardedKeys++ } log.Debugf("raft snapshot restore: discarded %+v keys", discardedKeys) existingKeysMap := inst.NewInstanceKeyMap() From 0d8d669fca9c3e7a1889a7abad87c907e34241b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Fri, 28 Aug 2026 17:12:11 +0700 Subject: [PATCH 2/3] test(raft): cover snapshot restore recency guard (issue #123) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unit tests (go/logic/snapshot_data_test.go) run Restore() against a real SQLite backend: - ReadRecentlySeenInstanceKeyMap returns only instances seen within the recency window (fresh/borderline included, stale/never-seen excluded). - Restore retains recently-seen instances that are absent from the snapshot and still forgets genuinely stale ones (fails without the fix). Functional test (tests/functional/test-raft.sh, Phase 5) reproduces the issue #123 scenario deterministically: forget mysql3, snapshot all nodes, let continuous discovery re-discover mysql3 locally (no raft command), stop mysql3 to block re-discovery, then rolling-restart all nodes with a leader change per round and assert every node's local backend still contains the recently-seen instance. Signed-off-by: René Cannaò --- go/logic/snapshot_data_test.go | 202 +++++++++++++++++++++++++++++++ tests/functional/test-raft.sh | 213 +++++++++++++++++++++++++++++++++ 2 files changed, 415 insertions(+) create mode 100644 go/logic/snapshot_data_test.go diff --git a/go/logic/snapshot_data_test.go b/go/logic/snapshot_data_test.go new file mode 100644 index 00000000..0cbb4644 --- /dev/null +++ b/go/logic/snapshot_data_test.go @@ -0,0 +1,202 @@ +/* + Copyright 2026 ProxySQL Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package logic + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "io" + "os" + "path/filepath" + "testing" + "time" + + test "github.com/proxysql/golib/tests" + "github.com/proxysql/orchestrator/go/config" + "github.com/proxysql/orchestrator/go/db" + "github.com/proxysql/orchestrator/go/inst" +) + +func TestMain(m *testing.M) { + config.MarkConfigurationLoaded() + // keep hostname resolution local-only for the whole test binary, so that + // ResolveHostname never spawns asynchronous backend writes + config.Config.HostnameResolveMethod = "none" + os.Exit(m.Run()) +} + +// waitForInstanceDaoInit blocks until the background initializeInstanceDao() +// goroutine has created the package-level caches, which are required by +// WriteInstance/ForgetInstance. +func waitForInstanceDaoInit(t *testing.T) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + ready := false + func() { + defer func() { + _ = recover() + }() + inst.InstanceIsForgotten(&inst.InstanceKey{}) + ready = true + }() + if ready { + return + } + if time.Now().After(deadline) { + t.Fatal("instance DAO caches were not initialized in time") + } + time.Sleep(10 * time.Millisecond) + } +} + +func setupSQLiteBackend(t *testing.T) { + t.Helper() + waitForInstanceDaoInit(t) + origBackendDB := config.Config.BackendDB + origSQLiteDataFile := config.Config.SQLite3DataFile + origHostnameResolveMethod := config.Config.HostnameResolveMethod + config.Config.BackendDB = "sqlite" + config.Config.SQLite3DataFile = filepath.Join(t.TempDir(), "orchestrator.sqlite3") + // avoid asynchronous hostname-resolve writes racing with the backend switch + config.Config.HostnameResolveMethod = "none" + t.Cleanup(func() { + config.Config.BackendDB = origBackendDB + config.Config.SQLite3DataFile = origSQLiteDataFile + config.Config.HostnameResolveMethod = origHostnameResolveMethod + }) + _, err := db.OpenOrchestrator() + test.S(t).ExpectNil(err) +} + +func writeTestInstance(t *testing.T, hostname string, port int) { + t.Helper() + instance := &inst.Instance{Key: inst.InstanceKey{Hostname: hostname, Port: port}} + test.S(t).ExpectNil(inst.WriteInstance(instance, false, nil)) +} + +func setInstanceLastSeen(t *testing.T, hostname string, port int, hoursAgo int) { + t.Helper() + _, err := db.ExecOrchestrator(` + update database_instance + set last_seen = NOW() - interval ? hour + where hostname = ? and port = ?`, + hoursAgo, hostname, port, + ) + test.S(t).ExpectNil(err) +} + +func readInstanceKeyMap(t *testing.T) *inst.InstanceKeyMap { + t.Helper() + keys, err := inst.ReadAllInstanceKeys() + test.S(t).ExpectNil(err) + keyMap := inst.NewInstanceKeyMap() + keyMap.AddKeys(keys) + return keyMap +} + +func mkSnapshotReader(t *testing.T, snapshotData *SnapshotData) io.ReadCloser { + t.Helper() + b, err := json.Marshal(snapshotData) + test.S(t).ExpectNil(err) + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + _, err = zw.Write(b) + test.S(t).ExpectNil(err) + test.S(t).ExpectNil(zw.Close()) + return io.NopCloser(bytes.NewReader(buf.Bytes())) +} + +func TestReadRecentlySeenInstanceKeyMap(t *testing.T) { + setupSQLiteBackend(t) + + writeTestInstance(t, "fresh", 3306) + writeTestInstance(t, "borderline", 3306) + writeTestInstance(t, "stale", 3306) + writeTestInstance(t, "never-seen", 3306) + setInstanceLastSeen(t, "borderline", 3306, 239) + setInstanceLastSeen(t, "stale", 3306, 241) + _, err := db.ExecOrchestrator(`update database_instance set last_seen = NULL where hostname = 'never-seen'`) + test.S(t).ExpectNil(err) + + keys, err := inst.ReadRecentlySeenInstanceKeyMap(240) + test.S(t).ExpectNil(err) + + test.S(t).ExpectTrue(keys.HasKey(inst.InstanceKey{Hostname: "fresh", Port: 3306})) + test.S(t).ExpectTrue(keys.HasKey(inst.InstanceKey{Hostname: "borderline", Port: 3306})) + test.S(t).ExpectFalse(keys.HasKey(inst.InstanceKey{Hostname: "stale", Port: 3306})) + test.S(t).ExpectFalse(keys.HasKey(inst.InstanceKey{Hostname: "never-seen", Port: 3306})) +} + +// TestSnapshotRestoreRetainsRecentlySeenInstances verifies the fix for issue #123: +// instances that exist in the local backend but are absent from a (stale) raft +// snapshot must not be purged at restore time while they are still within the +// UnseenInstanceForgetHours recency window. Genuinely stale instances must +// still be forgotten, and instances carried by the snapshot must be added. +func TestSnapshotRestoreRetainsRecentlySeenInstances(t *testing.T) { + setupSQLiteBackend(t) + + origUnseenHours := config.Config.UnseenInstanceForgetHours + config.Config.UnseenInstanceForgetHours = 240 + t.Cleanup(func() { config.Config.UnseenInstanceForgetHours = origUnseenHours }) + + writeTestInstance(t, "fresh-a", 3306) + writeTestInstance(t, "fresh-b", 3307) + writeTestInstance(t, "stale", 3306) + setInstanceLastSeen(t, "stale", 3306, 300) + + snapshotData := NewSnapshotData() + snapshotData.MinimalInstances = []inst.MinimalInstance{ + {Key: inst.InstanceKey{Hostname: "in-snapshot", Port: 3306}}, + } + + applier := NewSnapshotDataCreatorApplier() + test.S(t).ExpectNil(applier.Restore(mkSnapshotReader(t, snapshotData))) + + keyMap := readInstanceKeyMap(t) + test.S(t).ExpectTrue(keyMap.HasKey(inst.InstanceKey{Hostname: "fresh-a", Port: 3306})) + test.S(t).ExpectTrue(keyMap.HasKey(inst.InstanceKey{Hostname: "fresh-b", Port: 3307})) + test.S(t).ExpectTrue(keyMap.HasKey(inst.InstanceKey{Hostname: "in-snapshot", Port: 3306})) + test.S(t).ExpectFalse(keyMap.HasKey(inst.InstanceKey{Hostname: "stale", Port: 3306})) +} + +// TestSnapshotRestoreForgetsUnseenStaleInstances ensures the recency guard does +// not prevent restore from forgetting instances that are both absent from the +// snapshot and already stale locally (the pre-existing behavior for genuine +// decommissions). +func TestSnapshotRestoreForgetsUnseenStaleInstances(t *testing.T) { + setupSQLiteBackend(t) + + origUnseenHours := config.Config.UnseenInstanceForgetHours + config.Config.UnseenInstanceForgetHours = 1 + t.Cleanup(func() { config.Config.UnseenInstanceForgetHours = origUnseenHours }) + + writeTestInstance(t, "stale-a", 3306) + writeTestInstance(t, "stale-b", 3307) + setInstanceLastSeen(t, "stale-a", 3306, 2) + setInstanceLastSeen(t, "stale-b", 3307, 2) + + snapshotData := NewSnapshotData() + + applier := NewSnapshotDataCreatorApplier() + test.S(t).ExpectNil(applier.Restore(mkSnapshotReader(t, snapshotData))) + + keyMap := readInstanceKeyMap(t) + test.S(t).ExpectFalse(keyMap.HasKey(inst.InstanceKey{Hostname: "stale-a", Port: 3306})) + test.S(t).ExpectFalse(keyMap.HasKey(inst.InstanceKey{Hostname: "stale-b", Port: 3307})) +} diff --git a/tests/functional/test-raft.sh b/tests/functional/test-raft.sh index 2af3e895..3bdde983 100755 --- a/tests/functional/test-raft.sh +++ b/tests/functional/test-raft.sh @@ -339,6 +339,219 @@ else fi fi +# ============================================================ +# Phase 5: Snapshot Restore Retains Recently-Discovered Instances +# ============================================================ +# Regression test for issue #123 (fixed by PR #124). +# +# Restore() in go/logic/snapshot_data.go must not purge locally-known +# instances that are absent from a (stale) snapshot while they were still +# seen within UnseenInstanceForgetHours. +# +# Deterministic setup (reproduces the production race without timing luck): +# 1. forget mysql3 (raft-replicated), then force a snapshot on every node: +# each node now holds a snapshot WITHOUT mysql3. All earlier commands +# (including the discover/forget of mysql3) are at or below the snapshot +# index, so restart-time log replay cannot re-add or re-forget mysql3. +# 2. continuous discovery re-discovers mysql3 on every node independently +# (a local side effect of polling mysql1; no raft command is involved). +# mysql3 is now present in every local backend, but absent from every +# node's last snapshot -- exactly the state the bug report describes. +# 3. stop mysql3 so that nothing can re-discover it after a restore. +# 4. rolling restart of all 3 nodes, stopping the current leader each round +# (leader change per restart, as in the issue's reproduction steps): +# - unpatched: Restore() forgets mysql3 on every node -> cluster-wide loss. +# - patched: mysql3 was recently seen -> retained on every node. +echo "" +echo "--- Phase 5: Snapshot Restore Retains Recently-Discovered Instances (issue #123) ---" + +RAFT_DATA_DIRS=(/tmp/raft1 /tmp/raft2 /tmp/raft3) + +# backend_instances : list instance hostnames in the node's local backend +backend_instances() { + docker compose -f "$COMPOSE_FILE" exec -T "${RAFT_NODES[$1]}" \ + sqlite3 "${RAFT_DATA_DIRS[$1]}/orchestrator.sqlite3" \ + "select hostname from database_instance order by hostname" 2>/dev/null +} + +# wait_all_backends : poll until every node's +# backend holds exactly instances; when the expected count is 3, +# mysql3 must be among them +wait_all_backends() { + local expected="$1" deadline="$2" i idx rows count all_match + for i in $(seq 1 "$deadline"); do + all_match=true + for idx in 0 1 2; do + rows=$(backend_instances "$idx" | tr '\n' ' ') + count=$(echo "$rows" | wc -w | tr -d ' ') + if [ "$count" != "$expected" ]; then + all_match=false + break + fi + if [ "$expected" = "3" ] && ! echo "$rows" | grep -q "mysql3"; then + all_match=false + break + fi + done + if $all_match; then + return 0 + fi + sleep 1 + done + return 1 +} + +# index of the current raft leader ("" if none) +current_leader_index() { + local idx state + for idx in 0 1 2; do + state=$(curl -sf --max-time 10 "http://localhost:${RAFT_PORTS[$idx]}/api/raft-state" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin))" 2>/dev/null || echo "") + if [ "$state" = "Leader" ]; then + echo "$idx" + return 0 + fi + done + echo "" + return 1 +} + +LEADER_INDEX=$(current_leader_index) +if [ -z "$LEADER_INDEX" ] || ! wait_all_backends 3 10; then + skip "Raft cluster not healthy with full topology; skipping issue #123 regression phase" +else + LEADER_PORT="${RAFT_PORTS[$LEADER_INDEX]}" + PHASE5_OK=true + + # forget mysql3, then snapshot: the new snapshots do not contain mysql3 + curl -sf --max-time 10 "http://localhost:${LEADER_PORT}/api/forget/mysql3/3306" > /dev/null 2>&1 \ + || { fail "forget mysql3 failed"; PHASE5_OK=false; } + if $PHASE5_OK && wait_all_backends 2 30; then + pass "mysql3 forgotten on all nodes" + else + fail "mysql3 not forgotten on all nodes within 30s" + PHASE5_OK=false + fi + + if $PHASE5_OK; then + for port in "${RAFT_PORTS[@]}"; do + curl -sf --max-time 30 "http://localhost:${port}/api/raft-snapshot" > /dev/null 2>&1 || { fail "raft-snapshot failed on :${port}"; PHASE5_OK=false; } + done + sleep 2 + $PHASE5_OK && pass "Snapshots without mysql3 taken on all nodes" + fi + + if $PHASE5_OK; then + # continuous discovery re-discovers mysql3 on each node as a replica of + # mysql1 -- purely local writes, no raft commands in the log + echo "Waiting for continuous discovery to re-discover mysql3 on all nodes (up to 90s)..." + if wait_all_backends 3 90; then + pass "mysql3 re-discovered locally on all nodes (no raft command)" + else + fail "mysql3 not re-discovered on all nodes within 90s" + for idx in 0 1 2; do + echo " ${RAFT_NODES[$idx]} backend: $(backend_instances "$idx" | tr '\n' ' ')" + done + PHASE5_OK=false + fi + fi + + if $PHASE5_OK; then + # stop mysql3: without it, nothing can re-discover mysql3 after a + # restore, so any loss caused by Restore() becomes observable + echo "Stopping mysql3 to block re-discovery after restore..." + docker compose -f "$COMPOSE_FILE" stop mysql3 > /dev/null 2>&1 \ + && pass "mysql3 stopped" \ + || { fail "Could not stop mysql3"; PHASE5_OK=false; } + # let in-flight discovery attempts drain + sleep 3 + fi + + if $PHASE5_OK; then + # rolling restart: stop the current leader, wait for re-election, restart; + # repeat so that all 3 nodes restore from their snapshots + for ROUND in 1 2 3; do + LIDX=$(current_leader_index) + if [ -z "$LIDX" ]; then + fail "Rolling restart round ${ROUND}: no leader found" + PHASE5_OK=false + break + fi + NODE="${RAFT_NODES[$LIDX]}" + RPORT="${RAFT_PORTS[$LIDX]}" + echo "Rolling restart round ${ROUND}: stopping leader ${NODE}" + docker compose -f "$COMPOSE_FILE" stop "$NODE" > /dev/null 2>&1 + + REMAINING_PORTS=() + for idx in 0 1 2; do + [ "$idx" != "$LIDX" ] && REMAINING_PORTS+=("${RAFT_PORTS[$idx]}") + done + REELECTED=false + NEW_LEADER="" + for i in $(seq 1 60); do + L1=$(curl -sf --max-time 10 "http://localhost:${REMAINING_PORTS[0]}/api/raft-leader" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin))" 2>/dev/null || echo "") + L2=$(curl -sf --max-time 10 "http://localhost:${REMAINING_PORTS[1]}/api/raft-leader" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin))" 2>/dev/null || echo "") + if [ -n "$L1" ] && [ "$L1" = "$L2" ]; then + REELECTED=true + NEW_LEADER="$L1" + break + fi + sleep 1 + done + if ! $REELECTED; then + fail "Rolling restart round ${ROUND}: no re-election within 60s" + PHASE5_OK=false + docker compose -f "$COMPOSE_FILE" start "$NODE" > /dev/null 2>&1 + break + fi + + docker compose -f "$COMPOSE_FILE" start "$NODE" > /dev/null 2>&1 + REJOINED=false + for i in $(seq 1 60); do + RL=$(curl -sf --max-time 10 "http://localhost:${RPORT}/api/raft-leader" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin))" 2>/dev/null || echo "") + if [ -n "$RL" ] && [ "$RL" = "$NEW_LEADER" ]; then + REJOINED=true + break + fi + sleep 1 + done + if $REJOINED; then + pass "Round ${ROUND}: leader change + ${NODE} restarted and rejoined" + else + fail "Rolling restart round ${ROUND}: ${NODE} did not rejoin within 60s" + PHASE5_OK=false + break + fi + done + fi + + if $PHASE5_OK; then + # The actual regression check: every node's local backend must still + # contain mysql3, which was recently seen but absent from the snapshots + RETAINED=true + for idx in 0 1 2; do + ROWS=$(backend_instances "$idx" | tr '\n' ' ') + if ! echo "$ROWS" | grep -q "mysql3"; then + RETAINED=false + echo " ${RAFT_NODES[$idx]} backend after restart: ${ROWS}" + fi + done + if $RETAINED; then + pass "Recently-discovered instance retained on all nodes after rolling restart (issue #123)" + else + fail "Recently-discovered instance lost after rolling restart (issue #123 regression)" + fi + fi + + # restore the environment: mysql3 back up, topology healed + echo "Restarting mysql3..." + docker compose -f "$COMPOSE_FILE" start mysql3 > /dev/null 2>&1 + if wait_all_backends 3 90; then + pass "Topology healed after issue #123 regression phase" + else + skip "Topology not fully healed after issue #123 regression phase (mysql3 may need more time)" + fi +fi + # ============================================================ # Cleanup # ============================================================ From 68c3a3027999750c930ae7f21380b0af4f64d212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Fri, 28 Aug 2026 17:21:15 +0700 Subject: [PATCH 3/3] test(raft): fix re-election detection in issue #123 rolling restart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until the election completes, remaining nodes keep reporting the old (stopped) leader via /api/raft-leader, so the re-election wait must require the agreed leader to differ from the stopped node's address (as phase 3 already does). Also compare the rejoined node's leader against a live node's current leader instead of a fixed value. Signed-off-by: René Cannaò --- tests/functional/test-raft.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/functional/test-raft.sh b/tests/functional/test-raft.sh index 3bdde983..c0f0a66c 100755 --- a/tests/functional/test-raft.sh +++ b/tests/functional/test-raft.sh @@ -478,6 +478,7 @@ else fi NODE="${RAFT_NODES[$LIDX]}" RPORT="${RAFT_PORTS[$LIDX]}" + OLD_LEADER=$(curl -sf --max-time 10 "http://localhost:${RPORT}/api/raft-leader" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin))" 2>/dev/null || echo "") echo "Rolling restart round ${ROUND}: stopping leader ${NODE}" docker compose -f "$COMPOSE_FILE" stop "$NODE" > /dev/null 2>&1 @@ -485,14 +486,14 @@ else for idx in 0 1 2; do [ "$idx" != "$LIDX" ] && REMAINING_PORTS+=("${RAFT_PORTS[$idx]}") done + # wait for a NEW leader: until the election completes, the remaining + # nodes keep reporting the old (stopped) leader REELECTED=false - NEW_LEADER="" for i in $(seq 1 60); do L1=$(curl -sf --max-time 10 "http://localhost:${REMAINING_PORTS[0]}/api/raft-leader" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin))" 2>/dev/null || echo "") L2=$(curl -sf --max-time 10 "http://localhost:${REMAINING_PORTS[1]}/api/raft-leader" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin))" 2>/dev/null || echo "") - if [ -n "$L1" ] && [ "$L1" = "$L2" ]; then + if [ -n "$L1" ] && [ "$L1" = "$L2" ] && [ "$L1" != "$OLD_LEADER" ]; then REELECTED=true - NEW_LEADER="$L1" break fi sleep 1 @@ -505,10 +506,12 @@ else fi docker compose -f "$COMPOSE_FILE" start "$NODE" > /dev/null 2>&1 + # wait for the restarted node to agree on the leader with a running node REJOINED=false for i in $(seq 1 60); do RL=$(curl -sf --max-time 10 "http://localhost:${RPORT}/api/raft-leader" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin))" 2>/dev/null || echo "") - if [ -n "$RL" ] && [ "$RL" = "$NEW_LEADER" ]; then + CL=$(curl -sf --max-time 10 "http://localhost:${REMAINING_PORTS[0]}/api/raft-leader" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin))" 2>/dev/null || echo "") + if [ -n "$RL" ] && [ "$RL" = "$CL" ]; then REJOINED=true break fi