RATIS-1912. Fix infinity election when perform membership change.#954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
szetszwo merged 10 commits into from
Nov 3, 2023
Merged

RATIS-1912. Fix infinity election when perform membership change. #954

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
861cbdf
v0.1
Oct 26, 2023
8c00760
v0.2
Oct 27, 2023
ac6de8f
v0.3
Oct 30, 2023
88aa0af
v0.4
Oct 30, 2023
feeac8d
v0.5 remove unused import
Oct 31, 2023
d3b0134
v0.6
Nov 1, 2023
61744a7
v0.7
Nov 1, 2023
1080e99
v0.8 fix unit tests.
Nov 2, 2023
449d485
v0.9
Nov 3, 2023
f93e9cc
v0.10
Nov 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -104,7 +104,7 @@ enum Phase {
ELECTION
}

enum Result {PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}
enum Result {PASSED, SINGLE_MODE_PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}

private static class ResultAndTerm {
private final Result result;
Expand DownExpand Up@@ -331,6 +331,7 @@ private boolean askForVotes(Phase phase, int round) throws InterruptedException,

switch (r.getResult()) {
case PASSED:
case SINGLE_MODE_PASSED:
return true;
case NOT_IN_CONF:
case SHUTDOWN:
Expand DownExpand Up@@ -379,6 +380,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
Collection<RaftPeerId> votedPeers = new ArrayList<>();
Collection<RaftPeerId> rejectedPeers = new ArrayList<>();
Set<RaftPeerId> higherPriorityPeers = getHigherPriorityPeers(conf);
final boolean singleMode = conf.isSingleMode(server.getId());

while (waitForNum > 0 && shouldRun(electionTerm)) {
final TimeDuration waitTime = timeout.elapsedTime().apply(n -> -n);
Expand All@@ -387,6 +389,9 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
// if some higher priority peer did not response when timeout, but candidate get majority,
// candidate pass vote
return logAndReturn(phase, Result.PASSED, responses, exceptions);
} else if (singleMode) {
// if candidate is in single mode, candidate pass vote.
return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
} else {
return logAndReturn(phase, Result.TIMEOUT, responses, exceptions);
}
Expand DownExpand Up@@ -418,7 +423,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
}

// If any peer with higher priority rejects vote, candidate can not pass vote
if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId)) {
if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId) && !singleMode) {
return logAndReturn(phase, Result.REJECTED, responses, exceptions);
}

Expand DownExpand Up@@ -447,6 +452,8 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
// received all the responses
if (conf.hasMajority(votedPeers, server.getId())) {
return logAndReturn(phase, Result.PASSED, responses, exceptions);
} else if (singleMode) {
return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
} else {
return logAndReturn(phase, Result.REJECTED, responses, exceptions);
}
Expand Down
28 changes: 28 additions & 0 deletions ratis-server/src/main/java/org/apache/ratis/server/impl/RaftConfigurationImpl.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -232,6 +232,34 @@ Collection<RaftPeer> getOtherPeers(RaftPeerId selfId) {
return others;
}

/**
* @return true if the new peers number reaches half of new conf peers number or the group is
* changing from single mode to HA mode.
*/
boolean changeMajority(Collection<RaftPeer> newMembers) {
Preconditions.assertNull(oldConf, "oldConf");
final long newPeersCount = newMembers.stream().map(RaftPeer::getId).filter(id -> conf.getPeer(id) == null).count();

if (conf.size() == 1 && newMembers.size() == 2 && newPeersCount == 1) {
// Change from single peer to HA mode. This is a special case, skip majority verification.
return false;
}

// If newPeersCount reaches majority number of new conf size, the cluster may end with infinity
// election. See https://issues.apache.org/jira/browse/RATIS-1912 for more details.
final long oldPeersCount = newMembers.size() - newPeersCount;
return newPeersCount >= oldPeersCount;
}

/** @return True if the selfId is in single mode. */
boolean isSingleMode(RaftPeerId selfId) {
if (isStable()) {
return conf.size() == 1;
} else {
return oldConf.size() == 1 && oldConf.contains(selfId) && conf.size() == 2 && conf.contains(selfId);
}
}

/** @return true if the self id together with the others are in the majority. */
boolean hasMajority(Collection<RaftPeerId> others, RaftPeerId selfId) {
Preconditions.assertTrue(!others.contains(selfId));
Expand Down
4 changes: 4 additions & 0 deletions ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1311,6 +1311,10 @@ public CompletableFuture<RaftClientReply> setConfigurationAsync(SetConfiguration
pending.setReply(newSuccessReply(request));
return pending.getFuture();
}
if (current.changeMajority(serversInNewConf)) {
throw new SetConfigurationException("Failed to set configuration: request " + request
+ " changes a majority set of the current configuration " + current);
}

getRaftServer().addRaftPeers(serversInNewConf);
getRaftServer().addRaftPeers(listenersInNewConf);
Expand Down
4 changes: 3 additions & 1 deletion ratis-server/src/test/java/org/apache/ratis/InstallSnapshotFromLeaderTests.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.protocol.RaftClientReply;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.RaftServerConfigKeys;
Expand DownExpand Up@@ -111,7 +112,8 @@ private void testMultiFileInstallSnapshot(CLUSTER cluster) throws Exception {
final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
true);
// trigger setConfiguration
cluster.setConfiguration(change.allPeersInNewConf);
RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

RaftServerTestUtil
.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand Down
17 changes: 11 additions & 6 deletions ratis-server/src/test/java/org/apache/ratis/InstallSnapshotNotificationTests.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -49,6 +49,7 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
Expand DownExpand Up@@ -242,7 +243,8 @@ private void testAddNewFollowers(CLUSTER cluster, int numRequests) throws Except
final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
true);
// trigger setConfiguration
cluster.setConfiguration(change.allPeersInNewConf);
RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

RaftServerTestUtil
.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -391,7 +393,8 @@ private void testInstallSnapshotNotificationCount(CLUSTER cluster) throws Except
final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
true);
// trigger setConfiguration
cluster.setConfiguration(change.allPeersInNewConf);
RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));
RaftServerTestUtil
.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand DownExpand Up@@ -478,7 +481,8 @@ private void testInstallSnapshotInstalledEvent(CLUSTER cluster) throws Exception
// add one new peer
final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(1, true, true);
// trigger setConfiguration
cluster.setConfiguration(change.allPeersInNewConf);
RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

RaftServerTestUtil
.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -556,7 +560,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
true);
// trigger setConfiguration
cluster.setConfiguration(change.allPeersInNewConf);
RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

RaftServerTestUtil.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand All@@ -567,8 +572,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
RaftServerTestUtil.getLatestInstalledSnapshotIndex(follower));
}

// Make sure each new peer got one snapshot notification.
Assert.assertEquals(2, numSnapshotRequests.get());
// Make sure each new peer got at least one snapshot notification.
Assert.assertTrue(2 <= numSnapshotRequests.get());
} finally {
cluster.shutdown();
}
Expand Down
4 changes: 3 additions & 1 deletion ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.impl.RaftServerTestUtil;
import org.apache.ratis.server.impl.RetryCacheTestUtil;
import org.apache.ratis.server.raftlog.RaftLog;
import org.apache.ratis.server.raftlog.RaftLogIOException;
Expand DownExpand Up@@ -140,7 +141,8 @@ void runTestRetryOnNewLeader(CLUSTER cluster) throws Exception {
RaftPeer[] allPeers = cluster.removePeers(2, true,
asList(change.newPeers)).allPeersInNewConf;
// trigger setConfiguration
cluster.setConfiguration(allPeers);
RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(allPeers),
peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

final RaftPeerId newLeaderId = JavaUtils.attemptRepeatedly(() -> {
final RaftPeerId id = RaftTestUtil.waitForLeader(cluster).getId();
Expand Down
3 changes: 2 additions & 1 deletion ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -329,7 +329,8 @@ public static <T extends Throwable> void runMultiGroupTest(
}
LOG.info(chosen + ") setConfiguration: " + cluster.printServers(groups[chosen].getGroupId()));
try (final RaftClient client = cluster.createClient(groups[chosen])) {
client.admin().setConfiguration(allPeers.toArray(RaftPeer.emptyArray()));
RaftServerTestUtil.runWithMinorityPeers(cluster, allPeers,
peers -> client.admin().setConfiguration(peers.toArray(RaftPeer.emptyArray())));
}

Assert.assertNotNull(RaftTestUtil.waitForLeader(cluster));
Expand Down
65 changes: 64 additions & 1 deletion ratis-server/src/test/java/org/apache/ratis/server/impl/RaftReconfigurationBaseTest.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -61,6 +61,7 @@

import static java.util.Arrays.asList;
import static org.apache.ratis.server.impl.RaftServerTestUtil.waitAndCheckNewConf;
import static org.junit.Assert.assertThrows;

public abstract class RaftReconfigurationBaseTest<CLUSTER extends MiniRaftCluster>
extends BaseTest
Expand DownExpand Up@@ -145,6 +146,58 @@ public void testAddPeers() throws Exception {
});
}

/**
* Test leader election when changing cluster from single mode to HA mode.
*/
@Test
public void testLeaderElectionWhenChangeFromSingleToHA() throws Exception {
runWithNewCluster(1, cluster -> {
RaftTestUtil.waitForLeader(cluster);

RaftGroupId groupId = cluster.getGroup().getGroupId();
RaftPeer curPeer = cluster.getGroup().getPeers().iterator().next();
RaftPeer newPeer = cluster.addNewPeers(1, true, true).newPeers[0];

RaftServerProxy leaderServer = cluster.getServer(curPeer.getId());

// Update leader conf to transitional single mode.
RaftConfigurationImpl oldNewConf = RaftConfigurationImpl.newBuilder()
.setOldConf(new PeerConfiguration(Arrays.asList(curPeer)))
.setConf(new PeerConfiguration(Arrays.asList(curPeer, newPeer)))
.setLogEntryIndex(Long.MAX_VALUE / 2)
.build();
Assert.assertTrue(oldNewConf.isSingleMode(curPeer.getId()));
RaftServerTestUtil.setRaftConf(leaderServer, groupId, oldNewConf);
try(RaftClient client = cluster.createClient()) {
client.admin().transferLeadership(null, leaderServer.getId(), 1000);
}

final RaftServer.Division newLeader = RaftTestUtil.waitForLeader(cluster);
Assert.assertEquals(leaderServer.getId(), newLeader.getId());
Assert.assertEquals(oldNewConf, newLeader.getRaftConf());
});
}

@Test
public void testChangeMajority() throws Exception {
runWithNewCluster(1, cluster -> {
RaftTestUtil.waitForLeader(cluster);
final RaftPeerId leaderId = cluster.getLeader().getId();

try (final RaftClient client = cluster.createClient(leaderId)) {
final PeerChanges c1 = cluster.addNewPeers(2, true);

SetConfigurationRequest.Arguments arguments = SetConfigurationRequest.Arguments.newBuilder()
.setServersInCurrentConf(cluster.getPeers())
.setServersInNewConf(c1.allPeersInNewConf)
.setMode(SetConfigurationRequest.Mode.COMPARE_AND_SET)
.build();
assertThrows("Expect change majority error.", SetConfigurationException.class,
() -> client.admin().setConfiguration(arguments));
}
});
}

/**
* remove 2 peers (5 peers -> 3 peers), no leader change
*/
Expand DownExpand Up@@ -380,7 +433,17 @@ public void testBootstrapReconfWithSingleNodeAddOne() throws Exception {
@Test
public void testBootstrapReconfWithSingleNodeAddTwo() throws Exception {
// originally 1 peer, add 2 more
runWithNewCluster(1, cluster -> runTestBootstrapReconf(2, true, cluster));
runWithNewCluster(1, cluster -> {
RaftTestUtil.waitForLeader(cluster);
final RaftPeerId leaderId = cluster.getLeader().getId();

try (final RaftClient client = cluster.createClient(leaderId)) {
final PeerChanges c1 = cluster.addNewPeers(2, true);

assertThrows("Expect change majority error.", SetConfigurationException.class,
() -> client.admin().setConfiguration(c1.allPeersInNewConf));
}
});
}

@Test
Expand Down
47 changes: 47 additions & 0 deletions ratis-server/src/test/java/org/apache/ratis/server/impl/RaftServerTestUtil.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,6 +19,7 @@

import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftGroupMemberId;
import org.apache.ratis.protocol.RaftPeer;
Expand All@@ -35,16 +36,22 @@
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.Slf4jUtils;
import org.apache.ratis.util.TimeDuration;
import org.apache.ratis.util.function.CheckedConsumer;
import org.junit.Assert;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.event.Level;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class RaftServerTestUtil {
Expand DownExpand Up@@ -135,6 +142,10 @@ public static RaftConfiguration newRaftConfiguration(Collection<RaftPeer> peers)
return RaftConfigurationImpl.newBuilder().setConf(peers).build();
}

public static void setRaftConf(RaftServer proxy, RaftGroupId groupId, RaftConfiguration conf) {
((RaftServerImpl)getDivision(proxy, groupId)).getState().setRaftConf(conf);
}

public static RaftServerRpc getServerRpc(RaftServer.Division server) {
return ((RaftServerImpl)server).getRaftServer().getServerRpc();
}
Expand DownExpand Up@@ -196,4 +207,40 @@ public static SegmentedRaftLog newSegmentedRaftLog(RaftGroupMemberId memberId, D
public static boolean isHighestPriority(RaftConfiguration config, RaftPeerId peerId) {
return ((RaftConfigurationImpl)config).isHighestPriority(peerId);
}

public static void runWithMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf,
CheckedConsumer<Collection<RaftPeer>, IOException> consumer) throws IOException {
Collection<RaftPeer> peers = parseMinorityPeers(cluster, peersInNewConf);
while (peers != null) {
consumer.accept(peers);
peers = parseMinorityPeers(cluster, peersInNewConf);
}
}

private static Collection<RaftPeer> parseMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf) {
RaftConfigurationImpl conf = (RaftConfigurationImpl) cluster.getLeader().getRaftConf();
Set<RaftPeer> peers = new HashSet<>(conf.getCurrentPeers());

// Add new peers to construct minority conf peers.
List<RaftPeer> peersToAdd = peersInNewConf.stream().filter(
peer -> !conf.containsInConf(peer.getId(), RaftProtos.RaftPeerRole.FOLLOWER)).collect(Collectors.toList());
if (!peersToAdd.isEmpty()) {
for (RaftPeer peer : peersToAdd) {
if (peers.add(peer) && conf.changeMajority(peers)) {
peers.remove(peer);
break;
}
}
return peers;
}

// All new peers has been added. Handle the removed peers.
List<RaftPeer> peersToRemove = peers.stream().filter(peer -> !peersInNewConf.contains(peer)).collect(Collectors.toList());
if (!peersToRemove.isEmpty()) {
return peersInNewConf;
}

// The peers in new conf are the same as current conf, return null.
return null;
}
}
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
RATIS-1912. Fix infinity election when perform membership change.#954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

szetszwo
merged 10 commits into

from

Nov 3, 2023
 Merged


RATIS-1912. Fix infinity election when perform membership change.
#954
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
861cbdf

v0.1

Oct 26, 2023
8c00760

v0.2

Oct 27, 2023
ac6de8f

v0.3

Oct 30, 2023
88aa0af

v0.4

Oct 30, 2023
feeac8d

v0.5 remove unused import

Oct 31, 2023
d3b0134

v0.6

Nov 1, 2023
61744a7

v0.7

Nov 1, 2023
1080e99

v0.8 fix unit tests.

Nov 2, 2023
449d485

v0.9

Nov 3, 2023
f93e9cc

v0.10

Nov 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


11 changes: 9 additions & 2 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -104,7 +104,7 @@ enum Phase {
 ELECTION
 }

 enum Result {PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}
 enum Result {PASSED, SINGLE_MODE_PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}

 private static class ResultAndTerm {
 private final Result result;
Expand DownExpand Up@@ -331,6 +331,7 @@ private boolean askForVotes(Phase phase, int round) throws InterruptedException,

 switch (r.getResult()) {
 case PASSED:
 case SINGLE_MODE_PASSED:
 return true;
 case NOT_IN_CONF:
 case SHUTDOWN:
Expand DownExpand Up@@ -379,6 +380,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 Collection<RaftPeerId> votedPeers = new ArrayList<>();
 Collection<RaftPeerId> rejectedPeers = new ArrayList<>();
 Set<RaftPeerId> higherPriorityPeers = getHigherPriorityPeers(conf);
 final boolean singleMode = conf.isSingleMode(server.getId());

 while (waitForNum > 0 && shouldRun(electionTerm)) {
 final TimeDuration waitTime = timeout.elapsedTime().apply(n -> -n);
Expand All@@ -387,6 +389,9 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 // if some higher priority peer did not response when timeout, but candidate get majority,
 // candidate pass vote
 return logAndReturn(phase, Result.PASSED, responses, exceptions);
 } else if (singleMode) {
 // if candidate is in single mode, candidate pass vote.
 return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
 } else {
 return logAndReturn(phase, Result.TIMEOUT, responses, exceptions);
 }
Expand DownExpand Up@@ -418,7 +423,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 }

 // If any peer with higher priority rejects vote, candidate can not pass vote
 if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId)) {
 if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId) && !singleMode) {
 return logAndReturn(phase, Result.REJECTED, responses, exceptions);
 }

Expand DownExpand Up@@ -447,6 +452,8 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 // received all the responses
 if (conf.hasMajority(votedPeers, server.getId())) {
 return logAndReturn(phase, Result.PASSED, responses, exceptions);
 } else if (singleMode) {
 return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
 } else {
 return logAndReturn(phase, Result.REJECTED, responses, exceptions);
 }
Expand Down

28 changes: 28 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/RaftConfigurationImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -232,6 +232,34 @@ Collection<RaftPeer> getOtherPeers(RaftPeerId selfId) {
 return others;
 }

 /**
 * @return true if the new peers number reaches half of new conf peers number or the group is
 * changing from single mode to HA mode.
 */
 boolean changeMajority(Collection<RaftPeer> newMembers) {
 Preconditions.assertNull(oldConf, "oldConf");
 final long newPeersCount = newMembers.stream().map(RaftPeer::getId).filter(id -> conf.getPeer(id) == null).count();

 if (conf.size() == 1 && newMembers.size() == 2 && newPeersCount == 1) {
 // Change from single peer to HA mode. This is a special case, skip majority verification.
 return false;
 }

 // If newPeersCount reaches majority number of new conf size, the cluster may end with infinity
 // election. See https://issues.apache.org/jira/browse/RATIS-1912 for more details.
 final long oldPeersCount = newMembers.size() - newPeersCount;
 return newPeersCount >= oldPeersCount;
 }

 /** @return True if the selfId is in single mode. */
 boolean isSingleMode(RaftPeerId selfId) {
 if (isStable()) {
 return conf.size() == 1;
 } else {
 return oldConf.size() == 1 && oldConf.contains(selfId) && conf.size() == 2 && conf.contains(selfId);
 }
 }

 /** @return true if the self id together with the others are in the majority. */
 boolean hasMajority(Collection<RaftPeerId> others, RaftPeerId selfId) {
 Preconditions.assertTrue(!others.contains(selfId));
Expand Down

4 changes: 4 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1311,6 +1311,10 @@ public CompletableFuture<RaftClientReply> setConfigurationAsync(SetConfiguration
 pending.setReply(newSuccessReply(request));
 return pending.getFuture();
 }
 if (current.changeMajority(serversInNewConf)) {
 throw new SetConfigurationException("Failed to set configuration: request " + request
 + " changes a majority set of the current configuration " + current);
 }

 getRaftServer().addRaftPeers(serversInNewConf);
 getRaftServer().addRaftPeers(listenersInNewConf);
Expand Down

4 changes: 3 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/InstallSnapshotFromLeaderTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.protocol.RaftClientReply;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.RaftServerConfigKeys;
Expand DownExpand Up@@ -111,7 +112,8 @@ private void testMultiFileInstallSnapshot(CLUSTER cluster) throws Exception {
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand Down

17 changes: 11 additions & 6 deletions
ratis-server/src/test/java/org/apache/ratis/InstallSnapshotNotificationTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,6 +49,7 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
Expand DownExpand Up@@ -242,7 +243,8 @@ private void testAddNewFollowers(CLUSTER cluster, int numRequests) throws Except
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -391,7 +393,8 @@ private void testInstallSnapshotNotificationCount(CLUSTER cluster) throws Except
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));
 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand DownExpand Up@@ -478,7 +481,8 @@ private void testInstallSnapshotInstalledEvent(CLUSTER cluster) throws Exception
 // add one new peer
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(1, true, true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -556,7 +560,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand All@@ -567,8 +572,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
 RaftServerTestUtil.getLatestInstalledSnapshotIndex(follower));
 }

 // Make sure each new peer got one snapshot notification.
 Assert.assertEquals(2, numSnapshotRequests.get());
 // Make sure each new peer got at least one snapshot notification.
 Assert.assertTrue(2 <= numSnapshotRequests.get());
 } finally {
 cluster.shutdown();
 }
Expand Down

4 changes: 3 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.impl.RaftServerTestUtil;
import org.apache.ratis.server.impl.RetryCacheTestUtil;
import org.apache.ratis.server.raftlog.RaftLog;
import org.apache.ratis.server.raftlog.RaftLogIOException;
Expand DownExpand Up@@ -140,7 +141,8 @@ void runTestRetryOnNewLeader(CLUSTER cluster) throws Exception {
 RaftPeer[] allPeers = cluster.removePeers(2, true,
 asList(change.newPeers)).allPeersInNewConf;
 // trigger setConfiguration
 cluster.setConfiguration(allPeers);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(allPeers),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 final RaftPeerId newLeaderId = JavaUtils.attemptRepeatedly(() -> {
 final RaftPeerId id = RaftTestUtil.waitForLeader(cluster).getId();
Expand Down

3 changes: 2 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -329,7 +329,8 @@ public static <T extends Throwable> void runMultiGroupTest(
 }
 LOG.info(chosen + ") setConfiguration: " + cluster.printServers(groups[chosen].getGroupId()));
 try (final RaftClient client = cluster.createClient(groups[chosen])) {
 client.admin().setConfiguration(allPeers.toArray(RaftPeer.emptyArray()));
 RaftServerTestUtil.runWithMinorityPeers(cluster, allPeers,
 peers -> client.admin().setConfiguration(peers.toArray(RaftPeer.emptyArray())));
 }

 Assert.assertNotNull(RaftTestUtil.waitForLeader(cluster));
Expand Down

65 changes: 64 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/impl/RaftReconfigurationBaseTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -61,6 +61,7 @@

import static java.util.Arrays.asList;
import static org.apache.ratis.server.impl.RaftServerTestUtil.waitAndCheckNewConf;
import static org.junit.Assert.assertThrows;

public abstract class RaftReconfigurationBaseTest<CLUSTER extends MiniRaftCluster>
 extends BaseTest
Expand DownExpand Up@@ -145,6 +146,58 @@ public void testAddPeers() throws Exception {
 });
 }

 /**
 * Test leader election when changing cluster from single mode to HA mode.
 */
 @Test
 public void testLeaderElectionWhenChangeFromSingleToHA() throws Exception {
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);

 RaftGroupId groupId = cluster.getGroup().getGroupId();
 RaftPeer curPeer = cluster.getGroup().getPeers().iterator().next();
 RaftPeer newPeer = cluster.addNewPeers(1, true, true).newPeers[0];

 RaftServerProxy leaderServer = cluster.getServer(curPeer.getId());

 // Update leader conf to transitional single mode.
 RaftConfigurationImpl oldNewConf = RaftConfigurationImpl.newBuilder()
 .setOldConf(new PeerConfiguration(Arrays.asList(curPeer)))
 .setConf(new PeerConfiguration(Arrays.asList(curPeer, newPeer)))
 .setLogEntryIndex(Long.MAX_VALUE / 2)
 .build();
 Assert.assertTrue(oldNewConf.isSingleMode(curPeer.getId()));
 RaftServerTestUtil.setRaftConf(leaderServer, groupId, oldNewConf);
 try(RaftClient client = cluster.createClient()) {
 client.admin().transferLeadership(null, leaderServer.getId(), 1000);
 }

 final RaftServer.Division newLeader = RaftTestUtil.waitForLeader(cluster);
 Assert.assertEquals(leaderServer.getId(), newLeader.getId());
 Assert.assertEquals(oldNewConf, newLeader.getRaftConf());
 });
 }

 @Test
 public void testChangeMajority() throws Exception {
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);
 final RaftPeerId leaderId = cluster.getLeader().getId();

 try (final RaftClient client = cluster.createClient(leaderId)) {
 final PeerChanges c1 = cluster.addNewPeers(2, true);

 SetConfigurationRequest.Arguments arguments = SetConfigurationRequest.Arguments.newBuilder()
 .setServersInCurrentConf(cluster.getPeers())
 .setServersInNewConf(c1.allPeersInNewConf)
 .setMode(SetConfigurationRequest.Mode.COMPARE_AND_SET)
 .build();
 assertThrows("Expect change majority error.", SetConfigurationException.class,
 () -> client.admin().setConfiguration(arguments));
 }
 });
 }

 /**
 * remove 2 peers (5 peers -> 3 peers), no leader change
 */
Expand DownExpand Up@@ -380,7 +433,17 @@ public void testBootstrapReconfWithSingleNodeAddOne() throws Exception {
 @Test
 public void testBootstrapReconfWithSingleNodeAddTwo() throws Exception {
 // originally 1 peer, add 2 more
 runWithNewCluster(1, cluster -> runTestBootstrapReconf(2, true, cluster));
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);
 final RaftPeerId leaderId = cluster.getLeader().getId();

 try (final RaftClient client = cluster.createClient(leaderId)) {
 final PeerChanges c1 = cluster.addNewPeers(2, true);

 assertThrows("Expect change majority error.", SetConfigurationException.class,
 () -> client.admin().setConfiguration(c1.allPeersInNewConf));
 }
 });
 }

 @Test
Expand Down

47 changes: 47 additions & 0 deletions
ratis-server/src/test/java/org/apache/ratis/server/impl/RaftServerTestUtil.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,6 +19,7 @@

import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftGroupMemberId;
import org.apache.ratis.protocol.RaftPeer;
Expand All@@ -35,16 +36,22 @@
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.Slf4jUtils;
import org.apache.ratis.util.TimeDuration;
import org.apache.ratis.util.function.CheckedConsumer;
import org.junit.Assert;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.event.Level;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class RaftServerTestUtil {
Expand DownExpand Up@@ -135,6 +142,10 @@ public static RaftConfiguration newRaftConfiguration(Collection<RaftPeer> peers)
 return RaftConfigurationImpl.newBuilder().setConf(peers).build();
 }

 public static void setRaftConf(RaftServer proxy, RaftGroupId groupId, RaftConfiguration conf) {
 ((RaftServerImpl)getDivision(proxy, groupId)).getState().setRaftConf(conf);
 }

 public static RaftServerRpc getServerRpc(RaftServer.Division server) {
 return ((RaftServerImpl)server).getRaftServer().getServerRpc();
 }
Expand DownExpand Up@@ -196,4 +207,40 @@ public static SegmentedRaftLog newSegmentedRaftLog(RaftGroupMemberId memberId, D
 public static boolean isHighestPriority(RaftConfiguration config, RaftPeerId peerId) {
 return ((RaftConfigurationImpl)config).isHighestPriority(peerId);
 }

 public static void runWithMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf,
 CheckedConsumer<Collection<RaftPeer>, IOException> consumer) throws IOException {
 Collection<RaftPeer> peers = parseMinorityPeers(cluster, peersInNewConf);
 while (peers != null) {
 consumer.accept(peers);
 peers = parseMinorityPeers(cluster, peersInNewConf);
 }
 }

 private static Collection<RaftPeer> parseMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf) {
 RaftConfigurationImpl conf = (RaftConfigurationImpl) cluster.getLeader().getRaftConf();
 Set<RaftPeer> peers = new HashSet<>(conf.getCurrentPeers());

 // Add new peers to construct minority conf peers.
 List<RaftPeer> peersToAdd = peersInNewConf.stream().filter(
 peer -> !conf.containsInConf(peer.getId(), RaftProtos.RaftPeerRole.FOLLOWER)).collect(Collectors.toList());
 if (!peersToAdd.isEmpty()) {
 for (RaftPeer peer : peersToAdd) {
 if (peers.add(peer) && conf.changeMajority(peers)) {
 peers.remove(peer);
 break;
 }
 }
 return peers;
 }

 // All new peers has been added. Handle the removed peers.
 List<RaftPeer> peersToRemove = peers.stream().filter(peer -> !peersInNewConf.contains(peer)).collect(Collectors.toList());
 if (!peersToRemove.isEmpty()) {
 return peersInNewConf;
 }

 // The peers in new conf are the same as current conf, return null.
 return null;
 }
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
RATIS-1912. Fix infinity election when perform membership change.#954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

szetszwo
merged 10 commits into

from

Nov 3, 2023
 Merged


RATIS-1912. Fix infinity election when perform membership change.
#954
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
861cbdf

v0.1

Oct 26, 2023
8c00760

v0.2

Oct 27, 2023
ac6de8f

v0.3

Oct 30, 2023
88aa0af

v0.4

Oct 30, 2023
feeac8d

v0.5 remove unused import

Oct 31, 2023
d3b0134

v0.6

Nov 1, 2023
61744a7

v0.7

Nov 1, 2023
1080e99

v0.8 fix unit tests.

Nov 2, 2023
449d485

v0.9

Nov 3, 2023
f93e9cc

v0.10

Nov 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


11 changes: 9 additions & 2 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -104,7 +104,7 @@ enum Phase {
 ELECTION
 }

 enum Result {PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}
 enum Result {PASSED, SINGLE_MODE_PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}

 private static class ResultAndTerm {
 private final Result result;
Expand DownExpand Up@@ -331,6 +331,7 @@ private boolean askForVotes(Phase phase, int round) throws InterruptedException,

 switch (r.getResult()) {
 case PASSED:
 case SINGLE_MODE_PASSED:
 return true;
 case NOT_IN_CONF:
 case SHUTDOWN:
Expand DownExpand Up@@ -379,6 +380,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 Collection<RaftPeerId> votedPeers = new ArrayList<>();
 Collection<RaftPeerId> rejectedPeers = new ArrayList<>();
 Set<RaftPeerId> higherPriorityPeers = getHigherPriorityPeers(conf);
 final boolean singleMode = conf.isSingleMode(server.getId());

 while (waitForNum > 0 && shouldRun(electionTerm)) {
 final TimeDuration waitTime = timeout.elapsedTime().apply(n -> -n);
Expand All@@ -387,6 +389,9 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 // if some higher priority peer did not response when timeout, but candidate get majority,
 // candidate pass vote
 return logAndReturn(phase, Result.PASSED, responses, exceptions);
 } else if (singleMode) {
 // if candidate is in single mode, candidate pass vote.
 return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
 } else {
 return logAndReturn(phase, Result.TIMEOUT, responses, exceptions);
 }
Expand DownExpand Up@@ -418,7 +423,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 }

 // If any peer with higher priority rejects vote, candidate can not pass vote
 if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId)) {
 if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId) && !singleMode) {
 return logAndReturn(phase, Result.REJECTED, responses, exceptions);
 }

Expand DownExpand Up@@ -447,6 +452,8 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 // received all the responses
 if (conf.hasMajority(votedPeers, server.getId())) {
 return logAndReturn(phase, Result.PASSED, responses, exceptions);
 } else if (singleMode) {
 return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
 } else {
 return logAndReturn(phase, Result.REJECTED, responses, exceptions);
 }
Expand Down

28 changes: 28 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/RaftConfigurationImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -232,6 +232,34 @@ Collection<RaftPeer> getOtherPeers(RaftPeerId selfId) {
 return others;
 }

 /**
 * @return true if the new peers number reaches half of new conf peers number or the group is
 * changing from single mode to HA mode.
 */
 boolean changeMajority(Collection<RaftPeer> newMembers) {
 Preconditions.assertNull(oldConf, "oldConf");
 final long newPeersCount = newMembers.stream().map(RaftPeer::getId).filter(id -> conf.getPeer(id) == null).count();

 if (conf.size() == 1 && newMembers.size() == 2 && newPeersCount == 1) {
 // Change from single peer to HA mode. This is a special case, skip majority verification.
 return false;
 }

 // If newPeersCount reaches majority number of new conf size, the cluster may end with infinity
 // election. See https://issues.apache.org/jira/browse/RATIS-1912 for more details.
 final long oldPeersCount = newMembers.size() - newPeersCount;
 return newPeersCount >= oldPeersCount;
 }

 /** @return True if the selfId is in single mode. */
 boolean isSingleMode(RaftPeerId selfId) {
 if (isStable()) {
 return conf.size() == 1;
 } else {
 return oldConf.size() == 1 && oldConf.contains(selfId) && conf.size() == 2 && conf.contains(selfId);
 }
 }

 /** @return true if the self id together with the others are in the majority. */
 boolean hasMajority(Collection<RaftPeerId> others, RaftPeerId selfId) {
 Preconditions.assertTrue(!others.contains(selfId));
Expand Down

4 changes: 4 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1311,6 +1311,10 @@ public CompletableFuture<RaftClientReply> setConfigurationAsync(SetConfiguration
 pending.setReply(newSuccessReply(request));
 return pending.getFuture();
 }
 if (current.changeMajority(serversInNewConf)) {
 throw new SetConfigurationException("Failed to set configuration: request " + request
 + " changes a majority set of the current configuration " + current);
 }

 getRaftServer().addRaftPeers(serversInNewConf);
 getRaftServer().addRaftPeers(listenersInNewConf);
Expand Down

4 changes: 3 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/InstallSnapshotFromLeaderTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.protocol.RaftClientReply;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.RaftServerConfigKeys;
Expand DownExpand Up@@ -111,7 +112,8 @@ private void testMultiFileInstallSnapshot(CLUSTER cluster) throws Exception {
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand Down

17 changes: 11 additions & 6 deletions
ratis-server/src/test/java/org/apache/ratis/InstallSnapshotNotificationTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,6 +49,7 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
Expand DownExpand Up@@ -242,7 +243,8 @@ private void testAddNewFollowers(CLUSTER cluster, int numRequests) throws Except
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -391,7 +393,8 @@ private void testInstallSnapshotNotificationCount(CLUSTER cluster) throws Except
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));
 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand DownExpand Up@@ -478,7 +481,8 @@ private void testInstallSnapshotInstalledEvent(CLUSTER cluster) throws Exception
 // add one new peer
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(1, true, true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -556,7 +560,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand All@@ -567,8 +572,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
 RaftServerTestUtil.getLatestInstalledSnapshotIndex(follower));
 }

 // Make sure each new peer got one snapshot notification.
 Assert.assertEquals(2, numSnapshotRequests.get());
 // Make sure each new peer got at least one snapshot notification.
 Assert.assertTrue(2 <= numSnapshotRequests.get());
 } finally {
 cluster.shutdown();
 }
Expand Down

4 changes: 3 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.impl.RaftServerTestUtil;
import org.apache.ratis.server.impl.RetryCacheTestUtil;
import org.apache.ratis.server.raftlog.RaftLog;
import org.apache.ratis.server.raftlog.RaftLogIOException;
Expand DownExpand Up@@ -140,7 +141,8 @@ void runTestRetryOnNewLeader(CLUSTER cluster) throws Exception {
 RaftPeer[] allPeers = cluster.removePeers(2, true,
 asList(change.newPeers)).allPeersInNewConf;
 // trigger setConfiguration
 cluster.setConfiguration(allPeers);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(allPeers),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 final RaftPeerId newLeaderId = JavaUtils.attemptRepeatedly(() -> {
 final RaftPeerId id = RaftTestUtil.waitForLeader(cluster).getId();
Expand Down

3 changes: 2 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -329,7 +329,8 @@ public static <T extends Throwable> void runMultiGroupTest(
 }
 LOG.info(chosen + ") setConfiguration: " + cluster.printServers(groups[chosen].getGroupId()));
 try (final RaftClient client = cluster.createClient(groups[chosen])) {
 client.admin().setConfiguration(allPeers.toArray(RaftPeer.emptyArray()));
 RaftServerTestUtil.runWithMinorityPeers(cluster, allPeers,
 peers -> client.admin().setConfiguration(peers.toArray(RaftPeer.emptyArray())));
 }

 Assert.assertNotNull(RaftTestUtil.waitForLeader(cluster));
Expand Down

65 changes: 64 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/impl/RaftReconfigurationBaseTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -61,6 +61,7 @@

import static java.util.Arrays.asList;
import static org.apache.ratis.server.impl.RaftServerTestUtil.waitAndCheckNewConf;
import static org.junit.Assert.assertThrows;

public abstract class RaftReconfigurationBaseTest<CLUSTER extends MiniRaftCluster>
 extends BaseTest
Expand DownExpand Up@@ -145,6 +146,58 @@ public void testAddPeers() throws Exception {
 });
 }

 /**
 * Test leader election when changing cluster from single mode to HA mode.
 */
 @Test
 public void testLeaderElectionWhenChangeFromSingleToHA() throws Exception {
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);

 RaftGroupId groupId = cluster.getGroup().getGroupId();
 RaftPeer curPeer = cluster.getGroup().getPeers().iterator().next();
 RaftPeer newPeer = cluster.addNewPeers(1, true, true).newPeers[0];

 RaftServerProxy leaderServer = cluster.getServer(curPeer.getId());

 // Update leader conf to transitional single mode.
 RaftConfigurationImpl oldNewConf = RaftConfigurationImpl.newBuilder()
 .setOldConf(new PeerConfiguration(Arrays.asList(curPeer)))
 .setConf(new PeerConfiguration(Arrays.asList(curPeer, newPeer)))
 .setLogEntryIndex(Long.MAX_VALUE / 2)
 .build();
 Assert.assertTrue(oldNewConf.isSingleMode(curPeer.getId()));
 RaftServerTestUtil.setRaftConf(leaderServer, groupId, oldNewConf);
 try(RaftClient client = cluster.createClient()) {
 client.admin().transferLeadership(null, leaderServer.getId(), 1000);
 }

 final RaftServer.Division newLeader = RaftTestUtil.waitForLeader(cluster);
 Assert.assertEquals(leaderServer.getId(), newLeader.getId());
 Assert.assertEquals(oldNewConf, newLeader.getRaftConf());
 });
 }

 @Test
 public void testChangeMajority() throws Exception {
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);
 final RaftPeerId leaderId = cluster.getLeader().getId();

 try (final RaftClient client = cluster.createClient(leaderId)) {
 final PeerChanges c1 = cluster.addNewPeers(2, true);

 SetConfigurationRequest.Arguments arguments = SetConfigurationRequest.Arguments.newBuilder()
 .setServersInCurrentConf(cluster.getPeers())
 .setServersInNewConf(c1.allPeersInNewConf)
 .setMode(SetConfigurationRequest.Mode.COMPARE_AND_SET)
 .build();
 assertThrows("Expect change majority error.", SetConfigurationException.class,
 () -> client.admin().setConfiguration(arguments));
 }
 });
 }

 /**
 * remove 2 peers (5 peers -> 3 peers), no leader change
 */
Expand DownExpand Up@@ -380,7 +433,17 @@ public void testBootstrapReconfWithSingleNodeAddOne() throws Exception {
 @Test
 public void testBootstrapReconfWithSingleNodeAddTwo() throws Exception {
 // originally 1 peer, add 2 more
 runWithNewCluster(1, cluster -> runTestBootstrapReconf(2, true, cluster));
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);
 final RaftPeerId leaderId = cluster.getLeader().getId();

 try (final RaftClient client = cluster.createClient(leaderId)) {
 final PeerChanges c1 = cluster.addNewPeers(2, true);

 assertThrows("Expect change majority error.", SetConfigurationException.class,
 () -> client.admin().setConfiguration(c1.allPeersInNewConf));
 }
 });
 }

 @Test
Expand Down

47 changes: 47 additions & 0 deletions
ratis-server/src/test/java/org/apache/ratis/server/impl/RaftServerTestUtil.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,6 +19,7 @@

import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftGroupMemberId;
import org.apache.ratis.protocol.RaftPeer;
Expand All@@ -35,16 +36,22 @@
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.Slf4jUtils;
import org.apache.ratis.util.TimeDuration;
import org.apache.ratis.util.function.CheckedConsumer;
import org.junit.Assert;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.event.Level;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class RaftServerTestUtil {
Expand DownExpand Up@@ -135,6 +142,10 @@ public static RaftConfiguration newRaftConfiguration(Collection<RaftPeer> peers)
 return RaftConfigurationImpl.newBuilder().setConf(peers).build();
 }

 public static void setRaftConf(RaftServer proxy, RaftGroupId groupId, RaftConfiguration conf) {
 ((RaftServerImpl)getDivision(proxy, groupId)).getState().setRaftConf(conf);
 }

 public static RaftServerRpc getServerRpc(RaftServer.Division server) {
 return ((RaftServerImpl)server).getRaftServer().getServerRpc();
 }
Expand DownExpand Up@@ -196,4 +207,40 @@ public static SegmentedRaftLog newSegmentedRaftLog(RaftGroupMemberId memberId, D
 public static boolean isHighestPriority(RaftConfiguration config, RaftPeerId peerId) {
 return ((RaftConfigurationImpl)config).isHighestPriority(peerId);
 }

 public static void runWithMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf,
 CheckedConsumer<Collection<RaftPeer>, IOException> consumer) throws IOException {
 Collection<RaftPeer> peers = parseMinorityPeers(cluster, peersInNewConf);
 while (peers != null) {
 consumer.accept(peers);
 peers = parseMinorityPeers(cluster, peersInNewConf);
 }
 }

 private static Collection<RaftPeer> parseMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf) {
 RaftConfigurationImpl conf = (RaftConfigurationImpl) cluster.getLeader().getRaftConf();
 Set<RaftPeer> peers = new HashSet<>(conf.getCurrentPeers());

 // Add new peers to construct minority conf peers.
 List<RaftPeer> peersToAdd = peersInNewConf.stream().filter(
 peer -> !conf.containsInConf(peer.getId(), RaftProtos.RaftPeerRole.FOLLOWER)).collect(Collectors.toList());
 if (!peersToAdd.isEmpty()) {
 for (RaftPeer peer : peersToAdd) {
 if (peers.add(peer) && conf.changeMajority(peers)) {
 peers.remove(peer);
 break;
 }
 }
 return peers;
 }

 // All new peers has been added. Handle the removed peers.
 List<RaftPeer> peersToRemove = peers.stream().filter(peer -> !peersInNewConf.contains(peer)).collect(Collectors.toList());
 if (!peersToRemove.isEmpty()) {
 return peersInNewConf;
 }

 // The peers in new conf are the same as current conf, return null.
 return null;
 }
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
RATIS-1912. Fix infinity election when perform membership change.#954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

szetszwo
merged 10 commits into

from

Nov 3, 2023
 Merged


RATIS-1912. Fix infinity election when perform membership change.
#954
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
861cbdf

v0.1

Oct 26, 2023
8c00760

v0.2

Oct 27, 2023
ac6de8f

v0.3

Oct 30, 2023
88aa0af

v0.4

Oct 30, 2023
feeac8d

v0.5 remove unused import

Oct 31, 2023
d3b0134

v0.6

Nov 1, 2023
61744a7

v0.7

Nov 1, 2023
1080e99

v0.8 fix unit tests.

Nov 2, 2023
449d485

v0.9

Nov 3, 2023
f93e9cc

v0.10

Nov 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


11 changes: 9 additions & 2 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -104,7 +104,7 @@ enum Phase {
 ELECTION
 }

 enum Result {PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}
 enum Result {PASSED, SINGLE_MODE_PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}

 private static class ResultAndTerm {
 private final Result result;
Expand DownExpand Up@@ -331,6 +331,7 @@ private boolean askForVotes(Phase phase, int round) throws InterruptedException,

 switch (r.getResult()) {
 case PASSED:
 case SINGLE_MODE_PASSED:
 return true;
 case NOT_IN_CONF:
 case SHUTDOWN:
Expand DownExpand Up@@ -379,6 +380,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 Collection<RaftPeerId> votedPeers = new ArrayList<>();
 Collection<RaftPeerId> rejectedPeers = new ArrayList<>();
 Set<RaftPeerId> higherPriorityPeers = getHigherPriorityPeers(conf);
 final boolean singleMode = conf.isSingleMode(server.getId());

 while (waitForNum > 0 && shouldRun(electionTerm)) {
 final TimeDuration waitTime = timeout.elapsedTime().apply(n -> -n);
Expand All@@ -387,6 +389,9 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 // if some higher priority peer did not response when timeout, but candidate get majority,
 // candidate pass vote
 return logAndReturn(phase, Result.PASSED, responses, exceptions);
 } else if (singleMode) {
 // if candidate is in single mode, candidate pass vote.
 return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
 } else {
 return logAndReturn(phase, Result.TIMEOUT, responses, exceptions);
 }
Expand DownExpand Up@@ -418,7 +423,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 }

 // If any peer with higher priority rejects vote, candidate can not pass vote
 if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId)) {
 if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId) && !singleMode) {
 return logAndReturn(phase, Result.REJECTED, responses, exceptions);
 }

Expand DownExpand Up@@ -447,6 +452,8 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 // received all the responses
 if (conf.hasMajority(votedPeers, server.getId())) {
 return logAndReturn(phase, Result.PASSED, responses, exceptions);
 } else if (singleMode) {
 return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
 } else {
 return logAndReturn(phase, Result.REJECTED, responses, exceptions);
 }
Expand Down

28 changes: 28 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/RaftConfigurationImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -232,6 +232,34 @@ Collection<RaftPeer> getOtherPeers(RaftPeerId selfId) {
 return others;
 }

 /**
 * @return true if the new peers number reaches half of new conf peers number or the group is
 * changing from single mode to HA mode.
 */
 boolean changeMajority(Collection<RaftPeer> newMembers) {
 Preconditions.assertNull(oldConf, "oldConf");
 final long newPeersCount = newMembers.stream().map(RaftPeer::getId).filter(id -> conf.getPeer(id) == null).count();

 if (conf.size() == 1 && newMembers.size() == 2 && newPeersCount == 1) {
 // Change from single peer to HA mode. This is a special case, skip majority verification.
 return false;
 }

 // If newPeersCount reaches majority number of new conf size, the cluster may end with infinity
 // election. See https://issues.apache.org/jira/browse/RATIS-1912 for more details.
 final long oldPeersCount = newMembers.size() - newPeersCount;
 return newPeersCount >= oldPeersCount;
 }

 /** @return True if the selfId is in single mode. */
 boolean isSingleMode(RaftPeerId selfId) {
 if (isStable()) {
 return conf.size() == 1;
 } else {
 return oldConf.size() == 1 && oldConf.contains(selfId) && conf.size() == 2 && conf.contains(selfId);
 }
 }

 /** @return true if the self id together with the others are in the majority. */
 boolean hasMajority(Collection<RaftPeerId> others, RaftPeerId selfId) {
 Preconditions.assertTrue(!others.contains(selfId));
Expand Down

4 changes: 4 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1311,6 +1311,10 @@ public CompletableFuture<RaftClientReply> setConfigurationAsync(SetConfiguration
 pending.setReply(newSuccessReply(request));
 return pending.getFuture();
 }
 if (current.changeMajority(serversInNewConf)) {
 throw new SetConfigurationException("Failed to set configuration: request " + request
 + " changes a majority set of the current configuration " + current);
 }

 getRaftServer().addRaftPeers(serversInNewConf);
 getRaftServer().addRaftPeers(listenersInNewConf);
Expand Down

4 changes: 3 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/InstallSnapshotFromLeaderTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.protocol.RaftClientReply;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.RaftServerConfigKeys;
Expand DownExpand Up@@ -111,7 +112,8 @@ private void testMultiFileInstallSnapshot(CLUSTER cluster) throws Exception {
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand Down

17 changes: 11 additions & 6 deletions
ratis-server/src/test/java/org/apache/ratis/InstallSnapshotNotificationTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,6 +49,7 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
Expand DownExpand Up@@ -242,7 +243,8 @@ private void testAddNewFollowers(CLUSTER cluster, int numRequests) throws Except
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -391,7 +393,8 @@ private void testInstallSnapshotNotificationCount(CLUSTER cluster) throws Except
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));
 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand DownExpand Up@@ -478,7 +481,8 @@ private void testInstallSnapshotInstalledEvent(CLUSTER cluster) throws Exception
 // add one new peer
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(1, true, true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -556,7 +560,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand All@@ -567,8 +572,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
 RaftServerTestUtil.getLatestInstalledSnapshotIndex(follower));
 }

 // Make sure each new peer got one snapshot notification.
 Assert.assertEquals(2, numSnapshotRequests.get());
 // Make sure each new peer got at least one snapshot notification.
 Assert.assertTrue(2 <= numSnapshotRequests.get());
 } finally {
 cluster.shutdown();
 }
Expand Down

4 changes: 3 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.impl.RaftServerTestUtil;
import org.apache.ratis.server.impl.RetryCacheTestUtil;
import org.apache.ratis.server.raftlog.RaftLog;
import org.apache.ratis.server.raftlog.RaftLogIOException;
Expand DownExpand Up@@ -140,7 +141,8 @@ void runTestRetryOnNewLeader(CLUSTER cluster) throws Exception {
 RaftPeer[] allPeers = cluster.removePeers(2, true,
 asList(change.newPeers)).allPeersInNewConf;
 // trigger setConfiguration
 cluster.setConfiguration(allPeers);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(allPeers),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 final RaftPeerId newLeaderId = JavaUtils.attemptRepeatedly(() -> {
 final RaftPeerId id = RaftTestUtil.waitForLeader(cluster).getId();
Expand Down

3 changes: 2 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -329,7 +329,8 @@ public static <T extends Throwable> void runMultiGroupTest(
 }
 LOG.info(chosen + ") setConfiguration: " + cluster.printServers(groups[chosen].getGroupId()));
 try (final RaftClient client = cluster.createClient(groups[chosen])) {
 client.admin().setConfiguration(allPeers.toArray(RaftPeer.emptyArray()));
 RaftServerTestUtil.runWithMinorityPeers(cluster, allPeers,
 peers -> client.admin().setConfiguration(peers.toArray(RaftPeer.emptyArray())));
 }

 Assert.assertNotNull(RaftTestUtil.waitForLeader(cluster));
Expand Down

65 changes: 64 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/impl/RaftReconfigurationBaseTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -61,6 +61,7 @@

import static java.util.Arrays.asList;
import static org.apache.ratis.server.impl.RaftServerTestUtil.waitAndCheckNewConf;
import static org.junit.Assert.assertThrows;

public abstract class RaftReconfigurationBaseTest<CLUSTER extends MiniRaftCluster>
 extends BaseTest
Expand DownExpand Up@@ -145,6 +146,58 @@ public void testAddPeers() throws Exception {
 });
 }

 /**
 * Test leader election when changing cluster from single mode to HA mode.
 */
 @Test
 public void testLeaderElectionWhenChangeFromSingleToHA() throws Exception {
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);

 RaftGroupId groupId = cluster.getGroup().getGroupId();
 RaftPeer curPeer = cluster.getGroup().getPeers().iterator().next();
 RaftPeer newPeer = cluster.addNewPeers(1, true, true).newPeers[0];

 RaftServerProxy leaderServer = cluster.getServer(curPeer.getId());

 // Update leader conf to transitional single mode.
 RaftConfigurationImpl oldNewConf = RaftConfigurationImpl.newBuilder()
 .setOldConf(new PeerConfiguration(Arrays.asList(curPeer)))
 .setConf(new PeerConfiguration(Arrays.asList(curPeer, newPeer)))
 .setLogEntryIndex(Long.MAX_VALUE / 2)
 .build();
 Assert.assertTrue(oldNewConf.isSingleMode(curPeer.getId()));
 RaftServerTestUtil.setRaftConf(leaderServer, groupId, oldNewConf);
 try(RaftClient client = cluster.createClient()) {
 client.admin().transferLeadership(null, leaderServer.getId(), 1000);
 }

 final RaftServer.Division newLeader = RaftTestUtil.waitForLeader(cluster);
 Assert.assertEquals(leaderServer.getId(), newLeader.getId());
 Assert.assertEquals(oldNewConf, newLeader.getRaftConf());
 });
 }

 @Test
 public void testChangeMajority() throws Exception {
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);
 final RaftPeerId leaderId = cluster.getLeader().getId();

 try (final RaftClient client = cluster.createClient(leaderId)) {
 final PeerChanges c1 = cluster.addNewPeers(2, true);

 SetConfigurationRequest.Arguments arguments = SetConfigurationRequest.Arguments.newBuilder()
 .setServersInCurrentConf(cluster.getPeers())
 .setServersInNewConf(c1.allPeersInNewConf)
 .setMode(SetConfigurationRequest.Mode.COMPARE_AND_SET)
 .build();
 assertThrows("Expect change majority error.", SetConfigurationException.class,
 () -> client.admin().setConfiguration(arguments));
 }
 });
 }

 /**
 * remove 2 peers (5 peers -> 3 peers), no leader change
 */
Expand DownExpand Up@@ -380,7 +433,17 @@ public void testBootstrapReconfWithSingleNodeAddOne() throws Exception {
 @Test
 public void testBootstrapReconfWithSingleNodeAddTwo() throws Exception {
 // originally 1 peer, add 2 more
 runWithNewCluster(1, cluster -> runTestBootstrapReconf(2, true, cluster));
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);
 final RaftPeerId leaderId = cluster.getLeader().getId();

 try (final RaftClient client = cluster.createClient(leaderId)) {
 final PeerChanges c1 = cluster.addNewPeers(2, true);

 assertThrows("Expect change majority error.", SetConfigurationException.class,
 () -> client.admin().setConfiguration(c1.allPeersInNewConf));
 }
 });
 }

 @Test
Expand Down

47 changes: 47 additions & 0 deletions
ratis-server/src/test/java/org/apache/ratis/server/impl/RaftServerTestUtil.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,6 +19,7 @@

import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftGroupMemberId;
import org.apache.ratis.protocol.RaftPeer;
Expand All@@ -35,16 +36,22 @@
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.Slf4jUtils;
import org.apache.ratis.util.TimeDuration;
import org.apache.ratis.util.function.CheckedConsumer;
import org.junit.Assert;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.event.Level;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class RaftServerTestUtil {
Expand DownExpand Up@@ -135,6 +142,10 @@ public static RaftConfiguration newRaftConfiguration(Collection<RaftPeer> peers)
 return RaftConfigurationImpl.newBuilder().setConf(peers).build();
 }

 public static void setRaftConf(RaftServer proxy, RaftGroupId groupId, RaftConfiguration conf) {
 ((RaftServerImpl)getDivision(proxy, groupId)).getState().setRaftConf(conf);
 }

 public static RaftServerRpc getServerRpc(RaftServer.Division server) {
 return ((RaftServerImpl)server).getRaftServer().getServerRpc();
 }
Expand DownExpand Up@@ -196,4 +207,40 @@ public static SegmentedRaftLog newSegmentedRaftLog(RaftGroupMemberId memberId, D
 public static boolean isHighestPriority(RaftConfiguration config, RaftPeerId peerId) {
 return ((RaftConfigurationImpl)config).isHighestPriority(peerId);
 }

 public static void runWithMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf,
 CheckedConsumer<Collection<RaftPeer>, IOException> consumer) throws IOException {
 Collection<RaftPeer> peers = parseMinorityPeers(cluster, peersInNewConf);
 while (peers != null) {
 consumer.accept(peers);
 peers = parseMinorityPeers(cluster, peersInNewConf);
 }
 }

 private static Collection<RaftPeer> parseMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf) {
 RaftConfigurationImpl conf = (RaftConfigurationImpl) cluster.getLeader().getRaftConf();
 Set<RaftPeer> peers = new HashSet<>(conf.getCurrentPeers());

 // Add new peers to construct minority conf peers.
 List<RaftPeer> peersToAdd = peersInNewConf.stream().filter(
 peer -> !conf.containsInConf(peer.getId(), RaftProtos.RaftPeerRole.FOLLOWER)).collect(Collectors.toList());
 if (!peersToAdd.isEmpty()) {
 for (RaftPeer peer : peersToAdd) {
 if (peers.add(peer) && conf.changeMajority(peers)) {
 peers.remove(peer);
 break;
 }
 }
 return peers;
 }

 // All new peers has been added. Handle the removed peers.
 List<RaftPeer> peersToRemove = peers.stream().filter(peer -> !peersInNewConf.contains(peer)).collect(Collectors.toList());
 if (!peersToRemove.isEmpty()) {
 return peersInNewConf;
 }

 // The peers in new conf are the same as current conf, return null.
 return null;
 }
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
RATIS-1912. Fix infinity election when perform membership change.#954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

szetszwo
merged 10 commits into

from

Nov 3, 2023
 Merged


RATIS-1912. Fix infinity election when perform membership change.
#954
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
861cbdf

v0.1

Oct 26, 2023
8c00760

v0.2

Oct 27, 2023
ac6de8f

v0.3

Oct 30, 2023
88aa0af

v0.4

Oct 30, 2023
feeac8d

v0.5 remove unused import

Oct 31, 2023
d3b0134

v0.6

Nov 1, 2023
61744a7

v0.7

Nov 1, 2023
1080e99

v0.8 fix unit tests.

Nov 2, 2023
449d485

v0.9

Nov 3, 2023
f93e9cc

v0.10

Nov 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


11 changes: 9 additions & 2 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -104,7 +104,7 @@ enum Phase {
 ELECTION
 }

 enum Result {PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}
 enum Result {PASSED, SINGLE_MODE_PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}

 private static class ResultAndTerm {
 private final Result result;
Expand DownExpand Up@@ -331,6 +331,7 @@ private boolean askForVotes(Phase phase, int round) throws InterruptedException,

 switch (r.getResult()) {
 case PASSED:
 case SINGLE_MODE_PASSED:
 return true;
 case NOT_IN_CONF:
 case SHUTDOWN:
Expand DownExpand Up@@ -379,6 +380,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 Collection<RaftPeerId> votedPeers = new ArrayList<>();
 Collection<RaftPeerId> rejectedPeers = new ArrayList<>();
 Set<RaftPeerId> higherPriorityPeers = getHigherPriorityPeers(conf);
 final boolean singleMode = conf.isSingleMode(server.getId());

 while (waitForNum > 0 && shouldRun(electionTerm)) {
 final TimeDuration waitTime = timeout.elapsedTime().apply(n -> -n);
Expand All@@ -387,6 +389,9 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 // if some higher priority peer did not response when timeout, but candidate get majority,
 // candidate pass vote
 return logAndReturn(phase, Result.PASSED, responses, exceptions);
 } else if (singleMode) {
 // if candidate is in single mode, candidate pass vote.
 return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
 } else {
 return logAndReturn(phase, Result.TIMEOUT, responses, exceptions);
 }
Expand DownExpand Up@@ -418,7 +423,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 }

 // If any peer with higher priority rejects vote, candidate can not pass vote
 if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId)) {
 if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId) && !singleMode) {
 return logAndReturn(phase, Result.REJECTED, responses, exceptions);
 }

Expand DownExpand Up@@ -447,6 +452,8 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 // received all the responses
 if (conf.hasMajority(votedPeers, server.getId())) {
 return logAndReturn(phase, Result.PASSED, responses, exceptions);
 } else if (singleMode) {
 return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
 } else {
 return logAndReturn(phase, Result.REJECTED, responses, exceptions);
 }
Expand Down

28 changes: 28 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/RaftConfigurationImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -232,6 +232,34 @@ Collection<RaftPeer> getOtherPeers(RaftPeerId selfId) {
 return others;
 }

 /**
 * @return true if the new peers number reaches half of new conf peers number or the group is
 * changing from single mode to HA mode.
 */
 boolean changeMajority(Collection<RaftPeer> newMembers) {
 Preconditions.assertNull(oldConf, "oldConf");
 final long newPeersCount = newMembers.stream().map(RaftPeer::getId).filter(id -> conf.getPeer(id) == null).count();

 if (conf.size() == 1 && newMembers.size() == 2 && newPeersCount == 1) {
 // Change from single peer to HA mode. This is a special case, skip majority verification.
 return false;
 }

 // If newPeersCount reaches majority number of new conf size, the cluster may end with infinity
 // election. See https://issues.apache.org/jira/browse/RATIS-1912 for more details.
 final long oldPeersCount = newMembers.size() - newPeersCount;
 return newPeersCount >= oldPeersCount;
 }

 /** @return True if the selfId is in single mode. */
 boolean isSingleMode(RaftPeerId selfId) {
 if (isStable()) {
 return conf.size() == 1;
 } else {
 return oldConf.size() == 1 && oldConf.contains(selfId) && conf.size() == 2 && conf.contains(selfId);
 }
 }

 /** @return true if the self id together with the others are in the majority. */
 boolean hasMajority(Collection<RaftPeerId> others, RaftPeerId selfId) {
 Preconditions.assertTrue(!others.contains(selfId));
Expand Down

4 changes: 4 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1311,6 +1311,10 @@ public CompletableFuture<RaftClientReply> setConfigurationAsync(SetConfiguration
 pending.setReply(newSuccessReply(request));
 return pending.getFuture();
 }
 if (current.changeMajority(serversInNewConf)) {
 throw new SetConfigurationException("Failed to set configuration: request " + request
 + " changes a majority set of the current configuration " + current);
 }

 getRaftServer().addRaftPeers(serversInNewConf);
 getRaftServer().addRaftPeers(listenersInNewConf);
Expand Down

4 changes: 3 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/InstallSnapshotFromLeaderTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.protocol.RaftClientReply;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.RaftServerConfigKeys;
Expand DownExpand Up@@ -111,7 +112,8 @@ private void testMultiFileInstallSnapshot(CLUSTER cluster) throws Exception {
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand Down

17 changes: 11 additions & 6 deletions
ratis-server/src/test/java/org/apache/ratis/InstallSnapshotNotificationTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,6 +49,7 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
Expand DownExpand Up@@ -242,7 +243,8 @@ private void testAddNewFollowers(CLUSTER cluster, int numRequests) throws Except
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -391,7 +393,8 @@ private void testInstallSnapshotNotificationCount(CLUSTER cluster) throws Except
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));
 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand DownExpand Up@@ -478,7 +481,8 @@ private void testInstallSnapshotInstalledEvent(CLUSTER cluster) throws Exception
 // add one new peer
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(1, true, true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -556,7 +560,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand All@@ -567,8 +572,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
 RaftServerTestUtil.getLatestInstalledSnapshotIndex(follower));
 }

 // Make sure each new peer got one snapshot notification.
 Assert.assertEquals(2, numSnapshotRequests.get());
 // Make sure each new peer got at least one snapshot notification.
 Assert.assertTrue(2 <= numSnapshotRequests.get());
 } finally {
 cluster.shutdown();
 }
Expand Down

4 changes: 3 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.impl.RaftServerTestUtil;
import org.apache.ratis.server.impl.RetryCacheTestUtil;
import org.apache.ratis.server.raftlog.RaftLog;
import org.apache.ratis.server.raftlog.RaftLogIOException;
Expand DownExpand Up@@ -140,7 +141,8 @@ void runTestRetryOnNewLeader(CLUSTER cluster) throws Exception {
 RaftPeer[] allPeers = cluster.removePeers(2, true,
 asList(change.newPeers)).allPeersInNewConf;
 // trigger setConfiguration
 cluster.setConfiguration(allPeers);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(allPeers),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 final RaftPeerId newLeaderId = JavaUtils.attemptRepeatedly(() -> {
 final RaftPeerId id = RaftTestUtil.waitForLeader(cluster).getId();
Expand Down

3 changes: 2 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -329,7 +329,8 @@ public static <T extends Throwable> void runMultiGroupTest(
 }
 LOG.info(chosen + ") setConfiguration: " + cluster.printServers(groups[chosen].getGroupId()));
 try (final RaftClient client = cluster.createClient(groups[chosen])) {
 client.admin().setConfiguration(allPeers.toArray(RaftPeer.emptyArray()));
 RaftServerTestUtil.runWithMinorityPeers(cluster, allPeers,
 peers -> client.admin().setConfiguration(peers.toArray(RaftPeer.emptyArray())));
 }

 Assert.assertNotNull(RaftTestUtil.waitForLeader(cluster));
Expand Down

65 changes: 64 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/impl/RaftReconfigurationBaseTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -61,6 +61,7 @@

import static java.util.Arrays.asList;
import static org.apache.ratis.server.impl.RaftServerTestUtil.waitAndCheckNewConf;
import static org.junit.Assert.assertThrows;

public abstract class RaftReconfigurationBaseTest<CLUSTER extends MiniRaftCluster>
 extends BaseTest
Expand DownExpand Up@@ -145,6 +146,58 @@ public void testAddPeers() throws Exception {
 });
 }

 /**
 * Test leader election when changing cluster from single mode to HA mode.
 */
 @Test
 public void testLeaderElectionWhenChangeFromSingleToHA() throws Exception {
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);

 RaftGroupId groupId = cluster.getGroup().getGroupId();
 RaftPeer curPeer = cluster.getGroup().getPeers().iterator().next();
 RaftPeer newPeer = cluster.addNewPeers(1, true, true).newPeers[0];

 RaftServerProxy leaderServer = cluster.getServer(curPeer.getId());

 // Update leader conf to transitional single mode.
 RaftConfigurationImpl oldNewConf = RaftConfigurationImpl.newBuilder()
 .setOldConf(new PeerConfiguration(Arrays.asList(curPeer)))
 .setConf(new PeerConfiguration(Arrays.asList(curPeer, newPeer)))
 .setLogEntryIndex(Long.MAX_VALUE / 2)
 .build();
 Assert.assertTrue(oldNewConf.isSingleMode(curPeer.getId()));
 RaftServerTestUtil.setRaftConf(leaderServer, groupId, oldNewConf);
 try(RaftClient client = cluster.createClient()) {
 client.admin().transferLeadership(null, leaderServer.getId(), 1000);
 }

 final RaftServer.Division newLeader = RaftTestUtil.waitForLeader(cluster);
 Assert.assertEquals(leaderServer.getId(), newLeader.getId());
 Assert.assertEquals(oldNewConf, newLeader.getRaftConf());
 });
 }

 @Test
 public void testChangeMajority() throws Exception {
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);
 final RaftPeerId leaderId = cluster.getLeader().getId();

 try (final RaftClient client = cluster.createClient(leaderId)) {
 final PeerChanges c1 = cluster.addNewPeers(2, true);

 SetConfigurationRequest.Arguments arguments = SetConfigurationRequest.Arguments.newBuilder()
 .setServersInCurrentConf(cluster.getPeers())
 .setServersInNewConf(c1.allPeersInNewConf)
 .setMode(SetConfigurationRequest.Mode.COMPARE_AND_SET)
 .build();
 assertThrows("Expect change majority error.", SetConfigurationException.class,
 () -> client.admin().setConfiguration(arguments));
 }
 });
 }

 /**
 * remove 2 peers (5 peers -> 3 peers), no leader change
 */
Expand DownExpand Up@@ -380,7 +433,17 @@ public void testBootstrapReconfWithSingleNodeAddOne() throws Exception {
 @Test
 public void testBootstrapReconfWithSingleNodeAddTwo() throws Exception {
 // originally 1 peer, add 2 more
 runWithNewCluster(1, cluster -> runTestBootstrapReconf(2, true, cluster));
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);
 final RaftPeerId leaderId = cluster.getLeader().getId();

 try (final RaftClient client = cluster.createClient(leaderId)) {
 final PeerChanges c1 = cluster.addNewPeers(2, true);

 assertThrows("Expect change majority error.", SetConfigurationException.class,
 () -> client.admin().setConfiguration(c1.allPeersInNewConf));
 }
 });
 }

 @Test
Expand Down

47 changes: 47 additions & 0 deletions
ratis-server/src/test/java/org/apache/ratis/server/impl/RaftServerTestUtil.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,6 +19,7 @@

import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftGroupMemberId;
import org.apache.ratis.protocol.RaftPeer;
Expand All@@ -35,16 +36,22 @@
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.Slf4jUtils;
import org.apache.ratis.util.TimeDuration;
import org.apache.ratis.util.function.CheckedConsumer;
import org.junit.Assert;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.event.Level;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class RaftServerTestUtil {
Expand DownExpand Up@@ -135,6 +142,10 @@ public static RaftConfiguration newRaftConfiguration(Collection<RaftPeer> peers)
 return RaftConfigurationImpl.newBuilder().setConf(peers).build();
 }

 public static void setRaftConf(RaftServer proxy, RaftGroupId groupId, RaftConfiguration conf) {
 ((RaftServerImpl)getDivision(proxy, groupId)).getState().setRaftConf(conf);
 }

 public static RaftServerRpc getServerRpc(RaftServer.Division server) {
 return ((RaftServerImpl)server).getRaftServer().getServerRpc();
 }
Expand DownExpand Up@@ -196,4 +207,40 @@ public static SegmentedRaftLog newSegmentedRaftLog(RaftGroupMemberId memberId, D
 public static boolean isHighestPriority(RaftConfiguration config, RaftPeerId peerId) {
 return ((RaftConfigurationImpl)config).isHighestPriority(peerId);
 }

 public static void runWithMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf,
 CheckedConsumer<Collection<RaftPeer>, IOException> consumer) throws IOException {
 Collection<RaftPeer> peers = parseMinorityPeers(cluster, peersInNewConf);
 while (peers != null) {
 consumer.accept(peers);
 peers = parseMinorityPeers(cluster, peersInNewConf);
 }
 }

 private static Collection<RaftPeer> parseMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf) {
 RaftConfigurationImpl conf = (RaftConfigurationImpl) cluster.getLeader().getRaftConf();
 Set<RaftPeer> peers = new HashSet<>(conf.getCurrentPeers());

 // Add new peers to construct minority conf peers.
 List<RaftPeer> peersToAdd = peersInNewConf.stream().filter(
 peer -> !conf.containsInConf(peer.getId(), RaftProtos.RaftPeerRole.FOLLOWER)).collect(Collectors.toList());
 if (!peersToAdd.isEmpty()) {
 for (RaftPeer peer : peersToAdd) {
 if (peers.add(peer) && conf.changeMajority(peers)) {
 peers.remove(peer);
 break;
 }
 }
 return peers;
 }

 // All new peers has been added. Handle the removed peers.
 List<RaftPeer> peersToRemove = peers.stream().filter(peer -> !peersInNewConf.contains(peer)).collect(Collectors.toList());
 if (!peersToRemove.isEmpty()) {
 return peersInNewConf;
 }

 // The peers in new conf are the same as current conf, return null.
 return null;
 }
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
RATIS-1912. Fix infinity election when perform membership change.#954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

szetszwo
merged 10 commits into

from

Nov 3, 2023
 Merged


RATIS-1912. Fix infinity election when perform membership change.
#954
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
861cbdf

v0.1

Oct 26, 2023
8c00760

v0.2

Oct 27, 2023
ac6de8f

v0.3

Oct 30, 2023
88aa0af

v0.4

Oct 30, 2023
feeac8d

v0.5 remove unused import

Oct 31, 2023
d3b0134

v0.6

Nov 1, 2023
61744a7

v0.7

Nov 1, 2023
1080e99

v0.8 fix unit tests.

Nov 2, 2023
449d485

v0.9

Nov 3, 2023
f93e9cc

v0.10

Nov 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


11 changes: 9 additions & 2 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -104,7 +104,7 @@ enum Phase {
 ELECTION
 }

 enum Result {PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}
 enum Result {PASSED, SINGLE_MODE_PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}

 private static class ResultAndTerm {
 private final Result result;
Expand DownExpand Up@@ -331,6 +331,7 @@ private boolean askForVotes(Phase phase, int round) throws InterruptedException,

 switch (r.getResult()) {
 case PASSED:
 case SINGLE_MODE_PASSED:
 return true;
 case NOT_IN_CONF:
 case SHUTDOWN:
Expand DownExpand Up@@ -379,6 +380,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 Collection<RaftPeerId> votedPeers = new ArrayList<>();
 Collection<RaftPeerId> rejectedPeers = new ArrayList<>();
 Set<RaftPeerId> higherPriorityPeers = getHigherPriorityPeers(conf);
 final boolean singleMode = conf.isSingleMode(server.getId());

 while (waitForNum > 0 && shouldRun(electionTerm)) {
 final TimeDuration waitTime = timeout.elapsedTime().apply(n -> -n);
Expand All@@ -387,6 +389,9 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 // if some higher priority peer did not response when timeout, but candidate get majority,
 // candidate pass vote
 return logAndReturn(phase, Result.PASSED, responses, exceptions);
 } else if (singleMode) {
 // if candidate is in single mode, candidate pass vote.
 return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
 } else {
 return logAndReturn(phase, Result.TIMEOUT, responses, exceptions);
 }
Expand DownExpand Up@@ -418,7 +423,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 }

 // If any peer with higher priority rejects vote, candidate can not pass vote
 if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId)) {
 if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId) && !singleMode) {
 return logAndReturn(phase, Result.REJECTED, responses, exceptions);
 }

Expand DownExpand Up@@ -447,6 +452,8 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 // received all the responses
 if (conf.hasMajority(votedPeers, server.getId())) {
 return logAndReturn(phase, Result.PASSED, responses, exceptions);
 } else if (singleMode) {
 return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
 } else {
 return logAndReturn(phase, Result.REJECTED, responses, exceptions);
 }
Expand Down

28 changes: 28 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/RaftConfigurationImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -232,6 +232,34 @@ Collection<RaftPeer> getOtherPeers(RaftPeerId selfId) {
 return others;
 }

 /**
 * @return true if the new peers number reaches half of new conf peers number or the group is
 * changing from single mode to HA mode.
 */
 boolean changeMajority(Collection<RaftPeer> newMembers) {
 Preconditions.assertNull(oldConf, "oldConf");
 final long newPeersCount = newMembers.stream().map(RaftPeer::getId).filter(id -> conf.getPeer(id) == null).count();

 if (conf.size() == 1 && newMembers.size() == 2 && newPeersCount == 1) {
 // Change from single peer to HA mode. This is a special case, skip majority verification.
 return false;
 }

 // If newPeersCount reaches majority number of new conf size, the cluster may end with infinity
 // election. See https://issues.apache.org/jira/browse/RATIS-1912 for more details.
 final long oldPeersCount = newMembers.size() - newPeersCount;
 return newPeersCount >= oldPeersCount;
 }

 /** @return True if the selfId is in single mode. */
 boolean isSingleMode(RaftPeerId selfId) {
 if (isStable()) {
 return conf.size() == 1;
 } else {
 return oldConf.size() == 1 && oldConf.contains(selfId) && conf.size() == 2 && conf.contains(selfId);
 }
 }

 /** @return true if the self id together with the others are in the majority. */
 boolean hasMajority(Collection<RaftPeerId> others, RaftPeerId selfId) {
 Preconditions.assertTrue(!others.contains(selfId));
Expand Down

4 changes: 4 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1311,6 +1311,10 @@ public CompletableFuture<RaftClientReply> setConfigurationAsync(SetConfiguration
 pending.setReply(newSuccessReply(request));
 return pending.getFuture();
 }
 if (current.changeMajority(serversInNewConf)) {
 throw new SetConfigurationException("Failed to set configuration: request " + request
 + " changes a majority set of the current configuration " + current);
 }

 getRaftServer().addRaftPeers(serversInNewConf);
 getRaftServer().addRaftPeers(listenersInNewConf);
Expand Down

4 changes: 3 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/InstallSnapshotFromLeaderTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.protocol.RaftClientReply;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.RaftServerConfigKeys;
Expand DownExpand Up@@ -111,7 +112,8 @@ private void testMultiFileInstallSnapshot(CLUSTER cluster) throws Exception {
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand Down

17 changes: 11 additions & 6 deletions
ratis-server/src/test/java/org/apache/ratis/InstallSnapshotNotificationTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,6 +49,7 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
Expand DownExpand Up@@ -242,7 +243,8 @@ private void testAddNewFollowers(CLUSTER cluster, int numRequests) throws Except
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -391,7 +393,8 @@ private void testInstallSnapshotNotificationCount(CLUSTER cluster) throws Except
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));
 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand DownExpand Up@@ -478,7 +481,8 @@ private void testInstallSnapshotInstalledEvent(CLUSTER cluster) throws Exception
 // add one new peer
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(1, true, true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -556,7 +560,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand All@@ -567,8 +572,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
 RaftServerTestUtil.getLatestInstalledSnapshotIndex(follower));
 }

 // Make sure each new peer got one snapshot notification.
 Assert.assertEquals(2, numSnapshotRequests.get());
 // Make sure each new peer got at least one snapshot notification.
 Assert.assertTrue(2 <= numSnapshotRequests.get());
 } finally {
 cluster.shutdown();
 }
Expand Down

4 changes: 3 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.impl.RaftServerTestUtil;
import org.apache.ratis.server.impl.RetryCacheTestUtil;
import org.apache.ratis.server.raftlog.RaftLog;
import org.apache.ratis.server.raftlog.RaftLogIOException;
Expand DownExpand Up@@ -140,7 +141,8 @@ void runTestRetryOnNewLeader(CLUSTER cluster) throws Exception {
 RaftPeer[] allPeers = cluster.removePeers(2, true,
 asList(change.newPeers)).allPeersInNewConf;
 // trigger setConfiguration
 cluster.setConfiguration(allPeers);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(allPeers),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 final RaftPeerId newLeaderId = JavaUtils.attemptRepeatedly(() -> {
 final RaftPeerId id = RaftTestUtil.waitForLeader(cluster).getId();
Expand Down

3 changes: 2 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -329,7 +329,8 @@ public static <T extends Throwable> void runMultiGroupTest(
 }
 LOG.info(chosen + ") setConfiguration: " + cluster.printServers(groups[chosen].getGroupId()));
 try (final RaftClient client = cluster.createClient(groups[chosen])) {
 client.admin().setConfiguration(allPeers.toArray(RaftPeer.emptyArray()));
 RaftServerTestUtil.runWithMinorityPeers(cluster, allPeers,
 peers -> client.admin().setConfiguration(peers.toArray(RaftPeer.emptyArray())));
 }

 Assert.assertNotNull(RaftTestUtil.waitForLeader(cluster));
Expand Down

65 changes: 64 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/impl/RaftReconfigurationBaseTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -61,6 +61,7 @@

import static java.util.Arrays.asList;
import static org.apache.ratis.server.impl.RaftServerTestUtil.waitAndCheckNewConf;
import static org.junit.Assert.assertThrows;

public abstract class RaftReconfigurationBaseTest<CLUSTER extends MiniRaftCluster>
 extends BaseTest
Expand DownExpand Up@@ -145,6 +146,58 @@ public void testAddPeers() throws Exception {
 });
 }

 /**
 * Test leader election when changing cluster from single mode to HA mode.
 */
 @Test
 public void testLeaderElectionWhenChangeFromSingleToHA() throws Exception {
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);

 RaftGroupId groupId = cluster.getGroup().getGroupId();
 RaftPeer curPeer = cluster.getGroup().getPeers().iterator().next();
 RaftPeer newPeer = cluster.addNewPeers(1, true, true).newPeers[0];

 RaftServerProxy leaderServer = cluster.getServer(curPeer.getId());

 // Update leader conf to transitional single mode.
 RaftConfigurationImpl oldNewConf = RaftConfigurationImpl.newBuilder()
 .setOldConf(new PeerConfiguration(Arrays.asList(curPeer)))
 .setConf(new PeerConfiguration(Arrays.asList(curPeer, newPeer)))
 .setLogEntryIndex(Long.MAX_VALUE / 2)
 .build();
 Assert.assertTrue(oldNewConf.isSingleMode(curPeer.getId()));
 RaftServerTestUtil.setRaftConf(leaderServer, groupId, oldNewConf);
 try(RaftClient client = cluster.createClient()) {
 client.admin().transferLeadership(null, leaderServer.getId(), 1000);
 }

 final RaftServer.Division newLeader = RaftTestUtil.waitForLeader(cluster);
 Assert.assertEquals(leaderServer.getId(), newLeader.getId());
 Assert.assertEquals(oldNewConf, newLeader.getRaftConf());
 });
 }

 @Test
 public void testChangeMajority() throws Exception {
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);
 final RaftPeerId leaderId = cluster.getLeader().getId();

 try (final RaftClient client = cluster.createClient(leaderId)) {
 final PeerChanges c1 = cluster.addNewPeers(2, true);

 SetConfigurationRequest.Arguments arguments = SetConfigurationRequest.Arguments.newBuilder()
 .setServersInCurrentConf(cluster.getPeers())
 .setServersInNewConf(c1.allPeersInNewConf)
 .setMode(SetConfigurationRequest.Mode.COMPARE_AND_SET)
 .build();
 assertThrows("Expect change majority error.", SetConfigurationException.class,
 () -> client.admin().setConfiguration(arguments));
 }
 });
 }

 /**
 * remove 2 peers (5 peers -> 3 peers), no leader change
 */
Expand DownExpand Up@@ -380,7 +433,17 @@ public void testBootstrapReconfWithSingleNodeAddOne() throws Exception {
 @Test
 public void testBootstrapReconfWithSingleNodeAddTwo() throws Exception {
 // originally 1 peer, add 2 more
 runWithNewCluster(1, cluster -> runTestBootstrapReconf(2, true, cluster));
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);
 final RaftPeerId leaderId = cluster.getLeader().getId();

 try (final RaftClient client = cluster.createClient(leaderId)) {
 final PeerChanges c1 = cluster.addNewPeers(2, true);

 assertThrows("Expect change majority error.", SetConfigurationException.class,
 () -> client.admin().setConfiguration(c1.allPeersInNewConf));
 }
 });
 }

 @Test
Expand Down

47 changes: 47 additions & 0 deletions
ratis-server/src/test/java/org/apache/ratis/server/impl/RaftServerTestUtil.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,6 +19,7 @@

import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftGroupMemberId;
import org.apache.ratis.protocol.RaftPeer;
Expand All@@ -35,16 +36,22 @@
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.Slf4jUtils;
import org.apache.ratis.util.TimeDuration;
import org.apache.ratis.util.function.CheckedConsumer;
import org.junit.Assert;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.event.Level;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class RaftServerTestUtil {
Expand DownExpand Up@@ -135,6 +142,10 @@ public static RaftConfiguration newRaftConfiguration(Collection<RaftPeer> peers)
 return RaftConfigurationImpl.newBuilder().setConf(peers).build();
 }

 public static void setRaftConf(RaftServer proxy, RaftGroupId groupId, RaftConfiguration conf) {
 ((RaftServerImpl)getDivision(proxy, groupId)).getState().setRaftConf(conf);
 }

 public static RaftServerRpc getServerRpc(RaftServer.Division server) {
 return ((RaftServerImpl)server).getRaftServer().getServerRpc();
 }
Expand DownExpand Up@@ -196,4 +207,40 @@ public static SegmentedRaftLog newSegmentedRaftLog(RaftGroupMemberId memberId, D
 public static boolean isHighestPriority(RaftConfiguration config, RaftPeerId peerId) {
 return ((RaftConfigurationImpl)config).isHighestPriority(peerId);
 }

 public static void runWithMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf,
 CheckedConsumer<Collection<RaftPeer>, IOException> consumer) throws IOException {
 Collection<RaftPeer> peers = parseMinorityPeers(cluster, peersInNewConf);
 while (peers != null) {
 consumer.accept(peers);
 peers = parseMinorityPeers(cluster, peersInNewConf);
 }
 }

 private static Collection<RaftPeer> parseMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf) {
 RaftConfigurationImpl conf = (RaftConfigurationImpl) cluster.getLeader().getRaftConf();
 Set<RaftPeer> peers = new HashSet<>(conf.getCurrentPeers());

 // Add new peers to construct minority conf peers.
 List<RaftPeer> peersToAdd = peersInNewConf.stream().filter(
 peer -> !conf.containsInConf(peer.getId(), RaftProtos.RaftPeerRole.FOLLOWER)).collect(Collectors.toList());
 if (!peersToAdd.isEmpty()) {
 for (RaftPeer peer : peersToAdd) {
 if (peers.add(peer) && conf.changeMajority(peers)) {
 peers.remove(peer);
 break;
 }
 }
 return peers;
 }

 // All new peers has been added. Handle the removed peers.
 List<RaftPeer> peersToRemove = peers.stream().filter(peer -> !peersInNewConf.contains(peer)).collect(Collectors.toList());
 if (!peersToRemove.isEmpty()) {
 return peersInNewConf;
 }

 // The peers in new conf are the same as current conf, return null.
 return null;
 }
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
RATIS-1912. Fix infinity election when perform membership change.#954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

szetszwo
merged 10 commits into

from

Nov 3, 2023
 Merged


RATIS-1912. Fix infinity election when perform membership change.
#954
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
861cbdf

v0.1

Oct 26, 2023
8c00760

v0.2

Oct 27, 2023
ac6de8f

v0.3

Oct 30, 2023
88aa0af

v0.4

Oct 30, 2023
feeac8d

v0.5 remove unused import

Oct 31, 2023
d3b0134

v0.6

Nov 1, 2023
61744a7

v0.7

Nov 1, 2023
1080e99

v0.8 fix unit tests.

Nov 2, 2023
449d485

v0.9

Nov 3, 2023
f93e9cc

v0.10

Nov 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


11 changes: 9 additions & 2 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -104,7 +104,7 @@ enum Phase {
 ELECTION
 }

 enum Result {PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}
 enum Result {PASSED, SINGLE_MODE_PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}

 private static class ResultAndTerm {
 private final Result result;
Expand DownExpand Up@@ -331,6 +331,7 @@ private boolean askForVotes(Phase phase, int round) throws InterruptedException,

 switch (r.getResult()) {
 case PASSED:
 case SINGLE_MODE_PASSED:
 return true;
 case NOT_IN_CONF:
 case SHUTDOWN:
Expand DownExpand Up@@ -379,6 +380,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 Collection<RaftPeerId> votedPeers = new ArrayList<>();
 Collection<RaftPeerId> rejectedPeers = new ArrayList<>();
 Set<RaftPeerId> higherPriorityPeers = getHigherPriorityPeers(conf);
 final boolean singleMode = conf.isSingleMode(server.getId());

 while (waitForNum > 0 && shouldRun(electionTerm)) {
 final TimeDuration waitTime = timeout.elapsedTime().apply(n -> -n);
Expand All@@ -387,6 +389,9 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 // if some higher priority peer did not response when timeout, but candidate get majority,
 // candidate pass vote
 return logAndReturn(phase, Result.PASSED, responses, exceptions);
 } else if (singleMode) {
 // if candidate is in single mode, candidate pass vote.
 return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
 } else {
 return logAndReturn(phase, Result.TIMEOUT, responses, exceptions);
 }
Expand DownExpand Up@@ -418,7 +423,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 }

 // If any peer with higher priority rejects vote, candidate can not pass vote
 if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId)) {
 if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId) && !singleMode) {
 return logAndReturn(phase, Result.REJECTED, responses, exceptions);
 }

Expand DownExpand Up@@ -447,6 +452,8 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 // received all the responses
 if (conf.hasMajority(votedPeers, server.getId())) {
 return logAndReturn(phase, Result.PASSED, responses, exceptions);
 } else if (singleMode) {
 return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
 } else {
 return logAndReturn(phase, Result.REJECTED, responses, exceptions);
 }
Expand Down

28 changes: 28 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/RaftConfigurationImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -232,6 +232,34 @@ Collection<RaftPeer> getOtherPeers(RaftPeerId selfId) {
 return others;
 }

 /**
 * @return true if the new peers number reaches half of new conf peers number or the group is
 * changing from single mode to HA mode.
 */
 boolean changeMajority(Collection<RaftPeer> newMembers) {
 Preconditions.assertNull(oldConf, "oldConf");
 final long newPeersCount = newMembers.stream().map(RaftPeer::getId).filter(id -> conf.getPeer(id) == null).count();

 if (conf.size() == 1 && newMembers.size() == 2 && newPeersCount == 1) {
 // Change from single peer to HA mode. This is a special case, skip majority verification.
 return false;
 }

 // If newPeersCount reaches majority number of new conf size, the cluster may end with infinity
 // election. See https://issues.apache.org/jira/browse/RATIS-1912 for more details.
 final long oldPeersCount = newMembers.size() - newPeersCount;
 return newPeersCount >= oldPeersCount;
 }

 /** @return True if the selfId is in single mode. */
 boolean isSingleMode(RaftPeerId selfId) {
 if (isStable()) {
 return conf.size() == 1;
 } else {
 return oldConf.size() == 1 && oldConf.contains(selfId) && conf.size() == 2 && conf.contains(selfId);
 }
 }

 /** @return true if the self id together with the others are in the majority. */
 boolean hasMajority(Collection<RaftPeerId> others, RaftPeerId selfId) {
 Preconditions.assertTrue(!others.contains(selfId));
Expand Down

4 changes: 4 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1311,6 +1311,10 @@ public CompletableFuture<RaftClientReply> setConfigurationAsync(SetConfiguration
 pending.setReply(newSuccessReply(request));
 return pending.getFuture();
 }
 if (current.changeMajority(serversInNewConf)) {
 throw new SetConfigurationException("Failed to set configuration: request " + request
 + " changes a majority set of the current configuration " + current);
 }

 getRaftServer().addRaftPeers(serversInNewConf);
 getRaftServer().addRaftPeers(listenersInNewConf);
Expand Down

4 changes: 3 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/InstallSnapshotFromLeaderTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.protocol.RaftClientReply;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.RaftServerConfigKeys;
Expand DownExpand Up@@ -111,7 +112,8 @@ private void testMultiFileInstallSnapshot(CLUSTER cluster) throws Exception {
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand Down

17 changes: 11 additions & 6 deletions
ratis-server/src/test/java/org/apache/ratis/InstallSnapshotNotificationTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,6 +49,7 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
Expand DownExpand Up@@ -242,7 +243,8 @@ private void testAddNewFollowers(CLUSTER cluster, int numRequests) throws Except
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -391,7 +393,8 @@ private void testInstallSnapshotNotificationCount(CLUSTER cluster) throws Except
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));
 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand DownExpand Up@@ -478,7 +481,8 @@ private void testInstallSnapshotInstalledEvent(CLUSTER cluster) throws Exception
 // add one new peer
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(1, true, true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -556,7 +560,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand All@@ -567,8 +572,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
 RaftServerTestUtil.getLatestInstalledSnapshotIndex(follower));
 }

 // Make sure each new peer got one snapshot notification.
 Assert.assertEquals(2, numSnapshotRequests.get());
 // Make sure each new peer got at least one snapshot notification.
 Assert.assertTrue(2 <= numSnapshotRequests.get());
 } finally {
 cluster.shutdown();
 }
Expand Down

4 changes: 3 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.impl.RaftServerTestUtil;
import org.apache.ratis.server.impl.RetryCacheTestUtil;
import org.apache.ratis.server.raftlog.RaftLog;
import org.apache.ratis.server.raftlog.RaftLogIOException;
Expand DownExpand Up@@ -140,7 +141,8 @@ void runTestRetryOnNewLeader(CLUSTER cluster) throws Exception {
 RaftPeer[] allPeers = cluster.removePeers(2, true,
 asList(change.newPeers)).allPeersInNewConf;
 // trigger setConfiguration
 cluster.setConfiguration(allPeers);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(allPeers),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 final RaftPeerId newLeaderId = JavaUtils.attemptRepeatedly(() -> {
 final RaftPeerId id = RaftTestUtil.waitForLeader(cluster).getId();
Expand Down

3 changes: 2 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -329,7 +329,8 @@ public static <T extends Throwable> void runMultiGroupTest(
 }
 LOG.info(chosen + ") setConfiguration: " + cluster.printServers(groups[chosen].getGroupId()));
 try (final RaftClient client = cluster.createClient(groups[chosen])) {
 client.admin().setConfiguration(allPeers.toArray(RaftPeer.emptyArray()));
 RaftServerTestUtil.runWithMinorityPeers(cluster, allPeers,
 peers -> client.admin().setConfiguration(peers.toArray(RaftPeer.emptyArray())));
 }

 Assert.assertNotNull(RaftTestUtil.waitForLeader(cluster));
Expand Down

65 changes: 64 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/impl/RaftReconfigurationBaseTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -61,6 +61,7 @@

import static java.util.Arrays.asList;
import static org.apache.ratis.server.impl.RaftServerTestUtil.waitAndCheckNewConf;
import static org.junit.Assert.assertThrows;

public abstract class RaftReconfigurationBaseTest<CLUSTER extends MiniRaftCluster>
 extends BaseTest
Expand DownExpand Up@@ -145,6 +146,58 @@ public void testAddPeers() throws Exception {
 });
 }

 /**
 * Test leader election when changing cluster from single mode to HA mode.
 */
 @Test
 public void testLeaderElectionWhenChangeFromSingleToHA() throws Exception {
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);

 RaftGroupId groupId = cluster.getGroup().getGroupId();
 RaftPeer curPeer = cluster.getGroup().getPeers().iterator().next();
 RaftPeer newPeer = cluster.addNewPeers(1, true, true).newPeers[0];

 RaftServerProxy leaderServer = cluster.getServer(curPeer.getId());

 // Update leader conf to transitional single mode.
 RaftConfigurationImpl oldNewConf = RaftConfigurationImpl.newBuilder()
 .setOldConf(new PeerConfiguration(Arrays.asList(curPeer)))
 .setConf(new PeerConfiguration(Arrays.asList(curPeer, newPeer)))
 .setLogEntryIndex(Long.MAX_VALUE / 2)
 .build();
 Assert.assertTrue(oldNewConf.isSingleMode(curPeer.getId()));
 RaftServerTestUtil.setRaftConf(leaderServer, groupId, oldNewConf);
 try(RaftClient client = cluster.createClient()) {
 client.admin().transferLeadership(null, leaderServer.getId(), 1000);
 }

 final RaftServer.Division newLeader = RaftTestUtil.waitForLeader(cluster);
 Assert.assertEquals(leaderServer.getId(), newLeader.getId());
 Assert.assertEquals(oldNewConf, newLeader.getRaftConf());
 });
 }

 @Test
 public void testChangeMajority() throws Exception {
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);
 final RaftPeerId leaderId = cluster.getLeader().getId();

 try (final RaftClient client = cluster.createClient(leaderId)) {
 final PeerChanges c1 = cluster.addNewPeers(2, true);

 SetConfigurationRequest.Arguments arguments = SetConfigurationRequest.Arguments.newBuilder()
 .setServersInCurrentConf(cluster.getPeers())
 .setServersInNewConf(c1.allPeersInNewConf)
 .setMode(SetConfigurationRequest.Mode.COMPARE_AND_SET)
 .build();
 assertThrows("Expect change majority error.", SetConfigurationException.class,
 () -> client.admin().setConfiguration(arguments));
 }
 });
 }

 /**
 * remove 2 peers (5 peers -> 3 peers), no leader change
 */
Expand DownExpand Up@@ -380,7 +433,17 @@ public void testBootstrapReconfWithSingleNodeAddOne() throws Exception {
 @Test
 public void testBootstrapReconfWithSingleNodeAddTwo() throws Exception {
 // originally 1 peer, add 2 more
 runWithNewCluster(1, cluster -> runTestBootstrapReconf(2, true, cluster));
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);
 final RaftPeerId leaderId = cluster.getLeader().getId();

 try (final RaftClient client = cluster.createClient(leaderId)) {
 final PeerChanges c1 = cluster.addNewPeers(2, true);

 assertThrows("Expect change majority error.", SetConfigurationException.class,
 () -> client.admin().setConfiguration(c1.allPeersInNewConf));
 }
 });
 }

 @Test
Expand Down

47 changes: 47 additions & 0 deletions
ratis-server/src/test/java/org/apache/ratis/server/impl/RaftServerTestUtil.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,6 +19,7 @@

import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftGroupMemberId;
import org.apache.ratis.protocol.RaftPeer;
Expand All@@ -35,16 +36,22 @@
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.Slf4jUtils;
import org.apache.ratis.util.TimeDuration;
import org.apache.ratis.util.function.CheckedConsumer;
import org.junit.Assert;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.event.Level;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class RaftServerTestUtil {
Expand DownExpand Up@@ -135,6 +142,10 @@ public static RaftConfiguration newRaftConfiguration(Collection<RaftPeer> peers)
 return RaftConfigurationImpl.newBuilder().setConf(peers).build();
 }

 public static void setRaftConf(RaftServer proxy, RaftGroupId groupId, RaftConfiguration conf) {
 ((RaftServerImpl)getDivision(proxy, groupId)).getState().setRaftConf(conf);
 }

 public static RaftServerRpc getServerRpc(RaftServer.Division server) {
 return ((RaftServerImpl)server).getRaftServer().getServerRpc();
 }
Expand DownExpand Up@@ -196,4 +207,40 @@ public static SegmentedRaftLog newSegmentedRaftLog(RaftGroupMemberId memberId, D
 public static boolean isHighestPriority(RaftConfiguration config, RaftPeerId peerId) {
 return ((RaftConfigurationImpl)config).isHighestPriority(peerId);
 }

 public static void runWithMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf,
 CheckedConsumer<Collection<RaftPeer>, IOException> consumer) throws IOException {
 Collection<RaftPeer> peers = parseMinorityPeers(cluster, peersInNewConf);
 while (peers != null) {
 consumer.accept(peers);
 peers = parseMinorityPeers(cluster, peersInNewConf);
 }
 }

 private static Collection<RaftPeer> parseMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf) {
 RaftConfigurationImpl conf = (RaftConfigurationImpl) cluster.getLeader().getRaftConf();
 Set<RaftPeer> peers = new HashSet<>(conf.getCurrentPeers());

 // Add new peers to construct minority conf peers.
 List<RaftPeer> peersToAdd = peersInNewConf.stream().filter(
 peer -> !conf.containsInConf(peer.getId(), RaftProtos.RaftPeerRole.FOLLOWER)).collect(Collectors.toList());
 if (!peersToAdd.isEmpty()) {
 for (RaftPeer peer : peersToAdd) {
 if (peers.add(peer) && conf.changeMajority(peers)) {
 peers.remove(peer);
 break;
 }
 }
 return peers;
 }

 // All new peers has been added. Handle the removed peers.
 List<RaftPeer> peersToRemove = peers.stream().filter(peer -> !peersInNewConf.contains(peer)).collect(Collectors.toList());
 if (!peersToRemove.isEmpty()) {
 return peersInNewConf;
 }

 // The peers in new conf are the same as current conf, return null.
 return null;
 }
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
RATIS-1912. Fix infinity election when perform membership change.#954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

szetszwo
merged 10 commits into

from

Nov 3, 2023
 Merged


RATIS-1912. Fix infinity election when perform membership change.
#954
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
861cbdf

v0.1

Oct 26, 2023
8c00760

v0.2

Oct 27, 2023
ac6de8f

v0.3

Oct 30, 2023
88aa0af

v0.4

Oct 30, 2023
feeac8d

v0.5 remove unused import

Oct 31, 2023
d3b0134

v0.6

Nov 1, 2023
61744a7

v0.7

Nov 1, 2023
1080e99

v0.8 fix unit tests.

Nov 2, 2023
449d485

v0.9

Nov 3, 2023
f93e9cc

v0.10

Nov 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


11 changes: 9 additions & 2 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -104,7 +104,7 @@ enum Phase {
 ELECTION
 }

 enum Result {PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}
 enum Result {PASSED, SINGLE_MODE_PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF}

 private static class ResultAndTerm {
 private final Result result;
Expand DownExpand Up@@ -331,6 +331,7 @@ private boolean askForVotes(Phase phase, int round) throws InterruptedException,

 switch (r.getResult()) {
 case PASSED:
 case SINGLE_MODE_PASSED:
 return true;
 case NOT_IN_CONF:
 case SHUTDOWN:
Expand DownExpand Up@@ -379,6 +380,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 Collection<RaftPeerId> votedPeers = new ArrayList<>();
 Collection<RaftPeerId> rejectedPeers = new ArrayList<>();
 Set<RaftPeerId> higherPriorityPeers = getHigherPriorityPeers(conf);
 final boolean singleMode = conf.isSingleMode(server.getId());

 while (waitForNum > 0 && shouldRun(electionTerm)) {
 final TimeDuration waitTime = timeout.elapsedTime().apply(n -> -n);
Expand All@@ -387,6 +389,9 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 // if some higher priority peer did not response when timeout, but candidate get majority,
 // candidate pass vote
 return logAndReturn(phase, Result.PASSED, responses, exceptions);
 } else if (singleMode) {
 // if candidate is in single mode, candidate pass vote.
 return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
 } else {
 return logAndReturn(phase, Result.TIMEOUT, responses, exceptions);
 }
Expand DownExpand Up@@ -418,7 +423,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 }

 // If any peer with higher priority rejects vote, candidate can not pass vote
 if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId)) {
 if (!r.getServerReply().getSuccess() && higherPriorityPeers.contains(replierId) && !singleMode) {
 return logAndReturn(phase, Result.REJECTED, responses, exceptions);
 }

Expand DownExpand Up@@ -447,6 +452,8 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt
 // received all the responses
 if (conf.hasMajority(votedPeers, server.getId())) {
 return logAndReturn(phase, Result.PASSED, responses, exceptions);
 } else if (singleMode) {
 return logAndReturn(phase, Result.SINGLE_MODE_PASSED, responses, exceptions);
 } else {
 return logAndReturn(phase, Result.REJECTED, responses, exceptions);
 }
Expand Down

28 changes: 28 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/RaftConfigurationImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -232,6 +232,34 @@ Collection<RaftPeer> getOtherPeers(RaftPeerId selfId) {
 return others;
 }

 /**
 * @return true if the new peers number reaches half of new conf peers number or the group is
 * changing from single mode to HA mode.
 */
 boolean changeMajority(Collection<RaftPeer> newMembers) {
 Preconditions.assertNull(oldConf, "oldConf");
 final long newPeersCount = newMembers.stream().map(RaftPeer::getId).filter(id -> conf.getPeer(id) == null).count();

 if (conf.size() == 1 && newMembers.size() == 2 && newPeersCount == 1) {
 // Change from single peer to HA mode. This is a special case, skip majority verification.
 return false;
 }

 // If newPeersCount reaches majority number of new conf size, the cluster may end with infinity
 // election. See https://issues.apache.org/jira/browse/RATIS-1912 for more details.
 final long oldPeersCount = newMembers.size() - newPeersCount;
 return newPeersCount >= oldPeersCount;
 }

 /** @return True if the selfId is in single mode. */
 boolean isSingleMode(RaftPeerId selfId) {
 if (isStable()) {
 return conf.size() == 1;
 } else {
 return oldConf.size() == 1 && oldConf.contains(selfId) && conf.size() == 2 && conf.contains(selfId);
 }
 }

 /** @return true if the self id together with the others are in the majority. */
 boolean hasMajority(Collection<RaftPeerId> others, RaftPeerId selfId) {
 Preconditions.assertTrue(!others.contains(selfId));
Expand Down

4 changes: 4 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1311,6 +1311,10 @@ public CompletableFuture<RaftClientReply> setConfigurationAsync(SetConfiguration
 pending.setReply(newSuccessReply(request));
 return pending.getFuture();
 }
 if (current.changeMajority(serversInNewConf)) {
 throw new SetConfigurationException("Failed to set configuration: request " + request
 + " changes a majority set of the current configuration " + current);
 }

 getRaftServer().addRaftPeers(serversInNewConf);
 getRaftServer().addRaftPeers(listenersInNewConf);
Expand Down

4 changes: 3 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/InstallSnapshotFromLeaderTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.protocol.RaftClientReply;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.RaftServerConfigKeys;
Expand DownExpand Up@@ -111,7 +112,8 @@ private void testMultiFileInstallSnapshot(CLUSTER cluster) throws Exception {
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand Down

17 changes: 11 additions & 6 deletions
ratis-server/src/test/java/org/apache/ratis/InstallSnapshotNotificationTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,6 +49,7 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
Expand DownExpand Up@@ -242,7 +243,8 @@ private void testAddNewFollowers(CLUSTER cluster, int numRequests) throws Except
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -391,7 +393,8 @@ private void testInstallSnapshotNotificationCount(CLUSTER cluster) throws Except
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));
 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand DownExpand Up@@ -478,7 +481,8 @@ private void testInstallSnapshotInstalledEvent(CLUSTER cluster) throws Exception
 // add one new peer
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(1, true, true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil
 .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
Expand DownExpand Up@@ -556,7 +560,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
 final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true,
 true);
 // trigger setConfiguration
 cluster.setConfiguration(change.allPeersInNewConf);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(change.allPeersInNewConf),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 RaftServerTestUtil.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);

Expand All@@ -567,8 +572,8 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio
 RaftServerTestUtil.getLatestInstalledSnapshotIndex(follower));
 }

 // Make sure each new peer got one snapshot notification.
 Assert.assertEquals(2, numSnapshotRequests.get());
 // Make sure each new peer got at least one snapshot notification.
 Assert.assertTrue(2 <= numSnapshotRequests.get());
 } finally {
 cluster.shutdown();
 }
Expand Down

4 changes: 3 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.impl.RaftServerTestUtil;
import org.apache.ratis.server.impl.RetryCacheTestUtil;
import org.apache.ratis.server.raftlog.RaftLog;
import org.apache.ratis.server.raftlog.RaftLogIOException;
Expand DownExpand Up@@ -140,7 +141,8 @@ void runTestRetryOnNewLeader(CLUSTER cluster) throws Exception {
 RaftPeer[] allPeers = cluster.removePeers(2, true,
 asList(change.newPeers)).allPeersInNewConf;
 // trigger setConfiguration
 cluster.setConfiguration(allPeers);
 RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(allPeers),
 peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray())));

 final RaftPeerId newLeaderId = JavaUtils.attemptRepeatedly(() -> {
 final RaftPeerId id = RaftTestUtil.waitForLeader(cluster).getId();
Expand Down

3 changes: 2 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -329,7 +329,8 @@ public static <T extends Throwable> void runMultiGroupTest(
 }
 LOG.info(chosen + ") setConfiguration: " + cluster.printServers(groups[chosen].getGroupId()));
 try (final RaftClient client = cluster.createClient(groups[chosen])) {
 client.admin().setConfiguration(allPeers.toArray(RaftPeer.emptyArray()));
 RaftServerTestUtil.runWithMinorityPeers(cluster, allPeers,
 peers -> client.admin().setConfiguration(peers.toArray(RaftPeer.emptyArray())));
 }

 Assert.assertNotNull(RaftTestUtil.waitForLeader(cluster));
Expand Down

65 changes: 64 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/impl/RaftReconfigurationBaseTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -61,6 +61,7 @@

import static java.util.Arrays.asList;
import static org.apache.ratis.server.impl.RaftServerTestUtil.waitAndCheckNewConf;
import static org.junit.Assert.assertThrows;

public abstract class RaftReconfigurationBaseTest<CLUSTER extends MiniRaftCluster>
 extends BaseTest
Expand DownExpand Up@@ -145,6 +146,58 @@ public void testAddPeers() throws Exception {
 });
 }

 /**
 * Test leader election when changing cluster from single mode to HA mode.
 */
 @Test
 public void testLeaderElectionWhenChangeFromSingleToHA() throws Exception {
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);

 RaftGroupId groupId = cluster.getGroup().getGroupId();
 RaftPeer curPeer = cluster.getGroup().getPeers().iterator().next();
 RaftPeer newPeer = cluster.addNewPeers(1, true, true).newPeers[0];

 RaftServerProxy leaderServer = cluster.getServer(curPeer.getId());

 // Update leader conf to transitional single mode.
 RaftConfigurationImpl oldNewConf = RaftConfigurationImpl.newBuilder()
 .setOldConf(new PeerConfiguration(Arrays.asList(curPeer)))
 .setConf(new PeerConfiguration(Arrays.asList(curPeer, newPeer)))
 .setLogEntryIndex(Long.MAX_VALUE / 2)
 .build();
 Assert.assertTrue(oldNewConf.isSingleMode(curPeer.getId()));
 RaftServerTestUtil.setRaftConf(leaderServer, groupId, oldNewConf);
 try(RaftClient client = cluster.createClient()) {
 client.admin().transferLeadership(null, leaderServer.getId(), 1000);
 }

 final RaftServer.Division newLeader = RaftTestUtil.waitForLeader(cluster);
 Assert.assertEquals(leaderServer.getId(), newLeader.getId());
 Assert.assertEquals(oldNewConf, newLeader.getRaftConf());
 });
 }

 @Test
 public void testChangeMajority() throws Exception {
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);
 final RaftPeerId leaderId = cluster.getLeader().getId();

 try (final RaftClient client = cluster.createClient(leaderId)) {
 final PeerChanges c1 = cluster.addNewPeers(2, true);

 SetConfigurationRequest.Arguments arguments = SetConfigurationRequest.Arguments.newBuilder()
 .setServersInCurrentConf(cluster.getPeers())
 .setServersInNewConf(c1.allPeersInNewConf)
 .setMode(SetConfigurationRequest.Mode.COMPARE_AND_SET)
 .build();
 assertThrows("Expect change majority error.", SetConfigurationException.class,
 () -> client.admin().setConfiguration(arguments));
 }
 });
 }

 /**
 * remove 2 peers (5 peers -> 3 peers), no leader change
 */
Expand DownExpand Up@@ -380,7 +433,17 @@ public void testBootstrapReconfWithSingleNodeAddOne() throws Exception {
 @Test
 public void testBootstrapReconfWithSingleNodeAddTwo() throws Exception {
 // originally 1 peer, add 2 more
 runWithNewCluster(1, cluster -> runTestBootstrapReconf(2, true, cluster));
 runWithNewCluster(1, cluster -> {
 RaftTestUtil.waitForLeader(cluster);
 final RaftPeerId leaderId = cluster.getLeader().getId();

 try (final RaftClient client = cluster.createClient(leaderId)) {
 final PeerChanges c1 = cluster.addNewPeers(2, true);

 assertThrows("Expect change majority error.", SetConfigurationException.class,
 () -> client.admin().setConfiguration(c1.allPeersInNewConf));
 }
 });
 }

 @Test
Expand Down

47 changes: 47 additions & 0 deletions
ratis-server/src/test/java/org/apache/ratis/server/impl/RaftServerTestUtil.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,6 +19,7 @@

import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftGroupMemberId;
import org.apache.ratis.protocol.RaftPeer;
Expand All@@ -35,16 +36,22 @@
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.Slf4jUtils;
import org.apache.ratis.util.TimeDuration;
import org.apache.ratis.util.function.CheckedConsumer;
import org.junit.Assert;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.event.Level;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class RaftServerTestUtil {
Expand DownExpand Up@@ -135,6 +142,10 @@ public static RaftConfiguration newRaftConfiguration(Collection<RaftPeer> peers)
 return RaftConfigurationImpl.newBuilder().setConf(peers).build();
 }

 public static void setRaftConf(RaftServer proxy, RaftGroupId groupId, RaftConfiguration conf) {
 ((RaftServerImpl)getDivision(proxy, groupId)).getState().setRaftConf(conf);
 }

 public static RaftServerRpc getServerRpc(RaftServer.Division server) {
 return ((RaftServerImpl)server).getRaftServer().getServerRpc();
 }
Expand DownExpand Up@@ -196,4 +207,40 @@ public static SegmentedRaftLog newSegmentedRaftLog(RaftGroupMemberId memberId, D
 public static boolean isHighestPriority(RaftConfiguration config, RaftPeerId peerId) {
 return ((RaftConfigurationImpl)config).isHighestPriority(peerId);
 }

 public static void runWithMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf,
 CheckedConsumer<Collection<RaftPeer>, IOException> consumer) throws IOException {
 Collection<RaftPeer> peers = parseMinorityPeers(cluster, peersInNewConf);
 while (peers != null) {
 consumer.accept(peers);
 peers = parseMinorityPeers(cluster, peersInNewConf);
 }
 }

 private static Collection<RaftPeer> parseMinorityPeers(MiniRaftCluster cluster, Collection<RaftPeer> peersInNewConf) {
 RaftConfigurationImpl conf = (RaftConfigurationImpl) cluster.getLeader().getRaftConf();
 Set<RaftPeer> peers = new HashSet<>(conf.getCurrentPeers());

 // Add new peers to construct minority conf peers.
 List<RaftPeer> peersToAdd = peersInNewConf.stream().filter(
 peer -> !conf.containsInConf(peer.getId(), RaftProtos.RaftPeerRole.FOLLOWER)).collect(Collectors.toList());
 if (!peersToAdd.isEmpty()) {
 for (RaftPeer peer : peersToAdd) {
 if (peers.add(peer) && conf.changeMajority(peers)) {
 peers.remove(peer);
 break;
 }
 }
 return peers;
 }

 // All new peers has been added. Handle the removed peers.
 List<RaftPeer> peersToRemove = peers.stream().filter(peer -> !peersInNewConf.contains(peer)).collect(Collectors.toList());
 if (!peersToRemove.isEmpty()) {
 return peersInNewConf;
 }

 // The peers in new conf are the same as current conf, return null.
 return null;
 }
}
Loading