Uh oh!
There was an error while loading. Please reload this page.
HBASE-22642 Make move operations of RSGroup idempotent#387
Uh oh!
There was an error while loading. Please reload this page.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -21,6 +21,7 @@ | ||
| import static org.apache.hadoop.hbase.util.Threads.sleep; | ||
| import static org.junit.Assert.assertEquals; | ||
| import static org.junit.Assert.assertFalse; | ||
| import static org.junit.Assert.assertNotEquals; | ||
| import static org.junit.Assert.assertTrue; | ||
| import static org.junit.Assert.fail; | ||
| @@ -30,6 +31,7 @@ | ||
| import java.util.Iterator; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Random; | ||
| import java.util.Set; | ||
| import java.util.concurrent.atomic.AtomicBoolean; | ||
| import java.util.function.Function; | ||
| @@ -57,6 +59,7 @@ | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
| import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; | ||
| import org.apache.hbase.thirdparty.com.google.common.collect.Sets; | ||
| @Category({ LargeTests.class }) | ||
| @@ -341,17 +344,9 @@ public boolean evaluate() throws Exception { | ||
| assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); | ||
| } | ||
| // test move when src = dst | ||
| rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()), | ||
| Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP); | ||
| // verify default group info | ||
| Assert.assertEquals(oldDefaultGroupServerSize, | ||
| @@ -590,39 +585,6 @@ private <T> Thread recoverRegionStateThread(T owner, Function<T, List<RegionInfo | ||
| }); | ||
| } | ||
| private Pair<ServerName, RegionStateNode> createTableWithRegionSplitting(RSGroupInfo rsGroupInfo, | ||
| int tableRegionCount) throws Exception{ | ||
| final byte[] familyNameBytes = Bytes.toBytes("f"); | ||
| @@ -652,24 +614,171 @@ private Pair<ServerName, RegionStateNode> randomlySetOneRegionStateToSplitting( | ||
| RSGroupInfo newGroup) throws IOException{ | ||
| // get target server to move, which should has more than one regions | ||
| // randomly set a region state to SPLITTING to make move fail | ||
| return randomlySetRegionState(newGroup, RegionState.State.SPLITTING,tableName); | ||
| } | ||
| private Pair<ServerName, RegionStateNode> randomlySetRegionState(RSGroupInfo groupInfo, | ||
| RegionState.State state, TableName... tableNames) throws IOException { | ||
| Preconditions.checkArgument(tableNames.length == 1 || tableNames.length == 2, | ||
| "only support one or two tables"); | ||
| Map<ServerName, List<String>> assignMap = getTableServerRegionMap().get(tableNames[0]); | ||
| if(tableNames.length == 2) { | ||
| Map<ServerName, List<String>> assignMap2 = getTableServerRegionMap().get(tableNames[1]); | ||
| assignMap2.forEach((k ,v) -> { | ||
| if(!assignMap.containsKey(k)) { | ||
| assignMap.remove(k); | ||
| } | ||
| }); | ||
| } | ||
| String toCorrectRegionName = null; | ||
| ServerName srcServer = null; | ||
| for (ServerName server : assignMap.keySet()) { | ||
| toCorrectRegionName = assignMap.get(server).size() >= 1 && | ||
| !groupInfo.containsServer(server.getAddress()) ? assignMap.get(server).get(0) : null; | ||
| if (toCorrectRegionName != null) { | ||
| srcServer = server; | ||
| break; | ||
| } | ||
| } | ||
| assert srcServer != null; | ||
| RegionInfo toCorrectRegionInfo = TEST_UTIL.getMiniHBaseCluster().getMaster() | ||
| .getAssignmentManager().getRegionInfo(Bytes.toBytesBinary(toCorrectRegionName)); | ||
| RegionStateNode rsn = | ||
| TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates() | ||
| .getRegionStateNode(toCorrectRegionInfo); | ||
| rsn.setState(state); | ||
| return new Pair<>(srcServer, rsn); | ||
| } | ||
| @Test | ||
| public void testFailedMoveTablesAndRepair() throws Exception{ | ||
sunhelly marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||
| // This UT calls moveTables() twice to test the idempotency of it. | ||
| // The first time, movement fails because a region is made in SPLITTING state | ||
| // which will not be moved. | ||
| // The second time, the region state is OPEN and check if all | ||
| // regions on target group servers after the call. | ||
| final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1); | ||
| Iterator iterator = newGroup.getServers().iterator(); | ||
| Address newGroupServer1 = (Address) iterator.next(); | ||
| // create table | ||
| // randomly set a region state to SPLITTING to make move abort | ||
| Pair<ServerName, RegionStateNode> gotPair = createTableWithRegionSplitting(newGroup, | ||
| new Random().nextInt(8) + 4); | ||
| RegionStateNode rsn = gotPair.getSecond(); | ||
| // move table to newGroup and check regions | ||
| try { | ||
| rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName()); | ||
| fail("should get IOException when retry exhausted but there still exists failed moved " | ||
| + "regions"); | ||
| }catch (Exception e){ | ||
| assertTrue(e.getMessage().contains( | ||
| gotPair.getSecond().getRegionInfo().getRegionNameAsString())); | ||
| } | ||
| for(RegionInfo regionInfo : master.getAssignmentManager().getAssignedRegions()){ | ||
| if (regionInfo.getTable().equals(tableName) && regionInfo.equals(rsn.getRegionInfo())) { | ||
| assertNotEquals(master.getAssignmentManager().getRegionStates() | ||
| .getRegionServerOfRegion(regionInfo).getAddress(), newGroupServer1); | ||
| } | ||
| } | ||
| // retry move table to newGroup and check if all regions are corrected | ||
| rsn.setState(RegionState.State.OPEN); | ||
| rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName()); | ||
| for(RegionInfo regionInfo : master.getAssignmentManager().getAssignedRegions()){ | ||
| if (regionInfo.getTable().equals(tableName)) { | ||
| assertEquals(master.getAssignmentManager().getRegionStates() | ||
| .getRegionServerOfRegion(regionInfo).getAddress(), newGroupServer1); | ||
| } | ||
| } | ||
| } | ||
| @Test | ||
| public void testFailedMoveServersAndRepair() throws Exception{ | ||
sunhelly marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||
| // This UT calls moveServers() twice to test the idempotency of it. | ||
| // The first time, movement fails because a region is made in SPLITTING state | ||
| // which will not be moved. | ||
| // The second time, the region state is OPEN and check if all | ||
| // regions on target group servers after the call. | ||
| final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1); | ||
| // create table | ||
| // randomly set a region state to SPLITTING to make move abort | ||
| Pair<ServerName, RegionStateNode> gotPair = createTableWithRegionSplitting(newGroup, | ||
| new Random().nextInt(8) + 4); | ||
| RegionStateNode rsn = gotPair.getSecond(); | ||
| ServerName srcServer = rsn.getRegionLocation(); | ||
| // move server to newGroup and check regions | ||
| try { | ||
| rsGroupAdmin.moveServers(Sets.newHashSet(srcServer.getAddress()), newGroup.getName()); | ||
| fail("should get IOException when retry exhausted but there still exists failed moved " | ||
| + "regions"); | ||
| }catch (Exception e){ | ||
| assertTrue(e.getMessage().contains( | ||
| gotPair.getSecond().getRegionInfo().getRegionNameAsString())); | ||
| } | ||
| for(RegionInfo regionInfo : master.getAssignmentManager().getAssignedRegions()){ | ||
| if (regionInfo.getTable().equals(tableName) && regionInfo.equals(rsn.getRegionInfo())) { | ||
| assertEquals(master.getAssignmentManager().getRegionStates() | ||
| .getRegionServerOfRegion(regionInfo), srcServer); | ||
| } | ||
| } | ||
| // retry move server to newGroup and check if all regions on srcServer was moved | ||
| rsn.setState(RegionState.State.OPEN); | ||
| rsGroupAdmin.moveServers(Sets.newHashSet(srcServer.getAddress()), newGroup.getName()); | ||
| assertEquals(master.getAssignmentManager().getRegionsOnServer(srcServer).size(), 0); | ||
| } | ||
| @Test | ||
| public void testFailedMoveServersTablesAndRepair() throws Exception{ | ||
| // This UT calls moveTablesAndServers() twice to test the idempotency of it. | ||
| // The first time, movement fails because a region is made in SPLITTING state | ||
| // which will not be moved. | ||
| // The second time, the region state is OPEN and check if all | ||
| // regions on target group servers after the call. | ||
| final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1); | ||
| // create table | ||
| final byte[] familyNameBytes = Bytes.toBytes("f"); | ||
| TableName table1 = TableName.valueOf(tableName.getNameAsString() + "_1"); | ||
| TableName table2 = TableName.valueOf(tableName.getNameAsString() + "_2"); | ||
| TEST_UTIL.createMultiRegionTable(table1, familyNameBytes, | ||
| new Random().nextInt(12) + 4); | ||
| TEST_UTIL.createMultiRegionTable(table2, familyNameBytes, | ||
| new Random().nextInt(12) + 4); | ||
| // randomly set a region state to SPLITTING to make move abort | ||
| Pair<ServerName, RegionStateNode> gotPair = | ||
| randomlySetRegionState(newGroup, RegionState.State.SPLITTING, table1, table2); | ||
| RegionStateNode rsn = gotPair.getSecond(); | ||
| ServerName srcServer = rsn.getRegionLocation(); | ||
| // move server and table to newGroup and check regions | ||
| try { | ||
| rsGroupAdmin.moveServersAndTables(Sets.newHashSet(srcServer.getAddress()), | ||
| Sets.newHashSet(table2), newGroup.getName()); | ||
| fail("should get IOException when retry exhausted but there still exists failed moved " | ||
| + "regions"); | ||
| }catch (Exception e){ | ||
| assertTrue(e.getMessage().contains( | ||
| gotPair.getSecond().getRegionInfo().getRegionNameAsString())); | ||
| } | ||
| for(RegionInfo regionInfo : master.getAssignmentManager().getAssignedRegions()){ | ||
| if (regionInfo.getTable().equals(table1) && regionInfo.equals(rsn.getRegionInfo())) { | ||
| assertEquals(master.getAssignmentManager().getRegionStates() | ||
| .getRegionServerOfRegion(regionInfo), srcServer); | ||
| } | ||
| } | ||
| // retry moveServersAndTables to newGroup and check if all regions on srcServer belongs to | ||
| // table2 | ||
| rsn.setState(RegionState.State.OPEN); | ||
| rsGroupAdmin.moveServersAndTables(Sets.newHashSet(srcServer.getAddress()), | ||
| Sets.newHashSet(table2), newGroup.getName()); | ||
| for(RegionInfo regionsInfo : master.getAssignmentManager().getRegionsOnServer(srcServer)){ | ||
| assertEquals(regionsInfo.getTable(), table2); | ||
| } | ||
| } | ||
| } | ||
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.