RATIS-1588. Add a builder of RaftStorageImpl#652

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
szetszwo merged 7 commits into from
Jun 17, 2022
Merged

RATIS-1588. Add a builder of RaftStorageImpl #652

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e9a51de
Change the constructor of RaftStorageImpl for forward compatibility
codings-dan Jun 1, 2022
f2c2dde
Add a builder to the RaftStorageImpl
codings-dan Jun 7, 2022
6bf7e65
make code better
codings-dan Jun 13, 2022
59c8501
make code better
codings-dan Jun 14, 2022
b84c2e3
fix ut and remove extra code
codings-dan Jun 16, 2022
beb3ac9
fix checkstyle error
codings-dan Jun 16, 2022
046af27
fix ut error
codings-dan Jun 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions ratis-server-api/src/main/java/org/apache/ratis/server/storage/RaftStorage.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,10 +18,17 @@
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys.Log.CorruptionPolicy;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.ReflectionUtils;
import org.apache.ratis.util.SizeInBytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;

/** The storage of a raft server. */
public interface RaftStorage extends Closeable {
Expand All@@ -35,4 +42,77 @@ public interface RaftStorage extends Closeable {

/** @return the corruption policy for raft log. */
CorruptionPolicy getLogCorruptionPolicy();

static Builder newBuilder() {
return new Builder();
}

enum StartupOption {
/** Format the storage. */
FORMAT,
RECOVER
}

class Builder {

private static final Method NEW_RAFT_STORAGE_METHOD = initNewRaftStorageMethod();

private static Method initNewRaftStorageMethod() {
final String className = RaftStorage.class.getPackage().getName() + ".StorageImplUtils";
//final String className = "org.apache.ratis.server.storage.RaftStorageImpl";
final Class<?>[] argClasses = { File.class, CorruptionPolicy.class, StartupOption.class, long.class };
try {
final Class<?> clazz = ReflectionUtils.getClassByName(className);
return clazz.getMethod("newRaftStorage", argClasses);
} catch (Exception e) {
throw new IllegalStateException("Failed to initNewRaftStorageMethod", e);
}
}

private static RaftStorage newRaftStorage(File dir, CorruptionPolicy logCorruptionPolicy,
StartupOption option, SizeInBytes storageFreeSpaceMin) throws IOException {
try {
return (RaftStorage) NEW_RAFT_STORAGE_METHOD.invoke(null,
dir, logCorruptionPolicy, option, storageFreeSpaceMin.getSize());
} catch (IllegalAccessException e) {
throw new IllegalStateException("Failed to build " + dir, e);
} catch (InvocationTargetException e) {
Throwable t = e.getTargetException();
if (t.getCause() instanceof IOException) {
throw IOUtils.asIOException(t.getCause());
}
throw IOUtils.asIOException(e.getCause());
}
}


private File directory;
private CorruptionPolicy logCorruptionPolicy;
private StartupOption option;
private SizeInBytes storageFreeSpaceMin;

public Builder setDirectory(File directory) {
this.directory = directory;
return this;
}

public Builder setLogCorruptionPolicy(CorruptionPolicy logCorruptionPolicy) {
this.logCorruptionPolicy = logCorruptionPolicy;
return this;
}

public Builder setOption(StartupOption option) {
this.option = option;
return this;
}

public Builder setStorageFreeSpaceMin(SizeInBytes storageFreeSpaceMin) {
this.storageFreeSpaceMin = storageFreeSpaceMin;
return this;
}

public RaftStorage build() throws IOException {
return newRaftStorage(directory, logCorruptionPolicy, option, storageFreeSpaceMin);
}
}
}
8 changes: 6 additions & 2 deletions ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -112,8 +112,12 @@ class ServerState implements Closeable {
// use full uuid string to create a subdirectory
File dir = chooseStorageDir(directories, group.getGroupId().getUuid().toString());
try {
storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop),
RaftServerConfigKeys.storageFreeSpaceMin(prop).getSize());
storage = (RaftStorageImpl) RaftStorage.newBuilder()
.setDirectory(dir)
.setOption(RaftStorage.StartupOption.RECOVER)
.setLogCorruptionPolicy(RaftServerConfigKeys.Log.corruptionPolicy(prop))
.setStorageFreeSpaceMin(RaftServerConfigKeys.storageFreeSpaceMin(prop))
.build();
storageFound = true;
break;
} catch (IOException e) {
Expand Down
10 changes: 0 additions & 10 deletions ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -36,22 +36,12 @@
/** The storage of a {@link org.apache.ratis.server.RaftServer}. */
public class RaftStorageImpl implements RaftStorage {

public enum StartupOption {
/** Format the storage. */
FORMAT
}

// TODO support multiple storage directories
private final RaftStorageDirectoryImpl storageDir;
private final StorageState state;
private final CorruptionPolicy logCorruptionPolicy;
private volatile RaftStorageMetadataFileImpl metaFile;

public RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy,
long storageFeeSpaceMin) throws IOException {
this(dir, logCorruptionPolicy, null, storageFeeSpaceMin);
}

RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy, StartupOption option,
long storageFeeSpaceMin) throws IOException {
this.storageDir = new RaftStorageDirectoryImpl(dir, storageFeeSpaceMin);
Expand Down
56 changes: 56 additions & 0 deletions ratis-server/src/main/java/org/apache/ratis/server/storage/StorageImplUtils.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.TimeDuration;

import java.io.File;
import java.io.IOException;
import java.util.concurrent.TimeUnit;

public final class StorageImplUtils {

private StorageImplUtils() {
//Never constructed
}

/** Create a {@link RaftStorageImpl}. */
public static RaftStorageImpl newRaftStorage(File dir, RaftServerConfigKeys.Log.CorruptionPolicy logCorruptionPolicy,
RaftStorage.StartupOption option, long storageFeeSpaceMin) throws IOException {
RaftStorage.LOG.debug("newRaftStorage: {}, {}, {}, {}",dir, logCorruptionPolicy, option, storageFeeSpaceMin);

final TimeDuration sleepTime = TimeDuration.valueOf(500, TimeUnit.MILLISECONDS);
final RaftStorageImpl raftStorage;
try {
// attempt multiple times to avoid temporary bind exception
raftStorage = JavaUtils.attemptRepeatedly(
() -> new RaftStorageImpl(dir, logCorruptionPolicy, option, storageFeeSpaceMin),
5, sleepTime, "new RaftStorageImpl", RaftStorage.LOG);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw IOUtils.toInterruptedIOException(
"Interrupted when creating RaftStorage " + dir, e);
} catch (IOException e) {
throw new RuntimeException(e);
}
return raftStorage;
}
}
7 changes: 6 additions & 1 deletion ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import static org.apache.ratis.server.metrics.SegmentedRaftLogMetrics.RATIS_LOG_WORKER_METRICS;

import org.apache.ratis.metrics.RatisMetrics;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.raftlog.LogProtoUtils;
import org.apache.ratis.server.raftlog.RaftLogBase;
Expand All@@ -33,7 +34,11 @@

public interface RaftStorageTestUtils {
static RaftStorage newRaftStorage(File dir) throws IOException {
return new RaftStorageImpl(dir, null, 0L);
return RaftStorage.newBuilder()
.setDirectory(dir)
.setOption(RaftStorage.StartupOption.RECOVER)
.setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
.build();
}

static String getLogFlushTimeMetric(String memberId) {
Expand Down
10 changes: 7 additions & 3 deletions ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftServerWithGrpc.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -125,9 +125,13 @@ void runTestServerRestartOnException(MiniRaftClusterWithGrpc cluster) throws Exc
// the raft server proxy created earlier. Raft server proxy should close
// the rpc server on failure.
RaftServerConfigKeys.setStorageDir(p, Collections.singletonList(cluster.getStorageDir(leaderId)));
testFailureCase("start a new server with the same address",
() -> newRaftServer(cluster, leaderId, stateMachine, p).start(),
IOException.class, OverlappingFileLockException.class);
try {
LOG.info("start a new server with the same address");
newRaftServer(cluster, leaderId, stateMachine, p).start();
} catch (IOException e) {
Assert.assertTrue(e.getCause() instanceof OverlappingFileLockException);
Assert.assertTrue(e.getMessage().contains("directory is already locked"));
}
// Try to start a raft server rpc at the leader address.
cluster.getServerFactory(leaderId).newRaftServerRpc(cluster.getServer(leaderId));
}
Expand Down
14 changes: 12 additions & 2 deletions ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,11 +22,13 @@
import org.apache.ratis.BaseTest;
import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.storage.RaftStorageDirectoryImpl.StorageState;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
import org.apache.ratis.statemachine.SnapshotRetentionPolicy;
import org.apache.ratis.util.FileUtils;
import org.apache.ratis.util.SizeInBytes;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
Expand All@@ -47,7 +49,11 @@
*/
public class TestRaftStorage extends BaseTest {
static RaftStorageImpl newRaftStorage(File dir) throws IOException {
return new RaftStorageImpl(dir, null, 0);
return (RaftStorageImpl) RaftStorage.newBuilder()
.setDirectory(dir)
.setOption(RaftStorage.StartupOption.RECOVER)
.setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
.build();
}

private File storageDir;
Expand All@@ -65,7 +71,11 @@ public void tearDown() throws Exception {
}

static RaftStorageImpl formatRaftStorage(File dir) throws IOException {
return new RaftStorageImpl(dir, null, RaftStorageImpl.StartupOption.FORMAT, 0);
return (RaftStorageImpl) RaftStorage.newBuilder()
.setDirectory(dir)
.setOption(RaftStorage.StartupOption.FORMAT)
.setStorageFreeSpaceMin(SizeInBytes.valueOf(0))
.build();
}

@Test
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
RATIS-1588. Add a builder of RaftStorageImpl#652

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

szetszwo
merged 7 commits into

from

Jun 17, 2022
 Merged


RATIS-1588. Add a builder of RaftStorageImpl
#652
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
e9a51de

Change the constructor of RaftStorageImpl for forward compatibility


codings-dan Jun 1, 2022
f2c2dde

Add a builder to the RaftStorageImpl


codings-dan Jun 7, 2022
6bf7e65

make code better


codings-dan Jun 13, 2022
59c8501

make code better


codings-dan Jun 14, 2022
b84c2e3

fix ut and remove extra code


codings-dan Jun 16, 2022
beb3ac9

fix checkstyle error


codings-dan Jun 16, 2022
046af27

fix ut error


codings-dan Jun 17, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


80 changes: 80 additions & 0 deletions
ratis-server-api/src/main/java/org/apache/ratis/server/storage/RaftStorage.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,10 +18,17 @@
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys.Log.CorruptionPolicy;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.ReflectionUtils;
import org.apache.ratis.util.SizeInBytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;

/** The storage of a raft server. */
public interface RaftStorage extends Closeable {
Expand All@@ -35,4 +42,77 @@ public interface RaftStorage extends Closeable {

 /** @return the corruption policy for raft log. */
 CorruptionPolicy getLogCorruptionPolicy();

 static Builder newBuilder() {
 return new Builder();
 }

 enum StartupOption {
 /** Format the storage. */
 FORMAT,
 RECOVER
 }

 class Builder {

 private static final Method NEW_RAFT_STORAGE_METHOD = initNewRaftStorageMethod();

 private static Method initNewRaftStorageMethod() {
 final String className = RaftStorage.class.getPackage().getName() + ".StorageImplUtils";
 //final String className = "org.apache.ratis.server.storage.RaftStorageImpl";
 final Class<?>[] argClasses = { File.class, CorruptionPolicy.class, StartupOption.class, long.class };
 try {
 final Class<?> clazz = ReflectionUtils.getClassByName(className);
 return clazz.getMethod("newRaftStorage", argClasses);
 } catch (Exception e) {
 throw new IllegalStateException("Failed to initNewRaftStorageMethod", e);
 }
 }

 private static RaftStorage newRaftStorage(File dir, CorruptionPolicy logCorruptionPolicy,
 StartupOption option, SizeInBytes storageFreeSpaceMin) throws IOException {
 try {
 return (RaftStorage) NEW_RAFT_STORAGE_METHOD.invoke(null,
 dir, logCorruptionPolicy, option, storageFreeSpaceMin.getSize());
 } catch (IllegalAccessException e) {
 throw new IllegalStateException("Failed to build " + dir, e);
 } catch (InvocationTargetException e) {
 Throwable t = e.getTargetException();
 if (t.getCause() instanceof IOException) {
 throw IOUtils.asIOException(t.getCause());
 }
 throw IOUtils.asIOException(e.getCause());
 }
 }


 private File directory;
 private CorruptionPolicy logCorruptionPolicy;
 private StartupOption option;
 private SizeInBytes storageFreeSpaceMin;

 public Builder setDirectory(File directory) {
 this.directory = directory;
 return this;
 }

 public Builder setLogCorruptionPolicy(CorruptionPolicy logCorruptionPolicy) {
 this.logCorruptionPolicy = logCorruptionPolicy;
 return this;
 }

 public Builder setOption(StartupOption option) {
 this.option = option;
 return this;
 }

 public Builder setStorageFreeSpaceMin(SizeInBytes storageFreeSpaceMin) {
 this.storageFreeSpaceMin = storageFreeSpaceMin;
 return this;
 }

 public RaftStorage build() throws IOException {
 return newRaftStorage(directory, logCorruptionPolicy, option, storageFreeSpaceMin);
 }
 }
}

8 changes: 6 additions & 2 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -112,8 +112,12 @@ class ServerState implements Closeable {
 // use full uuid string to create a subdirectory
 File dir = chooseStorageDir(directories, group.getGroupId().getUuid().toString());
 try {
 storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop),
 RaftServerConfigKeys.storageFreeSpaceMin(prop).getSize());
 storage = (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setLogCorruptionPolicy(RaftServerConfigKeys.Log.corruptionPolicy(prop))
 .setStorageFreeSpaceMin(RaftServerConfigKeys.storageFreeSpaceMin(prop))
 .build();
 storageFound = true;
 break;
 } catch (IOException e) {
Expand Down

10 changes: 0 additions & 10 deletions
ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,22 +36,12 @@
/** The storage of a {@link org.apache.ratis.server.RaftServer}. */
public class RaftStorageImpl implements RaftStorage {

 public enum StartupOption {
 /** Format the storage. */
 FORMAT
 }

 // TODO support multiple storage directories
 private final RaftStorageDirectoryImpl storageDir;
 private final StorageState state;
 private final CorruptionPolicy logCorruptionPolicy;
 private volatile RaftStorageMetadataFileImpl metaFile;

 public RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy,
 long storageFeeSpaceMin) throws IOException {
 this(dir, logCorruptionPolicy, null, storageFeeSpaceMin);
 }

 RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy, StartupOption option,
 long storageFeeSpaceMin) throws IOException {
 this.storageDir = new RaftStorageDirectoryImpl(dir, storageFeeSpaceMin);
Expand Down

56 changes: 56 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/storage/StorageImplUtils.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.TimeDuration;

import java.io.File;
import java.io.IOException;
import java.util.concurrent.TimeUnit;

public final class StorageImplUtils {

 private StorageImplUtils() {
 //Never constructed
 }

 /** Create a {@link RaftStorageImpl}. */
 public static RaftStorageImpl newRaftStorage(File dir, RaftServerConfigKeys.Log.CorruptionPolicy logCorruptionPolicy,
 RaftStorage.StartupOption option, long storageFeeSpaceMin) throws IOException {
 RaftStorage.LOG.debug("newRaftStorage: {}, {}, {}, {}",dir, logCorruptionPolicy, option, storageFeeSpaceMin);

 final TimeDuration sleepTime = TimeDuration.valueOf(500, TimeUnit.MILLISECONDS);
 final RaftStorageImpl raftStorage;
 try {
 // attempt multiple times to avoid temporary bind exception
 raftStorage = JavaUtils.attemptRepeatedly(
 () -> new RaftStorageImpl(dir, logCorruptionPolicy, option, storageFeeSpaceMin),
 5, sleepTime, "new RaftStorageImpl", RaftStorage.LOG);
 } catch (InterruptedException e) {
 Thread.currentThread().interrupt();
 throw IOUtils.toInterruptedIOException(
 "Interrupted when creating RaftStorage " + dir, e);
 } catch (IOException e) {
 throw new RuntimeException(e);
 }
 return raftStorage;
 }
}

7 changes: 6 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import static org.apache.ratis.server.metrics.SegmentedRaftLogMetrics.RATIS_LOG_WORKER_METRICS;

import org.apache.ratis.metrics.RatisMetrics;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.raftlog.LogProtoUtils;
import org.apache.ratis.server.raftlog.RaftLogBase;
Expand All@@ -33,7 +34,11 @@

public interface RaftStorageTestUtils {
 static RaftStorage newRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, 0L);
 return RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
 .build();
 }

 static String getLogFlushTimeMetric(String memberId) {
Expand Down

10 changes: 7 additions & 3 deletions
ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftServerWithGrpc.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -125,9 +125,13 @@ void runTestServerRestartOnException(MiniRaftClusterWithGrpc cluster) throws Exc
 // the raft server proxy created earlier. Raft server proxy should close
 // the rpc server on failure.
 RaftServerConfigKeys.setStorageDir(p, Collections.singletonList(cluster.getStorageDir(leaderId)));
 testFailureCase("start a new server with the same address",
 () -> newRaftServer(cluster, leaderId, stateMachine, p).start(),
 IOException.class, OverlappingFileLockException.class);
 try {
 LOG.info("start a new server with the same address");
 newRaftServer(cluster, leaderId, stateMachine, p).start();
 } catch (IOException e) {
 Assert.assertTrue(e.getCause() instanceof OverlappingFileLockException);
 Assert.assertTrue(e.getMessage().contains("directory is already locked"));
 }
 // Try to start a raft server rpc at the leader address.
 cluster.getServerFactory(leaderId).newRaftServerRpc(cluster.getServer(leaderId));
 }
Expand Down

14 changes: 12 additions & 2 deletions
ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,11 +22,13 @@
import org.apache.ratis.BaseTest;
import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.storage.RaftStorageDirectoryImpl.StorageState;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
import org.apache.ratis.statemachine.SnapshotRetentionPolicy;
import org.apache.ratis.util.FileUtils;
import org.apache.ratis.util.SizeInBytes;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
Expand All@@ -47,7 +49,11 @@
 */
public class TestRaftStorage extends BaseTest {
 static RaftStorageImpl newRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, 0);
 return (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
 .build();
 }

 private File storageDir;
Expand All@@ -65,7 +71,11 @@ public void tearDown() throws Exception {
 }

 static RaftStorageImpl formatRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, RaftStorageImpl.StartupOption.FORMAT, 0);
 return (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.FORMAT)
 .setStorageFreeSpaceMin(SizeInBytes.valueOf(0))
 .build();
 }

 @Test
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
RATIS-1588. Add a builder of RaftStorageImpl#652

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

szetszwo
merged 7 commits into

from

Jun 17, 2022
 Merged


RATIS-1588. Add a builder of RaftStorageImpl
#652
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
e9a51de

Change the constructor of RaftStorageImpl for forward compatibility


codings-dan Jun 1, 2022
f2c2dde

Add a builder to the RaftStorageImpl


codings-dan Jun 7, 2022
6bf7e65

make code better


codings-dan Jun 13, 2022
59c8501

make code better


codings-dan Jun 14, 2022
b84c2e3

fix ut and remove extra code


codings-dan Jun 16, 2022
beb3ac9

fix checkstyle error


codings-dan Jun 16, 2022
046af27

fix ut error


codings-dan Jun 17, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


80 changes: 80 additions & 0 deletions
ratis-server-api/src/main/java/org/apache/ratis/server/storage/RaftStorage.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,10 +18,17 @@
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys.Log.CorruptionPolicy;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.ReflectionUtils;
import org.apache.ratis.util.SizeInBytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;

/** The storage of a raft server. */
public interface RaftStorage extends Closeable {
Expand All@@ -35,4 +42,77 @@ public interface RaftStorage extends Closeable {

 /** @return the corruption policy for raft log. */
 CorruptionPolicy getLogCorruptionPolicy();

 static Builder newBuilder() {
 return new Builder();
 }

 enum StartupOption {
 /** Format the storage. */
 FORMAT,
 RECOVER
 }

 class Builder {

 private static final Method NEW_RAFT_STORAGE_METHOD = initNewRaftStorageMethod();

 private static Method initNewRaftStorageMethod() {
 final String className = RaftStorage.class.getPackage().getName() + ".StorageImplUtils";
 //final String className = "org.apache.ratis.server.storage.RaftStorageImpl";
 final Class<?>[] argClasses = { File.class, CorruptionPolicy.class, StartupOption.class, long.class };
 try {
 final Class<?> clazz = ReflectionUtils.getClassByName(className);
 return clazz.getMethod("newRaftStorage", argClasses);
 } catch (Exception e) {
 throw new IllegalStateException("Failed to initNewRaftStorageMethod", e);
 }
 }

 private static RaftStorage newRaftStorage(File dir, CorruptionPolicy logCorruptionPolicy,
 StartupOption option, SizeInBytes storageFreeSpaceMin) throws IOException {
 try {
 return (RaftStorage) NEW_RAFT_STORAGE_METHOD.invoke(null,
 dir, logCorruptionPolicy, option, storageFreeSpaceMin.getSize());
 } catch (IllegalAccessException e) {
 throw new IllegalStateException("Failed to build " + dir, e);
 } catch (InvocationTargetException e) {
 Throwable t = e.getTargetException();
 if (t.getCause() instanceof IOException) {
 throw IOUtils.asIOException(t.getCause());
 }
 throw IOUtils.asIOException(e.getCause());
 }
 }


 private File directory;
 private CorruptionPolicy logCorruptionPolicy;
 private StartupOption option;
 private SizeInBytes storageFreeSpaceMin;

 public Builder setDirectory(File directory) {
 this.directory = directory;
 return this;
 }

 public Builder setLogCorruptionPolicy(CorruptionPolicy logCorruptionPolicy) {
 this.logCorruptionPolicy = logCorruptionPolicy;
 return this;
 }

 public Builder setOption(StartupOption option) {
 this.option = option;
 return this;
 }

 public Builder setStorageFreeSpaceMin(SizeInBytes storageFreeSpaceMin) {
 this.storageFreeSpaceMin = storageFreeSpaceMin;
 return this;
 }

 public RaftStorage build() throws IOException {
 return newRaftStorage(directory, logCorruptionPolicy, option, storageFreeSpaceMin);
 }
 }
}

8 changes: 6 additions & 2 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -112,8 +112,12 @@ class ServerState implements Closeable {
 // use full uuid string to create a subdirectory
 File dir = chooseStorageDir(directories, group.getGroupId().getUuid().toString());
 try {
 storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop),
 RaftServerConfigKeys.storageFreeSpaceMin(prop).getSize());
 storage = (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setLogCorruptionPolicy(RaftServerConfigKeys.Log.corruptionPolicy(prop))
 .setStorageFreeSpaceMin(RaftServerConfigKeys.storageFreeSpaceMin(prop))
 .build();
 storageFound = true;
 break;
 } catch (IOException e) {
Expand Down

10 changes: 0 additions & 10 deletions
ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,22 +36,12 @@
/** The storage of a {@link org.apache.ratis.server.RaftServer}. */
public class RaftStorageImpl implements RaftStorage {

 public enum StartupOption {
 /** Format the storage. */
 FORMAT
 }

 // TODO support multiple storage directories
 private final RaftStorageDirectoryImpl storageDir;
 private final StorageState state;
 private final CorruptionPolicy logCorruptionPolicy;
 private volatile RaftStorageMetadataFileImpl metaFile;

 public RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy,
 long storageFeeSpaceMin) throws IOException {
 this(dir, logCorruptionPolicy, null, storageFeeSpaceMin);
 }

 RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy, StartupOption option,
 long storageFeeSpaceMin) throws IOException {
 this.storageDir = new RaftStorageDirectoryImpl(dir, storageFeeSpaceMin);
Expand Down

56 changes: 56 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/storage/StorageImplUtils.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.TimeDuration;

import java.io.File;
import java.io.IOException;
import java.util.concurrent.TimeUnit;

public final class StorageImplUtils {

 private StorageImplUtils() {
 //Never constructed
 }

 /** Create a {@link RaftStorageImpl}. */
 public static RaftStorageImpl newRaftStorage(File dir, RaftServerConfigKeys.Log.CorruptionPolicy logCorruptionPolicy,
 RaftStorage.StartupOption option, long storageFeeSpaceMin) throws IOException {
 RaftStorage.LOG.debug("newRaftStorage: {}, {}, {}, {}",dir, logCorruptionPolicy, option, storageFeeSpaceMin);

 final TimeDuration sleepTime = TimeDuration.valueOf(500, TimeUnit.MILLISECONDS);
 final RaftStorageImpl raftStorage;
 try {
 // attempt multiple times to avoid temporary bind exception
 raftStorage = JavaUtils.attemptRepeatedly(
 () -> new RaftStorageImpl(dir, logCorruptionPolicy, option, storageFeeSpaceMin),
 5, sleepTime, "new RaftStorageImpl", RaftStorage.LOG);
 } catch (InterruptedException e) {
 Thread.currentThread().interrupt();
 throw IOUtils.toInterruptedIOException(
 "Interrupted when creating RaftStorage " + dir, e);
 } catch (IOException e) {
 throw new RuntimeException(e);
 }
 return raftStorage;
 }
}

7 changes: 6 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import static org.apache.ratis.server.metrics.SegmentedRaftLogMetrics.RATIS_LOG_WORKER_METRICS;

import org.apache.ratis.metrics.RatisMetrics;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.raftlog.LogProtoUtils;
import org.apache.ratis.server.raftlog.RaftLogBase;
Expand All@@ -33,7 +34,11 @@

public interface RaftStorageTestUtils {
 static RaftStorage newRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, 0L);
 return RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
 .build();
 }

 static String getLogFlushTimeMetric(String memberId) {
Expand Down

10 changes: 7 additions & 3 deletions
ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftServerWithGrpc.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -125,9 +125,13 @@ void runTestServerRestartOnException(MiniRaftClusterWithGrpc cluster) throws Exc
 // the raft server proxy created earlier. Raft server proxy should close
 // the rpc server on failure.
 RaftServerConfigKeys.setStorageDir(p, Collections.singletonList(cluster.getStorageDir(leaderId)));
 testFailureCase("start a new server with the same address",
 () -> newRaftServer(cluster, leaderId, stateMachine, p).start(),
 IOException.class, OverlappingFileLockException.class);
 try {
 LOG.info("start a new server with the same address");
 newRaftServer(cluster, leaderId, stateMachine, p).start();
 } catch (IOException e) {
 Assert.assertTrue(e.getCause() instanceof OverlappingFileLockException);
 Assert.assertTrue(e.getMessage().contains("directory is already locked"));
 }
 // Try to start a raft server rpc at the leader address.
 cluster.getServerFactory(leaderId).newRaftServerRpc(cluster.getServer(leaderId));
 }
Expand Down

14 changes: 12 additions & 2 deletions
ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,11 +22,13 @@
import org.apache.ratis.BaseTest;
import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.storage.RaftStorageDirectoryImpl.StorageState;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
import org.apache.ratis.statemachine.SnapshotRetentionPolicy;
import org.apache.ratis.util.FileUtils;
import org.apache.ratis.util.SizeInBytes;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
Expand All@@ -47,7 +49,11 @@
 */
public class TestRaftStorage extends BaseTest {
 static RaftStorageImpl newRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, 0);
 return (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
 .build();
 }

 private File storageDir;
Expand All@@ -65,7 +71,11 @@ public void tearDown() throws Exception {
 }

 static RaftStorageImpl formatRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, RaftStorageImpl.StartupOption.FORMAT, 0);
 return (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.FORMAT)
 .setStorageFreeSpaceMin(SizeInBytes.valueOf(0))
 .build();
 }

 @Test
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
RATIS-1588. Add a builder of RaftStorageImpl#652

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

szetszwo
merged 7 commits into

from

Jun 17, 2022
 Merged


RATIS-1588. Add a builder of RaftStorageImpl
#652
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
e9a51de

Change the constructor of RaftStorageImpl for forward compatibility


codings-dan Jun 1, 2022
f2c2dde

Add a builder to the RaftStorageImpl


codings-dan Jun 7, 2022
6bf7e65

make code better


codings-dan Jun 13, 2022
59c8501

make code better


codings-dan Jun 14, 2022
b84c2e3

fix ut and remove extra code


codings-dan Jun 16, 2022
beb3ac9

fix checkstyle error


codings-dan Jun 16, 2022
046af27

fix ut error


codings-dan Jun 17, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


80 changes: 80 additions & 0 deletions
ratis-server-api/src/main/java/org/apache/ratis/server/storage/RaftStorage.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,10 +18,17 @@
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys.Log.CorruptionPolicy;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.ReflectionUtils;
import org.apache.ratis.util.SizeInBytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;

/** The storage of a raft server. */
public interface RaftStorage extends Closeable {
Expand All@@ -35,4 +42,77 @@ public interface RaftStorage extends Closeable {

 /** @return the corruption policy for raft log. */
 CorruptionPolicy getLogCorruptionPolicy();

 static Builder newBuilder() {
 return new Builder();
 }

 enum StartupOption {
 /** Format the storage. */
 FORMAT,
 RECOVER
 }

 class Builder {

 private static final Method NEW_RAFT_STORAGE_METHOD = initNewRaftStorageMethod();

 private static Method initNewRaftStorageMethod() {
 final String className = RaftStorage.class.getPackage().getName() + ".StorageImplUtils";
 //final String className = "org.apache.ratis.server.storage.RaftStorageImpl";
 final Class<?>[] argClasses = { File.class, CorruptionPolicy.class, StartupOption.class, long.class };
 try {
 final Class<?> clazz = ReflectionUtils.getClassByName(className);
 return clazz.getMethod("newRaftStorage", argClasses);
 } catch (Exception e) {
 throw new IllegalStateException("Failed to initNewRaftStorageMethod", e);
 }
 }

 private static RaftStorage newRaftStorage(File dir, CorruptionPolicy logCorruptionPolicy,
 StartupOption option, SizeInBytes storageFreeSpaceMin) throws IOException {
 try {
 return (RaftStorage) NEW_RAFT_STORAGE_METHOD.invoke(null,
 dir, logCorruptionPolicy, option, storageFreeSpaceMin.getSize());
 } catch (IllegalAccessException e) {
 throw new IllegalStateException("Failed to build " + dir, e);
 } catch (InvocationTargetException e) {
 Throwable t = e.getTargetException();
 if (t.getCause() instanceof IOException) {
 throw IOUtils.asIOException(t.getCause());
 }
 throw IOUtils.asIOException(e.getCause());
 }
 }


 private File directory;
 private CorruptionPolicy logCorruptionPolicy;
 private StartupOption option;
 private SizeInBytes storageFreeSpaceMin;

 public Builder setDirectory(File directory) {
 this.directory = directory;
 return this;
 }

 public Builder setLogCorruptionPolicy(CorruptionPolicy logCorruptionPolicy) {
 this.logCorruptionPolicy = logCorruptionPolicy;
 return this;
 }

 public Builder setOption(StartupOption option) {
 this.option = option;
 return this;
 }

 public Builder setStorageFreeSpaceMin(SizeInBytes storageFreeSpaceMin) {
 this.storageFreeSpaceMin = storageFreeSpaceMin;
 return this;
 }

 public RaftStorage build() throws IOException {
 return newRaftStorage(directory, logCorruptionPolicy, option, storageFreeSpaceMin);
 }
 }
}

8 changes: 6 additions & 2 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -112,8 +112,12 @@ class ServerState implements Closeable {
 // use full uuid string to create a subdirectory
 File dir = chooseStorageDir(directories, group.getGroupId().getUuid().toString());
 try {
 storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop),
 RaftServerConfigKeys.storageFreeSpaceMin(prop).getSize());
 storage = (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setLogCorruptionPolicy(RaftServerConfigKeys.Log.corruptionPolicy(prop))
 .setStorageFreeSpaceMin(RaftServerConfigKeys.storageFreeSpaceMin(prop))
 .build();
 storageFound = true;
 break;
 } catch (IOException e) {
Expand Down

10 changes: 0 additions & 10 deletions
ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,22 +36,12 @@
/** The storage of a {@link org.apache.ratis.server.RaftServer}. */
public class RaftStorageImpl implements RaftStorage {

 public enum StartupOption {
 /** Format the storage. */
 FORMAT
 }

 // TODO support multiple storage directories
 private final RaftStorageDirectoryImpl storageDir;
 private final StorageState state;
 private final CorruptionPolicy logCorruptionPolicy;
 private volatile RaftStorageMetadataFileImpl metaFile;

 public RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy,
 long storageFeeSpaceMin) throws IOException {
 this(dir, logCorruptionPolicy, null, storageFeeSpaceMin);
 }

 RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy, StartupOption option,
 long storageFeeSpaceMin) throws IOException {
 this.storageDir = new RaftStorageDirectoryImpl(dir, storageFeeSpaceMin);
Expand Down

56 changes: 56 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/storage/StorageImplUtils.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.TimeDuration;

import java.io.File;
import java.io.IOException;
import java.util.concurrent.TimeUnit;

public final class StorageImplUtils {

 private StorageImplUtils() {
 //Never constructed
 }

 /** Create a {@link RaftStorageImpl}. */
 public static RaftStorageImpl newRaftStorage(File dir, RaftServerConfigKeys.Log.CorruptionPolicy logCorruptionPolicy,
 RaftStorage.StartupOption option, long storageFeeSpaceMin) throws IOException {
 RaftStorage.LOG.debug("newRaftStorage: {}, {}, {}, {}",dir, logCorruptionPolicy, option, storageFeeSpaceMin);

 final TimeDuration sleepTime = TimeDuration.valueOf(500, TimeUnit.MILLISECONDS);
 final RaftStorageImpl raftStorage;
 try {
 // attempt multiple times to avoid temporary bind exception
 raftStorage = JavaUtils.attemptRepeatedly(
 () -> new RaftStorageImpl(dir, logCorruptionPolicy, option, storageFeeSpaceMin),
 5, sleepTime, "new RaftStorageImpl", RaftStorage.LOG);
 } catch (InterruptedException e) {
 Thread.currentThread().interrupt();
 throw IOUtils.toInterruptedIOException(
 "Interrupted when creating RaftStorage " + dir, e);
 } catch (IOException e) {
 throw new RuntimeException(e);
 }
 return raftStorage;
 }
}

7 changes: 6 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import static org.apache.ratis.server.metrics.SegmentedRaftLogMetrics.RATIS_LOG_WORKER_METRICS;

import org.apache.ratis.metrics.RatisMetrics;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.raftlog.LogProtoUtils;
import org.apache.ratis.server.raftlog.RaftLogBase;
Expand All@@ -33,7 +34,11 @@

public interface RaftStorageTestUtils {
 static RaftStorage newRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, 0L);
 return RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
 .build();
 }

 static String getLogFlushTimeMetric(String memberId) {
Expand Down

10 changes: 7 additions & 3 deletions
ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftServerWithGrpc.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -125,9 +125,13 @@ void runTestServerRestartOnException(MiniRaftClusterWithGrpc cluster) throws Exc
 // the raft server proxy created earlier. Raft server proxy should close
 // the rpc server on failure.
 RaftServerConfigKeys.setStorageDir(p, Collections.singletonList(cluster.getStorageDir(leaderId)));
 testFailureCase("start a new server with the same address",
 () -> newRaftServer(cluster, leaderId, stateMachine, p).start(),
 IOException.class, OverlappingFileLockException.class);
 try {
 LOG.info("start a new server with the same address");
 newRaftServer(cluster, leaderId, stateMachine, p).start();
 } catch (IOException e) {
 Assert.assertTrue(e.getCause() instanceof OverlappingFileLockException);
 Assert.assertTrue(e.getMessage().contains("directory is already locked"));
 }
 // Try to start a raft server rpc at the leader address.
 cluster.getServerFactory(leaderId).newRaftServerRpc(cluster.getServer(leaderId));
 }
Expand Down

14 changes: 12 additions & 2 deletions
ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,11 +22,13 @@
import org.apache.ratis.BaseTest;
import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.storage.RaftStorageDirectoryImpl.StorageState;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
import org.apache.ratis.statemachine.SnapshotRetentionPolicy;
import org.apache.ratis.util.FileUtils;
import org.apache.ratis.util.SizeInBytes;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
Expand All@@ -47,7 +49,11 @@
 */
public class TestRaftStorage extends BaseTest {
 static RaftStorageImpl newRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, 0);
 return (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
 .build();
 }

 private File storageDir;
Expand All@@ -65,7 +71,11 @@ public void tearDown() throws Exception {
 }

 static RaftStorageImpl formatRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, RaftStorageImpl.StartupOption.FORMAT, 0);
 return (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.FORMAT)
 .setStorageFreeSpaceMin(SizeInBytes.valueOf(0))
 .build();
 }

 @Test
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
RATIS-1588. Add a builder of RaftStorageImpl#652

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

szetszwo
merged 7 commits into

from

Jun 17, 2022
 Merged


RATIS-1588. Add a builder of RaftStorageImpl
#652
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
e9a51de

Change the constructor of RaftStorageImpl for forward compatibility


codings-dan Jun 1, 2022
f2c2dde

Add a builder to the RaftStorageImpl


codings-dan Jun 7, 2022
6bf7e65

make code better


codings-dan Jun 13, 2022
59c8501

make code better


codings-dan Jun 14, 2022
b84c2e3

fix ut and remove extra code


codings-dan Jun 16, 2022
beb3ac9

fix checkstyle error


codings-dan Jun 16, 2022
046af27

fix ut error


codings-dan Jun 17, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


80 changes: 80 additions & 0 deletions
ratis-server-api/src/main/java/org/apache/ratis/server/storage/RaftStorage.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,10 +18,17 @@
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys.Log.CorruptionPolicy;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.ReflectionUtils;
import org.apache.ratis.util.SizeInBytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;

/** The storage of a raft server. */
public interface RaftStorage extends Closeable {
Expand All@@ -35,4 +42,77 @@ public interface RaftStorage extends Closeable {

 /** @return the corruption policy for raft log. */
 CorruptionPolicy getLogCorruptionPolicy();

 static Builder newBuilder() {
 return new Builder();
 }

 enum StartupOption {
 /** Format the storage. */
 FORMAT,
 RECOVER
 }

 class Builder {

 private static final Method NEW_RAFT_STORAGE_METHOD = initNewRaftStorageMethod();

 private static Method initNewRaftStorageMethod() {
 final String className = RaftStorage.class.getPackage().getName() + ".StorageImplUtils";
 //final String className = "org.apache.ratis.server.storage.RaftStorageImpl";
 final Class<?>[] argClasses = { File.class, CorruptionPolicy.class, StartupOption.class, long.class };
 try {
 final Class<?> clazz = ReflectionUtils.getClassByName(className);
 return clazz.getMethod("newRaftStorage", argClasses);
 } catch (Exception e) {
 throw new IllegalStateException("Failed to initNewRaftStorageMethod", e);
 }
 }

 private static RaftStorage newRaftStorage(File dir, CorruptionPolicy logCorruptionPolicy,
 StartupOption option, SizeInBytes storageFreeSpaceMin) throws IOException {
 try {
 return (RaftStorage) NEW_RAFT_STORAGE_METHOD.invoke(null,
 dir, logCorruptionPolicy, option, storageFreeSpaceMin.getSize());
 } catch (IllegalAccessException e) {
 throw new IllegalStateException("Failed to build " + dir, e);
 } catch (InvocationTargetException e) {
 Throwable t = e.getTargetException();
 if (t.getCause() instanceof IOException) {
 throw IOUtils.asIOException(t.getCause());
 }
 throw IOUtils.asIOException(e.getCause());
 }
 }


 private File directory;
 private CorruptionPolicy logCorruptionPolicy;
 private StartupOption option;
 private SizeInBytes storageFreeSpaceMin;

 public Builder setDirectory(File directory) {
 this.directory = directory;
 return this;
 }

 public Builder setLogCorruptionPolicy(CorruptionPolicy logCorruptionPolicy) {
 this.logCorruptionPolicy = logCorruptionPolicy;
 return this;
 }

 public Builder setOption(StartupOption option) {
 this.option = option;
 return this;
 }

 public Builder setStorageFreeSpaceMin(SizeInBytes storageFreeSpaceMin) {
 this.storageFreeSpaceMin = storageFreeSpaceMin;
 return this;
 }

 public RaftStorage build() throws IOException {
 return newRaftStorage(directory, logCorruptionPolicy, option, storageFreeSpaceMin);
 }
 }
}

8 changes: 6 additions & 2 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -112,8 +112,12 @@ class ServerState implements Closeable {
 // use full uuid string to create a subdirectory
 File dir = chooseStorageDir(directories, group.getGroupId().getUuid().toString());
 try {
 storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop),
 RaftServerConfigKeys.storageFreeSpaceMin(prop).getSize());
 storage = (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setLogCorruptionPolicy(RaftServerConfigKeys.Log.corruptionPolicy(prop))
 .setStorageFreeSpaceMin(RaftServerConfigKeys.storageFreeSpaceMin(prop))
 .build();
 storageFound = true;
 break;
 } catch (IOException e) {
Expand Down

10 changes: 0 additions & 10 deletions
ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,22 +36,12 @@
/** The storage of a {@link org.apache.ratis.server.RaftServer}. */
public class RaftStorageImpl implements RaftStorage {

 public enum StartupOption {
 /** Format the storage. */
 FORMAT
 }

 // TODO support multiple storage directories
 private final RaftStorageDirectoryImpl storageDir;
 private final StorageState state;
 private final CorruptionPolicy logCorruptionPolicy;
 private volatile RaftStorageMetadataFileImpl metaFile;

 public RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy,
 long storageFeeSpaceMin) throws IOException {
 this(dir, logCorruptionPolicy, null, storageFeeSpaceMin);
 }

 RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy, StartupOption option,
 long storageFeeSpaceMin) throws IOException {
 this.storageDir = new RaftStorageDirectoryImpl(dir, storageFeeSpaceMin);
Expand Down

56 changes: 56 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/storage/StorageImplUtils.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.TimeDuration;

import java.io.File;
import java.io.IOException;
import java.util.concurrent.TimeUnit;

public final class StorageImplUtils {

 private StorageImplUtils() {
 //Never constructed
 }

 /** Create a {@link RaftStorageImpl}. */
 public static RaftStorageImpl newRaftStorage(File dir, RaftServerConfigKeys.Log.CorruptionPolicy logCorruptionPolicy,
 RaftStorage.StartupOption option, long storageFeeSpaceMin) throws IOException {
 RaftStorage.LOG.debug("newRaftStorage: {}, {}, {}, {}",dir, logCorruptionPolicy, option, storageFeeSpaceMin);

 final TimeDuration sleepTime = TimeDuration.valueOf(500, TimeUnit.MILLISECONDS);
 final RaftStorageImpl raftStorage;
 try {
 // attempt multiple times to avoid temporary bind exception
 raftStorage = JavaUtils.attemptRepeatedly(
 () -> new RaftStorageImpl(dir, logCorruptionPolicy, option, storageFeeSpaceMin),
 5, sleepTime, "new RaftStorageImpl", RaftStorage.LOG);
 } catch (InterruptedException e) {
 Thread.currentThread().interrupt();
 throw IOUtils.toInterruptedIOException(
 "Interrupted when creating RaftStorage " + dir, e);
 } catch (IOException e) {
 throw new RuntimeException(e);
 }
 return raftStorage;
 }
}

7 changes: 6 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import static org.apache.ratis.server.metrics.SegmentedRaftLogMetrics.RATIS_LOG_WORKER_METRICS;

import org.apache.ratis.metrics.RatisMetrics;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.raftlog.LogProtoUtils;
import org.apache.ratis.server.raftlog.RaftLogBase;
Expand All@@ -33,7 +34,11 @@

public interface RaftStorageTestUtils {
 static RaftStorage newRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, 0L);
 return RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
 .build();
 }

 static String getLogFlushTimeMetric(String memberId) {
Expand Down

10 changes: 7 additions & 3 deletions
ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftServerWithGrpc.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -125,9 +125,13 @@ void runTestServerRestartOnException(MiniRaftClusterWithGrpc cluster) throws Exc
 // the raft server proxy created earlier. Raft server proxy should close
 // the rpc server on failure.
 RaftServerConfigKeys.setStorageDir(p, Collections.singletonList(cluster.getStorageDir(leaderId)));
 testFailureCase("start a new server with the same address",
 () -> newRaftServer(cluster, leaderId, stateMachine, p).start(),
 IOException.class, OverlappingFileLockException.class);
 try {
 LOG.info("start a new server with the same address");
 newRaftServer(cluster, leaderId, stateMachine, p).start();
 } catch (IOException e) {
 Assert.assertTrue(e.getCause() instanceof OverlappingFileLockException);
 Assert.assertTrue(e.getMessage().contains("directory is already locked"));
 }
 // Try to start a raft server rpc at the leader address.
 cluster.getServerFactory(leaderId).newRaftServerRpc(cluster.getServer(leaderId));
 }
Expand Down

14 changes: 12 additions & 2 deletions
ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,11 +22,13 @@
import org.apache.ratis.BaseTest;
import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.storage.RaftStorageDirectoryImpl.StorageState;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
import org.apache.ratis.statemachine.SnapshotRetentionPolicy;
import org.apache.ratis.util.FileUtils;
import org.apache.ratis.util.SizeInBytes;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
Expand All@@ -47,7 +49,11 @@
 */
public class TestRaftStorage extends BaseTest {
 static RaftStorageImpl newRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, 0);
 return (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
 .build();
 }

 private File storageDir;
Expand All@@ -65,7 +71,11 @@ public void tearDown() throws Exception {
 }

 static RaftStorageImpl formatRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, RaftStorageImpl.StartupOption.FORMAT, 0);
 return (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.FORMAT)
 .setStorageFreeSpaceMin(SizeInBytes.valueOf(0))
 .build();
 }

 @Test
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
RATIS-1588. Add a builder of RaftStorageImpl#652

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

szetszwo
merged 7 commits into

from

Jun 17, 2022
 Merged


RATIS-1588. Add a builder of RaftStorageImpl
#652
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
e9a51de

Change the constructor of RaftStorageImpl for forward compatibility


codings-dan Jun 1, 2022
f2c2dde

Add a builder to the RaftStorageImpl


codings-dan Jun 7, 2022
6bf7e65

make code better


codings-dan Jun 13, 2022
59c8501

make code better


codings-dan Jun 14, 2022
b84c2e3

fix ut and remove extra code


codings-dan Jun 16, 2022
beb3ac9

fix checkstyle error


codings-dan Jun 16, 2022
046af27

fix ut error


codings-dan Jun 17, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


80 changes: 80 additions & 0 deletions
ratis-server-api/src/main/java/org/apache/ratis/server/storage/RaftStorage.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,10 +18,17 @@
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys.Log.CorruptionPolicy;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.ReflectionUtils;
import org.apache.ratis.util.SizeInBytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;

/** The storage of a raft server. */
public interface RaftStorage extends Closeable {
Expand All@@ -35,4 +42,77 @@ public interface RaftStorage extends Closeable {

 /** @return the corruption policy for raft log. */
 CorruptionPolicy getLogCorruptionPolicy();

 static Builder newBuilder() {
 return new Builder();
 }

 enum StartupOption {
 /** Format the storage. */
 FORMAT,
 RECOVER
 }

 class Builder {

 private static final Method NEW_RAFT_STORAGE_METHOD = initNewRaftStorageMethod();

 private static Method initNewRaftStorageMethod() {
 final String className = RaftStorage.class.getPackage().getName() + ".StorageImplUtils";
 //final String className = "org.apache.ratis.server.storage.RaftStorageImpl";
 final Class<?>[] argClasses = { File.class, CorruptionPolicy.class, StartupOption.class, long.class };
 try {
 final Class<?> clazz = ReflectionUtils.getClassByName(className);
 return clazz.getMethod("newRaftStorage", argClasses);
 } catch (Exception e) {
 throw new IllegalStateException("Failed to initNewRaftStorageMethod", e);
 }
 }

 private static RaftStorage newRaftStorage(File dir, CorruptionPolicy logCorruptionPolicy,
 StartupOption option, SizeInBytes storageFreeSpaceMin) throws IOException {
 try {
 return (RaftStorage) NEW_RAFT_STORAGE_METHOD.invoke(null,
 dir, logCorruptionPolicy, option, storageFreeSpaceMin.getSize());
 } catch (IllegalAccessException e) {
 throw new IllegalStateException("Failed to build " + dir, e);
 } catch (InvocationTargetException e) {
 Throwable t = e.getTargetException();
 if (t.getCause() instanceof IOException) {
 throw IOUtils.asIOException(t.getCause());
 }
 throw IOUtils.asIOException(e.getCause());
 }
 }


 private File directory;
 private CorruptionPolicy logCorruptionPolicy;
 private StartupOption option;
 private SizeInBytes storageFreeSpaceMin;

 public Builder setDirectory(File directory) {
 this.directory = directory;
 return this;
 }

 public Builder setLogCorruptionPolicy(CorruptionPolicy logCorruptionPolicy) {
 this.logCorruptionPolicy = logCorruptionPolicy;
 return this;
 }

 public Builder setOption(StartupOption option) {
 this.option = option;
 return this;
 }

 public Builder setStorageFreeSpaceMin(SizeInBytes storageFreeSpaceMin) {
 this.storageFreeSpaceMin = storageFreeSpaceMin;
 return this;
 }

 public RaftStorage build() throws IOException {
 return newRaftStorage(directory, logCorruptionPolicy, option, storageFreeSpaceMin);
 }
 }
}

8 changes: 6 additions & 2 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -112,8 +112,12 @@ class ServerState implements Closeable {
 // use full uuid string to create a subdirectory
 File dir = chooseStorageDir(directories, group.getGroupId().getUuid().toString());
 try {
 storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop),
 RaftServerConfigKeys.storageFreeSpaceMin(prop).getSize());
 storage = (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setLogCorruptionPolicy(RaftServerConfigKeys.Log.corruptionPolicy(prop))
 .setStorageFreeSpaceMin(RaftServerConfigKeys.storageFreeSpaceMin(prop))
 .build();
 storageFound = true;
 break;
 } catch (IOException e) {
Expand Down

10 changes: 0 additions & 10 deletions
ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,22 +36,12 @@
/** The storage of a {@link org.apache.ratis.server.RaftServer}. */
public class RaftStorageImpl implements RaftStorage {

 public enum StartupOption {
 /** Format the storage. */
 FORMAT
 }

 // TODO support multiple storage directories
 private final RaftStorageDirectoryImpl storageDir;
 private final StorageState state;
 private final CorruptionPolicy logCorruptionPolicy;
 private volatile RaftStorageMetadataFileImpl metaFile;

 public RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy,
 long storageFeeSpaceMin) throws IOException {
 this(dir, logCorruptionPolicy, null, storageFeeSpaceMin);
 }

 RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy, StartupOption option,
 long storageFeeSpaceMin) throws IOException {
 this.storageDir = new RaftStorageDirectoryImpl(dir, storageFeeSpaceMin);
Expand Down

56 changes: 56 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/storage/StorageImplUtils.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.TimeDuration;

import java.io.File;
import java.io.IOException;
import java.util.concurrent.TimeUnit;

public final class StorageImplUtils {

 private StorageImplUtils() {
 //Never constructed
 }

 /** Create a {@link RaftStorageImpl}. */
 public static RaftStorageImpl newRaftStorage(File dir, RaftServerConfigKeys.Log.CorruptionPolicy logCorruptionPolicy,
 RaftStorage.StartupOption option, long storageFeeSpaceMin) throws IOException {
 RaftStorage.LOG.debug("newRaftStorage: {}, {}, {}, {}",dir, logCorruptionPolicy, option, storageFeeSpaceMin);

 final TimeDuration sleepTime = TimeDuration.valueOf(500, TimeUnit.MILLISECONDS);
 final RaftStorageImpl raftStorage;
 try {
 // attempt multiple times to avoid temporary bind exception
 raftStorage = JavaUtils.attemptRepeatedly(
 () -> new RaftStorageImpl(dir, logCorruptionPolicy, option, storageFeeSpaceMin),
 5, sleepTime, "new RaftStorageImpl", RaftStorage.LOG);
 } catch (InterruptedException e) {
 Thread.currentThread().interrupt();
 throw IOUtils.toInterruptedIOException(
 "Interrupted when creating RaftStorage " + dir, e);
 } catch (IOException e) {
 throw new RuntimeException(e);
 }
 return raftStorage;
 }
}

7 changes: 6 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import static org.apache.ratis.server.metrics.SegmentedRaftLogMetrics.RATIS_LOG_WORKER_METRICS;

import org.apache.ratis.metrics.RatisMetrics;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.raftlog.LogProtoUtils;
import org.apache.ratis.server.raftlog.RaftLogBase;
Expand All@@ -33,7 +34,11 @@

public interface RaftStorageTestUtils {
 static RaftStorage newRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, 0L);
 return RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
 .build();
 }

 static String getLogFlushTimeMetric(String memberId) {
Expand Down

10 changes: 7 additions & 3 deletions
ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftServerWithGrpc.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -125,9 +125,13 @@ void runTestServerRestartOnException(MiniRaftClusterWithGrpc cluster) throws Exc
 // the raft server proxy created earlier. Raft server proxy should close
 // the rpc server on failure.
 RaftServerConfigKeys.setStorageDir(p, Collections.singletonList(cluster.getStorageDir(leaderId)));
 testFailureCase("start a new server with the same address",
 () -> newRaftServer(cluster, leaderId, stateMachine, p).start(),
 IOException.class, OverlappingFileLockException.class);
 try {
 LOG.info("start a new server with the same address");
 newRaftServer(cluster, leaderId, stateMachine, p).start();
 } catch (IOException e) {
 Assert.assertTrue(e.getCause() instanceof OverlappingFileLockException);
 Assert.assertTrue(e.getMessage().contains("directory is already locked"));
 }
 // Try to start a raft server rpc at the leader address.
 cluster.getServerFactory(leaderId).newRaftServerRpc(cluster.getServer(leaderId));
 }
Expand Down

14 changes: 12 additions & 2 deletions
ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,11 +22,13 @@
import org.apache.ratis.BaseTest;
import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.storage.RaftStorageDirectoryImpl.StorageState;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
import org.apache.ratis.statemachine.SnapshotRetentionPolicy;
import org.apache.ratis.util.FileUtils;
import org.apache.ratis.util.SizeInBytes;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
Expand All@@ -47,7 +49,11 @@
 */
public class TestRaftStorage extends BaseTest {
 static RaftStorageImpl newRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, 0);
 return (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
 .build();
 }

 private File storageDir;
Expand All@@ -65,7 +71,11 @@ public void tearDown() throws Exception {
 }

 static RaftStorageImpl formatRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, RaftStorageImpl.StartupOption.FORMAT, 0);
 return (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.FORMAT)
 .setStorageFreeSpaceMin(SizeInBytes.valueOf(0))
 .build();
 }

 @Test
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
RATIS-1588. Add a builder of RaftStorageImpl#652

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

szetszwo
merged 7 commits into

from

Jun 17, 2022
 Merged


RATIS-1588. Add a builder of RaftStorageImpl
#652
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
e9a51de

Change the constructor of RaftStorageImpl for forward compatibility


codings-dan Jun 1, 2022
f2c2dde

Add a builder to the RaftStorageImpl


codings-dan Jun 7, 2022
6bf7e65

make code better


codings-dan Jun 13, 2022
59c8501

make code better


codings-dan Jun 14, 2022
b84c2e3

fix ut and remove extra code


codings-dan Jun 16, 2022
beb3ac9

fix checkstyle error


codings-dan Jun 16, 2022
046af27

fix ut error


codings-dan Jun 17, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


80 changes: 80 additions & 0 deletions
ratis-server-api/src/main/java/org/apache/ratis/server/storage/RaftStorage.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,10 +18,17 @@
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys.Log.CorruptionPolicy;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.ReflectionUtils;
import org.apache.ratis.util.SizeInBytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;

/** The storage of a raft server. */
public interface RaftStorage extends Closeable {
Expand All@@ -35,4 +42,77 @@ public interface RaftStorage extends Closeable {

 /** @return the corruption policy for raft log. */
 CorruptionPolicy getLogCorruptionPolicy();

 static Builder newBuilder() {
 return new Builder();
 }

 enum StartupOption {
 /** Format the storage. */
 FORMAT,
 RECOVER
 }

 class Builder {

 private static final Method NEW_RAFT_STORAGE_METHOD = initNewRaftStorageMethod();

 private static Method initNewRaftStorageMethod() {
 final String className = RaftStorage.class.getPackage().getName() + ".StorageImplUtils";
 //final String className = "org.apache.ratis.server.storage.RaftStorageImpl";
 final Class<?>[] argClasses = { File.class, CorruptionPolicy.class, StartupOption.class, long.class };
 try {
 final Class<?> clazz = ReflectionUtils.getClassByName(className);
 return clazz.getMethod("newRaftStorage", argClasses);
 } catch (Exception e) {
 throw new IllegalStateException("Failed to initNewRaftStorageMethod", e);
 }
 }

 private static RaftStorage newRaftStorage(File dir, CorruptionPolicy logCorruptionPolicy,
 StartupOption option, SizeInBytes storageFreeSpaceMin) throws IOException {
 try {
 return (RaftStorage) NEW_RAFT_STORAGE_METHOD.invoke(null,
 dir, logCorruptionPolicy, option, storageFreeSpaceMin.getSize());
 } catch (IllegalAccessException e) {
 throw new IllegalStateException("Failed to build " + dir, e);
 } catch (InvocationTargetException e) {
 Throwable t = e.getTargetException();
 if (t.getCause() instanceof IOException) {
 throw IOUtils.asIOException(t.getCause());
 }
 throw IOUtils.asIOException(e.getCause());
 }
 }


 private File directory;
 private CorruptionPolicy logCorruptionPolicy;
 private StartupOption option;
 private SizeInBytes storageFreeSpaceMin;

 public Builder setDirectory(File directory) {
 this.directory = directory;
 return this;
 }

 public Builder setLogCorruptionPolicy(CorruptionPolicy logCorruptionPolicy) {
 this.logCorruptionPolicy = logCorruptionPolicy;
 return this;
 }

 public Builder setOption(StartupOption option) {
 this.option = option;
 return this;
 }

 public Builder setStorageFreeSpaceMin(SizeInBytes storageFreeSpaceMin) {
 this.storageFreeSpaceMin = storageFreeSpaceMin;
 return this;
 }

 public RaftStorage build() throws IOException {
 return newRaftStorage(directory, logCorruptionPolicy, option, storageFreeSpaceMin);
 }
 }
}

8 changes: 6 additions & 2 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -112,8 +112,12 @@ class ServerState implements Closeable {
 // use full uuid string to create a subdirectory
 File dir = chooseStorageDir(directories, group.getGroupId().getUuid().toString());
 try {
 storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop),
 RaftServerConfigKeys.storageFreeSpaceMin(prop).getSize());
 storage = (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setLogCorruptionPolicy(RaftServerConfigKeys.Log.corruptionPolicy(prop))
 .setStorageFreeSpaceMin(RaftServerConfigKeys.storageFreeSpaceMin(prop))
 .build();
 storageFound = true;
 break;
 } catch (IOException e) {
Expand Down

10 changes: 0 additions & 10 deletions
ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,22 +36,12 @@
/** The storage of a {@link org.apache.ratis.server.RaftServer}. */
public class RaftStorageImpl implements RaftStorage {

 public enum StartupOption {
 /** Format the storage. */
 FORMAT
 }

 // TODO support multiple storage directories
 private final RaftStorageDirectoryImpl storageDir;
 private final StorageState state;
 private final CorruptionPolicy logCorruptionPolicy;
 private volatile RaftStorageMetadataFileImpl metaFile;

 public RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy,
 long storageFeeSpaceMin) throws IOException {
 this(dir, logCorruptionPolicy, null, storageFeeSpaceMin);
 }

 RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy, StartupOption option,
 long storageFeeSpaceMin) throws IOException {
 this.storageDir = new RaftStorageDirectoryImpl(dir, storageFeeSpaceMin);
Expand Down

56 changes: 56 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/storage/StorageImplUtils.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.TimeDuration;

import java.io.File;
import java.io.IOException;
import java.util.concurrent.TimeUnit;

public final class StorageImplUtils {

 private StorageImplUtils() {
 //Never constructed
 }

 /** Create a {@link RaftStorageImpl}. */
 public static RaftStorageImpl newRaftStorage(File dir, RaftServerConfigKeys.Log.CorruptionPolicy logCorruptionPolicy,
 RaftStorage.StartupOption option, long storageFeeSpaceMin) throws IOException {
 RaftStorage.LOG.debug("newRaftStorage: {}, {}, {}, {}",dir, logCorruptionPolicy, option, storageFeeSpaceMin);

 final TimeDuration sleepTime = TimeDuration.valueOf(500, TimeUnit.MILLISECONDS);
 final RaftStorageImpl raftStorage;
 try {
 // attempt multiple times to avoid temporary bind exception
 raftStorage = JavaUtils.attemptRepeatedly(
 () -> new RaftStorageImpl(dir, logCorruptionPolicy, option, storageFeeSpaceMin),
 5, sleepTime, "new RaftStorageImpl", RaftStorage.LOG);
 } catch (InterruptedException e) {
 Thread.currentThread().interrupt();
 throw IOUtils.toInterruptedIOException(
 "Interrupted when creating RaftStorage " + dir, e);
 } catch (IOException e) {
 throw new RuntimeException(e);
 }
 return raftStorage;
 }
}

7 changes: 6 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import static org.apache.ratis.server.metrics.SegmentedRaftLogMetrics.RATIS_LOG_WORKER_METRICS;

import org.apache.ratis.metrics.RatisMetrics;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.raftlog.LogProtoUtils;
import org.apache.ratis.server.raftlog.RaftLogBase;
Expand All@@ -33,7 +34,11 @@

public interface RaftStorageTestUtils {
 static RaftStorage newRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, 0L);
 return RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
 .build();
 }

 static String getLogFlushTimeMetric(String memberId) {
Expand Down

10 changes: 7 additions & 3 deletions
ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftServerWithGrpc.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -125,9 +125,13 @@ void runTestServerRestartOnException(MiniRaftClusterWithGrpc cluster) throws Exc
 // the raft server proxy created earlier. Raft server proxy should close
 // the rpc server on failure.
 RaftServerConfigKeys.setStorageDir(p, Collections.singletonList(cluster.getStorageDir(leaderId)));
 testFailureCase("start a new server with the same address",
 () -> newRaftServer(cluster, leaderId, stateMachine, p).start(),
 IOException.class, OverlappingFileLockException.class);
 try {
 LOG.info("start a new server with the same address");
 newRaftServer(cluster, leaderId, stateMachine, p).start();
 } catch (IOException e) {
 Assert.assertTrue(e.getCause() instanceof OverlappingFileLockException);
 Assert.assertTrue(e.getMessage().contains("directory is already locked"));
 }
 // Try to start a raft server rpc at the leader address.
 cluster.getServerFactory(leaderId).newRaftServerRpc(cluster.getServer(leaderId));
 }
Expand Down

14 changes: 12 additions & 2 deletions
ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,11 +22,13 @@
import org.apache.ratis.BaseTest;
import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.storage.RaftStorageDirectoryImpl.StorageState;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
import org.apache.ratis.statemachine.SnapshotRetentionPolicy;
import org.apache.ratis.util.FileUtils;
import org.apache.ratis.util.SizeInBytes;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
Expand All@@ -47,7 +49,11 @@
 */
public class TestRaftStorage extends BaseTest {
 static RaftStorageImpl newRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, 0);
 return (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
 .build();
 }

 private File storageDir;
Expand All@@ -65,7 +71,11 @@ public void tearDown() throws Exception {
 }

 static RaftStorageImpl formatRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, RaftStorageImpl.StartupOption.FORMAT, 0);
 return (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.FORMAT)
 .setStorageFreeSpaceMin(SizeInBytes.valueOf(0))
 .build();
 }

 @Test
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
RATIS-1588. Add a builder of RaftStorageImpl#652

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

szetszwo
merged 7 commits into

from

Jun 17, 2022
 Merged


RATIS-1588. Add a builder of RaftStorageImpl
#652
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
e9a51de

Change the constructor of RaftStorageImpl for forward compatibility


codings-dan Jun 1, 2022
f2c2dde

Add a builder to the RaftStorageImpl


codings-dan Jun 7, 2022
6bf7e65

make code better


codings-dan Jun 13, 2022
59c8501

make code better


codings-dan Jun 14, 2022
b84c2e3

fix ut and remove extra code


codings-dan Jun 16, 2022
beb3ac9

fix checkstyle error


codings-dan Jun 16, 2022
046af27

fix ut error


codings-dan Jun 17, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


80 changes: 80 additions & 0 deletions
ratis-server-api/src/main/java/org/apache/ratis/server/storage/RaftStorage.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,10 +18,17 @@
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys.Log.CorruptionPolicy;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.ReflectionUtils;
import org.apache.ratis.util.SizeInBytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;

/** The storage of a raft server. */
public interface RaftStorage extends Closeable {
Expand All@@ -35,4 +42,77 @@ public interface RaftStorage extends Closeable {

 /** @return the corruption policy for raft log. */
 CorruptionPolicy getLogCorruptionPolicy();

 static Builder newBuilder() {
 return new Builder();
 }

 enum StartupOption {
 /** Format the storage. */
 FORMAT,
 RECOVER
 }

 class Builder {

 private static final Method NEW_RAFT_STORAGE_METHOD = initNewRaftStorageMethod();

 private static Method initNewRaftStorageMethod() {
 final String className = RaftStorage.class.getPackage().getName() + ".StorageImplUtils";
 //final String className = "org.apache.ratis.server.storage.RaftStorageImpl";
 final Class<?>[] argClasses = { File.class, CorruptionPolicy.class, StartupOption.class, long.class };
 try {
 final Class<?> clazz = ReflectionUtils.getClassByName(className);
 return clazz.getMethod("newRaftStorage", argClasses);
 } catch (Exception e) {
 throw new IllegalStateException("Failed to initNewRaftStorageMethod", e);
 }
 }

 private static RaftStorage newRaftStorage(File dir, CorruptionPolicy logCorruptionPolicy,
 StartupOption option, SizeInBytes storageFreeSpaceMin) throws IOException {
 try {
 return (RaftStorage) NEW_RAFT_STORAGE_METHOD.invoke(null,
 dir, logCorruptionPolicy, option, storageFreeSpaceMin.getSize());
 } catch (IllegalAccessException e) {
 throw new IllegalStateException("Failed to build " + dir, e);
 } catch (InvocationTargetException e) {
 Throwable t = e.getTargetException();
 if (t.getCause() instanceof IOException) {
 throw IOUtils.asIOException(t.getCause());
 }
 throw IOUtils.asIOException(e.getCause());
 }
 }


 private File directory;
 private CorruptionPolicy logCorruptionPolicy;
 private StartupOption option;
 private SizeInBytes storageFreeSpaceMin;

 public Builder setDirectory(File directory) {
 this.directory = directory;
 return this;
 }

 public Builder setLogCorruptionPolicy(CorruptionPolicy logCorruptionPolicy) {
 this.logCorruptionPolicy = logCorruptionPolicy;
 return this;
 }

 public Builder setOption(StartupOption option) {
 this.option = option;
 return this;
 }

 public Builder setStorageFreeSpaceMin(SizeInBytes storageFreeSpaceMin) {
 this.storageFreeSpaceMin = storageFreeSpaceMin;
 return this;
 }

 public RaftStorage build() throws IOException {
 return newRaftStorage(directory, logCorruptionPolicy, option, storageFreeSpaceMin);
 }
 }
}

8 changes: 6 additions & 2 deletions
ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -112,8 +112,12 @@ class ServerState implements Closeable {
 // use full uuid string to create a subdirectory
 File dir = chooseStorageDir(directories, group.getGroupId().getUuid().toString());
 try {
 storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop),
 RaftServerConfigKeys.storageFreeSpaceMin(prop).getSize());
 storage = (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setLogCorruptionPolicy(RaftServerConfigKeys.Log.corruptionPolicy(prop))
 .setStorageFreeSpaceMin(RaftServerConfigKeys.storageFreeSpaceMin(prop))
 .build();
 storageFound = true;
 break;
 } catch (IOException e) {
Expand Down

10 changes: 0 additions & 10 deletions
ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,22 +36,12 @@
/** The storage of a {@link org.apache.ratis.server.RaftServer}. */
public class RaftStorageImpl implements RaftStorage {

 public enum StartupOption {
 /** Format the storage. */
 FORMAT
 }

 // TODO support multiple storage directories
 private final RaftStorageDirectoryImpl storageDir;
 private final StorageState state;
 private final CorruptionPolicy logCorruptionPolicy;
 private volatile RaftStorageMetadataFileImpl metaFile;

 public RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy,
 long storageFeeSpaceMin) throws IOException {
 this(dir, logCorruptionPolicy, null, storageFeeSpaceMin);
 }

 RaftStorageImpl(File dir, CorruptionPolicy logCorruptionPolicy, StartupOption option,
 long storageFeeSpaceMin) throws IOException {
 this.storageDir = new RaftStorageDirectoryImpl(dir, storageFeeSpaceMin);
Expand Down

56 changes: 56 additions & 0 deletions
ratis-server/src/main/java/org/apache/ratis/server/storage/StorageImplUtils.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.ratis.server.storage;

import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.TimeDuration;

import java.io.File;
import java.io.IOException;
import java.util.concurrent.TimeUnit;

public final class StorageImplUtils {

 private StorageImplUtils() {
 //Never constructed
 }

 /** Create a {@link RaftStorageImpl}. */
 public static RaftStorageImpl newRaftStorage(File dir, RaftServerConfigKeys.Log.CorruptionPolicy logCorruptionPolicy,
 RaftStorage.StartupOption option, long storageFeeSpaceMin) throws IOException {
 RaftStorage.LOG.debug("newRaftStorage: {}, {}, {}, {}",dir, logCorruptionPolicy, option, storageFeeSpaceMin);

 final TimeDuration sleepTime = TimeDuration.valueOf(500, TimeUnit.MILLISECONDS);
 final RaftStorageImpl raftStorage;
 try {
 // attempt multiple times to avoid temporary bind exception
 raftStorage = JavaUtils.attemptRepeatedly(
 () -> new RaftStorageImpl(dir, logCorruptionPolicy, option, storageFeeSpaceMin),
 5, sleepTime, "new RaftStorageImpl", RaftStorage.LOG);
 } catch (InterruptedException e) {
 Thread.currentThread().interrupt();
 throw IOUtils.toInterruptedIOException(
 "Interrupted when creating RaftStorage " + dir, e);
 } catch (IOException e) {
 throw new RuntimeException(e);
 }
 return raftStorage;
 }
}

7 changes: 6 additions & 1 deletion
ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
import static org.apache.ratis.server.metrics.SegmentedRaftLogMetrics.RATIS_LOG_WORKER_METRICS;

import org.apache.ratis.metrics.RatisMetrics;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.raftlog.LogProtoUtils;
import org.apache.ratis.server.raftlog.RaftLogBase;
Expand All@@ -33,7 +34,11 @@

public interface RaftStorageTestUtils {
 static RaftStorage newRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, 0L);
 return RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
 .build();
 }

 static String getLogFlushTimeMetric(String memberId) {
Expand Down

10 changes: 7 additions & 3 deletions
ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftServerWithGrpc.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -125,9 +125,13 @@ void runTestServerRestartOnException(MiniRaftClusterWithGrpc cluster) throws Exc
 // the raft server proxy created earlier. Raft server proxy should close
 // the rpc server on failure.
 RaftServerConfigKeys.setStorageDir(p, Collections.singletonList(cluster.getStorageDir(leaderId)));
 testFailureCase("start a new server with the same address",
 () -> newRaftServer(cluster, leaderId, stateMachine, p).start(),
 IOException.class, OverlappingFileLockException.class);
 try {
 LOG.info("start a new server with the same address");
 newRaftServer(cluster, leaderId, stateMachine, p).start();
 } catch (IOException e) {
 Assert.assertTrue(e.getCause() instanceof OverlappingFileLockException);
 Assert.assertTrue(e.getMessage().contains("directory is already locked"));
 }
 // Try to start a raft server rpc at the leader address.
 cluster.getServerFactory(leaderId).newRaftServerRpc(cluster.getServer(leaderId));
 }
Expand Down

14 changes: 12 additions & 2 deletions
ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,11 +22,13 @@
import org.apache.ratis.BaseTest;
import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.storage.RaftStorageDirectoryImpl.StorageState;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
import org.apache.ratis.statemachine.SnapshotRetentionPolicy;
import org.apache.ratis.util.FileUtils;
import org.apache.ratis.util.SizeInBytes;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
Expand All@@ -47,7 +49,11 @@
 */
public class TestRaftStorage extends BaseTest {
 static RaftStorageImpl newRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, 0);
 return (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.RECOVER)
 .setStorageFreeSpaceMin(RaftServerConfigKeys.STORAGE_FREE_SPACE_MIN_DEFAULT)
 .build();
 }

 private File storageDir;
Expand All@@ -65,7 +71,11 @@ public void tearDown() throws Exception {
 }

 static RaftStorageImpl formatRaftStorage(File dir) throws IOException {
 return new RaftStorageImpl(dir, null, RaftStorageImpl.StartupOption.FORMAT, 0);
 return (RaftStorageImpl) RaftStorage.newBuilder()
 .setDirectory(dir)
 .setOption(RaftStorage.StartupOption.FORMAT)
 .setStorageFreeSpaceMin(SizeInBytes.valueOf(0))
 .build();
 }

 @Test
Expand Down