HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.#2866

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bshashikant merged 6 commits into from
Dec 1, 2021
Merged

HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes. #2866

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0bfe67d
HDDS-5851.Define a PutBlock/maxBuffer fixed boundary for streaming wr…
Nov 23, 2021
3b57efb
edit description
Nov 23, 2021
518d1e1
fix test
Nov 23, 2021
b7164cd
address comment
Nov 30, 2021
90cb0ad
address comment
Nov 30, 2021
47e3393
trigger new CI check
Nov 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,21 @@ public class OzoneClientConfig {
tags = ConfigTag.CLIENT)
private int streamBufferSize = 4 * 1024 * 1024;

@Config(key = "datastream.max.buffer.size",
defaultValue = "4MB",
type = ConfigType.SIZE,
description = "The maximum size of the ByteBuffer "
Comment thread
sadanand48 marked this conversation as resolved.
Outdated
+ "(used via ratis streaming)",
tags = ConfigTag.CLIENT)
private int dataStreamMaxBufferSize = 4 * 1024 * 1024;

@Config(key = "datastream.buffer.flush.size",
defaultValue = "16MB",
type = ConfigType.SIZE,
description = "The boundary at which putBlock is executed",
tags = ConfigTag.CLIENT)
private long dataStreamBufferFlushSize = 16 * 1024 * 1024;

@Config(key = "stream.buffer.increment",
Comment thread
bshashikant marked this conversation as resolved.
defaultValue = "0B",
type = ConfigType.SIZE,
Expand DownExpand Up@@ -168,6 +183,14 @@ public void setStreamBufferSize(int streamBufferSize) {
this.streamBufferSize = streamBufferSize;
}

public int getDataStreamMaxBufferSize() {
return dataStreamMaxBufferSize;
}

public void setDataStreamMaxBufferSize(int dataStreamMaxBufferSize) {
this.dataStreamMaxBufferSize = dataStreamMaxBufferSize;
}

public boolean isStreamBufferFlushDelay() {
return streamBufferFlushDelay;
}
Expand DownExpand Up@@ -227,4 +250,12 @@ public void setChecksumVerify(boolean checksumVerify) {
public int getBufferIncrement() {
return bufferIncrement;
}

public long getDataStreamBufferFlushSize() {
return dataStreamBufferFlushSize;
}

public void setDataStreamBufferFlushSize(long dataStreamBufferFlushSize) {
this.dataStreamBufferFlushSize = dataStreamBufferFlushSize;
}
}
29 changes: 23 additions & 6 deletions ...p-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -257,13 +257,30 @@ public void write(ByteBuffer b, int off, int len) throws IOException {
if (len == 0) {
return;
}
int curLen = len;
// set limit on the number of bytes that a ByteBuffer(StreamBuffer) can hold
int maxBufferLen = config.getDataStreamMaxBufferSize();
while (curLen > 0) {
int writeLen = Math.min(curLen, maxBufferLen);
final StreamBuffer buf = new StreamBuffer(b, off, writeLen);
off += writeLen;
bufferList.add(buf);
writeChunkToContainer(buf.duplicate());
curLen -= writeLen;
writtenDataLength += writeLen;
doFlushIfNeeded();
}
}

final StreamBuffer buf = new StreamBuffer(b, off, len);
bufferList.add(buf);

writeChunkToContainer(buf.duplicate());

writtenDataLength += len;
private void doFlushIfNeeded() throws IOException {
Preconditions.checkArgument(config.getDataStreamBufferFlushSize() > config
.getDataStreamMaxBufferSize());
long boundary = config.getDataStreamBufferFlushSize() / config
.getDataStreamMaxBufferSize();
if (bufferList.size() % boundary == 0) {
updateFlushLength();
executePutBlock(false, false);

@captainzmccaptainzmcNov 30, 2021

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did we need add updateFlushLength(), waitOnFlushFutures() and watchForCommit(false) here? Just as what we did in handleFlush. Currently we only releaseBuffers after watchForCommit.

@sadanand48sadanand48Nov 30, 2021

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @captainzmc for the comment. The plan here is to execute a putBlock at 32MB and not block the client right away by waiting for the flush futures and calling watchForCommit. We will introduce another boundary say at 64MB to check whether atleast 32MB is flushed and committed to all datanodes i.e watchForCommit() and waitForFlushFutures() will occur at 64mb. (I will add this is in a different patch after this PR)

@captainzmccaptainzmcNov 30, 2021

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for explaining, make sense.

}
}

private void updateFlushLength() {
Expand Down
12 changes: 12 additions & 0 deletions hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -320,6 +320,8 @@ abstract class Builder {
protected Optional<Integer> chunkSize = Optional.empty();
protected OptionalInt streamBufferSize = OptionalInt.empty();
protected Optional<Long> streamBufferFlushSize = Optional.empty();
protected Optional<Long> dataStreamBufferFlushSize= Optional.empty();
protected OptionalInt dataStreamMaxBufferSize = OptionalInt.empty();
protected Optional<Long> streamBufferMaxSize = Optional.empty();
protected Optional<Long> blockSize = Optional.empty();
protected Optional<StorageUnit> streamBufferSizeUnit = Optional.empty();
Expand DownExpand Up@@ -553,6 +555,16 @@ public Builder setStreamBufferMaxSize(long size) {
return this;
}

public Builder setDataStreamBufferMaxSize(int size) {
dataStreamMaxBufferSize = OptionalInt.of(size);
return this;
}

public Builder setDataStreamBufferFlushize(long size) {
dataStreamBufferFlushSize = Optional.of(size);
return this;
}

/**
* Sets the block size for stream buffer.
*
Expand Down
11 changes: 11 additions & 0 deletions ...op-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -658,6 +658,12 @@ protected void initializeConfiguration() throws IOException {
if (!streamBufferMaxSize.isPresent()) {
streamBufferMaxSize = Optional.of(2 * streamBufferFlushSize.get());
}
if (!dataStreamBufferFlushSize.isPresent()) {
dataStreamBufferFlushSize = Optional.of((long) 4 * chunkSize.get());
}
if (!dataStreamMaxBufferSize.isPresent()) {
dataStreamMaxBufferSize = OptionalInt.of(chunkSize.get());
}
if (!blockSize.isPresent()) {
blockSize = Optional.of(2 * streamBufferMaxSize.get());
}
Expand All@@ -674,6 +680,11 @@ protected void initializeConfiguration() throws IOException {
streamBufferSizeUnit.get().toBytes(streamBufferMaxSize.get())));
clientConfig.setStreamBufferFlushSize(Math.round(
streamBufferSizeUnit.get().toBytes(streamBufferFlushSize.get())));
clientConfig.setDataStreamBufferFlushSize(Math.round(
streamBufferSizeUnit.get().toBytes(dataStreamBufferFlushSize.get())));
clientConfig.setDataStreamMaxBufferSize((int) Math.round(
streamBufferSizeUnit.get()
.toBytes(dataStreamMaxBufferSize.getAsInt())));
conf.setFromObject(clientConfig);

conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
Expand Down
34 changes: 34 additions & 0 deletions ...tion-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,10 @@
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientMetrics;
import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
import org.apache.hadoop.ozone.MiniOzoneCluster;
Expand DownExpand Up@@ -101,6 +104,8 @@ public static void init() throws Exception {
.setChunkSize(chunkSize)
.setStreamBufferFlushSize(flushSize)
.setStreamBufferMaxSize(maxFlushSize)
.setDataStreamBufferFlushize(maxFlushSize)
.setDataStreamBufferMaxSize(chunkSize)
.setStreamBufferSizeUnit(StorageUnit.BYTES)
.build();
cluster.waitForClusterToBeReady();
Expand DownExpand Up@@ -186,6 +191,35 @@ private void testWriteWithFailure(int dataLength) throws Exception {
validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
}

@Test
public void testPutBlockAtBoundary() throws Exception {
int dataLength = 500;
XceiverClientMetrics metrics =
XceiverClientManager.getXceiverClientMetrics();
long putBlockCount = metrics.getContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
String keyName = getKeyName();
OzoneDataStreamOutput key = createKey(
keyName, ReplicationType.RATIS, 0);
byte[] data =
ContainerTestHelper.getFixedLengthString(keyString, dataLength)
.getBytes(UTF_8);
key.write(ByteBuffer.wrap(data));
Assert.assertTrue(
metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
<= pendingPutBlockCount + 1);
key.close();
// Since data length is 500 , first putBlock will be at 400(flush boundary)
// and the other at 500
Assert.assertTrue(
metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
== putBlockCount + 2);
validateData(keyName, data);
}


private OzoneDataStreamOutput createKey(String keyName, ReplicationType type,
long size) throws Exception {
return TestHelper.createStreamKey(
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.#2866

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bshashikant
merged 6 commits into

from

Dec 1, 2021
 Merged


HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.
#2866
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
0bfe67d

HDDS-5851.Define a PutBlock/maxBuffer fixed boundary for streaming wr…

Nov 23, 2021
3b57efb

edit description

Nov 23, 2021
518d1e1

fix test

Nov 23, 2021
b7164cd

address comment

Nov 30, 2021
90cb0ad

address comment

Nov 30, 2021
47e3393

trigger new CI check

Nov 30, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


31 changes: 31 additions & 0 deletions
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,21 @@ public class OzoneClientConfig {
 tags = ConfigTag.CLIENT)
 private int streamBufferSize = 4 * 1024 * 1024;

 @Config(key = "datastream.max.buffer.size",
 defaultValue = "4MB",
 type = ConfigType.SIZE,
 description = "The maximum size of the ByteBuffer "
Comment thread
sadanand48 marked this conversation as resolved.


Outdated

 + "(used via ratis streaming)",
 tags = ConfigTag.CLIENT)
 private int dataStreamMaxBufferSize = 4 * 1024 * 1024;

 @Config(key = "datastream.buffer.flush.size",
 defaultValue = "16MB",
 type = ConfigType.SIZE,
 description = "The boundary at which putBlock is executed",
 tags = ConfigTag.CLIENT)
 private long dataStreamBufferFlushSize = 16 * 1024 * 1024;

 @Config(key = "stream.buffer.increment",
Comment thread
bshashikant marked this conversation as resolved.

 defaultValue = "0B",
 type = ConfigType.SIZE,
Expand DownExpand Up@@ -168,6 +183,14 @@ public void setStreamBufferSize(int streamBufferSize) {
 this.streamBufferSize = streamBufferSize;
 }

 public int getDataStreamMaxBufferSize() {
 return dataStreamMaxBufferSize;
 }

 public void setDataStreamMaxBufferSize(int dataStreamMaxBufferSize) {
 this.dataStreamMaxBufferSize = dataStreamMaxBufferSize;
 }

 public boolean isStreamBufferFlushDelay() {
 return streamBufferFlushDelay;
 }
Expand DownExpand Up@@ -227,4 +250,12 @@ public void setChecksumVerify(boolean checksumVerify) {
 public int getBufferIncrement() {
 return bufferIncrement;
 }

 public long getDataStreamBufferFlushSize() {
 return dataStreamBufferFlushSize;
 }

 public void setDataStreamBufferFlushSize(long dataStreamBufferFlushSize) {
 this.dataStreamBufferFlushSize = dataStreamBufferFlushSize;
 }
}

29 changes: 23 additions & 6 deletions
...p-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -257,13 +257,30 @@ public void write(ByteBuffer b, int off, int len) throws IOException {
 if (len == 0) {
 return;
 }
 int curLen = len;
 // set limit on the number of bytes that a ByteBuffer(StreamBuffer) can hold
 int maxBufferLen = config.getDataStreamMaxBufferSize();
 while (curLen > 0) {
 int writeLen = Math.min(curLen, maxBufferLen);
 final StreamBuffer buf = new StreamBuffer(b, off, writeLen);
 off += writeLen;
 bufferList.add(buf);
 writeChunkToContainer(buf.duplicate());
 curLen -= writeLen;
 writtenDataLength += writeLen;
 doFlushIfNeeded();
 }
 }

 final StreamBuffer buf = new StreamBuffer(b, off, len);
 bufferList.add(buf);

 writeChunkToContainer(buf.duplicate());

 writtenDataLength += len;
 private void doFlushIfNeeded() throws IOException {
 Preconditions.checkArgument(config.getDataStreamBufferFlushSize() > config
 .getDataStreamMaxBufferSize());
 long boundary = config.getDataStreamBufferFlushSize() / config
 .getDataStreamMaxBufferSize();
 if (bufferList.size() % boundary == 0) {
 updateFlushLength();
 executePutBlock(false, false);
@captainzmccaptainzmcNov 30, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did we need add updateFlushLength(), waitOnFlushFutures() and watchForCommit(false) here? Just as what we did in handleFlush. Currently we only releaseBuffers after watchForCommit.
@sadanand48sadanand48Nov 30, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @captainzmc for the comment. The plan here is to execute a putBlock at 32MB and not block the client right away by waiting for the flush futures and calling watchForCommit. We will introduce another boundary say at 64MB to check whether atleast 32MB is flushed and committed to all datanodes i.e watchForCommit() and waitForFlushFutures() will occur at 64mb. (I will add this is in a different patch after this PR)
@captainzmccaptainzmcNov 30, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for explaining, make sense.
 }
 }

 private void updateFlushLength() {
Expand Down

12 changes: 12 additions & 0 deletions
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -320,6 +320,8 @@ abstract class Builder {
 protected Optional<Integer> chunkSize = Optional.empty();
 protected OptionalInt streamBufferSize = OptionalInt.empty();
 protected Optional<Long> streamBufferFlushSize = Optional.empty();
 protected Optional<Long> dataStreamBufferFlushSize= Optional.empty();
 protected OptionalInt dataStreamMaxBufferSize = OptionalInt.empty();
 protected Optional<Long> streamBufferMaxSize = Optional.empty();
 protected Optional<Long> blockSize = Optional.empty();
 protected Optional<StorageUnit> streamBufferSizeUnit = Optional.empty();
Expand DownExpand Up@@ -553,6 +555,16 @@ public Builder setStreamBufferMaxSize(long size) {
 return this;
 }

 public Builder setDataStreamBufferMaxSize(int size) {
 dataStreamMaxBufferSize = OptionalInt.of(size);
 return this;
 }

 public Builder setDataStreamBufferFlushize(long size) {
 dataStreamBufferFlushSize = Optional.of(size);
 return this;
 }

 /**
 * Sets the block size for stream buffer.
 *
Expand Down

11 changes: 11 additions & 0 deletions
...op-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -658,6 +658,12 @@ protected void initializeConfiguration() throws IOException {
 if (!streamBufferMaxSize.isPresent()) {
 streamBufferMaxSize = Optional.of(2 * streamBufferFlushSize.get());
 }
 if (!dataStreamBufferFlushSize.isPresent()) {
 dataStreamBufferFlushSize = Optional.of((long) 4 * chunkSize.get());
 }
 if (!dataStreamMaxBufferSize.isPresent()) {
 dataStreamMaxBufferSize = OptionalInt.of(chunkSize.get());
 }
 if (!blockSize.isPresent()) {
 blockSize = Optional.of(2 * streamBufferMaxSize.get());
 }
Expand All@@ -674,6 +680,11 @@ protected void initializeConfiguration() throws IOException {
 streamBufferSizeUnit.get().toBytes(streamBufferMaxSize.get())));
 clientConfig.setStreamBufferFlushSize(Math.round(
 streamBufferSizeUnit.get().toBytes(streamBufferFlushSize.get())));
 clientConfig.setDataStreamBufferFlushSize(Math.round(
 streamBufferSizeUnit.get().toBytes(dataStreamBufferFlushSize.get())));
 clientConfig.setDataStreamMaxBufferSize((int) Math.round(
 streamBufferSizeUnit.get()
 .toBytes(dataStreamMaxBufferSize.getAsInt())));
 conf.setFromObject(clientConfig);

 conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
Expand Down

34 changes: 34 additions & 0 deletions
...tion-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,10 @@
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientMetrics;
import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
import org.apache.hadoop.ozone.MiniOzoneCluster;
Expand DownExpand Up@@ -101,6 +104,8 @@ public static void init() throws Exception {
 .setChunkSize(chunkSize)
 .setStreamBufferFlushSize(flushSize)
 .setStreamBufferMaxSize(maxFlushSize)
 .setDataStreamBufferFlushize(maxFlushSize)
 .setDataStreamBufferMaxSize(chunkSize)
 .setStreamBufferSizeUnit(StorageUnit.BYTES)
 .build();
 cluster.waitForClusterToBeReady();
Expand DownExpand Up@@ -186,6 +191,35 @@ private void testWriteWithFailure(int dataLength) throws Exception {
 validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
 }

 @Test
 public void testPutBlockAtBoundary() throws Exception {
 int dataLength = 500;
 XceiverClientMetrics metrics =
 XceiverClientManager.getXceiverClientMetrics();
 long putBlockCount = metrics.getContainerOpCountMetrics(
 ContainerProtos.Type.PutBlock);
 long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
 ContainerProtos.Type.PutBlock);
 String keyName = getKeyName();
 OzoneDataStreamOutput key = createKey(
 keyName, ReplicationType.RATIS, 0);
 byte[] data =
 ContainerTestHelper.getFixedLengthString(keyString, dataLength)
 .getBytes(UTF_8);
 key.write(ByteBuffer.wrap(data));
 Assert.assertTrue(
 metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
 <= pendingPutBlockCount + 1);
 key.close();
 // Since data length is 500 , first putBlock will be at 400(flush boundary)
 // and the other at 500
 Assert.assertTrue(
 metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
 == putBlockCount + 2);
 validateData(keyName, data);
 }


 private OzoneDataStreamOutput createKey(String keyName, ReplicationType type,
 long size) throws Exception {
 return TestHelper.createStreamKey(
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.#2866

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bshashikant
merged 6 commits into

from

Dec 1, 2021
 Merged


HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.
#2866
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
0bfe67d

HDDS-5851.Define a PutBlock/maxBuffer fixed boundary for streaming wr…

Nov 23, 2021
3b57efb

edit description

Nov 23, 2021
518d1e1

fix test

Nov 23, 2021
b7164cd

address comment

Nov 30, 2021
90cb0ad

address comment

Nov 30, 2021
47e3393

trigger new CI check

Nov 30, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


31 changes: 31 additions & 0 deletions
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,21 @@ public class OzoneClientConfig {
 tags = ConfigTag.CLIENT)
 private int streamBufferSize = 4 * 1024 * 1024;

 @Config(key = "datastream.max.buffer.size",
 defaultValue = "4MB",
 type = ConfigType.SIZE,
 description = "The maximum size of the ByteBuffer "
Comment thread
sadanand48 marked this conversation as resolved.


Outdated

 + "(used via ratis streaming)",
 tags = ConfigTag.CLIENT)
 private int dataStreamMaxBufferSize = 4 * 1024 * 1024;

 @Config(key = "datastream.buffer.flush.size",
 defaultValue = "16MB",
 type = ConfigType.SIZE,
 description = "The boundary at which putBlock is executed",
 tags = ConfigTag.CLIENT)
 private long dataStreamBufferFlushSize = 16 * 1024 * 1024;

 @Config(key = "stream.buffer.increment",
Comment thread
bshashikant marked this conversation as resolved.

 defaultValue = "0B",
 type = ConfigType.SIZE,
Expand DownExpand Up@@ -168,6 +183,14 @@ public void setStreamBufferSize(int streamBufferSize) {
 this.streamBufferSize = streamBufferSize;
 }

 public int getDataStreamMaxBufferSize() {
 return dataStreamMaxBufferSize;
 }

 public void setDataStreamMaxBufferSize(int dataStreamMaxBufferSize) {
 this.dataStreamMaxBufferSize = dataStreamMaxBufferSize;
 }

 public boolean isStreamBufferFlushDelay() {
 return streamBufferFlushDelay;
 }
Expand DownExpand Up@@ -227,4 +250,12 @@ public void setChecksumVerify(boolean checksumVerify) {
 public int getBufferIncrement() {
 return bufferIncrement;
 }

 public long getDataStreamBufferFlushSize() {
 return dataStreamBufferFlushSize;
 }

 public void setDataStreamBufferFlushSize(long dataStreamBufferFlushSize) {
 this.dataStreamBufferFlushSize = dataStreamBufferFlushSize;
 }
}

29 changes: 23 additions & 6 deletions
...p-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -257,13 +257,30 @@ public void write(ByteBuffer b, int off, int len) throws IOException {
 if (len == 0) {
 return;
 }
 int curLen = len;
 // set limit on the number of bytes that a ByteBuffer(StreamBuffer) can hold
 int maxBufferLen = config.getDataStreamMaxBufferSize();
 while (curLen > 0) {
 int writeLen = Math.min(curLen, maxBufferLen);
 final StreamBuffer buf = new StreamBuffer(b, off, writeLen);
 off += writeLen;
 bufferList.add(buf);
 writeChunkToContainer(buf.duplicate());
 curLen -= writeLen;
 writtenDataLength += writeLen;
 doFlushIfNeeded();
 }
 }

 final StreamBuffer buf = new StreamBuffer(b, off, len);
 bufferList.add(buf);

 writeChunkToContainer(buf.duplicate());

 writtenDataLength += len;
 private void doFlushIfNeeded() throws IOException {
 Preconditions.checkArgument(config.getDataStreamBufferFlushSize() > config
 .getDataStreamMaxBufferSize());
 long boundary = config.getDataStreamBufferFlushSize() / config
 .getDataStreamMaxBufferSize();
 if (bufferList.size() % boundary == 0) {
 updateFlushLength();
 executePutBlock(false, false);
@captainzmccaptainzmcNov 30, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did we need add updateFlushLength(), waitOnFlushFutures() and watchForCommit(false) here? Just as what we did in handleFlush. Currently we only releaseBuffers after watchForCommit.
@sadanand48sadanand48Nov 30, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @captainzmc for the comment. The plan here is to execute a putBlock at 32MB and not block the client right away by waiting for the flush futures and calling watchForCommit. We will introduce another boundary say at 64MB to check whether atleast 32MB is flushed and committed to all datanodes i.e watchForCommit() and waitForFlushFutures() will occur at 64mb. (I will add this is in a different patch after this PR)
@captainzmccaptainzmcNov 30, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for explaining, make sense.
 }
 }

 private void updateFlushLength() {
Expand Down

12 changes: 12 additions & 0 deletions
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -320,6 +320,8 @@ abstract class Builder {
 protected Optional<Integer> chunkSize = Optional.empty();
 protected OptionalInt streamBufferSize = OptionalInt.empty();
 protected Optional<Long> streamBufferFlushSize = Optional.empty();
 protected Optional<Long> dataStreamBufferFlushSize= Optional.empty();
 protected OptionalInt dataStreamMaxBufferSize = OptionalInt.empty();
 protected Optional<Long> streamBufferMaxSize = Optional.empty();
 protected Optional<Long> blockSize = Optional.empty();
 protected Optional<StorageUnit> streamBufferSizeUnit = Optional.empty();
Expand DownExpand Up@@ -553,6 +555,16 @@ public Builder setStreamBufferMaxSize(long size) {
 return this;
 }

 public Builder setDataStreamBufferMaxSize(int size) {
 dataStreamMaxBufferSize = OptionalInt.of(size);
 return this;
 }

 public Builder setDataStreamBufferFlushize(long size) {
 dataStreamBufferFlushSize = Optional.of(size);
 return this;
 }

 /**
 * Sets the block size for stream buffer.
 *
Expand Down

11 changes: 11 additions & 0 deletions
...op-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -658,6 +658,12 @@ protected void initializeConfiguration() throws IOException {
 if (!streamBufferMaxSize.isPresent()) {
 streamBufferMaxSize = Optional.of(2 * streamBufferFlushSize.get());
 }
 if (!dataStreamBufferFlushSize.isPresent()) {
 dataStreamBufferFlushSize = Optional.of((long) 4 * chunkSize.get());
 }
 if (!dataStreamMaxBufferSize.isPresent()) {
 dataStreamMaxBufferSize = OptionalInt.of(chunkSize.get());
 }
 if (!blockSize.isPresent()) {
 blockSize = Optional.of(2 * streamBufferMaxSize.get());
 }
Expand All@@ -674,6 +680,11 @@ protected void initializeConfiguration() throws IOException {
 streamBufferSizeUnit.get().toBytes(streamBufferMaxSize.get())));
 clientConfig.setStreamBufferFlushSize(Math.round(
 streamBufferSizeUnit.get().toBytes(streamBufferFlushSize.get())));
 clientConfig.setDataStreamBufferFlushSize(Math.round(
 streamBufferSizeUnit.get().toBytes(dataStreamBufferFlushSize.get())));
 clientConfig.setDataStreamMaxBufferSize((int) Math.round(
 streamBufferSizeUnit.get()
 .toBytes(dataStreamMaxBufferSize.getAsInt())));
 conf.setFromObject(clientConfig);

 conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
Expand Down

34 changes: 34 additions & 0 deletions
...tion-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,10 @@
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientMetrics;
import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
import org.apache.hadoop.ozone.MiniOzoneCluster;
Expand DownExpand Up@@ -101,6 +104,8 @@ public static void init() throws Exception {
 .setChunkSize(chunkSize)
 .setStreamBufferFlushSize(flushSize)
 .setStreamBufferMaxSize(maxFlushSize)
 .setDataStreamBufferFlushize(maxFlushSize)
 .setDataStreamBufferMaxSize(chunkSize)
 .setStreamBufferSizeUnit(StorageUnit.BYTES)
 .build();
 cluster.waitForClusterToBeReady();
Expand DownExpand Up@@ -186,6 +191,35 @@ private void testWriteWithFailure(int dataLength) throws Exception {
 validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
 }

 @Test
 public void testPutBlockAtBoundary() throws Exception {
 int dataLength = 500;
 XceiverClientMetrics metrics =
 XceiverClientManager.getXceiverClientMetrics();
 long putBlockCount = metrics.getContainerOpCountMetrics(
 ContainerProtos.Type.PutBlock);
 long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
 ContainerProtos.Type.PutBlock);
 String keyName = getKeyName();
 OzoneDataStreamOutput key = createKey(
 keyName, ReplicationType.RATIS, 0);
 byte[] data =
 ContainerTestHelper.getFixedLengthString(keyString, dataLength)
 .getBytes(UTF_8);
 key.write(ByteBuffer.wrap(data));
 Assert.assertTrue(
 metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
 <= pendingPutBlockCount + 1);
 key.close();
 // Since data length is 500 , first putBlock will be at 400(flush boundary)
 // and the other at 500
 Assert.assertTrue(
 metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
 == putBlockCount + 2);
 validateData(keyName, data);
 }


 private OzoneDataStreamOutput createKey(String keyName, ReplicationType type,
 long size) throws Exception {
 return TestHelper.createStreamKey(
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.#2866

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bshashikant
merged 6 commits into

from

Dec 1, 2021
 Merged


HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.
#2866
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
0bfe67d

HDDS-5851.Define a PutBlock/maxBuffer fixed boundary for streaming wr…

Nov 23, 2021
3b57efb

edit description

Nov 23, 2021
518d1e1

fix test

Nov 23, 2021
b7164cd

address comment

Nov 30, 2021
90cb0ad

address comment

Nov 30, 2021
47e3393

trigger new CI check

Nov 30, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


31 changes: 31 additions & 0 deletions
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,21 @@ public class OzoneClientConfig {
 tags = ConfigTag.CLIENT)
 private int streamBufferSize = 4 * 1024 * 1024;

 @Config(key = "datastream.max.buffer.size",
 defaultValue = "4MB",
 type = ConfigType.SIZE,
 description = "The maximum size of the ByteBuffer "
Comment thread
sadanand48 marked this conversation as resolved.


Outdated

 + "(used via ratis streaming)",
 tags = ConfigTag.CLIENT)
 private int dataStreamMaxBufferSize = 4 * 1024 * 1024;

 @Config(key = "datastream.buffer.flush.size",
 defaultValue = "16MB",
 type = ConfigType.SIZE,
 description = "The boundary at which putBlock is executed",
 tags = ConfigTag.CLIENT)
 private long dataStreamBufferFlushSize = 16 * 1024 * 1024;

 @Config(key = "stream.buffer.increment",
Comment thread
bshashikant marked this conversation as resolved.

 defaultValue = "0B",
 type = ConfigType.SIZE,
Expand DownExpand Up@@ -168,6 +183,14 @@ public void setStreamBufferSize(int streamBufferSize) {
 this.streamBufferSize = streamBufferSize;
 }

 public int getDataStreamMaxBufferSize() {
 return dataStreamMaxBufferSize;
 }

 public void setDataStreamMaxBufferSize(int dataStreamMaxBufferSize) {
 this.dataStreamMaxBufferSize = dataStreamMaxBufferSize;
 }

 public boolean isStreamBufferFlushDelay() {
 return streamBufferFlushDelay;
 }
Expand DownExpand Up@@ -227,4 +250,12 @@ public void setChecksumVerify(boolean checksumVerify) {
 public int getBufferIncrement() {
 return bufferIncrement;
 }

 public long getDataStreamBufferFlushSize() {
 return dataStreamBufferFlushSize;
 }

 public void setDataStreamBufferFlushSize(long dataStreamBufferFlushSize) {
 this.dataStreamBufferFlushSize = dataStreamBufferFlushSize;
 }
}

29 changes: 23 additions & 6 deletions
...p-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -257,13 +257,30 @@ public void write(ByteBuffer b, int off, int len) throws IOException {
 if (len == 0) {
 return;
 }
 int curLen = len;
 // set limit on the number of bytes that a ByteBuffer(StreamBuffer) can hold
 int maxBufferLen = config.getDataStreamMaxBufferSize();
 while (curLen > 0) {
 int writeLen = Math.min(curLen, maxBufferLen);
 final StreamBuffer buf = new StreamBuffer(b, off, writeLen);
 off += writeLen;
 bufferList.add(buf);
 writeChunkToContainer(buf.duplicate());
 curLen -= writeLen;
 writtenDataLength += writeLen;
 doFlushIfNeeded();
 }
 }

 final StreamBuffer buf = new StreamBuffer(b, off, len);
 bufferList.add(buf);

 writeChunkToContainer(buf.duplicate());

 writtenDataLength += len;
 private void doFlushIfNeeded() throws IOException {
 Preconditions.checkArgument(config.getDataStreamBufferFlushSize() > config
 .getDataStreamMaxBufferSize());
 long boundary = config.getDataStreamBufferFlushSize() / config
 .getDataStreamMaxBufferSize();
 if (bufferList.size() % boundary == 0) {
 updateFlushLength();
 executePutBlock(false, false);
@captainzmccaptainzmcNov 30, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did we need add updateFlushLength(), waitOnFlushFutures() and watchForCommit(false) here? Just as what we did in handleFlush. Currently we only releaseBuffers after watchForCommit.
@sadanand48sadanand48Nov 30, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @captainzmc for the comment. The plan here is to execute a putBlock at 32MB and not block the client right away by waiting for the flush futures and calling watchForCommit. We will introduce another boundary say at 64MB to check whether atleast 32MB is flushed and committed to all datanodes i.e watchForCommit() and waitForFlushFutures() will occur at 64mb. (I will add this is in a different patch after this PR)
@captainzmccaptainzmcNov 30, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for explaining, make sense.
 }
 }

 private void updateFlushLength() {
Expand Down

12 changes: 12 additions & 0 deletions
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -320,6 +320,8 @@ abstract class Builder {
 protected Optional<Integer> chunkSize = Optional.empty();
 protected OptionalInt streamBufferSize = OptionalInt.empty();
 protected Optional<Long> streamBufferFlushSize = Optional.empty();
 protected Optional<Long> dataStreamBufferFlushSize= Optional.empty();
 protected OptionalInt dataStreamMaxBufferSize = OptionalInt.empty();
 protected Optional<Long> streamBufferMaxSize = Optional.empty();
 protected Optional<Long> blockSize = Optional.empty();
 protected Optional<StorageUnit> streamBufferSizeUnit = Optional.empty();
Expand DownExpand Up@@ -553,6 +555,16 @@ public Builder setStreamBufferMaxSize(long size) {
 return this;
 }

 public Builder setDataStreamBufferMaxSize(int size) {
 dataStreamMaxBufferSize = OptionalInt.of(size);
 return this;
 }

 public Builder setDataStreamBufferFlushize(long size) {
 dataStreamBufferFlushSize = Optional.of(size);
 return this;
 }

 /**
 * Sets the block size for stream buffer.
 *
Expand Down

11 changes: 11 additions & 0 deletions
...op-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -658,6 +658,12 @@ protected void initializeConfiguration() throws IOException {
 if (!streamBufferMaxSize.isPresent()) {
 streamBufferMaxSize = Optional.of(2 * streamBufferFlushSize.get());
 }
 if (!dataStreamBufferFlushSize.isPresent()) {
 dataStreamBufferFlushSize = Optional.of((long) 4 * chunkSize.get());
 }
 if (!dataStreamMaxBufferSize.isPresent()) {
 dataStreamMaxBufferSize = OptionalInt.of(chunkSize.get());
 }
 if (!blockSize.isPresent()) {
 blockSize = Optional.of(2 * streamBufferMaxSize.get());
 }
Expand All@@ -674,6 +680,11 @@ protected void initializeConfiguration() throws IOException {
 streamBufferSizeUnit.get().toBytes(streamBufferMaxSize.get())));
 clientConfig.setStreamBufferFlushSize(Math.round(
 streamBufferSizeUnit.get().toBytes(streamBufferFlushSize.get())));
 clientConfig.setDataStreamBufferFlushSize(Math.round(
 streamBufferSizeUnit.get().toBytes(dataStreamBufferFlushSize.get())));
 clientConfig.setDataStreamMaxBufferSize((int) Math.round(
 streamBufferSizeUnit.get()
 .toBytes(dataStreamMaxBufferSize.getAsInt())));
 conf.setFromObject(clientConfig);

 conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
Expand Down

34 changes: 34 additions & 0 deletions
...tion-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,10 @@
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientMetrics;
import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
import org.apache.hadoop.ozone.MiniOzoneCluster;
Expand DownExpand Up@@ -101,6 +104,8 @@ public static void init() throws Exception {
 .setChunkSize(chunkSize)
 .setStreamBufferFlushSize(flushSize)
 .setStreamBufferMaxSize(maxFlushSize)
 .setDataStreamBufferFlushize(maxFlushSize)
 .setDataStreamBufferMaxSize(chunkSize)
 .setStreamBufferSizeUnit(StorageUnit.BYTES)
 .build();
 cluster.waitForClusterToBeReady();
Expand DownExpand Up@@ -186,6 +191,35 @@ private void testWriteWithFailure(int dataLength) throws Exception {
 validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
 }

 @Test
 public void testPutBlockAtBoundary() throws Exception {
 int dataLength = 500;
 XceiverClientMetrics metrics =
 XceiverClientManager.getXceiverClientMetrics();
 long putBlockCount = metrics.getContainerOpCountMetrics(
 ContainerProtos.Type.PutBlock);
 long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
 ContainerProtos.Type.PutBlock);
 String keyName = getKeyName();
 OzoneDataStreamOutput key = createKey(
 keyName, ReplicationType.RATIS, 0);
 byte[] data =
 ContainerTestHelper.getFixedLengthString(keyString, dataLength)
 .getBytes(UTF_8);
 key.write(ByteBuffer.wrap(data));
 Assert.assertTrue(
 metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
 <= pendingPutBlockCount + 1);
 key.close();
 // Since data length is 500 , first putBlock will be at 400(flush boundary)
 // and the other at 500
 Assert.assertTrue(
 metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
 == putBlockCount + 2);
 validateData(keyName, data);
 }


 private OzoneDataStreamOutput createKey(String keyName, ReplicationType type,
 long size) throws Exception {
 return TestHelper.createStreamKey(
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.#2866

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bshashikant
merged 6 commits into

from

Dec 1, 2021
 Merged


HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.
#2866
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
0bfe67d

HDDS-5851.Define a PutBlock/maxBuffer fixed boundary for streaming wr…

Nov 23, 2021
3b57efb

edit description

Nov 23, 2021
518d1e1

fix test

Nov 23, 2021
b7164cd

address comment

Nov 30, 2021
90cb0ad

address comment

Nov 30, 2021
47e3393

trigger new CI check

Nov 30, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


31 changes: 31 additions & 0 deletions
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,21 @@ public class OzoneClientConfig {
 tags = ConfigTag.CLIENT)
 private int streamBufferSize = 4 * 1024 * 1024;

 @Config(key = "datastream.max.buffer.size",
 defaultValue = "4MB",
 type = ConfigType.SIZE,
 description = "The maximum size of the ByteBuffer "
Comment thread
sadanand48 marked this conversation as resolved.


Outdated

 + "(used via ratis streaming)",
 tags = ConfigTag.CLIENT)
 private int dataStreamMaxBufferSize = 4 * 1024 * 1024;

 @Config(key = "datastream.buffer.flush.size",
 defaultValue = "16MB",
 type = ConfigType.SIZE,
 description = "The boundary at which putBlock is executed",
 tags = ConfigTag.CLIENT)
 private long dataStreamBufferFlushSize = 16 * 1024 * 1024;

 @Config(key = "stream.buffer.increment",
Comment thread
bshashikant marked this conversation as resolved.

 defaultValue = "0B",
 type = ConfigType.SIZE,
Expand DownExpand Up@@ -168,6 +183,14 @@ public void setStreamBufferSize(int streamBufferSize) {
 this.streamBufferSize = streamBufferSize;
 }

 public int getDataStreamMaxBufferSize() {
 return dataStreamMaxBufferSize;
 }

 public void setDataStreamMaxBufferSize(int dataStreamMaxBufferSize) {
 this.dataStreamMaxBufferSize = dataStreamMaxBufferSize;
 }

 public boolean isStreamBufferFlushDelay() {
 return streamBufferFlushDelay;
 }
Expand DownExpand Up@@ -227,4 +250,12 @@ public void setChecksumVerify(boolean checksumVerify) {
 public int getBufferIncrement() {
 return bufferIncrement;
 }

 public long getDataStreamBufferFlushSize() {
 return dataStreamBufferFlushSize;
 }

 public void setDataStreamBufferFlushSize(long dataStreamBufferFlushSize) {
 this.dataStreamBufferFlushSize = dataStreamBufferFlushSize;
 }
}

29 changes: 23 additions & 6 deletions
...p-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -257,13 +257,30 @@ public void write(ByteBuffer b, int off, int len) throws IOException {
 if (len == 0) {
 return;
 }
 int curLen = len;
 // set limit on the number of bytes that a ByteBuffer(StreamBuffer) can hold
 int maxBufferLen = config.getDataStreamMaxBufferSize();
 while (curLen > 0) {
 int writeLen = Math.min(curLen, maxBufferLen);
 final StreamBuffer buf = new StreamBuffer(b, off, writeLen);
 off += writeLen;
 bufferList.add(buf);
 writeChunkToContainer(buf.duplicate());
 curLen -= writeLen;
 writtenDataLength += writeLen;
 doFlushIfNeeded();
 }
 }

 final StreamBuffer buf = new StreamBuffer(b, off, len);
 bufferList.add(buf);

 writeChunkToContainer(buf.duplicate());

 writtenDataLength += len;
 private void doFlushIfNeeded() throws IOException {
 Preconditions.checkArgument(config.getDataStreamBufferFlushSize() > config
 .getDataStreamMaxBufferSize());
 long boundary = config.getDataStreamBufferFlushSize() / config
 .getDataStreamMaxBufferSize();
 if (bufferList.size() % boundary == 0) {
 updateFlushLength();
 executePutBlock(false, false);
@captainzmccaptainzmcNov 30, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did we need add updateFlushLength(), waitOnFlushFutures() and watchForCommit(false) here? Just as what we did in handleFlush. Currently we only releaseBuffers after watchForCommit.
@sadanand48sadanand48Nov 30, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @captainzmc for the comment. The plan here is to execute a putBlock at 32MB and not block the client right away by waiting for the flush futures and calling watchForCommit. We will introduce another boundary say at 64MB to check whether atleast 32MB is flushed and committed to all datanodes i.e watchForCommit() and waitForFlushFutures() will occur at 64mb. (I will add this is in a different patch after this PR)
@captainzmccaptainzmcNov 30, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for explaining, make sense.
 }
 }

 private void updateFlushLength() {
Expand Down

12 changes: 12 additions & 0 deletions
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -320,6 +320,8 @@ abstract class Builder {
 protected Optional<Integer> chunkSize = Optional.empty();
 protected OptionalInt streamBufferSize = OptionalInt.empty();
 protected Optional<Long> streamBufferFlushSize = Optional.empty();
 protected Optional<Long> dataStreamBufferFlushSize= Optional.empty();
 protected OptionalInt dataStreamMaxBufferSize = OptionalInt.empty();
 protected Optional<Long> streamBufferMaxSize = Optional.empty();
 protected Optional<Long> blockSize = Optional.empty();
 protected Optional<StorageUnit> streamBufferSizeUnit = Optional.empty();
Expand DownExpand Up@@ -553,6 +555,16 @@ public Builder setStreamBufferMaxSize(long size) {
 return this;
 }

 public Builder setDataStreamBufferMaxSize(int size) {
 dataStreamMaxBufferSize = OptionalInt.of(size);
 return this;
 }

 public Builder setDataStreamBufferFlushize(long size) {
 dataStreamBufferFlushSize = Optional.of(size);
 return this;
 }

 /**
 * Sets the block size for stream buffer.
 *
Expand Down

11 changes: 11 additions & 0 deletions
...op-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -658,6 +658,12 @@ protected void initializeConfiguration() throws IOException {
 if (!streamBufferMaxSize.isPresent()) {
 streamBufferMaxSize = Optional.of(2 * streamBufferFlushSize.get());
 }
 if (!dataStreamBufferFlushSize.isPresent()) {
 dataStreamBufferFlushSize = Optional.of((long) 4 * chunkSize.get());
 }
 if (!dataStreamMaxBufferSize.isPresent()) {
 dataStreamMaxBufferSize = OptionalInt.of(chunkSize.get());
 }
 if (!blockSize.isPresent()) {
 blockSize = Optional.of(2 * streamBufferMaxSize.get());
 }
Expand All@@ -674,6 +680,11 @@ protected void initializeConfiguration() throws IOException {
 streamBufferSizeUnit.get().toBytes(streamBufferMaxSize.get())));
 clientConfig.setStreamBufferFlushSize(Math.round(
 streamBufferSizeUnit.get().toBytes(streamBufferFlushSize.get())));
 clientConfig.setDataStreamBufferFlushSize(Math.round(
 streamBufferSizeUnit.get().toBytes(dataStreamBufferFlushSize.get())));
 clientConfig.setDataStreamMaxBufferSize((int) Math.round(
 streamBufferSizeUnit.get()
 .toBytes(dataStreamMaxBufferSize.getAsInt())));
 conf.setFromObject(clientConfig);

 conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
Expand Down

34 changes: 34 additions & 0 deletions
...tion-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,10 @@
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientMetrics;
import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
import org.apache.hadoop.ozone.MiniOzoneCluster;
Expand DownExpand Up@@ -101,6 +104,8 @@ public static void init() throws Exception {
 .setChunkSize(chunkSize)
 .setStreamBufferFlushSize(flushSize)
 .setStreamBufferMaxSize(maxFlushSize)
 .setDataStreamBufferFlushize(maxFlushSize)
 .setDataStreamBufferMaxSize(chunkSize)
 .setStreamBufferSizeUnit(StorageUnit.BYTES)
 .build();
 cluster.waitForClusterToBeReady();
Expand DownExpand Up@@ -186,6 +191,35 @@ private void testWriteWithFailure(int dataLength) throws Exception {
 validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
 }

 @Test
 public void testPutBlockAtBoundary() throws Exception {
 int dataLength = 500;
 XceiverClientMetrics metrics =
 XceiverClientManager.getXceiverClientMetrics();
 long putBlockCount = metrics.getContainerOpCountMetrics(
 ContainerProtos.Type.PutBlock);
 long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
 ContainerProtos.Type.PutBlock);
 String keyName = getKeyName();
 OzoneDataStreamOutput key = createKey(
 keyName, ReplicationType.RATIS, 0);
 byte[] data =
 ContainerTestHelper.getFixedLengthString(keyString, dataLength)
 .getBytes(UTF_8);
 key.write(ByteBuffer.wrap(data));
 Assert.assertTrue(
 metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
 <= pendingPutBlockCount + 1);
 key.close();
 // Since data length is 500 , first putBlock will be at 400(flush boundary)
 // and the other at 500
 Assert.assertTrue(
 metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
 == putBlockCount + 2);
 validateData(keyName, data);
 }


 private OzoneDataStreamOutput createKey(String keyName, ReplicationType type,
 long size) throws Exception {
 return TestHelper.createStreamKey(
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.#2866

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bshashikant
merged 6 commits into

from

Dec 1, 2021
 Merged


HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.
#2866
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
0bfe67d

HDDS-5851.Define a PutBlock/maxBuffer fixed boundary for streaming wr…

Nov 23, 2021
3b57efb

edit description

Nov 23, 2021
518d1e1

fix test

Nov 23, 2021
b7164cd

address comment

Nov 30, 2021
90cb0ad

address comment

Nov 30, 2021
47e3393

trigger new CI check

Nov 30, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


31 changes: 31 additions & 0 deletions
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,21 @@ public class OzoneClientConfig {
 tags = ConfigTag.CLIENT)
 private int streamBufferSize = 4 * 1024 * 1024;

 @Config(key = "datastream.max.buffer.size",
 defaultValue = "4MB",
 type = ConfigType.SIZE,
 description = "The maximum size of the ByteBuffer "
Comment thread
sadanand48 marked this conversation as resolved.


Outdated

 + "(used via ratis streaming)",
 tags = ConfigTag.CLIENT)
 private int dataStreamMaxBufferSize = 4 * 1024 * 1024;

 @Config(key = "datastream.buffer.flush.size",
 defaultValue = "16MB",
 type = ConfigType.SIZE,
 description = "The boundary at which putBlock is executed",
 tags = ConfigTag.CLIENT)
 private long dataStreamBufferFlushSize = 16 * 1024 * 1024;

 @Config(key = "stream.buffer.increment",
Comment thread
bshashikant marked this conversation as resolved.

 defaultValue = "0B",
 type = ConfigType.SIZE,
Expand DownExpand Up@@ -168,6 +183,14 @@ public void setStreamBufferSize(int streamBufferSize) {
 this.streamBufferSize = streamBufferSize;
 }

 public int getDataStreamMaxBufferSize() {
 return dataStreamMaxBufferSize;
 }

 public void setDataStreamMaxBufferSize(int dataStreamMaxBufferSize) {
 this.dataStreamMaxBufferSize = dataStreamMaxBufferSize;
 }

 public boolean isStreamBufferFlushDelay() {
 return streamBufferFlushDelay;
 }
Expand DownExpand Up@@ -227,4 +250,12 @@ public void setChecksumVerify(boolean checksumVerify) {
 public int getBufferIncrement() {
 return bufferIncrement;
 }

 public long getDataStreamBufferFlushSize() {
 return dataStreamBufferFlushSize;
 }

 public void setDataStreamBufferFlushSize(long dataStreamBufferFlushSize) {
 this.dataStreamBufferFlushSize = dataStreamBufferFlushSize;
 }
}

29 changes: 23 additions & 6 deletions
...p-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -257,13 +257,30 @@ public void write(ByteBuffer b, int off, int len) throws IOException {
 if (len == 0) {
 return;
 }
 int curLen = len;
 // set limit on the number of bytes that a ByteBuffer(StreamBuffer) can hold
 int maxBufferLen = config.getDataStreamMaxBufferSize();
 while (curLen > 0) {
 int writeLen = Math.min(curLen, maxBufferLen);
 final StreamBuffer buf = new StreamBuffer(b, off, writeLen);
 off += writeLen;
 bufferList.add(buf);
 writeChunkToContainer(buf.duplicate());
 curLen -= writeLen;
 writtenDataLength += writeLen;
 doFlushIfNeeded();
 }
 }

 final StreamBuffer buf = new StreamBuffer(b, off, len);
 bufferList.add(buf);

 writeChunkToContainer(buf.duplicate());

 writtenDataLength += len;
 private void doFlushIfNeeded() throws IOException {
 Preconditions.checkArgument(config.getDataStreamBufferFlushSize() > config
 .getDataStreamMaxBufferSize());
 long boundary = config.getDataStreamBufferFlushSize() / config
 .getDataStreamMaxBufferSize();
 if (bufferList.size() % boundary == 0) {
 updateFlushLength();
 executePutBlock(false, false);
@captainzmccaptainzmcNov 30, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did we need add updateFlushLength(), waitOnFlushFutures() and watchForCommit(false) here? Just as what we did in handleFlush. Currently we only releaseBuffers after watchForCommit.
@sadanand48sadanand48Nov 30, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @captainzmc for the comment. The plan here is to execute a putBlock at 32MB and not block the client right away by waiting for the flush futures and calling watchForCommit. We will introduce another boundary say at 64MB to check whether atleast 32MB is flushed and committed to all datanodes i.e watchForCommit() and waitForFlushFutures() will occur at 64mb. (I will add this is in a different patch after this PR)
@captainzmccaptainzmcNov 30, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for explaining, make sense.
 }
 }

 private void updateFlushLength() {
Expand Down

12 changes: 12 additions & 0 deletions
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -320,6 +320,8 @@ abstract class Builder {
 protected Optional<Integer> chunkSize = Optional.empty();
 protected OptionalInt streamBufferSize = OptionalInt.empty();
 protected Optional<Long> streamBufferFlushSize = Optional.empty();
 protected Optional<Long> dataStreamBufferFlushSize= Optional.empty();
 protected OptionalInt dataStreamMaxBufferSize = OptionalInt.empty();
 protected Optional<Long> streamBufferMaxSize = Optional.empty();
 protected Optional<Long> blockSize = Optional.empty();
 protected Optional<StorageUnit> streamBufferSizeUnit = Optional.empty();
Expand DownExpand Up@@ -553,6 +555,16 @@ public Builder setStreamBufferMaxSize(long size) {
 return this;
 }

 public Builder setDataStreamBufferMaxSize(int size) {
 dataStreamMaxBufferSize = OptionalInt.of(size);
 return this;
 }

 public Builder setDataStreamBufferFlushize(long size) {
 dataStreamBufferFlushSize = Optional.of(size);
 return this;
 }

 /**
 * Sets the block size for stream buffer.
 *
Expand Down

11 changes: 11 additions & 0 deletions
...op-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -658,6 +658,12 @@ protected void initializeConfiguration() throws IOException {
 if (!streamBufferMaxSize.isPresent()) {
 streamBufferMaxSize = Optional.of(2 * streamBufferFlushSize.get());
 }
 if (!dataStreamBufferFlushSize.isPresent()) {
 dataStreamBufferFlushSize = Optional.of((long) 4 * chunkSize.get());
 }
 if (!dataStreamMaxBufferSize.isPresent()) {
 dataStreamMaxBufferSize = OptionalInt.of(chunkSize.get());
 }
 if (!blockSize.isPresent()) {
 blockSize = Optional.of(2 * streamBufferMaxSize.get());
 }
Expand All@@ -674,6 +680,11 @@ protected void initializeConfiguration() throws IOException {
 streamBufferSizeUnit.get().toBytes(streamBufferMaxSize.get())));
 clientConfig.setStreamBufferFlushSize(Math.round(
 streamBufferSizeUnit.get().toBytes(streamBufferFlushSize.get())));
 clientConfig.setDataStreamBufferFlushSize(Math.round(
 streamBufferSizeUnit.get().toBytes(dataStreamBufferFlushSize.get())));
 clientConfig.setDataStreamMaxBufferSize((int) Math.round(
 streamBufferSizeUnit.get()
 .toBytes(dataStreamMaxBufferSize.getAsInt())));
 conf.setFromObject(clientConfig);

 conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
Expand Down

34 changes: 34 additions & 0 deletions
...tion-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,10 @@
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientMetrics;
import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
import org.apache.hadoop.ozone.MiniOzoneCluster;
Expand DownExpand Up@@ -101,6 +104,8 @@ public static void init() throws Exception {
 .setChunkSize(chunkSize)
 .setStreamBufferFlushSize(flushSize)
 .setStreamBufferMaxSize(maxFlushSize)
 .setDataStreamBufferFlushize(maxFlushSize)
 .setDataStreamBufferMaxSize(chunkSize)
 .setStreamBufferSizeUnit(StorageUnit.BYTES)
 .build();
 cluster.waitForClusterToBeReady();
Expand DownExpand Up@@ -186,6 +191,35 @@ private void testWriteWithFailure(int dataLength) throws Exception {
 validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
 }

 @Test
 public void testPutBlockAtBoundary() throws Exception {
 int dataLength = 500;
 XceiverClientMetrics metrics =
 XceiverClientManager.getXceiverClientMetrics();
 long putBlockCount = metrics.getContainerOpCountMetrics(
 ContainerProtos.Type.PutBlock);
 long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
 ContainerProtos.Type.PutBlock);
 String keyName = getKeyName();
 OzoneDataStreamOutput key = createKey(
 keyName, ReplicationType.RATIS, 0);
 byte[] data =
 ContainerTestHelper.getFixedLengthString(keyString, dataLength)
 .getBytes(UTF_8);
 key.write(ByteBuffer.wrap(data));
 Assert.assertTrue(
 metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
 <= pendingPutBlockCount + 1);
 key.close();
 // Since data length is 500 , first putBlock will be at 400(flush boundary)
 // and the other at 500
 Assert.assertTrue(
 metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
 == putBlockCount + 2);
 validateData(keyName, data);
 }


 private OzoneDataStreamOutput createKey(String keyName, ReplicationType type,
 long size) throws Exception {
 return TestHelper.createStreamKey(
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.#2866

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bshashikant
merged 6 commits into

from

Dec 1, 2021
 Merged


HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.
#2866
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
0bfe67d

HDDS-5851.Define a PutBlock/maxBuffer fixed boundary for streaming wr…

Nov 23, 2021
3b57efb

edit description

Nov 23, 2021
518d1e1

fix test

Nov 23, 2021
b7164cd

address comment

Nov 30, 2021
90cb0ad

address comment

Nov 30, 2021
47e3393

trigger new CI check

Nov 30, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


31 changes: 31 additions & 0 deletions
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,21 @@ public class OzoneClientConfig {
 tags = ConfigTag.CLIENT)
 private int streamBufferSize = 4 * 1024 * 1024;

 @Config(key = "datastream.max.buffer.size",
 defaultValue = "4MB",
 type = ConfigType.SIZE,
 description = "The maximum size of the ByteBuffer "
Comment thread
sadanand48 marked this conversation as resolved.


Outdated

 + "(used via ratis streaming)",
 tags = ConfigTag.CLIENT)
 private int dataStreamMaxBufferSize = 4 * 1024 * 1024;

 @Config(key = "datastream.buffer.flush.size",
 defaultValue = "16MB",
 type = ConfigType.SIZE,
 description = "The boundary at which putBlock is executed",
 tags = ConfigTag.CLIENT)
 private long dataStreamBufferFlushSize = 16 * 1024 * 1024;

 @Config(key = "stream.buffer.increment",
Comment thread
bshashikant marked this conversation as resolved.

 defaultValue = "0B",
 type = ConfigType.SIZE,
Expand DownExpand Up@@ -168,6 +183,14 @@ public void setStreamBufferSize(int streamBufferSize) {
 this.streamBufferSize = streamBufferSize;
 }

 public int getDataStreamMaxBufferSize() {
 return dataStreamMaxBufferSize;
 }

 public void setDataStreamMaxBufferSize(int dataStreamMaxBufferSize) {
 this.dataStreamMaxBufferSize = dataStreamMaxBufferSize;
 }

 public boolean isStreamBufferFlushDelay() {
 return streamBufferFlushDelay;
 }
Expand DownExpand Up@@ -227,4 +250,12 @@ public void setChecksumVerify(boolean checksumVerify) {
 public int getBufferIncrement() {
 return bufferIncrement;
 }

 public long getDataStreamBufferFlushSize() {
 return dataStreamBufferFlushSize;
 }

 public void setDataStreamBufferFlushSize(long dataStreamBufferFlushSize) {
 this.dataStreamBufferFlushSize = dataStreamBufferFlushSize;
 }
}

29 changes: 23 additions & 6 deletions
...p-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -257,13 +257,30 @@ public void write(ByteBuffer b, int off, int len) throws IOException {
 if (len == 0) {
 return;
 }
 int curLen = len;
 // set limit on the number of bytes that a ByteBuffer(StreamBuffer) can hold
 int maxBufferLen = config.getDataStreamMaxBufferSize();
 while (curLen > 0) {
 int writeLen = Math.min(curLen, maxBufferLen);
 final StreamBuffer buf = new StreamBuffer(b, off, writeLen);
 off += writeLen;
 bufferList.add(buf);
 writeChunkToContainer(buf.duplicate());
 curLen -= writeLen;
 writtenDataLength += writeLen;
 doFlushIfNeeded();
 }
 }

 final StreamBuffer buf = new StreamBuffer(b, off, len);
 bufferList.add(buf);

 writeChunkToContainer(buf.duplicate());

 writtenDataLength += len;
 private void doFlushIfNeeded() throws IOException {
 Preconditions.checkArgument(config.getDataStreamBufferFlushSize() > config
 .getDataStreamMaxBufferSize());
 long boundary = config.getDataStreamBufferFlushSize() / config
 .getDataStreamMaxBufferSize();
 if (bufferList.size() % boundary == 0) {
 updateFlushLength();
 executePutBlock(false, false);
@captainzmccaptainzmcNov 30, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did we need add updateFlushLength(), waitOnFlushFutures() and watchForCommit(false) here? Just as what we did in handleFlush. Currently we only releaseBuffers after watchForCommit.
@sadanand48sadanand48Nov 30, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @captainzmc for the comment. The plan here is to execute a putBlock at 32MB and not block the client right away by waiting for the flush futures and calling watchForCommit. We will introduce another boundary say at 64MB to check whether atleast 32MB is flushed and committed to all datanodes i.e watchForCommit() and waitForFlushFutures() will occur at 64mb. (I will add this is in a different patch after this PR)
@captainzmccaptainzmcNov 30, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for explaining, make sense.
 }
 }

 private void updateFlushLength() {
Expand Down

12 changes: 12 additions & 0 deletions
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -320,6 +320,8 @@ abstract class Builder {
 protected Optional<Integer> chunkSize = Optional.empty();
 protected OptionalInt streamBufferSize = OptionalInt.empty();
 protected Optional<Long> streamBufferFlushSize = Optional.empty();
 protected Optional<Long> dataStreamBufferFlushSize= Optional.empty();
 protected OptionalInt dataStreamMaxBufferSize = OptionalInt.empty();
 protected Optional<Long> streamBufferMaxSize = Optional.empty();
 protected Optional<Long> blockSize = Optional.empty();
 protected Optional<StorageUnit> streamBufferSizeUnit = Optional.empty();
Expand DownExpand Up@@ -553,6 +555,16 @@ public Builder setStreamBufferMaxSize(long size) {
 return this;
 }

 public Builder setDataStreamBufferMaxSize(int size) {
 dataStreamMaxBufferSize = OptionalInt.of(size);
 return this;
 }

 public Builder setDataStreamBufferFlushize(long size) {
 dataStreamBufferFlushSize = Optional.of(size);
 return this;
 }

 /**
 * Sets the block size for stream buffer.
 *
Expand Down

11 changes: 11 additions & 0 deletions
...op-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -658,6 +658,12 @@ protected void initializeConfiguration() throws IOException {
 if (!streamBufferMaxSize.isPresent()) {
 streamBufferMaxSize = Optional.of(2 * streamBufferFlushSize.get());
 }
 if (!dataStreamBufferFlushSize.isPresent()) {
 dataStreamBufferFlushSize = Optional.of((long) 4 * chunkSize.get());
 }
 if (!dataStreamMaxBufferSize.isPresent()) {
 dataStreamMaxBufferSize = OptionalInt.of(chunkSize.get());
 }
 if (!blockSize.isPresent()) {
 blockSize = Optional.of(2 * streamBufferMaxSize.get());
 }
Expand All@@ -674,6 +680,11 @@ protected void initializeConfiguration() throws IOException {
 streamBufferSizeUnit.get().toBytes(streamBufferMaxSize.get())));
 clientConfig.setStreamBufferFlushSize(Math.round(
 streamBufferSizeUnit.get().toBytes(streamBufferFlushSize.get())));
 clientConfig.setDataStreamBufferFlushSize(Math.round(
 streamBufferSizeUnit.get().toBytes(dataStreamBufferFlushSize.get())));
 clientConfig.setDataStreamMaxBufferSize((int) Math.round(
 streamBufferSizeUnit.get()
 .toBytes(dataStreamMaxBufferSize.getAsInt())));
 conf.setFromObject(clientConfig);

 conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
Expand Down

34 changes: 34 additions & 0 deletions
...tion-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,10 @@
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientMetrics;
import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
import org.apache.hadoop.ozone.MiniOzoneCluster;
Expand DownExpand Up@@ -101,6 +104,8 @@ public static void init() throws Exception {
 .setChunkSize(chunkSize)
 .setStreamBufferFlushSize(flushSize)
 .setStreamBufferMaxSize(maxFlushSize)
 .setDataStreamBufferFlushize(maxFlushSize)
 .setDataStreamBufferMaxSize(chunkSize)
 .setStreamBufferSizeUnit(StorageUnit.BYTES)
 .build();
 cluster.waitForClusterToBeReady();
Expand DownExpand Up@@ -186,6 +191,35 @@ private void testWriteWithFailure(int dataLength) throws Exception {
 validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
 }

 @Test
 public void testPutBlockAtBoundary() throws Exception {
 int dataLength = 500;
 XceiverClientMetrics metrics =
 XceiverClientManager.getXceiverClientMetrics();
 long putBlockCount = metrics.getContainerOpCountMetrics(
 ContainerProtos.Type.PutBlock);
 long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
 ContainerProtos.Type.PutBlock);
 String keyName = getKeyName();
 OzoneDataStreamOutput key = createKey(
 keyName, ReplicationType.RATIS, 0);
 byte[] data =
 ContainerTestHelper.getFixedLengthString(keyString, dataLength)
 .getBytes(UTF_8);
 key.write(ByteBuffer.wrap(data));
 Assert.assertTrue(
 metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
 <= pendingPutBlockCount + 1);
 key.close();
 // Since data length is 500 , first putBlock will be at 400(flush boundary)
 // and the other at 500
 Assert.assertTrue(
 metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
 == putBlockCount + 2);
 validateData(keyName, data);
 }


 private OzoneDataStreamOutput createKey(String keyName, ReplicationType type,
 long size) throws Exception {
 return TestHelper.createStreamKey(
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.#2866

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bshashikant
merged 6 commits into

from

Dec 1, 2021
 Merged


HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.
#2866
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
0bfe67d

HDDS-5851.Define a PutBlock/maxBuffer fixed boundary for streaming wr…

Nov 23, 2021
3b57efb

edit description

Nov 23, 2021
518d1e1

fix test

Nov 23, 2021
b7164cd

address comment

Nov 30, 2021
90cb0ad

address comment

Nov 30, 2021
47e3393

trigger new CI check

Nov 30, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


31 changes: 31 additions & 0 deletions
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,21 @@ public class OzoneClientConfig {
 tags = ConfigTag.CLIENT)
 private int streamBufferSize = 4 * 1024 * 1024;

 @Config(key = "datastream.max.buffer.size",
 defaultValue = "4MB",
 type = ConfigType.SIZE,
 description = "The maximum size of the ByteBuffer "
Comment thread
sadanand48 marked this conversation as resolved.


Outdated

 + "(used via ratis streaming)",
 tags = ConfigTag.CLIENT)
 private int dataStreamMaxBufferSize = 4 * 1024 * 1024;

 @Config(key = "datastream.buffer.flush.size",
 defaultValue = "16MB",
 type = ConfigType.SIZE,
 description = "The boundary at which putBlock is executed",
 tags = ConfigTag.CLIENT)
 private long dataStreamBufferFlushSize = 16 * 1024 * 1024;

 @Config(key = "stream.buffer.increment",
Comment thread
bshashikant marked this conversation as resolved.

 defaultValue = "0B",
 type = ConfigType.SIZE,
Expand DownExpand Up@@ -168,6 +183,14 @@ public void setStreamBufferSize(int streamBufferSize) {
 this.streamBufferSize = streamBufferSize;
 }

 public int getDataStreamMaxBufferSize() {
 return dataStreamMaxBufferSize;
 }

 public void setDataStreamMaxBufferSize(int dataStreamMaxBufferSize) {
 this.dataStreamMaxBufferSize = dataStreamMaxBufferSize;
 }

 public boolean isStreamBufferFlushDelay() {
 return streamBufferFlushDelay;
 }
Expand DownExpand Up@@ -227,4 +250,12 @@ public void setChecksumVerify(boolean checksumVerify) {
 public int getBufferIncrement() {
 return bufferIncrement;
 }

 public long getDataStreamBufferFlushSize() {
 return dataStreamBufferFlushSize;
 }

 public void setDataStreamBufferFlushSize(long dataStreamBufferFlushSize) {
 this.dataStreamBufferFlushSize = dataStreamBufferFlushSize;
 }
}

29 changes: 23 additions & 6 deletions
...p-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -257,13 +257,30 @@ public void write(ByteBuffer b, int off, int len) throws IOException {
 if (len == 0) {
 return;
 }
 int curLen = len;
 // set limit on the number of bytes that a ByteBuffer(StreamBuffer) can hold
 int maxBufferLen = config.getDataStreamMaxBufferSize();
 while (curLen > 0) {
 int writeLen = Math.min(curLen, maxBufferLen);
 final StreamBuffer buf = new StreamBuffer(b, off, writeLen);
 off += writeLen;
 bufferList.add(buf);
 writeChunkToContainer(buf.duplicate());
 curLen -= writeLen;
 writtenDataLength += writeLen;
 doFlushIfNeeded();
 }
 }

 final StreamBuffer buf = new StreamBuffer(b, off, len);
 bufferList.add(buf);

 writeChunkToContainer(buf.duplicate());

 writtenDataLength += len;
 private void doFlushIfNeeded() throws IOException {
 Preconditions.checkArgument(config.getDataStreamBufferFlushSize() > config
 .getDataStreamMaxBufferSize());
 long boundary = config.getDataStreamBufferFlushSize() / config
 .getDataStreamMaxBufferSize();
 if (bufferList.size() % boundary == 0) {
 updateFlushLength();
 executePutBlock(false, false);
@captainzmccaptainzmcNov 30, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did we need add updateFlushLength(), waitOnFlushFutures() and watchForCommit(false) here? Just as what we did in handleFlush. Currently we only releaseBuffers after watchForCommit.
@sadanand48sadanand48Nov 30, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @captainzmc for the comment. The plan here is to execute a putBlock at 32MB and not block the client right away by waiting for the flush futures and calling watchForCommit. We will introduce another boundary say at 64MB to check whether atleast 32MB is flushed and committed to all datanodes i.e watchForCommit() and waitForFlushFutures() will occur at 64mb. (I will add this is in a different patch after this PR)
@captainzmccaptainzmcNov 30, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for explaining, make sense.
 }
 }

 private void updateFlushLength() {
Expand Down

12 changes: 12 additions & 0 deletions
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -320,6 +320,8 @@ abstract class Builder {
 protected Optional<Integer> chunkSize = Optional.empty();
 protected OptionalInt streamBufferSize = OptionalInt.empty();
 protected Optional<Long> streamBufferFlushSize = Optional.empty();
 protected Optional<Long> dataStreamBufferFlushSize= Optional.empty();
 protected OptionalInt dataStreamMaxBufferSize = OptionalInt.empty();
 protected Optional<Long> streamBufferMaxSize = Optional.empty();
 protected Optional<Long> blockSize = Optional.empty();
 protected Optional<StorageUnit> streamBufferSizeUnit = Optional.empty();
Expand DownExpand Up@@ -553,6 +555,16 @@ public Builder setStreamBufferMaxSize(long size) {
 return this;
 }

 public Builder setDataStreamBufferMaxSize(int size) {
 dataStreamMaxBufferSize = OptionalInt.of(size);
 return this;
 }

 public Builder setDataStreamBufferFlushize(long size) {
 dataStreamBufferFlushSize = Optional.of(size);
 return this;
 }

 /**
 * Sets the block size for stream buffer.
 *
Expand Down

11 changes: 11 additions & 0 deletions
...op-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -658,6 +658,12 @@ protected void initializeConfiguration() throws IOException {
 if (!streamBufferMaxSize.isPresent()) {
 streamBufferMaxSize = Optional.of(2 * streamBufferFlushSize.get());
 }
 if (!dataStreamBufferFlushSize.isPresent()) {
 dataStreamBufferFlushSize = Optional.of((long) 4 * chunkSize.get());
 }
 if (!dataStreamMaxBufferSize.isPresent()) {
 dataStreamMaxBufferSize = OptionalInt.of(chunkSize.get());
 }
 if (!blockSize.isPresent()) {
 blockSize = Optional.of(2 * streamBufferMaxSize.get());
 }
Expand All@@ -674,6 +680,11 @@ protected void initializeConfiguration() throws IOException {
 streamBufferSizeUnit.get().toBytes(streamBufferMaxSize.get())));
 clientConfig.setStreamBufferFlushSize(Math.round(
 streamBufferSizeUnit.get().toBytes(streamBufferFlushSize.get())));
 clientConfig.setDataStreamBufferFlushSize(Math.round(
 streamBufferSizeUnit.get().toBytes(dataStreamBufferFlushSize.get())));
 clientConfig.setDataStreamMaxBufferSize((int) Math.round(
 streamBufferSizeUnit.get()
 .toBytes(dataStreamMaxBufferSize.getAsInt())));
 conf.setFromObject(clientConfig);

 conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
Expand Down

34 changes: 34 additions & 0 deletions
...tion-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,10 @@
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientMetrics;
import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
import org.apache.hadoop.ozone.MiniOzoneCluster;
Expand DownExpand Up@@ -101,6 +104,8 @@ public static void init() throws Exception {
 .setChunkSize(chunkSize)
 .setStreamBufferFlushSize(flushSize)
 .setStreamBufferMaxSize(maxFlushSize)
 .setDataStreamBufferFlushize(maxFlushSize)
 .setDataStreamBufferMaxSize(chunkSize)
 .setStreamBufferSizeUnit(StorageUnit.BYTES)
 .build();
 cluster.waitForClusterToBeReady();
Expand DownExpand Up@@ -186,6 +191,35 @@ private void testWriteWithFailure(int dataLength) throws Exception {
 validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
 }

 @Test
 public void testPutBlockAtBoundary() throws Exception {
 int dataLength = 500;
 XceiverClientMetrics metrics =
 XceiverClientManager.getXceiverClientMetrics();
 long putBlockCount = metrics.getContainerOpCountMetrics(
 ContainerProtos.Type.PutBlock);
 long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
 ContainerProtos.Type.PutBlock);
 String keyName = getKeyName();
 OzoneDataStreamOutput key = createKey(
 keyName, ReplicationType.RATIS, 0);
 byte[] data =
 ContainerTestHelper.getFixedLengthString(keyString, dataLength)
 .getBytes(UTF_8);
 key.write(ByteBuffer.wrap(data));
 Assert.assertTrue(
 metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
 <= pendingPutBlockCount + 1);
 key.close();
 // Since data length is 500 , first putBlock will be at 400(flush boundary)
 // and the other at 500
 Assert.assertTrue(
 metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
 == putBlockCount + 2);
 validateData(keyName, data);
 }


 private OzoneDataStreamOutput createKey(String keyName, ReplicationType type,
 long size) throws Exception {
 return TestHelper.createStreamKey(
Expand Down