') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes. by sadanand48 · Pull Request #2866 · apache/ozone · GitHub
Skip to content

HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes. - #2866

Merged
bshashikant merged 6 commits into
apache:HDDS-4454from
sadanand48:HDDS-5851
Dec 1, 2021
Merged

HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.#2866
bshashikant merged 6 commits into
apache:HDDS-4454from
sadanand48:HDDS-5851

Conversation

@sadanand48

@sadanand48sadanand48 commented Nov 23, 2021

Copy link
Copy Markdown
Contributor

What changes were proposed in this pull request?

Currently, putBlock call gets executed only during close() when a block is written. The aim here is to have a putBlock boundary similar to async path where the putBlock gets executed which updates the key length and help reduce some memory on client. The default boundary here is set at 32 MB where a putBlock is executed. Also a limit is set on the ByteBuffer size the client can send here (4mb).

What is the link to the Apache JIRA

https://issues.apache.org/jira/browse/HDDS-5851

How was this patch tested?

Unit test

@sadanand48
sadanand48 marked this pull request as ready for review November 23, 2021 11:49
bshashikant
bshashikant reviewed Nov 29, 2021
View reviewed changes
Comment threadhadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java Outdated
captainzmc
captainzmc reviewed Nov 30, 2021
View reviewed changes
Comment threadhadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java Outdated

@captainzmccaptainzmcNov 30, 2021

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did we need add updateFlushLength(), waitOnFlushFutures() and watchForCommit(false) here? Just as what we did in handleFlush. Currently we only releaseBuffers after watchForCommit.

@sadanand48sadanand48Nov 30, 2021

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @captainzmc for the comment. The plan here is to execute a putBlock at 32MB and not block the client right away by waiting for the flush futures and calling watchForCommit. We will introduce another boundary say at 64MB to check whether atleast 32MB is flushed and committed to all datanodes i.e watchForCommit() and waitForFlushFutures() will occur at 64mb. (I will add this is in a different patch after this PR)

@captainzmccaptainzmcNov 30, 2021

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for explaining, make sense.

Sadanand Shenoy added 4 commits November 30, 2021 14:59
Sadanand Shenoy
HDDS-5851.Define a PutBlock/maxBuffer fixed boundary for streaming wr…
0bfe67d
…ites.
(cherry picked from commit 4406ecf99c38b14c6a0260a933df2fde954e5b99)
Sadanand Shenoy
edit description
3b57efb
(cherry picked from commit 1fcac2916ce5c1b1060edcef3714c8ec43bc6605)
Sadanand Shenoy
fix test
518d1e1
(cherry picked from commit a0eca8d0b1daa04bd0d9c1850584edf935b16162)
Sadanand Shenoy
address comment
b7164cd
(cherry picked from commit f0858e58cb214c1f5792ff54f1d26b72066359d1)
@sadanand48sadanand48 changed the title HDDS-5851. Define a PutBlock/maxBuffer fixed boundary for streaming writes.HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes.Nov 30, 2021
captainzmc
captainzmc approved these changes Nov 30, 2021
View reviewed changes

@captainzmccaptainzmc left a comment

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1 The change looks good.

@bshashikant
bshashikant merged commit 99ddf5a into apache:HDDS-4454Dec 1, 2021
@bshashikant

bshashikant commented Dec 1, 2021

Copy link
Copy Markdown
Contributor

Thanks @sadanand48 for the contribution.

captainzmc pushed a commit that referenced this pull request Dec 20, 2021
@sadanand48@captainzmc
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
89a12ad
…ry for streaming writes. (#2866)
szetszwo pushed a commit that referenced this pull request Dec 30, 2021
@sadanand48@szetszwo
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
bf634f3
…ry for streaming writes. (#2866)
szetszwo pushed a commit that referenced this pull request Jan 20, 2022
@sadanand48@szetszwo
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
f42c10a
…ry for streaming writes. (#2866)
captainzmc pushed a commit that referenced this pull request Feb 9, 2022
@sadanand48@captainzmc
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
fd5af78
…ry for streaming writes. (#2866)
szetszwo pushed a commit that referenced this pull request Feb 16, 2022
@sadanand48@szetszwo
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
e1f1546
…ry for streaming writes. (#2866)
szetszwo pushed a commit that referenced this pull request Mar 15, 2022
@sadanand48@szetszwo
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
d75ea44
…ry for streaming writes. (#2866)
szetszwo pushed a commit that referenced this pull request Mar 24, 2022
@sadanand48@szetszwo
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
3722393
…ry for streaming writes. (#2866)
szetszwo pushed a commit to szetszwo/ozone that referenced this pull request May 6, 2022
@sadanand48@szetszwo
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
4771a50
…ry for streaming writes. (apache#2866)
szetszwo pushed a commit that referenced this pull request May 13, 2022
@sadanand48@szetszwo
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
de79a78
…ry for streaming writes. (#2866)
szetszwo pushed a commit that referenced this pull request May 24, 2022
@sadanand48@szetszwo
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
2ef3b47
…ry for streaming writes. (#2866)
szetszwo pushed a commit that referenced this pull request Jun 9, 2022
@sadanand48@szetszwo
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
5804f22
…ry for streaming writes. (#2866)
captainzmc pushed a commit to captainzmc/hadoop-ozone that referenced this pull request Jul 4, 2022
@sadanand48@captainzmc
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
7a6ea80
…ry for streaming writes. (apache#2866)
szetszwo pushed a commit that referenced this pull request Oct 25, 2022
@sadanand48@szetszwo
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
eb441a8
…ry for streaming writes. (#2866)
(cherry picked from commit 7a6ea80)
(cherry picked from commit 3b9141e18d42555ccc62f7c9718b96fd8b88e55f)
szetszwo pushed a commit that referenced this pull request Nov 7, 2022
@sadanand48@szetszwo
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
9d5ccb3
…ry for streaming writes. (#2866)
(cherry picked from commit 7a6ea80)
(cherry picked from commit 3b9141e18d42555ccc62f7c9718b96fd8b88e55f)
(cherry picked from commit eb441a8)
szetszwo pushed a commit that referenced this pull request Dec 1, 2022
@sadanand48@szetszwo
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
2a56a60
…ry for streaming writes. (#2866)
(cherry picked from commit 9d5ccb3)
szetszwo pushed a commit that referenced this pull request Dec 16, 2022
@sadanand48@szetszwo
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
9a2de9a
…ry for streaming writes. (#2866)
(cherry picked from commit 9d5ccb3)
nishitpatira pushed a commit to nishitpatira/ozone that referenced this pull request Dec 16, 2022
@sadanand48Nishit Patira
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed bounda…
2d5ff0a
…ry for streaming writes. (apache#2866)
(cherry picked from commit 9d5ccb3)
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@captainzmccaptainzmccaptainzmc approved these changes
@bshashikantbshashikantbshashikant approved these changes
@szetszwoszetszwoAwaiting requested review from szetszwo

Assignees

No one assigned

Labels

None yet

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

3 participants

@sadanand48@bshashikant@captainzmc