Commit a46ddad

Browse files
jasnelladuh95
authored andcommitted
stream: refine the stream/iter backpressure
In `Writer`, there are two queues that matter: the slot queue and the pending queue. The sync `writeSync`/`writevSync` only use the slot queue. If writes cannot be accepted directly into slots, they return `false`. The sync methods *never* enqueue into the pending queue. The async `write`/`writev` will first attempt to add to the slot queue; if it is full, then it will attempt to add to the pending queue; if that is also full, the backpressure policy kicks in. Signed-off-by: James M Snell <jasnell@gmail.com> PR-URL: #63697 Backport-PR-URL: #64675 Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day>
1 parent 557bf99 commit a46ddad

9 files changed

Lines changed: 185 additions & 201 deletions

File tree

β€Ždoc/api/stream_iter.mdβ€Ž

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1649,7 +1649,7 @@ Creates a classic [`stream.Writable`][] backed by a stream/iter Writer.
16491649

16501650
Each `_write()` / `_writev()` call attempts the Writer's synchronous method
16511651
first (`writeSync` / `writevSync`), falling back to the async method if the
1652-
sync path returns `false` or throws. Similarly, `_final()` tries `endSync()`
1652+
sync path returns `false`. Similarly, `_final()` tries `endSync()`
16531653
before `end()`. When the sync path succeeds, the callback is deferred via
16541654
`queueMicrotask` to preserve the async resolution contract.
16551655

β€Žlib/internal/streams/iter/classic.jsβ€Ž

Lines changed: 16 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ const {
5757
const{
5858
toAsyncStreamable: kToAsyncStreamable,
5959
kValidatedSource,
60-
kSyncWriteAccepted,
6160
drainableProtocol,
6261
}=require('internal/streams/iter/types');
6362

@@ -765,41 +764,11 @@ function toWritable(writer) {
765764
consthasEndSync=hasEnd&&
766765
typeofwriter.endSync==='function';
767766
consthasFail=typeofwriter.fail==='function';
768-
consthasSyncWriteAccepted=
769-
typeofwriter[kSyncWriteAccepted]==='function';
770-
771-
functionsyncWriteAccepted(){
772-
returnhasSyncWriteAccepted&&writer[kSyncWriteAccepted]();
773-
}
774-
775-
functionfinishAfterSyncBackpressure(cb){
776-
letondrain;
777-
try{
778-
if(typeofwriter[drainableProtocol]==='function'){
779-
ondrain=writer[drainableProtocol]();
780-
}
781-
}catch(err){
782-
cb(err);
783-
return;
784-
}
785-
if(ondrain!==null&&ondrain!==undefined){
786-
PromisePrototypeThen(ondrain,(drained)=>{
787-
if(drained===false){
788-
cb(newERR_INVALID_STATE.TypeError('Stream closed by consumer'));
789-
return;
790-
}
791-
cb();
792-
},cb);
793-
return;
794-
}
795-
queueMicrotask(cb);
796-
}
797-
798767
// Try-sync-first pattern: attempt the synchronous method and fall back to the
799-
// async method if it returns false without accepting the data, or if it
800-
// throws. When the sync path succeeds, the callback is deferred via
801-
// queueMicrotask to preserve the async resolution contract that Writable
802-
// internals expect from _write/_writev/_final callbacks.
768+
// async method if it returns false (data not accepted synchronously).
769+
// When the sync path succeeds, the callback is deferred via queueMicrotask
770+
// to preserve the async resolution contract that Writable internals expect
771+
// from _write/_writev/_final callbacks.
803772

804773
function_write(chunk,encoding,cb){
805774
constbytes=typeofchunk==='string' ?
@@ -810,13 +779,10 @@ function toWritable(writer) {
810779
queueMicrotask(cb);
811780
return;
812781
}
813-
if(syncWriteAccepted()){
814-
// The chunk was accepted; false only signaled backpressure.
815-
finishAfterSyncBackpressure(cb);
816-
return;
817-
}
818-
}catch{
819-
// Sync path threw -- fall through to async.
782+
// WriteSync returned false: not accepted, fall through to async.
783+
}catch(err){
784+
cb(err);
785+
return;
820786
}
821787
}
822788
try{
@@ -839,13 +805,10 @@ function toWritable(writer) {
839805
queueMicrotask(cb);
840806
return;
841807
}
842-
if(syncWriteAccepted()){
843-
// The chunks were accepted; false only signaled backpressure.
844-
finishAfterSyncBackpressure(cb);
845-
return;
846-
}
847-
}catch{
848-
// Sync path threw -- fall through to async.
808+
// WritevSync returned false: not accepted, fall through to async.
809+
}catch(err){
810+
cb(err);
811+
return;
849812
}
850813
}
851814
try{
@@ -867,8 +830,10 @@ function toWritable(writer) {
867830
queueMicrotask(cb);
868831
return;
869832
}
870-
}catch{
871-
// Sync path threw -- fall through to async.
833+
// Result < 0: can't end synchronously, fall through to async.
834+
}catch(err){
835+
cb(err);
836+
return;
872837
}
873838
}
874839
try{

β€Žlib/internal/streams/iter/consumers.jsβ€Ž

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,7 @@ function merge(...args) {
480480
);
481481
}
482482

483+
letprimaryError;
483484
try{
484485
while(activeCount>0||ready.length>0){
485486
signal?.throwIfAborted();
@@ -500,22 +501,46 @@ function merge(...args) {
500501
});
501502
}
502503
}
504+
}catch(err){
505+
primaryError=err;
503506
}finally{
504-
// Clean up: return all iterators
505-
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
506-
if(iterator.return){
507-
try{
508-
awaititerator.return();
509-
}catch{
510-
// Ignore return errors
511-
}
512-
}
513-
});
507+
// Clean up: return all iterators. Cleanup errors are not
508+
// swallowed - a broken iterator.return() (e.g., failing to
509+
// release a resource) should be visible to the caller.
510+
awaitcleanupIterators(iterators,primaryError);
514511
}
515512
},
516513
};
517514
}
518515

516+
asyncfunctioncleanupIterators(iterators,primaryError){
517+
letcleanupError;
518+
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
519+
if(iterator.return){
520+
try{
521+
awaititerator.return();
522+
}catch(err){
523+
// Keep the first cleanup error encountered.
524+
cleanupError??=err;
525+
}
526+
}
527+
});
528+
if(cleanupError!==undefined){
529+
if(primaryError!==undefined){
530+
// Both a primary error and a cleanup error occurred.
531+
// Wrap in SuppressedError so neither is lost:
532+
// .error = primaryError, .suppressed = cleanupError.
533+
// eslint-disable-next-line no-restricted-syntax
534+
thrownewSuppressedError(primaryError,cleanupError);
535+
}
536+
// No primary error - the cleanup error is the only error.
537+
throwcleanupError;
538+
}
539+
if(primaryError!==undefined){
540+
throwprimaryError;
541+
}
542+
}
543+
519544
module.exports={
520545
array,
521546
arrayBuffer,

β€Žlib/internal/streams/iter/pull.jsβ€Ž

Lines changed: 5 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,6 @@ const {
6363
}=require('internal/streams/iter/utils');
6464

6565
const{
66-
drainableProtocol,
67-
kSyncWriteAcceptedOnFalse,
6866
kValidatedSource,
6967
kValidatedTransform,
7068
toAsyncStreamable,
@@ -863,18 +861,6 @@ async function* createAsyncPipeline(source, transforms, signal) {
863861
}
864862
}
865863

866-
/**
867-
* Check if a false sync write result means accepted backpressure.
868-
* @param {object} writer - The writer whose sync method returned.
869-
* @param {*} result - The return value from writeSync() or writevSync().
870-
* @returns {boolean}
871-
*/
872-
functionisAcceptedSyncWriteBackpressure(writer,result){
873-
returnresult===false&&
874-
writer[kSyncWriteAcceptedOnFalse]===true&&
875-
writer.desiredSize===0;
876-
}
877-
878864
// =============================================================================
879865
// Public API: pull() and pullSync()
880866
// =============================================================================
@@ -963,9 +949,7 @@ function pipeToSync(source, ...args) {
963949
break;
964950
}
965951
if(hasWritevSync&&batch.length>1){
966-
constresult=writer.writevSync(batch);
967-
if(result===false&&
968-
!isAcceptedSyncWriteBackpressure(writer,result)){
952+
if(writer.writevSync(batch)===false){
969953
break;
970954
}
971955
for(leti=0;i<batch.length;i++){
@@ -974,9 +958,7 @@ function pipeToSync(source, ...args) {
974958
}else{
975959
for(leti=0;i<batch.length;i++){
976960
constchunk=batch[i];
977-
constresult=writer.writeSync(chunk);
978-
if(result===false&&
979-
!isAcceptedSyncWriteBackpressure(writer,result)){
961+
if(writer.writeSync(chunk)===false){
980962
canContinue=false;
981963
break;
982964
}
@@ -1027,28 +1009,13 @@ async function pipeTo(source, ...args) {
10271009
consthasWritevSync=typeofwriter.writevSync==='function';
10281010
consthasEndSync=typeofwriter.endSync==='function';
10291011

1030-
functionwaitForSyncBackpressure(){
1031-
constondrain=writer[drainableProtocol];
1032-
returnondrain?.call(writer);
1033-
}
1034-
1035-
asyncfunctionwriteBatchAfterAcceptedBackpressure(batch,startIndex){
1036-
awaitwaitForSyncBackpressure();
1037-
awaitwriteBatchAsyncFallback(batch,startIndex);
1038-
}
1039-
10401012
// Async fallback for writeBatch when sync write fails partway through.
10411013
// Continues writing from batch[startIndex] using async write().
10421014
asyncfunctionwriteBatchAsyncFallback(batch,startIndex){
10431015
for(leti=startIndex;i<batch.length;i++){
10441016
constchunk=batch[i];
1045-
constresult=hasWriteSync&&writer.writeSync(chunk);
1046-
if(result){
1017+
if(hasWriteSync&&writer.writeSync(chunk)){
10471018
// Sync retry succeeded
1048-
}elseif(isAcceptedSyncWriteBackpressure(writer,result)){
1049-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1050-
awaitwaitForSyncBackpressure();
1051-
continue;
10521019
}else{
10531020
constresult=writer.write(
10541021
chunk,signal ? {__proto__: null, signal } : undefined);
@@ -1065,14 +1032,7 @@ async function pipeTo(source, ...args) {
10651032
// is required. Callers must check: const p = writeBatch(b); if (p) await p;
10661033
functionwriteBatch(batch){
10671034
if(hasWritev&&batch.length>1){
1068-
constresult=hasWritevSync&&writer.writevSync(batch);
1069-
if(!result){
1070-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1071-
for(leti=0;i<batch.length;i++){
1072-
totalBytes+=TypedArrayPrototypeGetByteLength(batch[i]);
1073-
}
1074-
returnwaitForSyncBackpressure();
1075-
}
1035+
if(!hasWritevSync||!writer.writevSync(batch)){
10761036
constopts=signal ? {__proto__: null, signal } : undefined;
10771037
constwritevResult=writer.writev(batch,opts);
10781038
if(writevResult===undefined){
@@ -1094,14 +1054,8 @@ async function pipeTo(source, ...args) {
10941054
}
10951055
for(leti=0;i<batch.length;i++){
10961056
constchunk=batch[i];
1097-
constresult=hasWriteSync&&writer.writeSync(chunk);
1098-
if(!result){
1099-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1100-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1101-
returnwriteBatchAfterAcceptedBackpressure(batch,i+1);
1102-
}
1057+
if(!hasWriteSync||!writer.writeSync(chunk)){
11031058
// Sync path failed at index i - fall back to async for the rest.
1104-
// Count bytes for chunks already written synchronously (0..i-1).
11051059
returnwriteBatchAsyncFallback(batch,i);
11061060
}
11071061
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);

β€Žlib/internal/streams/iter/push.jsβ€Ž

Lines changed: 2 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@ const {
3232

3333
const{
3434
drainableProtocol,
35-
kSyncWriteAccepted,
36-
kSyncWriteAcceptedOnFalse,
3735
}=require('internal/streams/iter/types');
3836

3937
const{
@@ -364,19 +362,6 @@ class PushQueue {
364362
this.#pendingEnd =pending;
365363
}
366364

367-
/**
368-
* Force-enqueue chunks into the slots buffer, bypassing capacity checks.
369-
* Used by PushWriter.writeSync() for 'block' policy where the data is
370-
* accepted but false is returned as a backpressure signal.
371-
*/
372-
forceEnqueue(chunks){
373-
this.#slots.push(chunks);
374-
for(leti=0;i<chunks.length;i++){
375-
this.#bytesWritten +=TypedArrayPrototypeGetByteLength(chunks[i]);
376-
}
377-
this.#resolvePendingReads();
378-
}
379-
380365
/**
381366
* Wait for backpressure to clear (desiredSize > 0).
382367
* @returns {Promise<void>}
@@ -558,16 +543,11 @@ class PushQueue {
558543

559544
classPushWriter{
560545
#queue;
561-
#syncWriteAccepted =false;
562546

563547
constructor(queue){
564548
this.#queue =queue;
565549
}
566550

567-
[kSyncWriteAccepted](){
568-
returnthis.#syncWriteAccepted;
569-
}
570-
571551
[drainableProtocol](){
572552
constdesired=this.desiredSize;
573553
if(desired===null)returnnull;
@@ -579,10 +559,6 @@ class PushWriter {
579559
returnthis.#queue.desiredSize;
580560
}
581561

582-
get[kSyncWriteAcceptedOnFalse](){
583-
returnthis.#queue.backpressurePolicy==='block';
584-
}
585-
586562
write(chunk,options){
587563
if(!options?.signal&&this.#queue.canWriteSync()){
588564
constbytes=toUint8Array(chunk);
@@ -607,36 +583,16 @@ class PushWriter {
607583
}
608584

609585
writeSync(chunk){
610-
this.#syncWriteAccepted =false;
611586
constbytes=toUint8Array(chunk);
612-
constresult=this.#queue.writeSync([bytes]);
613-
if(!result&&this.#queue.backpressurePolicy==='block'&&
614-
this.#queue.desiredSize===0){
615-
// Block policy: force-enqueue and return false as backpressure signal.
616-
// Data IS accepted; false tells caller to slow down.
617-
this.#queue.forceEnqueue([bytes]);
618-
this.#syncWriteAccepted =true;
619-
returnfalse;
620-
}
621-
this.#syncWriteAccepted =result;
622-
returnresult;
587+
returnthis.#queue.writeSync([bytes]);
623588
}
624589

625590
writevSync(chunks){
626-
this.#syncWriteAccepted =false;
627591
if(!ArrayIsArray(chunks)){
628592
thrownewERR_INVALID_ARG_TYPE('chunks','Array',chunks);
629593
}
630594
constbytes=convertChunks(chunks);
631-
constresult=this.#queue.writeSync(bytes);
632-
if(!result&&this.#queue.backpressurePolicy==='block'&&
633-
this.#queue.desiredSize===0){
634-
this.#queue.forceEnqueue(bytes);
635-
this.#syncWriteAccepted =true;
636-
returnfalse;
637-
}
638-
this.#syncWriteAccepted =result;
639-
returnresult;
595+
returnthis.#queue.writeSync(bytes);
640596
}
641597

642598
end(options){

0 commit comments

Comments
Β (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
Skip to content

Commit a46ddad

Browse files
jasnelladuh95
authored andcommitted
stream: refine the stream/iter backpressure
In `Writer`, there are two queues that matter: the slot queue and the pending queue. The sync `writeSync`/`writevSync` only use the slot queue. If writes cannot be accepted directly into slots, they return `false`. The sync methods *never* enqueue into the pending queue. The async `write`/`writev` will first attempt to add to the slot queue; if it is full, then it will attempt to add to the pending queue; if that is also full, the backpressure policy kicks in. Signed-off-by: James M Snell <jasnell@gmail.com> PR-URL: #63697 Backport-PR-URL: #64675 Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day>
1 parent 557bf99 commit a46ddad

9 files changed

Lines changed: 185 additions & 201 deletions

File tree

β€Ždoc/api/stream_iter.mdβ€Ž

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1649,7 +1649,7 @@ Creates a classic [`stream.Writable`][] backed by a stream/iter Writer.
16491649

16501650
Each `_write()` / `_writev()` call attempts the Writer's synchronous method
16511651
first (`writeSync` / `writevSync`), falling back to the async method if the
1652-
sync path returns `false` or throws. Similarly, `_final()` tries `endSync()`
1652+
sync path returns `false`. Similarly, `_final()` tries `endSync()`
16531653
before `end()`. When the sync path succeeds, the callback is deferred via
16541654
`queueMicrotask` to preserve the async resolution contract.
16551655

β€Žlib/internal/streams/iter/classic.jsβ€Ž

Lines changed: 16 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ const {
5757
const{
5858
toAsyncStreamable: kToAsyncStreamable,
5959
kValidatedSource,
60-
kSyncWriteAccepted,
6160
drainableProtocol,
6261
}=require('internal/streams/iter/types');
6362

@@ -765,41 +764,11 @@ function toWritable(writer) {
765764
consthasEndSync=hasEnd&&
766765
typeofwriter.endSync==='function';
767766
consthasFail=typeofwriter.fail==='function';
768-
consthasSyncWriteAccepted=
769-
typeofwriter[kSyncWriteAccepted]==='function';
770-
771-
functionsyncWriteAccepted(){
772-
returnhasSyncWriteAccepted&&writer[kSyncWriteAccepted]();
773-
}
774-
775-
functionfinishAfterSyncBackpressure(cb){
776-
letondrain;
777-
try{
778-
if(typeofwriter[drainableProtocol]==='function'){
779-
ondrain=writer[drainableProtocol]();
780-
}
781-
}catch(err){
782-
cb(err);
783-
return;
784-
}
785-
if(ondrain!==null&&ondrain!==undefined){
786-
PromisePrototypeThen(ondrain,(drained)=>{
787-
if(drained===false){
788-
cb(newERR_INVALID_STATE.TypeError('Stream closed by consumer'));
789-
return;
790-
}
791-
cb();
792-
},cb);
793-
return;
794-
}
795-
queueMicrotask(cb);
796-
}
797-
798767
// Try-sync-first pattern: attempt the synchronous method and fall back to the
799-
// async method if it returns false without accepting the data, or if it
800-
// throws. When the sync path succeeds, the callback is deferred via
801-
// queueMicrotask to preserve the async resolution contract that Writable
802-
// internals expect from _write/_writev/_final callbacks.
768+
// async method if it returns false (data not accepted synchronously).
769+
// When the sync path succeeds, the callback is deferred via queueMicrotask
770+
// to preserve the async resolution contract that Writable internals expect
771+
// from _write/_writev/_final callbacks.
803772

804773
function_write(chunk,encoding,cb){
805774
constbytes=typeofchunk==='string' ?
@@ -810,13 +779,10 @@ function toWritable(writer) {
810779
queueMicrotask(cb);
811780
return;
812781
}
813-
if(syncWriteAccepted()){
814-
// The chunk was accepted; false only signaled backpressure.
815-
finishAfterSyncBackpressure(cb);
816-
return;
817-
}
818-
}catch{
819-
// Sync path threw -- fall through to async.
782+
// WriteSync returned false: not accepted, fall through to async.
783+
}catch(err){
784+
cb(err);
785+
return;
820786
}
821787
}
822788
try{
@@ -839,13 +805,10 @@ function toWritable(writer) {
839805
queueMicrotask(cb);
840806
return;
841807
}
842-
if(syncWriteAccepted()){
843-
// The chunks were accepted; false only signaled backpressure.
844-
finishAfterSyncBackpressure(cb);
845-
return;
846-
}
847-
}catch{
848-
// Sync path threw -- fall through to async.
808+
// WritevSync returned false: not accepted, fall through to async.
809+
}catch(err){
810+
cb(err);
811+
return;
849812
}
850813
}
851814
try{
@@ -867,8 +830,10 @@ function toWritable(writer) {
867830
queueMicrotask(cb);
868831
return;
869832
}
870-
}catch{
871-
// Sync path threw -- fall through to async.
833+
// Result < 0: can't end synchronously, fall through to async.
834+
}catch(err){
835+
cb(err);
836+
return;
872837
}
873838
}
874839
try{

β€Žlib/internal/streams/iter/consumers.jsβ€Ž

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,7 @@ function merge(...args) {
480480
);
481481
}
482482

483+
letprimaryError;
483484
try{
484485
while(activeCount>0||ready.length>0){
485486
signal?.throwIfAborted();
@@ -500,22 +501,46 @@ function merge(...args) {
500501
});
501502
}
502503
}
504+
}catch(err){
505+
primaryError=err;
503506
}finally{
504-
// Clean up: return all iterators
505-
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
506-
if(iterator.return){
507-
try{
508-
awaititerator.return();
509-
}catch{
510-
// Ignore return errors
511-
}
512-
}
513-
});
507+
// Clean up: return all iterators. Cleanup errors are not
508+
// swallowed - a broken iterator.return() (e.g., failing to
509+
// release a resource) should be visible to the caller.
510+
awaitcleanupIterators(iterators,primaryError);
514511
}
515512
},
516513
};
517514
}
518515

516+
asyncfunctioncleanupIterators(iterators,primaryError){
517+
letcleanupError;
518+
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
519+
if(iterator.return){
520+
try{
521+
awaititerator.return();
522+
}catch(err){
523+
// Keep the first cleanup error encountered.
524+
cleanupError??=err;
525+
}
526+
}
527+
});
528+
if(cleanupError!==undefined){
529+
if(primaryError!==undefined){
530+
// Both a primary error and a cleanup error occurred.
531+
// Wrap in SuppressedError so neither is lost:
532+
// .error = primaryError, .suppressed = cleanupError.
533+
// eslint-disable-next-line no-restricted-syntax
534+
thrownewSuppressedError(primaryError,cleanupError);
535+
}
536+
// No primary error - the cleanup error is the only error.
537+
throwcleanupError;
538+
}
539+
if(primaryError!==undefined){
540+
throwprimaryError;
541+
}
542+
}
543+
519544
module.exports={
520545
array,
521546
arrayBuffer,

β€Žlib/internal/streams/iter/pull.jsβ€Ž

Lines changed: 5 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,6 @@ const {
6363
}=require('internal/streams/iter/utils');
6464

6565
const{
66-
drainableProtocol,
67-
kSyncWriteAcceptedOnFalse,
6866
kValidatedSource,
6967
kValidatedTransform,
7068
toAsyncStreamable,
@@ -863,18 +861,6 @@ async function* createAsyncPipeline(source, transforms, signal) {
863861
}
864862
}
865863

866-
/**
867-
* Check if a false sync write result means accepted backpressure.
868-
* @param {object} writer - The writer whose sync method returned.
869-
* @param {*} result - The return value from writeSync() or writevSync().
870-
* @returns {boolean}
871-
*/
872-
functionisAcceptedSyncWriteBackpressure(writer,result){
873-
returnresult===false&&
874-
writer[kSyncWriteAcceptedOnFalse]===true&&
875-
writer.desiredSize===0;
876-
}
877-
878864
// =============================================================================
879865
// Public API: pull() and pullSync()
880866
// =============================================================================
@@ -963,9 +949,7 @@ function pipeToSync(source, ...args) {
963949
break;
964950
}
965951
if(hasWritevSync&&batch.length>1){
966-
constresult=writer.writevSync(batch);
967-
if(result===false&&
968-
!isAcceptedSyncWriteBackpressure(writer,result)){
952+
if(writer.writevSync(batch)===false){
969953
break;
970954
}
971955
for(leti=0;i<batch.length;i++){
@@ -974,9 +958,7 @@ function pipeToSync(source, ...args) {
974958
}else{
975959
for(leti=0;i<batch.length;i++){
976960
constchunk=batch[i];
977-
constresult=writer.writeSync(chunk);
978-
if(result===false&&
979-
!isAcceptedSyncWriteBackpressure(writer,result)){
961+
if(writer.writeSync(chunk)===false){
980962
canContinue=false;
981963
break;
982964
}
@@ -1027,28 +1009,13 @@ async function pipeTo(source, ...args) {
10271009
consthasWritevSync=typeofwriter.writevSync==='function';
10281010
consthasEndSync=typeofwriter.endSync==='function';
10291011

1030-
functionwaitForSyncBackpressure(){
1031-
constondrain=writer[drainableProtocol];
1032-
returnondrain?.call(writer);
1033-
}
1034-
1035-
asyncfunctionwriteBatchAfterAcceptedBackpressure(batch,startIndex){
1036-
awaitwaitForSyncBackpressure();
1037-
awaitwriteBatchAsyncFallback(batch,startIndex);
1038-
}
1039-
10401012
// Async fallback for writeBatch when sync write fails partway through.
10411013
// Continues writing from batch[startIndex] using async write().
10421014
asyncfunctionwriteBatchAsyncFallback(batch,startIndex){
10431015
for(leti=startIndex;i<batch.length;i++){
10441016
constchunk=batch[i];
1045-
constresult=hasWriteSync&&writer.writeSync(chunk);
1046-
if(result){
1017+
if(hasWriteSync&&writer.writeSync(chunk)){
10471018
// Sync retry succeeded
1048-
}elseif(isAcceptedSyncWriteBackpressure(writer,result)){
1049-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1050-
awaitwaitForSyncBackpressure();
1051-
continue;
10521019
}else{
10531020
constresult=writer.write(
10541021
chunk,signal ? {__proto__: null, signal } : undefined);
@@ -1065,14 +1032,7 @@ async function pipeTo(source, ...args) {
10651032
// is required. Callers must check: const p = writeBatch(b); if (p) await p;
10661033
functionwriteBatch(batch){
10671034
if(hasWritev&&batch.length>1){
1068-
constresult=hasWritevSync&&writer.writevSync(batch);
1069-
if(!result){
1070-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1071-
for(leti=0;i<batch.length;i++){
1072-
totalBytes+=TypedArrayPrototypeGetByteLength(batch[i]);
1073-
}
1074-
returnwaitForSyncBackpressure();
1075-
}
1035+
if(!hasWritevSync||!writer.writevSync(batch)){
10761036
constopts=signal ? {__proto__: null, signal } : undefined;
10771037
constwritevResult=writer.writev(batch,opts);
10781038
if(writevResult===undefined){
@@ -1094,14 +1054,8 @@ async function pipeTo(source, ...args) {
10941054
}
10951055
for(leti=0;i<batch.length;i++){
10961056
constchunk=batch[i];
1097-
constresult=hasWriteSync&&writer.writeSync(chunk);
1098-
if(!result){
1099-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1100-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1101-
returnwriteBatchAfterAcceptedBackpressure(batch,i+1);
1102-
}
1057+
if(!hasWriteSync||!writer.writeSync(chunk)){
11031058
// Sync path failed at index i - fall back to async for the rest.
1104-
// Count bytes for chunks already written synchronously (0..i-1).
11051059
returnwriteBatchAsyncFallback(batch,i);
11061060
}
11071061
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);

β€Žlib/internal/streams/iter/push.jsβ€Ž

Lines changed: 2 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@ const {
3232

3333
const{
3434
drainableProtocol,
35-
kSyncWriteAccepted,
36-
kSyncWriteAcceptedOnFalse,
3735
}=require('internal/streams/iter/types');
3836

3937
const{
@@ -364,19 +362,6 @@ class PushQueue {
364362
this.#pendingEnd =pending;
365363
}
366364

367-
/**
368-
* Force-enqueue chunks into the slots buffer, bypassing capacity checks.
369-
* Used by PushWriter.writeSync() for 'block' policy where the data is
370-
* accepted but false is returned as a backpressure signal.
371-
*/
372-
forceEnqueue(chunks){
373-
this.#slots.push(chunks);
374-
for(leti=0;i<chunks.length;i++){
375-
this.#bytesWritten +=TypedArrayPrototypeGetByteLength(chunks[i]);
376-
}
377-
this.#resolvePendingReads();
378-
}
379-
380365
/**
381366
* Wait for backpressure to clear (desiredSize > 0).
382367
* @returns {Promise<void>}
@@ -558,16 +543,11 @@ class PushQueue {
558543

559544
classPushWriter{
560545
#queue;
561-
#syncWriteAccepted =false;
562546

563547
constructor(queue){
564548
this.#queue =queue;
565549
}
566550

567-
[kSyncWriteAccepted](){
568-
returnthis.#syncWriteAccepted;
569-
}
570-
571551
[drainableProtocol](){
572552
constdesired=this.desiredSize;
573553
if(desired===null)returnnull;
@@ -579,10 +559,6 @@ class PushWriter {
579559
returnthis.#queue.desiredSize;
580560
}
581561

582-
get[kSyncWriteAcceptedOnFalse](){
583-
returnthis.#queue.backpressurePolicy==='block';
584-
}
585-
586562
write(chunk,options){
587563
if(!options?.signal&&this.#queue.canWriteSync()){
588564
constbytes=toUint8Array(chunk);
@@ -607,36 +583,16 @@ class PushWriter {
607583
}
608584

609585
writeSync(chunk){
610-
this.#syncWriteAccepted =false;
611586
constbytes=toUint8Array(chunk);
612-
constresult=this.#queue.writeSync([bytes]);
613-
if(!result&&this.#queue.backpressurePolicy==='block'&&
614-
this.#queue.desiredSize===0){
615-
// Block policy: force-enqueue and return false as backpressure signal.
616-
// Data IS accepted; false tells caller to slow down.
617-
this.#queue.forceEnqueue([bytes]);
618-
this.#syncWriteAccepted =true;
619-
returnfalse;
620-
}
621-
this.#syncWriteAccepted =result;
622-
returnresult;
587+
returnthis.#queue.writeSync([bytes]);
623588
}
624589

625590
writevSync(chunks){
626-
this.#syncWriteAccepted =false;
627591
if(!ArrayIsArray(chunks)){
628592
thrownewERR_INVALID_ARG_TYPE('chunks','Array',chunks);
629593
}
630594
constbytes=convertChunks(chunks);
631-
constresult=this.#queue.writeSync(bytes);
632-
if(!result&&this.#queue.backpressurePolicy==='block'&&
633-
this.#queue.desiredSize===0){
634-
this.#queue.forceEnqueue(bytes);
635-
this.#syncWriteAccepted =true;
636-
returnfalse;
637-
}
638-
this.#syncWriteAccepted =result;
639-
returnresult;
595+
returnthis.#queue.writeSync(bytes);
640596
}
641597

642598
end(options){

0 commit comments

Comments
Β (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix"); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Commit a46ddad

Browse files
jasnelladuh95
authored andcommitted
stream: refine the stream/iter backpressure
In `Writer`, there are two queues that matter: the slot queue and the pending queue. The sync `writeSync`/`writevSync` only use the slot queue. If writes cannot be accepted directly into slots, they return `false`. The sync methods *never* enqueue into the pending queue. The async `write`/`writev` will first attempt to add to the slot queue; if it is full, then it will attempt to add to the pending queue; if that is also full, the backpressure policy kicks in. Signed-off-by: James M Snell <jasnell@gmail.com> PR-URL: #63697 Backport-PR-URL: #64675 Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day>
1 parent 557bf99 commit a46ddad

9 files changed

Lines changed: 185 additions & 201 deletions

File tree

β€Ždoc/api/stream_iter.mdβ€Ž

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1649,7 +1649,7 @@ Creates a classic [`stream.Writable`][] backed by a stream/iter Writer.
16491649

16501650
Each `_write()` / `_writev()` call attempts the Writer's synchronous method
16511651
first (`writeSync` / `writevSync`), falling back to the async method if the
1652-
sync path returns `false` or throws. Similarly, `_final()` tries `endSync()`
1652+
sync path returns `false`. Similarly, `_final()` tries `endSync()`
16531653
before `end()`. When the sync path succeeds, the callback is deferred via
16541654
`queueMicrotask` to preserve the async resolution contract.
16551655

β€Žlib/internal/streams/iter/classic.jsβ€Ž

Lines changed: 16 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ const {
5757
const{
5858
toAsyncStreamable: kToAsyncStreamable,
5959
kValidatedSource,
60-
kSyncWriteAccepted,
6160
drainableProtocol,
6261
}=require('internal/streams/iter/types');
6362

@@ -765,41 +764,11 @@ function toWritable(writer) {
765764
consthasEndSync=hasEnd&&
766765
typeofwriter.endSync==='function';
767766
consthasFail=typeofwriter.fail==='function';
768-
consthasSyncWriteAccepted=
769-
typeofwriter[kSyncWriteAccepted]==='function';
770-
771-
functionsyncWriteAccepted(){
772-
returnhasSyncWriteAccepted&&writer[kSyncWriteAccepted]();
773-
}
774-
775-
functionfinishAfterSyncBackpressure(cb){
776-
letondrain;
777-
try{
778-
if(typeofwriter[drainableProtocol]==='function'){
779-
ondrain=writer[drainableProtocol]();
780-
}
781-
}catch(err){
782-
cb(err);
783-
return;
784-
}
785-
if(ondrain!==null&&ondrain!==undefined){
786-
PromisePrototypeThen(ondrain,(drained)=>{
787-
if(drained===false){
788-
cb(newERR_INVALID_STATE.TypeError('Stream closed by consumer'));
789-
return;
790-
}
791-
cb();
792-
},cb);
793-
return;
794-
}
795-
queueMicrotask(cb);
796-
}
797-
798767
// Try-sync-first pattern: attempt the synchronous method and fall back to the
799-
// async method if it returns false without accepting the data, or if it
800-
// throws. When the sync path succeeds, the callback is deferred via
801-
// queueMicrotask to preserve the async resolution contract that Writable
802-
// internals expect from _write/_writev/_final callbacks.
768+
// async method if it returns false (data not accepted synchronously).
769+
// When the sync path succeeds, the callback is deferred via queueMicrotask
770+
// to preserve the async resolution contract that Writable internals expect
771+
// from _write/_writev/_final callbacks.
803772

804773
function_write(chunk,encoding,cb){
805774
constbytes=typeofchunk==='string' ?
@@ -810,13 +779,10 @@ function toWritable(writer) {
810779
queueMicrotask(cb);
811780
return;
812781
}
813-
if(syncWriteAccepted()){
814-
// The chunk was accepted; false only signaled backpressure.
815-
finishAfterSyncBackpressure(cb);
816-
return;
817-
}
818-
}catch{
819-
// Sync path threw -- fall through to async.
782+
// WriteSync returned false: not accepted, fall through to async.
783+
}catch(err){
784+
cb(err);
785+
return;
820786
}
821787
}
822788
try{
@@ -839,13 +805,10 @@ function toWritable(writer) {
839805
queueMicrotask(cb);
840806
return;
841807
}
842-
if(syncWriteAccepted()){
843-
// The chunks were accepted; false only signaled backpressure.
844-
finishAfterSyncBackpressure(cb);
845-
return;
846-
}
847-
}catch{
848-
// Sync path threw -- fall through to async.
808+
// WritevSync returned false: not accepted, fall through to async.
809+
}catch(err){
810+
cb(err);
811+
return;
849812
}
850813
}
851814
try{
@@ -867,8 +830,10 @@ function toWritable(writer) {
867830
queueMicrotask(cb);
868831
return;
869832
}
870-
}catch{
871-
// Sync path threw -- fall through to async.
833+
// Result < 0: can't end synchronously, fall through to async.
834+
}catch(err){
835+
cb(err);
836+
return;
872837
}
873838
}
874839
try{

β€Žlib/internal/streams/iter/consumers.jsβ€Ž

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,7 @@ function merge(...args) {
480480
);
481481
}
482482

483+
letprimaryError;
483484
try{
484485
while(activeCount>0||ready.length>0){
485486
signal?.throwIfAborted();
@@ -500,22 +501,46 @@ function merge(...args) {
500501
});
501502
}
502503
}
504+
}catch(err){
505+
primaryError=err;
503506
}finally{
504-
// Clean up: return all iterators
505-
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
506-
if(iterator.return){
507-
try{
508-
awaititerator.return();
509-
}catch{
510-
// Ignore return errors
511-
}
512-
}
513-
});
507+
// Clean up: return all iterators. Cleanup errors are not
508+
// swallowed - a broken iterator.return() (e.g., failing to
509+
// release a resource) should be visible to the caller.
510+
awaitcleanupIterators(iterators,primaryError);
514511
}
515512
},
516513
};
517514
}
518515

516+
asyncfunctioncleanupIterators(iterators,primaryError){
517+
letcleanupError;
518+
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
519+
if(iterator.return){
520+
try{
521+
awaititerator.return();
522+
}catch(err){
523+
// Keep the first cleanup error encountered.
524+
cleanupError??=err;
525+
}
526+
}
527+
});
528+
if(cleanupError!==undefined){
529+
if(primaryError!==undefined){
530+
// Both a primary error and a cleanup error occurred.
531+
// Wrap in SuppressedError so neither is lost:
532+
// .error = primaryError, .suppressed = cleanupError.
533+
// eslint-disable-next-line no-restricted-syntax
534+
thrownewSuppressedError(primaryError,cleanupError);
535+
}
536+
// No primary error - the cleanup error is the only error.
537+
throwcleanupError;
538+
}
539+
if(primaryError!==undefined){
540+
throwprimaryError;
541+
}
542+
}
543+
519544
module.exports={
520545
array,
521546
arrayBuffer,

β€Žlib/internal/streams/iter/pull.jsβ€Ž

Lines changed: 5 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,6 @@ const {
6363
}=require('internal/streams/iter/utils');
6464

6565
const{
66-
drainableProtocol,
67-
kSyncWriteAcceptedOnFalse,
6866
kValidatedSource,
6967
kValidatedTransform,
7068
toAsyncStreamable,
@@ -863,18 +861,6 @@ async function* createAsyncPipeline(source, transforms, signal) {
863861
}
864862
}
865863

866-
/**
867-
* Check if a false sync write result means accepted backpressure.
868-
* @param {object} writer - The writer whose sync method returned.
869-
* @param {*} result - The return value from writeSync() or writevSync().
870-
* @returns {boolean}
871-
*/
872-
functionisAcceptedSyncWriteBackpressure(writer,result){
873-
returnresult===false&&
874-
writer[kSyncWriteAcceptedOnFalse]===true&&
875-
writer.desiredSize===0;
876-
}
877-
878864
// =============================================================================
879865
// Public API: pull() and pullSync()
880866
// =============================================================================
@@ -963,9 +949,7 @@ function pipeToSync(source, ...args) {
963949
break;
964950
}
965951
if(hasWritevSync&&batch.length>1){
966-
constresult=writer.writevSync(batch);
967-
if(result===false&&
968-
!isAcceptedSyncWriteBackpressure(writer,result)){
952+
if(writer.writevSync(batch)===false){
969953
break;
970954
}
971955
for(leti=0;i<batch.length;i++){
@@ -974,9 +958,7 @@ function pipeToSync(source, ...args) {
974958
}else{
975959
for(leti=0;i<batch.length;i++){
976960
constchunk=batch[i];
977-
constresult=writer.writeSync(chunk);
978-
if(result===false&&
979-
!isAcceptedSyncWriteBackpressure(writer,result)){
961+
if(writer.writeSync(chunk)===false){
980962
canContinue=false;
981963
break;
982964
}
@@ -1027,28 +1009,13 @@ async function pipeTo(source, ...args) {
10271009
consthasWritevSync=typeofwriter.writevSync==='function';
10281010
consthasEndSync=typeofwriter.endSync==='function';
10291011

1030-
functionwaitForSyncBackpressure(){
1031-
constondrain=writer[drainableProtocol];
1032-
returnondrain?.call(writer);
1033-
}
1034-
1035-
asyncfunctionwriteBatchAfterAcceptedBackpressure(batch,startIndex){
1036-
awaitwaitForSyncBackpressure();
1037-
awaitwriteBatchAsyncFallback(batch,startIndex);
1038-
}
1039-
10401012
// Async fallback for writeBatch when sync write fails partway through.
10411013
// Continues writing from batch[startIndex] using async write().
10421014
asyncfunctionwriteBatchAsyncFallback(batch,startIndex){
10431015
for(leti=startIndex;i<batch.length;i++){
10441016
constchunk=batch[i];
1045-
constresult=hasWriteSync&&writer.writeSync(chunk);
1046-
if(result){
1017+
if(hasWriteSync&&writer.writeSync(chunk)){
10471018
// Sync retry succeeded
1048-
}elseif(isAcceptedSyncWriteBackpressure(writer,result)){
1049-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1050-
awaitwaitForSyncBackpressure();
1051-
continue;
10521019
}else{
10531020
constresult=writer.write(
10541021
chunk,signal ? {__proto__: null, signal } : undefined);
@@ -1065,14 +1032,7 @@ async function pipeTo(source, ...args) {
10651032
// is required. Callers must check: const p = writeBatch(b); if (p) await p;
10661033
functionwriteBatch(batch){
10671034
if(hasWritev&&batch.length>1){
1068-
constresult=hasWritevSync&&writer.writevSync(batch);
1069-
if(!result){
1070-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1071-
for(leti=0;i<batch.length;i++){
1072-
totalBytes+=TypedArrayPrototypeGetByteLength(batch[i]);
1073-
}
1074-
returnwaitForSyncBackpressure();
1075-
}
1035+
if(!hasWritevSync||!writer.writevSync(batch)){
10761036
constopts=signal ? {__proto__: null, signal } : undefined;
10771037
constwritevResult=writer.writev(batch,opts);
10781038
if(writevResult===undefined){
@@ -1094,14 +1054,8 @@ async function pipeTo(source, ...args) {
10941054
}
10951055
for(leti=0;i<batch.length;i++){
10961056
constchunk=batch[i];
1097-
constresult=hasWriteSync&&writer.writeSync(chunk);
1098-
if(!result){
1099-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1100-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1101-
returnwriteBatchAfterAcceptedBackpressure(batch,i+1);
1102-
}
1057+
if(!hasWriteSync||!writer.writeSync(chunk)){
11031058
// Sync path failed at index i - fall back to async for the rest.
1104-
// Count bytes for chunks already written synchronously (0..i-1).
11051059
returnwriteBatchAsyncFallback(batch,i);
11061060
}
11071061
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);

β€Žlib/internal/streams/iter/push.jsβ€Ž

Lines changed: 2 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@ const {
3232

3333
const{
3434
drainableProtocol,
35-
kSyncWriteAccepted,
36-
kSyncWriteAcceptedOnFalse,
3735
}=require('internal/streams/iter/types');
3836

3937
const{
@@ -364,19 +362,6 @@ class PushQueue {
364362
this.#pendingEnd =pending;
365363
}
366364

367-
/**
368-
* Force-enqueue chunks into the slots buffer, bypassing capacity checks.
369-
* Used by PushWriter.writeSync() for 'block' policy where the data is
370-
* accepted but false is returned as a backpressure signal.
371-
*/
372-
forceEnqueue(chunks){
373-
this.#slots.push(chunks);
374-
for(leti=0;i<chunks.length;i++){
375-
this.#bytesWritten +=TypedArrayPrototypeGetByteLength(chunks[i]);
376-
}
377-
this.#resolvePendingReads();
378-
}
379-
380365
/**
381366
* Wait for backpressure to clear (desiredSize > 0).
382367
* @returns {Promise<void>}
@@ -558,16 +543,11 @@ class PushQueue {
558543

559544
classPushWriter{
560545
#queue;
561-
#syncWriteAccepted =false;
562546

563547
constructor(queue){
564548
this.#queue =queue;
565549
}
566550

567-
[kSyncWriteAccepted](){
568-
returnthis.#syncWriteAccepted;
569-
}
570-
571551
[drainableProtocol](){
572552
constdesired=this.desiredSize;
573553
if(desired===null)returnnull;
@@ -579,10 +559,6 @@ class PushWriter {
579559
returnthis.#queue.desiredSize;
580560
}
581561

582-
get[kSyncWriteAcceptedOnFalse](){
583-
returnthis.#queue.backpressurePolicy==='block';
584-
}
585-
586562
write(chunk,options){
587563
if(!options?.signal&&this.#queue.canWriteSync()){
588564
constbytes=toUint8Array(chunk);
@@ -607,36 +583,16 @@ class PushWriter {
607583
}
608584

609585
writeSync(chunk){
610-
this.#syncWriteAccepted =false;
611586
constbytes=toUint8Array(chunk);
612-
constresult=this.#queue.writeSync([bytes]);
613-
if(!result&&this.#queue.backpressurePolicy==='block'&&
614-
this.#queue.desiredSize===0){
615-
// Block policy: force-enqueue and return false as backpressure signal.
616-
// Data IS accepted; false tells caller to slow down.
617-
this.#queue.forceEnqueue([bytes]);
618-
this.#syncWriteAccepted =true;
619-
returnfalse;
620-
}
621-
this.#syncWriteAccepted =result;
622-
returnresult;
587+
returnthis.#queue.writeSync([bytes]);
623588
}
624589

625590
writevSync(chunks){
626-
this.#syncWriteAccepted =false;
627591
if(!ArrayIsArray(chunks)){
628592
thrownewERR_INVALID_ARG_TYPE('chunks','Array',chunks);
629593
}
630594
constbytes=convertChunks(chunks);
631-
constresult=this.#queue.writeSync(bytes);
632-
if(!result&&this.#queue.backpressurePolicy==='block'&&
633-
this.#queue.desiredSize===0){
634-
this.#queue.forceEnqueue(bytes);
635-
this.#syncWriteAccepted =true;
636-
returnfalse;
637-
}
638-
this.#syncWriteAccepted =result;
639-
returnresult;
595+
returnthis.#queue.writeSync(bytes);
640596
}
641597

642598
end(options){

0 commit comments

Comments
Β (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms"); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Commit a46ddad

Browse files
jasnelladuh95
authored andcommitted
stream: refine the stream/iter backpressure
In `Writer`, there are two queues that matter: the slot queue and the pending queue. The sync `writeSync`/`writevSync` only use the slot queue. If writes cannot be accepted directly into slots, they return `false`. The sync methods *never* enqueue into the pending queue. The async `write`/`writev` will first attempt to add to the slot queue; if it is full, then it will attempt to add to the pending queue; if that is also full, the backpressure policy kicks in. Signed-off-by: James M Snell <jasnell@gmail.com> PR-URL: #63697 Backport-PR-URL: #64675 Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day>
1 parent 557bf99 commit a46ddad

9 files changed

Lines changed: 185 additions & 201 deletions

File tree

β€Ždoc/api/stream_iter.mdβ€Ž

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1649,7 +1649,7 @@ Creates a classic [`stream.Writable`][] backed by a stream/iter Writer.
16491649

16501650
Each `_write()` / `_writev()` call attempts the Writer's synchronous method
16511651
first (`writeSync` / `writevSync`), falling back to the async method if the
1652-
sync path returns `false` or throws. Similarly, `_final()` tries `endSync()`
1652+
sync path returns `false`. Similarly, `_final()` tries `endSync()`
16531653
before `end()`. When the sync path succeeds, the callback is deferred via
16541654
`queueMicrotask` to preserve the async resolution contract.
16551655

β€Žlib/internal/streams/iter/classic.jsβ€Ž

Lines changed: 16 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ const {
5757
const{
5858
toAsyncStreamable: kToAsyncStreamable,
5959
kValidatedSource,
60-
kSyncWriteAccepted,
6160
drainableProtocol,
6261
}=require('internal/streams/iter/types');
6362

@@ -765,41 +764,11 @@ function toWritable(writer) {
765764
consthasEndSync=hasEnd&&
766765
typeofwriter.endSync==='function';
767766
consthasFail=typeofwriter.fail==='function';
768-
consthasSyncWriteAccepted=
769-
typeofwriter[kSyncWriteAccepted]==='function';
770-
771-
functionsyncWriteAccepted(){
772-
returnhasSyncWriteAccepted&&writer[kSyncWriteAccepted]();
773-
}
774-
775-
functionfinishAfterSyncBackpressure(cb){
776-
letondrain;
777-
try{
778-
if(typeofwriter[drainableProtocol]==='function'){
779-
ondrain=writer[drainableProtocol]();
780-
}
781-
}catch(err){
782-
cb(err);
783-
return;
784-
}
785-
if(ondrain!==null&&ondrain!==undefined){
786-
PromisePrototypeThen(ondrain,(drained)=>{
787-
if(drained===false){
788-
cb(newERR_INVALID_STATE.TypeError('Stream closed by consumer'));
789-
return;
790-
}
791-
cb();
792-
},cb);
793-
return;
794-
}
795-
queueMicrotask(cb);
796-
}
797-
798767
// Try-sync-first pattern: attempt the synchronous method and fall back to the
799-
// async method if it returns false without accepting the data, or if it
800-
// throws. When the sync path succeeds, the callback is deferred via
801-
// queueMicrotask to preserve the async resolution contract that Writable
802-
// internals expect from _write/_writev/_final callbacks.
768+
// async method if it returns false (data not accepted synchronously).
769+
// When the sync path succeeds, the callback is deferred via queueMicrotask
770+
// to preserve the async resolution contract that Writable internals expect
771+
// from _write/_writev/_final callbacks.
803772

804773
function_write(chunk,encoding,cb){
805774
constbytes=typeofchunk==='string' ?
@@ -810,13 +779,10 @@ function toWritable(writer) {
810779
queueMicrotask(cb);
811780
return;
812781
}
813-
if(syncWriteAccepted()){
814-
// The chunk was accepted; false only signaled backpressure.
815-
finishAfterSyncBackpressure(cb);
816-
return;
817-
}
818-
}catch{
819-
// Sync path threw -- fall through to async.
782+
// WriteSync returned false: not accepted, fall through to async.
783+
}catch(err){
784+
cb(err);
785+
return;
820786
}
821787
}
822788
try{
@@ -839,13 +805,10 @@ function toWritable(writer) {
839805
queueMicrotask(cb);
840806
return;
841807
}
842-
if(syncWriteAccepted()){
843-
// The chunks were accepted; false only signaled backpressure.
844-
finishAfterSyncBackpressure(cb);
845-
return;
846-
}
847-
}catch{
848-
// Sync path threw -- fall through to async.
808+
// WritevSync returned false: not accepted, fall through to async.
809+
}catch(err){
810+
cb(err);
811+
return;
849812
}
850813
}
851814
try{
@@ -867,8 +830,10 @@ function toWritable(writer) {
867830
queueMicrotask(cb);
868831
return;
869832
}
870-
}catch{
871-
// Sync path threw -- fall through to async.
833+
// Result < 0: can't end synchronously, fall through to async.
834+
}catch(err){
835+
cb(err);
836+
return;
872837
}
873838
}
874839
try{

β€Žlib/internal/streams/iter/consumers.jsβ€Ž

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,7 @@ function merge(...args) {
480480
);
481481
}
482482

483+
letprimaryError;
483484
try{
484485
while(activeCount>0||ready.length>0){
485486
signal?.throwIfAborted();
@@ -500,22 +501,46 @@ function merge(...args) {
500501
});
501502
}
502503
}
504+
}catch(err){
505+
primaryError=err;
503506
}finally{
504-
// Clean up: return all iterators
505-
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
506-
if(iterator.return){
507-
try{
508-
awaititerator.return();
509-
}catch{
510-
// Ignore return errors
511-
}
512-
}
513-
});
507+
// Clean up: return all iterators. Cleanup errors are not
508+
// swallowed - a broken iterator.return() (e.g., failing to
509+
// release a resource) should be visible to the caller.
510+
awaitcleanupIterators(iterators,primaryError);
514511
}
515512
},
516513
};
517514
}
518515

516+
asyncfunctioncleanupIterators(iterators,primaryError){
517+
letcleanupError;
518+
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
519+
if(iterator.return){
520+
try{
521+
awaititerator.return();
522+
}catch(err){
523+
// Keep the first cleanup error encountered.
524+
cleanupError??=err;
525+
}
526+
}
527+
});
528+
if(cleanupError!==undefined){
529+
if(primaryError!==undefined){
530+
// Both a primary error and a cleanup error occurred.
531+
// Wrap in SuppressedError so neither is lost:
532+
// .error = primaryError, .suppressed = cleanupError.
533+
// eslint-disable-next-line no-restricted-syntax
534+
thrownewSuppressedError(primaryError,cleanupError);
535+
}
536+
// No primary error - the cleanup error is the only error.
537+
throwcleanupError;
538+
}
539+
if(primaryError!==undefined){
540+
throwprimaryError;
541+
}
542+
}
543+
519544
module.exports={
520545
array,
521546
arrayBuffer,

β€Žlib/internal/streams/iter/pull.jsβ€Ž

Lines changed: 5 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,6 @@ const {
6363
}=require('internal/streams/iter/utils');
6464

6565
const{
66-
drainableProtocol,
67-
kSyncWriteAcceptedOnFalse,
6866
kValidatedSource,
6967
kValidatedTransform,
7068
toAsyncStreamable,
@@ -863,18 +861,6 @@ async function* createAsyncPipeline(source, transforms, signal) {
863861
}
864862
}
865863

866-
/**
867-
* Check if a false sync write result means accepted backpressure.
868-
* @param {object} writer - The writer whose sync method returned.
869-
* @param {*} result - The return value from writeSync() or writevSync().
870-
* @returns {boolean}
871-
*/
872-
functionisAcceptedSyncWriteBackpressure(writer,result){
873-
returnresult===false&&
874-
writer[kSyncWriteAcceptedOnFalse]===true&&
875-
writer.desiredSize===0;
876-
}
877-
878864
// =============================================================================
879865
// Public API: pull() and pullSync()
880866
// =============================================================================
@@ -963,9 +949,7 @@ function pipeToSync(source, ...args) {
963949
break;
964950
}
965951
if(hasWritevSync&&batch.length>1){
966-
constresult=writer.writevSync(batch);
967-
if(result===false&&
968-
!isAcceptedSyncWriteBackpressure(writer,result)){
952+
if(writer.writevSync(batch)===false){
969953
break;
970954
}
971955
for(leti=0;i<batch.length;i++){
@@ -974,9 +958,7 @@ function pipeToSync(source, ...args) {
974958
}else{
975959
for(leti=0;i<batch.length;i++){
976960
constchunk=batch[i];
977-
constresult=writer.writeSync(chunk);
978-
if(result===false&&
979-
!isAcceptedSyncWriteBackpressure(writer,result)){
961+
if(writer.writeSync(chunk)===false){
980962
canContinue=false;
981963
break;
982964
}
@@ -1027,28 +1009,13 @@ async function pipeTo(source, ...args) {
10271009
consthasWritevSync=typeofwriter.writevSync==='function';
10281010
consthasEndSync=typeofwriter.endSync==='function';
10291011

1030-
functionwaitForSyncBackpressure(){
1031-
constondrain=writer[drainableProtocol];
1032-
returnondrain?.call(writer);
1033-
}
1034-
1035-
asyncfunctionwriteBatchAfterAcceptedBackpressure(batch,startIndex){
1036-
awaitwaitForSyncBackpressure();
1037-
awaitwriteBatchAsyncFallback(batch,startIndex);
1038-
}
1039-
10401012
// Async fallback for writeBatch when sync write fails partway through.
10411013
// Continues writing from batch[startIndex] using async write().
10421014
asyncfunctionwriteBatchAsyncFallback(batch,startIndex){
10431015
for(leti=startIndex;i<batch.length;i++){
10441016
constchunk=batch[i];
1045-
constresult=hasWriteSync&&writer.writeSync(chunk);
1046-
if(result){
1017+
if(hasWriteSync&&writer.writeSync(chunk)){
10471018
// Sync retry succeeded
1048-
}elseif(isAcceptedSyncWriteBackpressure(writer,result)){
1049-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1050-
awaitwaitForSyncBackpressure();
1051-
continue;
10521019
}else{
10531020
constresult=writer.write(
10541021
chunk,signal ? {__proto__: null, signal } : undefined);
@@ -1065,14 +1032,7 @@ async function pipeTo(source, ...args) {
10651032
// is required. Callers must check: const p = writeBatch(b); if (p) await p;
10661033
functionwriteBatch(batch){
10671034
if(hasWritev&&batch.length>1){
1068-
constresult=hasWritevSync&&writer.writevSync(batch);
1069-
if(!result){
1070-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1071-
for(leti=0;i<batch.length;i++){
1072-
totalBytes+=TypedArrayPrototypeGetByteLength(batch[i]);
1073-
}
1074-
returnwaitForSyncBackpressure();
1075-
}
1035+
if(!hasWritevSync||!writer.writevSync(batch)){
10761036
constopts=signal ? {__proto__: null, signal } : undefined;
10771037
constwritevResult=writer.writev(batch,opts);
10781038
if(writevResult===undefined){
@@ -1094,14 +1054,8 @@ async function pipeTo(source, ...args) {
10941054
}
10951055
for(leti=0;i<batch.length;i++){
10961056
constchunk=batch[i];
1097-
constresult=hasWriteSync&&writer.writeSync(chunk);
1098-
if(!result){
1099-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1100-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1101-
returnwriteBatchAfterAcceptedBackpressure(batch,i+1);
1102-
}
1057+
if(!hasWriteSync||!writer.writeSync(chunk)){
11031058
// Sync path failed at index i - fall back to async for the rest.
1104-
// Count bytes for chunks already written synchronously (0..i-1).
11051059
returnwriteBatchAsyncFallback(batch,i);
11061060
}
11071061
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);

β€Žlib/internal/streams/iter/push.jsβ€Ž

Lines changed: 2 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@ const {
3232

3333
const{
3434
drainableProtocol,
35-
kSyncWriteAccepted,
36-
kSyncWriteAcceptedOnFalse,
3735
}=require('internal/streams/iter/types');
3836

3937
const{
@@ -364,19 +362,6 @@ class PushQueue {
364362
this.#pendingEnd =pending;
365363
}
366364

367-
/**
368-
* Force-enqueue chunks into the slots buffer, bypassing capacity checks.
369-
* Used by PushWriter.writeSync() for 'block' policy where the data is
370-
* accepted but false is returned as a backpressure signal.
371-
*/
372-
forceEnqueue(chunks){
373-
this.#slots.push(chunks);
374-
for(leti=0;i<chunks.length;i++){
375-
this.#bytesWritten +=TypedArrayPrototypeGetByteLength(chunks[i]);
376-
}
377-
this.#resolvePendingReads();
378-
}
379-
380365
/**
381366
* Wait for backpressure to clear (desiredSize > 0).
382367
* @returns {Promise<void>}
@@ -558,16 +543,11 @@ class PushQueue {
558543

559544
classPushWriter{
560545
#queue;
561-
#syncWriteAccepted =false;
562546

563547
constructor(queue){
564548
this.#queue =queue;
565549
}
566550

567-
[kSyncWriteAccepted](){
568-
returnthis.#syncWriteAccepted;
569-
}
570-
571551
[drainableProtocol](){
572552
constdesired=this.desiredSize;
573553
if(desired===null)returnnull;
@@ -579,10 +559,6 @@ class PushWriter {
579559
returnthis.#queue.desiredSize;
580560
}
581561

582-
get[kSyncWriteAcceptedOnFalse](){
583-
returnthis.#queue.backpressurePolicy==='block';
584-
}
585-
586562
write(chunk,options){
587563
if(!options?.signal&&this.#queue.canWriteSync()){
588564
constbytes=toUint8Array(chunk);
@@ -607,36 +583,16 @@ class PushWriter {
607583
}
608584

609585
writeSync(chunk){
610-
this.#syncWriteAccepted =false;
611586
constbytes=toUint8Array(chunk);
612-
constresult=this.#queue.writeSync([bytes]);
613-
if(!result&&this.#queue.backpressurePolicy==='block'&&
614-
this.#queue.desiredSize===0){
615-
// Block policy: force-enqueue and return false as backpressure signal.
616-
// Data IS accepted; false tells caller to slow down.
617-
this.#queue.forceEnqueue([bytes]);
618-
this.#syncWriteAccepted =true;
619-
returnfalse;
620-
}
621-
this.#syncWriteAccepted =result;
622-
returnresult;
587+
returnthis.#queue.writeSync([bytes]);
623588
}
624589

625590
writevSync(chunks){
626-
this.#syncWriteAccepted =false;
627591
if(!ArrayIsArray(chunks)){
628592
thrownewERR_INVALID_ARG_TYPE('chunks','Array',chunks);
629593
}
630594
constbytes=convertChunks(chunks);
631-
constresult=this.#queue.writeSync(bytes);
632-
if(!result&&this.#queue.backpressurePolicy==='block'&&
633-
this.#queue.desiredSize===0){
634-
this.#queue.forceEnqueue(bytes);
635-
this.#syncWriteAccepted =true;
636-
returnfalse;
637-
}
638-
this.#syncWriteAccepted =result;
639-
returnresult;
595+
returnthis.#queue.writeSync(bytes);
640596
}
641597

642598
end(options){

0 commit comments

Comments
Β (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links"); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + '
Skip to content

Commit a46ddad

Browse files
jasnelladuh95
authored andcommitted
stream: refine the stream/iter backpressure
In `Writer`, there are two queues that matter: the slot queue and the pending queue. The sync `writeSync`/`writevSync` only use the slot queue. If writes cannot be accepted directly into slots, they return `false`. The sync methods *never* enqueue into the pending queue. The async `write`/`writev` will first attempt to add to the slot queue; if it is full, then it will attempt to add to the pending queue; if that is also full, the backpressure policy kicks in. Signed-off-by: James M Snell <jasnell@gmail.com> PR-URL: #63697 Backport-PR-URL: #64675 Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day>
1 parent 557bf99 commit a46ddad

9 files changed

Lines changed: 185 additions & 201 deletions

File tree

β€Ždoc/api/stream_iter.mdβ€Ž

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1649,7 +1649,7 @@ Creates a classic [`stream.Writable`][] backed by a stream/iter Writer.
16491649

16501650
Each `_write()` / `_writev()` call attempts the Writer's synchronous method
16511651
first (`writeSync` / `writevSync`), falling back to the async method if the
1652-
sync path returns `false` or throws. Similarly, `_final()` tries `endSync()`
1652+
sync path returns `false`. Similarly, `_final()` tries `endSync()`
16531653
before `end()`. When the sync path succeeds, the callback is deferred via
16541654
`queueMicrotask` to preserve the async resolution contract.
16551655

β€Žlib/internal/streams/iter/classic.jsβ€Ž

Lines changed: 16 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ const {
5757
const{
5858
toAsyncStreamable: kToAsyncStreamable,
5959
kValidatedSource,
60-
kSyncWriteAccepted,
6160
drainableProtocol,
6261
}=require('internal/streams/iter/types');
6362

@@ -765,41 +764,11 @@ function toWritable(writer) {
765764
consthasEndSync=hasEnd&&
766765
typeofwriter.endSync==='function';
767766
consthasFail=typeofwriter.fail==='function';
768-
consthasSyncWriteAccepted=
769-
typeofwriter[kSyncWriteAccepted]==='function';
770-
771-
functionsyncWriteAccepted(){
772-
returnhasSyncWriteAccepted&&writer[kSyncWriteAccepted]();
773-
}
774-
775-
functionfinishAfterSyncBackpressure(cb){
776-
letondrain;
777-
try{
778-
if(typeofwriter[drainableProtocol]==='function'){
779-
ondrain=writer[drainableProtocol]();
780-
}
781-
}catch(err){
782-
cb(err);
783-
return;
784-
}
785-
if(ondrain!==null&&ondrain!==undefined){
786-
PromisePrototypeThen(ondrain,(drained)=>{
787-
if(drained===false){
788-
cb(newERR_INVALID_STATE.TypeError('Stream closed by consumer'));
789-
return;
790-
}
791-
cb();
792-
},cb);
793-
return;
794-
}
795-
queueMicrotask(cb);
796-
}
797-
798767
// Try-sync-first pattern: attempt the synchronous method and fall back to the
799-
// async method if it returns false without accepting the data, or if it
800-
// throws. When the sync path succeeds, the callback is deferred via
801-
// queueMicrotask to preserve the async resolution contract that Writable
802-
// internals expect from _write/_writev/_final callbacks.
768+
// async method if it returns false (data not accepted synchronously).
769+
// When the sync path succeeds, the callback is deferred via queueMicrotask
770+
// to preserve the async resolution contract that Writable internals expect
771+
// from _write/_writev/_final callbacks.
803772

804773
function_write(chunk,encoding,cb){
805774
constbytes=typeofchunk==='string' ?
@@ -810,13 +779,10 @@ function toWritable(writer) {
810779
queueMicrotask(cb);
811780
return;
812781
}
813-
if(syncWriteAccepted()){
814-
// The chunk was accepted; false only signaled backpressure.
815-
finishAfterSyncBackpressure(cb);
816-
return;
817-
}
818-
}catch{
819-
// Sync path threw -- fall through to async.
782+
// WriteSync returned false: not accepted, fall through to async.
783+
}catch(err){
784+
cb(err);
785+
return;
820786
}
821787
}
822788
try{
@@ -839,13 +805,10 @@ function toWritable(writer) {
839805
queueMicrotask(cb);
840806
return;
841807
}
842-
if(syncWriteAccepted()){
843-
// The chunks were accepted; false only signaled backpressure.
844-
finishAfterSyncBackpressure(cb);
845-
return;
846-
}
847-
}catch{
848-
// Sync path threw -- fall through to async.
808+
// WritevSync returned false: not accepted, fall through to async.
809+
}catch(err){
810+
cb(err);
811+
return;
849812
}
850813
}
851814
try{
@@ -867,8 +830,10 @@ function toWritable(writer) {
867830
queueMicrotask(cb);
868831
return;
869832
}
870-
}catch{
871-
// Sync path threw -- fall through to async.
833+
// Result < 0: can't end synchronously, fall through to async.
834+
}catch(err){
835+
cb(err);
836+
return;
872837
}
873838
}
874839
try{

β€Žlib/internal/streams/iter/consumers.jsβ€Ž

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,7 @@ function merge(...args) {
480480
);
481481
}
482482

483+
letprimaryError;
483484
try{
484485
while(activeCount>0||ready.length>0){
485486
signal?.throwIfAborted();
@@ -500,22 +501,46 @@ function merge(...args) {
500501
});
501502
}
502503
}
504+
}catch(err){
505+
primaryError=err;
503506
}finally{
504-
// Clean up: return all iterators
505-
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
506-
if(iterator.return){
507-
try{
508-
awaititerator.return();
509-
}catch{
510-
// Ignore return errors
511-
}
512-
}
513-
});
507+
// Clean up: return all iterators. Cleanup errors are not
508+
// swallowed - a broken iterator.return() (e.g., failing to
509+
// release a resource) should be visible to the caller.
510+
awaitcleanupIterators(iterators,primaryError);
514511
}
515512
},
516513
};
517514
}
518515

516+
asyncfunctioncleanupIterators(iterators,primaryError){
517+
letcleanupError;
518+
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
519+
if(iterator.return){
520+
try{
521+
awaititerator.return();
522+
}catch(err){
523+
// Keep the first cleanup error encountered.
524+
cleanupError??=err;
525+
}
526+
}
527+
});
528+
if(cleanupError!==undefined){
529+
if(primaryError!==undefined){
530+
// Both a primary error and a cleanup error occurred.
531+
// Wrap in SuppressedError so neither is lost:
532+
// .error = primaryError, .suppressed = cleanupError.
533+
// eslint-disable-next-line no-restricted-syntax
534+
thrownewSuppressedError(primaryError,cleanupError);
535+
}
536+
// No primary error - the cleanup error is the only error.
537+
throwcleanupError;
538+
}
539+
if(primaryError!==undefined){
540+
throwprimaryError;
541+
}
542+
}
543+
519544
module.exports={
520545
array,
521546
arrayBuffer,

β€Žlib/internal/streams/iter/pull.jsβ€Ž

Lines changed: 5 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,6 @@ const {
6363
}=require('internal/streams/iter/utils');
6464

6565
const{
66-
drainableProtocol,
67-
kSyncWriteAcceptedOnFalse,
6866
kValidatedSource,
6967
kValidatedTransform,
7068
toAsyncStreamable,
@@ -863,18 +861,6 @@ async function* createAsyncPipeline(source, transforms, signal) {
863861
}
864862
}
865863

866-
/**
867-
* Check if a false sync write result means accepted backpressure.
868-
* @param {object} writer - The writer whose sync method returned.
869-
* @param {*} result - The return value from writeSync() or writevSync().
870-
* @returns {boolean}
871-
*/
872-
functionisAcceptedSyncWriteBackpressure(writer,result){
873-
returnresult===false&&
874-
writer[kSyncWriteAcceptedOnFalse]===true&&
875-
writer.desiredSize===0;
876-
}
877-
878864
// =============================================================================
879865
// Public API: pull() and pullSync()
880866
// =============================================================================
@@ -963,9 +949,7 @@ function pipeToSync(source, ...args) {
963949
break;
964950
}
965951
if(hasWritevSync&&batch.length>1){
966-
constresult=writer.writevSync(batch);
967-
if(result===false&&
968-
!isAcceptedSyncWriteBackpressure(writer,result)){
952+
if(writer.writevSync(batch)===false){
969953
break;
970954
}
971955
for(leti=0;i<batch.length;i++){
@@ -974,9 +958,7 @@ function pipeToSync(source, ...args) {
974958
}else{
975959
for(leti=0;i<batch.length;i++){
976960
constchunk=batch[i];
977-
constresult=writer.writeSync(chunk);
978-
if(result===false&&
979-
!isAcceptedSyncWriteBackpressure(writer,result)){
961+
if(writer.writeSync(chunk)===false){
980962
canContinue=false;
981963
break;
982964
}
@@ -1027,28 +1009,13 @@ async function pipeTo(source, ...args) {
10271009
consthasWritevSync=typeofwriter.writevSync==='function';
10281010
consthasEndSync=typeofwriter.endSync==='function';
10291011

1030-
functionwaitForSyncBackpressure(){
1031-
constondrain=writer[drainableProtocol];
1032-
returnondrain?.call(writer);
1033-
}
1034-
1035-
asyncfunctionwriteBatchAfterAcceptedBackpressure(batch,startIndex){
1036-
awaitwaitForSyncBackpressure();
1037-
awaitwriteBatchAsyncFallback(batch,startIndex);
1038-
}
1039-
10401012
// Async fallback for writeBatch when sync write fails partway through.
10411013
// Continues writing from batch[startIndex] using async write().
10421014
asyncfunctionwriteBatchAsyncFallback(batch,startIndex){
10431015
for(leti=startIndex;i<batch.length;i++){
10441016
constchunk=batch[i];
1045-
constresult=hasWriteSync&&writer.writeSync(chunk);
1046-
if(result){
1017+
if(hasWriteSync&&writer.writeSync(chunk)){
10471018
// Sync retry succeeded
1048-
}elseif(isAcceptedSyncWriteBackpressure(writer,result)){
1049-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1050-
awaitwaitForSyncBackpressure();
1051-
continue;
10521019
}else{
10531020
constresult=writer.write(
10541021
chunk,signal ? {__proto__: null, signal } : undefined);
@@ -1065,14 +1032,7 @@ async function pipeTo(source, ...args) {
10651032
// is required. Callers must check: const p = writeBatch(b); if (p) await p;
10661033
functionwriteBatch(batch){
10671034
if(hasWritev&&batch.length>1){
1068-
constresult=hasWritevSync&&writer.writevSync(batch);
1069-
if(!result){
1070-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1071-
for(leti=0;i<batch.length;i++){
1072-
totalBytes+=TypedArrayPrototypeGetByteLength(batch[i]);
1073-
}
1074-
returnwaitForSyncBackpressure();
1075-
}
1035+
if(!hasWritevSync||!writer.writevSync(batch)){
10761036
constopts=signal ? {__proto__: null, signal } : undefined;
10771037
constwritevResult=writer.writev(batch,opts);
10781038
if(writevResult===undefined){
@@ -1094,14 +1054,8 @@ async function pipeTo(source, ...args) {
10941054
}
10951055
for(leti=0;i<batch.length;i++){
10961056
constchunk=batch[i];
1097-
constresult=hasWriteSync&&writer.writeSync(chunk);
1098-
if(!result){
1099-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1100-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1101-
returnwriteBatchAfterAcceptedBackpressure(batch,i+1);
1102-
}
1057+
if(!hasWriteSync||!writer.writeSync(chunk)){
11031058
// Sync path failed at index i - fall back to async for the rest.
1104-
// Count bytes for chunks already written synchronously (0..i-1).
11051059
returnwriteBatchAsyncFallback(batch,i);
11061060
}
11071061
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);

β€Žlib/internal/streams/iter/push.jsβ€Ž

Lines changed: 2 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@ const {
3232

3333
const{
3434
drainableProtocol,
35-
kSyncWriteAccepted,
36-
kSyncWriteAcceptedOnFalse,
3735
}=require('internal/streams/iter/types');
3836

3937
const{
@@ -364,19 +362,6 @@ class PushQueue {
364362
this.#pendingEnd =pending;
365363
}
366364

367-
/**
368-
* Force-enqueue chunks into the slots buffer, bypassing capacity checks.
369-
* Used by PushWriter.writeSync() for 'block' policy where the data is
370-
* accepted but false is returned as a backpressure signal.
371-
*/
372-
forceEnqueue(chunks){
373-
this.#slots.push(chunks);
374-
for(leti=0;i<chunks.length;i++){
375-
this.#bytesWritten +=TypedArrayPrototypeGetByteLength(chunks[i]);
376-
}
377-
this.#resolvePendingReads();
378-
}
379-
380365
/**
381366
* Wait for backpressure to clear (desiredSize > 0).
382367
* @returns {Promise<void>}
@@ -558,16 +543,11 @@ class PushQueue {
558543

559544
classPushWriter{
560545
#queue;
561-
#syncWriteAccepted =false;
562546

563547
constructor(queue){
564548
this.#queue =queue;
565549
}
566550

567-
[kSyncWriteAccepted](){
568-
returnthis.#syncWriteAccepted;
569-
}
570-
571551
[drainableProtocol](){
572552
constdesired=this.desiredSize;
573553
if(desired===null)returnnull;
@@ -579,10 +559,6 @@ class PushWriter {
579559
returnthis.#queue.desiredSize;
580560
}
581561

582-
get[kSyncWriteAcceptedOnFalse](){
583-
returnthis.#queue.backpressurePolicy==='block';
584-
}
585-
586562
write(chunk,options){
587563
if(!options?.signal&&this.#queue.canWriteSync()){
588564
constbytes=toUint8Array(chunk);
@@ -607,36 +583,16 @@ class PushWriter {
607583
}
608584

609585
writeSync(chunk){
610-
this.#syncWriteAccepted =false;
611586
constbytes=toUint8Array(chunk);
612-
constresult=this.#queue.writeSync([bytes]);
613-
if(!result&&this.#queue.backpressurePolicy==='block'&&
614-
this.#queue.desiredSize===0){
615-
// Block policy: force-enqueue and return false as backpressure signal.
616-
// Data IS accepted; false tells caller to slow down.
617-
this.#queue.forceEnqueue([bytes]);
618-
this.#syncWriteAccepted =true;
619-
returnfalse;
620-
}
621-
this.#syncWriteAccepted =result;
622-
returnresult;
587+
returnthis.#queue.writeSync([bytes]);
623588
}
624589

625590
writevSync(chunks){
626-
this.#syncWriteAccepted =false;
627591
if(!ArrayIsArray(chunks)){
628592
thrownewERR_INVALID_ARG_TYPE('chunks','Array',chunks);
629593
}
630594
constbytes=convertChunks(chunks);
631-
constresult=this.#queue.writeSync(bytes);
632-
if(!result&&this.#queue.backpressurePolicy==='block'&&
633-
this.#queue.desiredSize===0){
634-
this.#queue.forceEnqueue(bytes);
635-
this.#syncWriteAccepted =true;
636-
returnfalse;
637-
}
638-
this.#syncWriteAccepted =result;
639-
returnresult;
595+
returnthis.#queue.writeSync(bytes);
640596
}
641597

642598
end(options){

0 commit comments

Comments
Β (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default"); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Commit a46ddad

Browse files
jasnelladuh95
authored andcommitted
stream: refine the stream/iter backpressure
In `Writer`, there are two queues that matter: the slot queue and the pending queue. The sync `writeSync`/`writevSync` only use the slot queue. If writes cannot be accepted directly into slots, they return `false`. The sync methods *never* enqueue into the pending queue. The async `write`/`writev` will first attempt to add to the slot queue; if it is full, then it will attempt to add to the pending queue; if that is also full, the backpressure policy kicks in. Signed-off-by: James M Snell <jasnell@gmail.com> PR-URL: #63697 Backport-PR-URL: #64675 Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day>
1 parent 557bf99 commit a46ddad

9 files changed

Lines changed: 185 additions & 201 deletions

File tree

β€Ždoc/api/stream_iter.mdβ€Ž

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1649,7 +1649,7 @@ Creates a classic [`stream.Writable`][] backed by a stream/iter Writer.
16491649

16501650
Each `_write()` / `_writev()` call attempts the Writer's synchronous method
16511651
first (`writeSync` / `writevSync`), falling back to the async method if the
1652-
sync path returns `false` or throws. Similarly, `_final()` tries `endSync()`
1652+
sync path returns `false`. Similarly, `_final()` tries `endSync()`
16531653
before `end()`. When the sync path succeeds, the callback is deferred via
16541654
`queueMicrotask` to preserve the async resolution contract.
16551655

β€Žlib/internal/streams/iter/classic.jsβ€Ž

Lines changed: 16 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ const {
5757
const{
5858
toAsyncStreamable: kToAsyncStreamable,
5959
kValidatedSource,
60-
kSyncWriteAccepted,
6160
drainableProtocol,
6261
}=require('internal/streams/iter/types');
6362

@@ -765,41 +764,11 @@ function toWritable(writer) {
765764
consthasEndSync=hasEnd&&
766765
typeofwriter.endSync==='function';
767766
consthasFail=typeofwriter.fail==='function';
768-
consthasSyncWriteAccepted=
769-
typeofwriter[kSyncWriteAccepted]==='function';
770-
771-
functionsyncWriteAccepted(){
772-
returnhasSyncWriteAccepted&&writer[kSyncWriteAccepted]();
773-
}
774-
775-
functionfinishAfterSyncBackpressure(cb){
776-
letondrain;
777-
try{
778-
if(typeofwriter[drainableProtocol]==='function'){
779-
ondrain=writer[drainableProtocol]();
780-
}
781-
}catch(err){
782-
cb(err);
783-
return;
784-
}
785-
if(ondrain!==null&&ondrain!==undefined){
786-
PromisePrototypeThen(ondrain,(drained)=>{
787-
if(drained===false){
788-
cb(newERR_INVALID_STATE.TypeError('Stream closed by consumer'));
789-
return;
790-
}
791-
cb();
792-
},cb);
793-
return;
794-
}
795-
queueMicrotask(cb);
796-
}
797-
798767
// Try-sync-first pattern: attempt the synchronous method and fall back to the
799-
// async method if it returns false without accepting the data, or if it
800-
// throws. When the sync path succeeds, the callback is deferred via
801-
// queueMicrotask to preserve the async resolution contract that Writable
802-
// internals expect from _write/_writev/_final callbacks.
768+
// async method if it returns false (data not accepted synchronously).
769+
// When the sync path succeeds, the callback is deferred via queueMicrotask
770+
// to preserve the async resolution contract that Writable internals expect
771+
// from _write/_writev/_final callbacks.
803772

804773
function_write(chunk,encoding,cb){
805774
constbytes=typeofchunk==='string' ?
@@ -810,13 +779,10 @@ function toWritable(writer) {
810779
queueMicrotask(cb);
811780
return;
812781
}
813-
if(syncWriteAccepted()){
814-
// The chunk was accepted; false only signaled backpressure.
815-
finishAfterSyncBackpressure(cb);
816-
return;
817-
}
818-
}catch{
819-
// Sync path threw -- fall through to async.
782+
// WriteSync returned false: not accepted, fall through to async.
783+
}catch(err){
784+
cb(err);
785+
return;
820786
}
821787
}
822788
try{
@@ -839,13 +805,10 @@ function toWritable(writer) {
839805
queueMicrotask(cb);
840806
return;
841807
}
842-
if(syncWriteAccepted()){
843-
// The chunks were accepted; false only signaled backpressure.
844-
finishAfterSyncBackpressure(cb);
845-
return;
846-
}
847-
}catch{
848-
// Sync path threw -- fall through to async.
808+
// WritevSync returned false: not accepted, fall through to async.
809+
}catch(err){
810+
cb(err);
811+
return;
849812
}
850813
}
851814
try{
@@ -867,8 +830,10 @@ function toWritable(writer) {
867830
queueMicrotask(cb);
868831
return;
869832
}
870-
}catch{
871-
// Sync path threw -- fall through to async.
833+
// Result < 0: can't end synchronously, fall through to async.
834+
}catch(err){
835+
cb(err);
836+
return;
872837
}
873838
}
874839
try{

β€Žlib/internal/streams/iter/consumers.jsβ€Ž

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,7 @@ function merge(...args) {
480480
);
481481
}
482482

483+
letprimaryError;
483484
try{
484485
while(activeCount>0||ready.length>0){
485486
signal?.throwIfAborted();
@@ -500,22 +501,46 @@ function merge(...args) {
500501
});
501502
}
502503
}
504+
}catch(err){
505+
primaryError=err;
503506
}finally{
504-
// Clean up: return all iterators
505-
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
506-
if(iterator.return){
507-
try{
508-
awaititerator.return();
509-
}catch{
510-
// Ignore return errors
511-
}
512-
}
513-
});
507+
// Clean up: return all iterators. Cleanup errors are not
508+
// swallowed - a broken iterator.return() (e.g., failing to
509+
// release a resource) should be visible to the caller.
510+
awaitcleanupIterators(iterators,primaryError);
514511
}
515512
},
516513
};
517514
}
518515

516+
asyncfunctioncleanupIterators(iterators,primaryError){
517+
letcleanupError;
518+
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
519+
if(iterator.return){
520+
try{
521+
awaititerator.return();
522+
}catch(err){
523+
// Keep the first cleanup error encountered.
524+
cleanupError??=err;
525+
}
526+
}
527+
});
528+
if(cleanupError!==undefined){
529+
if(primaryError!==undefined){
530+
// Both a primary error and a cleanup error occurred.
531+
// Wrap in SuppressedError so neither is lost:
532+
// .error = primaryError, .suppressed = cleanupError.
533+
// eslint-disable-next-line no-restricted-syntax
534+
thrownewSuppressedError(primaryError,cleanupError);
535+
}
536+
// No primary error - the cleanup error is the only error.
537+
throwcleanupError;
538+
}
539+
if(primaryError!==undefined){
540+
throwprimaryError;
541+
}
542+
}
543+
519544
module.exports={
520545
array,
521546
arrayBuffer,

β€Žlib/internal/streams/iter/pull.jsβ€Ž

Lines changed: 5 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,6 @@ const {
6363
}=require('internal/streams/iter/utils');
6464

6565
const{
66-
drainableProtocol,
67-
kSyncWriteAcceptedOnFalse,
6866
kValidatedSource,
6967
kValidatedTransform,
7068
toAsyncStreamable,
@@ -863,18 +861,6 @@ async function* createAsyncPipeline(source, transforms, signal) {
863861
}
864862
}
865863

866-
/**
867-
* Check if a false sync write result means accepted backpressure.
868-
* @param {object} writer - The writer whose sync method returned.
869-
* @param {*} result - The return value from writeSync() or writevSync().
870-
* @returns {boolean}
871-
*/
872-
functionisAcceptedSyncWriteBackpressure(writer,result){
873-
returnresult===false&&
874-
writer[kSyncWriteAcceptedOnFalse]===true&&
875-
writer.desiredSize===0;
876-
}
877-
878864
// =============================================================================
879865
// Public API: pull() and pullSync()
880866
// =============================================================================
@@ -963,9 +949,7 @@ function pipeToSync(source, ...args) {
963949
break;
964950
}
965951
if(hasWritevSync&&batch.length>1){
966-
constresult=writer.writevSync(batch);
967-
if(result===false&&
968-
!isAcceptedSyncWriteBackpressure(writer,result)){
952+
if(writer.writevSync(batch)===false){
969953
break;
970954
}
971955
for(leti=0;i<batch.length;i++){
@@ -974,9 +958,7 @@ function pipeToSync(source, ...args) {
974958
}else{
975959
for(leti=0;i<batch.length;i++){
976960
constchunk=batch[i];
977-
constresult=writer.writeSync(chunk);
978-
if(result===false&&
979-
!isAcceptedSyncWriteBackpressure(writer,result)){
961+
if(writer.writeSync(chunk)===false){
980962
canContinue=false;
981963
break;
982964
}
@@ -1027,28 +1009,13 @@ async function pipeTo(source, ...args) {
10271009
consthasWritevSync=typeofwriter.writevSync==='function';
10281010
consthasEndSync=typeofwriter.endSync==='function';
10291011

1030-
functionwaitForSyncBackpressure(){
1031-
constondrain=writer[drainableProtocol];
1032-
returnondrain?.call(writer);
1033-
}
1034-
1035-
asyncfunctionwriteBatchAfterAcceptedBackpressure(batch,startIndex){
1036-
awaitwaitForSyncBackpressure();
1037-
awaitwriteBatchAsyncFallback(batch,startIndex);
1038-
}
1039-
10401012
// Async fallback for writeBatch when sync write fails partway through.
10411013
// Continues writing from batch[startIndex] using async write().
10421014
asyncfunctionwriteBatchAsyncFallback(batch,startIndex){
10431015
for(leti=startIndex;i<batch.length;i++){
10441016
constchunk=batch[i];
1045-
constresult=hasWriteSync&&writer.writeSync(chunk);
1046-
if(result){
1017+
if(hasWriteSync&&writer.writeSync(chunk)){
10471018
// Sync retry succeeded
1048-
}elseif(isAcceptedSyncWriteBackpressure(writer,result)){
1049-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1050-
awaitwaitForSyncBackpressure();
1051-
continue;
10521019
}else{
10531020
constresult=writer.write(
10541021
chunk,signal ? {__proto__: null, signal } : undefined);
@@ -1065,14 +1032,7 @@ async function pipeTo(source, ...args) {
10651032
// is required. Callers must check: const p = writeBatch(b); if (p) await p;
10661033
functionwriteBatch(batch){
10671034
if(hasWritev&&batch.length>1){
1068-
constresult=hasWritevSync&&writer.writevSync(batch);
1069-
if(!result){
1070-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1071-
for(leti=0;i<batch.length;i++){
1072-
totalBytes+=TypedArrayPrototypeGetByteLength(batch[i]);
1073-
}
1074-
returnwaitForSyncBackpressure();
1075-
}
1035+
if(!hasWritevSync||!writer.writevSync(batch)){
10761036
constopts=signal ? {__proto__: null, signal } : undefined;
10771037
constwritevResult=writer.writev(batch,opts);
10781038
if(writevResult===undefined){
@@ -1094,14 +1054,8 @@ async function pipeTo(source, ...args) {
10941054
}
10951055
for(leti=0;i<batch.length;i++){
10961056
constchunk=batch[i];
1097-
constresult=hasWriteSync&&writer.writeSync(chunk);
1098-
if(!result){
1099-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1100-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1101-
returnwriteBatchAfterAcceptedBackpressure(batch,i+1);
1102-
}
1057+
if(!hasWriteSync||!writer.writeSync(chunk)){
11031058
// Sync path failed at index i - fall back to async for the rest.
1104-
// Count bytes for chunks already written synchronously (0..i-1).
11051059
returnwriteBatchAsyncFallback(batch,i);
11061060
}
11071061
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);

β€Žlib/internal/streams/iter/push.jsβ€Ž

Lines changed: 2 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@ const {
3232

3333
const{
3434
drainableProtocol,
35-
kSyncWriteAccepted,
36-
kSyncWriteAcceptedOnFalse,
3735
}=require('internal/streams/iter/types');
3836

3937
const{
@@ -364,19 +362,6 @@ class PushQueue {
364362
this.#pendingEnd =pending;
365363
}
366364

367-
/**
368-
* Force-enqueue chunks into the slots buffer, bypassing capacity checks.
369-
* Used by PushWriter.writeSync() for 'block' policy where the data is
370-
* accepted but false is returned as a backpressure signal.
371-
*/
372-
forceEnqueue(chunks){
373-
this.#slots.push(chunks);
374-
for(leti=0;i<chunks.length;i++){
375-
this.#bytesWritten +=TypedArrayPrototypeGetByteLength(chunks[i]);
376-
}
377-
this.#resolvePendingReads();
378-
}
379-
380365
/**
381366
* Wait for backpressure to clear (desiredSize > 0).
382367
* @returns {Promise<void>}
@@ -558,16 +543,11 @@ class PushQueue {
558543

559544
classPushWriter{
560545
#queue;
561-
#syncWriteAccepted =false;
562546

563547
constructor(queue){
564548
this.#queue =queue;
565549
}
566550

567-
[kSyncWriteAccepted](){
568-
returnthis.#syncWriteAccepted;
569-
}
570-
571551
[drainableProtocol](){
572552
constdesired=this.desiredSize;
573553
if(desired===null)returnnull;
@@ -579,10 +559,6 @@ class PushWriter {
579559
returnthis.#queue.desiredSize;
580560
}
581561

582-
get[kSyncWriteAcceptedOnFalse](){
583-
returnthis.#queue.backpressurePolicy==='block';
584-
}
585-
586562
write(chunk,options){
587563
if(!options?.signal&&this.#queue.canWriteSync()){
588564
constbytes=toUint8Array(chunk);
@@ -607,36 +583,16 @@ class PushWriter {
607583
}
608584

609585
writeSync(chunk){
610-
this.#syncWriteAccepted =false;
611586
constbytes=toUint8Array(chunk);
612-
constresult=this.#queue.writeSync([bytes]);
613-
if(!result&&this.#queue.backpressurePolicy==='block'&&
614-
this.#queue.desiredSize===0){
615-
// Block policy: force-enqueue and return false as backpressure signal.
616-
// Data IS accepted; false tells caller to slow down.
617-
this.#queue.forceEnqueue([bytes]);
618-
this.#syncWriteAccepted =true;
619-
returnfalse;
620-
}
621-
this.#syncWriteAccepted =result;
622-
returnresult;
587+
returnthis.#queue.writeSync([bytes]);
623588
}
624589

625590
writevSync(chunks){
626-
this.#syncWriteAccepted =false;
627591
if(!ArrayIsArray(chunks)){
628592
thrownewERR_INVALID_ARG_TYPE('chunks','Array',chunks);
629593
}
630594
constbytes=convertChunks(chunks);
631-
constresult=this.#queue.writeSync(bytes);
632-
if(!result&&this.#queue.backpressurePolicy==='block'&&
633-
this.#queue.desiredSize===0){
634-
this.#queue.forceEnqueue(bytes);
635-
this.#syncWriteAccepted =true;
636-
returnfalse;
637-
}
638-
this.#syncWriteAccepted =result;
639-
returnresult;
595+
returnthis.#queue.writeSync(bytes);
640596
}
641597

642598
end(options){

0 commit comments

Comments
Β (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers"); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Commit a46ddad

Browse files
jasnelladuh95
authored andcommitted
stream: refine the stream/iter backpressure
In `Writer`, there are two queues that matter: the slot queue and the pending queue. The sync `writeSync`/`writevSync` only use the slot queue. If writes cannot be accepted directly into slots, they return `false`. The sync methods *never* enqueue into the pending queue. The async `write`/`writev` will first attempt to add to the slot queue; if it is full, then it will attempt to add to the pending queue; if that is also full, the backpressure policy kicks in. Signed-off-by: James M Snell <jasnell@gmail.com> PR-URL: #63697 Backport-PR-URL: #64675 Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day>
1 parent 557bf99 commit a46ddad

9 files changed

Lines changed: 185 additions & 201 deletions

File tree

β€Ždoc/api/stream_iter.mdβ€Ž

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1649,7 +1649,7 @@ Creates a classic [`stream.Writable`][] backed by a stream/iter Writer.
16491649

16501650
Each `_write()` / `_writev()` call attempts the Writer's synchronous method
16511651
first (`writeSync` / `writevSync`), falling back to the async method if the
1652-
sync path returns `false` or throws. Similarly, `_final()` tries `endSync()`
1652+
sync path returns `false`. Similarly, `_final()` tries `endSync()`
16531653
before `end()`. When the sync path succeeds, the callback is deferred via
16541654
`queueMicrotask` to preserve the async resolution contract.
16551655

β€Žlib/internal/streams/iter/classic.jsβ€Ž

Lines changed: 16 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ const {
5757
const{
5858
toAsyncStreamable: kToAsyncStreamable,
5959
kValidatedSource,
60-
kSyncWriteAccepted,
6160
drainableProtocol,
6261
}=require('internal/streams/iter/types');
6362

@@ -765,41 +764,11 @@ function toWritable(writer) {
765764
consthasEndSync=hasEnd&&
766765
typeofwriter.endSync==='function';
767766
consthasFail=typeofwriter.fail==='function';
768-
consthasSyncWriteAccepted=
769-
typeofwriter[kSyncWriteAccepted]==='function';
770-
771-
functionsyncWriteAccepted(){
772-
returnhasSyncWriteAccepted&&writer[kSyncWriteAccepted]();
773-
}
774-
775-
functionfinishAfterSyncBackpressure(cb){
776-
letondrain;
777-
try{
778-
if(typeofwriter[drainableProtocol]==='function'){
779-
ondrain=writer[drainableProtocol]();
780-
}
781-
}catch(err){
782-
cb(err);
783-
return;
784-
}
785-
if(ondrain!==null&&ondrain!==undefined){
786-
PromisePrototypeThen(ondrain,(drained)=>{
787-
if(drained===false){
788-
cb(newERR_INVALID_STATE.TypeError('Stream closed by consumer'));
789-
return;
790-
}
791-
cb();
792-
},cb);
793-
return;
794-
}
795-
queueMicrotask(cb);
796-
}
797-
798767
// Try-sync-first pattern: attempt the synchronous method and fall back to the
799-
// async method if it returns false without accepting the data, or if it
800-
// throws. When the sync path succeeds, the callback is deferred via
801-
// queueMicrotask to preserve the async resolution contract that Writable
802-
// internals expect from _write/_writev/_final callbacks.
768+
// async method if it returns false (data not accepted synchronously).
769+
// When the sync path succeeds, the callback is deferred via queueMicrotask
770+
// to preserve the async resolution contract that Writable internals expect
771+
// from _write/_writev/_final callbacks.
803772

804773
function_write(chunk,encoding,cb){
805774
constbytes=typeofchunk==='string' ?
@@ -810,13 +779,10 @@ function toWritable(writer) {
810779
queueMicrotask(cb);
811780
return;
812781
}
813-
if(syncWriteAccepted()){
814-
// The chunk was accepted; false only signaled backpressure.
815-
finishAfterSyncBackpressure(cb);
816-
return;
817-
}
818-
}catch{
819-
// Sync path threw -- fall through to async.
782+
// WriteSync returned false: not accepted, fall through to async.
783+
}catch(err){
784+
cb(err);
785+
return;
820786
}
821787
}
822788
try{
@@ -839,13 +805,10 @@ function toWritable(writer) {
839805
queueMicrotask(cb);
840806
return;
841807
}
842-
if(syncWriteAccepted()){
843-
// The chunks were accepted; false only signaled backpressure.
844-
finishAfterSyncBackpressure(cb);
845-
return;
846-
}
847-
}catch{
848-
// Sync path threw -- fall through to async.
808+
// WritevSync returned false: not accepted, fall through to async.
809+
}catch(err){
810+
cb(err);
811+
return;
849812
}
850813
}
851814
try{
@@ -867,8 +830,10 @@ function toWritable(writer) {
867830
queueMicrotask(cb);
868831
return;
869832
}
870-
}catch{
871-
// Sync path threw -- fall through to async.
833+
// Result < 0: can't end synchronously, fall through to async.
834+
}catch(err){
835+
cb(err);
836+
return;
872837
}
873838
}
874839
try{

β€Žlib/internal/streams/iter/consumers.jsβ€Ž

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,7 @@ function merge(...args) {
480480
);
481481
}
482482

483+
letprimaryError;
483484
try{
484485
while(activeCount>0||ready.length>0){
485486
signal?.throwIfAborted();
@@ -500,22 +501,46 @@ function merge(...args) {
500501
});
501502
}
502503
}
504+
}catch(err){
505+
primaryError=err;
503506
}finally{
504-
// Clean up: return all iterators
505-
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
506-
if(iterator.return){
507-
try{
508-
awaititerator.return();
509-
}catch{
510-
// Ignore return errors
511-
}
512-
}
513-
});
507+
// Clean up: return all iterators. Cleanup errors are not
508+
// swallowed - a broken iterator.return() (e.g., failing to
509+
// release a resource) should be visible to the caller.
510+
awaitcleanupIterators(iterators,primaryError);
514511
}
515512
},
516513
};
517514
}
518515

516+
asyncfunctioncleanupIterators(iterators,primaryError){
517+
letcleanupError;
518+
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
519+
if(iterator.return){
520+
try{
521+
awaititerator.return();
522+
}catch(err){
523+
// Keep the first cleanup error encountered.
524+
cleanupError??=err;
525+
}
526+
}
527+
});
528+
if(cleanupError!==undefined){
529+
if(primaryError!==undefined){
530+
// Both a primary error and a cleanup error occurred.
531+
// Wrap in SuppressedError so neither is lost:
532+
// .error = primaryError, .suppressed = cleanupError.
533+
// eslint-disable-next-line no-restricted-syntax
534+
thrownewSuppressedError(primaryError,cleanupError);
535+
}
536+
// No primary error - the cleanup error is the only error.
537+
throwcleanupError;
538+
}
539+
if(primaryError!==undefined){
540+
throwprimaryError;
541+
}
542+
}
543+
519544
module.exports={
520545
array,
521546
arrayBuffer,

β€Žlib/internal/streams/iter/pull.jsβ€Ž

Lines changed: 5 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,6 @@ const {
6363
}=require('internal/streams/iter/utils');
6464

6565
const{
66-
drainableProtocol,
67-
kSyncWriteAcceptedOnFalse,
6866
kValidatedSource,
6967
kValidatedTransform,
7068
toAsyncStreamable,
@@ -863,18 +861,6 @@ async function* createAsyncPipeline(source, transforms, signal) {
863861
}
864862
}
865863

866-
/**
867-
* Check if a false sync write result means accepted backpressure.
868-
* @param {object} writer - The writer whose sync method returned.
869-
* @param {*} result - The return value from writeSync() or writevSync().
870-
* @returns {boolean}
871-
*/
872-
functionisAcceptedSyncWriteBackpressure(writer,result){
873-
returnresult===false&&
874-
writer[kSyncWriteAcceptedOnFalse]===true&&
875-
writer.desiredSize===0;
876-
}
877-
878864
// =============================================================================
879865
// Public API: pull() and pullSync()
880866
// =============================================================================
@@ -963,9 +949,7 @@ function pipeToSync(source, ...args) {
963949
break;
964950
}
965951
if(hasWritevSync&&batch.length>1){
966-
constresult=writer.writevSync(batch);
967-
if(result===false&&
968-
!isAcceptedSyncWriteBackpressure(writer,result)){
952+
if(writer.writevSync(batch)===false){
969953
break;
970954
}
971955
for(leti=0;i<batch.length;i++){
@@ -974,9 +958,7 @@ function pipeToSync(source, ...args) {
974958
}else{
975959
for(leti=0;i<batch.length;i++){
976960
constchunk=batch[i];
977-
constresult=writer.writeSync(chunk);
978-
if(result===false&&
979-
!isAcceptedSyncWriteBackpressure(writer,result)){
961+
if(writer.writeSync(chunk)===false){
980962
canContinue=false;
981963
break;
982964
}
@@ -1027,28 +1009,13 @@ async function pipeTo(source, ...args) {
10271009
consthasWritevSync=typeofwriter.writevSync==='function';
10281010
consthasEndSync=typeofwriter.endSync==='function';
10291011

1030-
functionwaitForSyncBackpressure(){
1031-
constondrain=writer[drainableProtocol];
1032-
returnondrain?.call(writer);
1033-
}
1034-
1035-
asyncfunctionwriteBatchAfterAcceptedBackpressure(batch,startIndex){
1036-
awaitwaitForSyncBackpressure();
1037-
awaitwriteBatchAsyncFallback(batch,startIndex);
1038-
}
1039-
10401012
// Async fallback for writeBatch when sync write fails partway through.
10411013
// Continues writing from batch[startIndex] using async write().
10421014
asyncfunctionwriteBatchAsyncFallback(batch,startIndex){
10431015
for(leti=startIndex;i<batch.length;i++){
10441016
constchunk=batch[i];
1045-
constresult=hasWriteSync&&writer.writeSync(chunk);
1046-
if(result){
1017+
if(hasWriteSync&&writer.writeSync(chunk)){
10471018
// Sync retry succeeded
1048-
}elseif(isAcceptedSyncWriteBackpressure(writer,result)){
1049-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1050-
awaitwaitForSyncBackpressure();
1051-
continue;
10521019
}else{
10531020
constresult=writer.write(
10541021
chunk,signal ? {__proto__: null, signal } : undefined);
@@ -1065,14 +1032,7 @@ async function pipeTo(source, ...args) {
10651032
// is required. Callers must check: const p = writeBatch(b); if (p) await p;
10661033
functionwriteBatch(batch){
10671034
if(hasWritev&&batch.length>1){
1068-
constresult=hasWritevSync&&writer.writevSync(batch);
1069-
if(!result){
1070-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1071-
for(leti=0;i<batch.length;i++){
1072-
totalBytes+=TypedArrayPrototypeGetByteLength(batch[i]);
1073-
}
1074-
returnwaitForSyncBackpressure();
1075-
}
1035+
if(!hasWritevSync||!writer.writevSync(batch)){
10761036
constopts=signal ? {__proto__: null, signal } : undefined;
10771037
constwritevResult=writer.writev(batch,opts);
10781038
if(writevResult===undefined){
@@ -1094,14 +1054,8 @@ async function pipeTo(source, ...args) {
10941054
}
10951055
for(leti=0;i<batch.length;i++){
10961056
constchunk=batch[i];
1097-
constresult=hasWriteSync&&writer.writeSync(chunk);
1098-
if(!result){
1099-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1100-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1101-
returnwriteBatchAfterAcceptedBackpressure(batch,i+1);
1102-
}
1057+
if(!hasWriteSync||!writer.writeSync(chunk)){
11031058
// Sync path failed at index i - fall back to async for the rest.
1104-
// Count bytes for chunks already written synchronously (0..i-1).
11051059
returnwriteBatchAsyncFallback(batch,i);
11061060
}
11071061
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);

β€Žlib/internal/streams/iter/push.jsβ€Ž

Lines changed: 2 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@ const {
3232

3333
const{
3434
drainableProtocol,
35-
kSyncWriteAccepted,
36-
kSyncWriteAcceptedOnFalse,
3735
}=require('internal/streams/iter/types');
3836

3937
const{
@@ -364,19 +362,6 @@ class PushQueue {
364362
this.#pendingEnd =pending;
365363
}
366364

367-
/**
368-
* Force-enqueue chunks into the slots buffer, bypassing capacity checks.
369-
* Used by PushWriter.writeSync() for 'block' policy where the data is
370-
* accepted but false is returned as a backpressure signal.
371-
*/
372-
forceEnqueue(chunks){
373-
this.#slots.push(chunks);
374-
for(leti=0;i<chunks.length;i++){
375-
this.#bytesWritten +=TypedArrayPrototypeGetByteLength(chunks[i]);
376-
}
377-
this.#resolvePendingReads();
378-
}
379-
380365
/**
381366
* Wait for backpressure to clear (desiredSize > 0).
382367
* @returns {Promise<void>}
@@ -558,16 +543,11 @@ class PushQueue {
558543

559544
classPushWriter{
560545
#queue;
561-
#syncWriteAccepted =false;
562546

563547
constructor(queue){
564548
this.#queue =queue;
565549
}
566550

567-
[kSyncWriteAccepted](){
568-
returnthis.#syncWriteAccepted;
569-
}
570-
571551
[drainableProtocol](){
572552
constdesired=this.desiredSize;
573553
if(desired===null)returnnull;
@@ -579,10 +559,6 @@ class PushWriter {
579559
returnthis.#queue.desiredSize;
580560
}
581561

582-
get[kSyncWriteAcceptedOnFalse](){
583-
returnthis.#queue.backpressurePolicy==='block';
584-
}
585-
586562
write(chunk,options){
587563
if(!options?.signal&&this.#queue.canWriteSync()){
588564
constbytes=toUint8Array(chunk);
@@ -607,36 +583,16 @@ class PushWriter {
607583
}
608584

609585
writeSync(chunk){
610-
this.#syncWriteAccepted =false;
611586
constbytes=toUint8Array(chunk);
612-
constresult=this.#queue.writeSync([bytes]);
613-
if(!result&&this.#queue.backpressurePolicy==='block'&&
614-
this.#queue.desiredSize===0){
615-
// Block policy: force-enqueue and return false as backpressure signal.
616-
// Data IS accepted; false tells caller to slow down.
617-
this.#queue.forceEnqueue([bytes]);
618-
this.#syncWriteAccepted =true;
619-
returnfalse;
620-
}
621-
this.#syncWriteAccepted =result;
622-
returnresult;
587+
returnthis.#queue.writeSync([bytes]);
623588
}
624589

625590
writevSync(chunks){
626-
this.#syncWriteAccepted =false;
627591
if(!ArrayIsArray(chunks)){
628592
thrownewERR_INVALID_ARG_TYPE('chunks','Array',chunks);
629593
}
630594
constbytes=convertChunks(chunks);
631-
constresult=this.#queue.writeSync(bytes);
632-
if(!result&&this.#queue.backpressurePolicy==='block'&&
633-
this.#queue.desiredSize===0){
634-
this.#queue.forceEnqueue(bytes);
635-
this.#syncWriteAccepted =true;
636-
returnfalse;
637-
}
638-
this.#syncWriteAccepted =result;
639-
returnresult;
595+
returnthis.#queue.writeSync(bytes);
640596
}
641597

642598
end(options){

0 commit comments

Comments
Β (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode"); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })();
Skip to content

Commit a46ddad

Browse files
jasnelladuh95
authored andcommitted
stream: refine the stream/iter backpressure
In `Writer`, there are two queues that matter: the slot queue and the pending queue. The sync `writeSync`/`writevSync` only use the slot queue. If writes cannot be accepted directly into slots, they return `false`. The sync methods *never* enqueue into the pending queue. The async `write`/`writev` will first attempt to add to the slot queue; if it is full, then it will attempt to add to the pending queue; if that is also full, the backpressure policy kicks in. Signed-off-by: James M Snell <jasnell@gmail.com> PR-URL: #63697 Backport-PR-URL: #64675 Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day>
1 parent 557bf99 commit a46ddad

9 files changed

Lines changed: 185 additions & 201 deletions

File tree

β€Ždoc/api/stream_iter.mdβ€Ž

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1649,7 +1649,7 @@ Creates a classic [`stream.Writable`][] backed by a stream/iter Writer.
16491649

16501650
Each `_write()` / `_writev()` call attempts the Writer's synchronous method
16511651
first (`writeSync` / `writevSync`), falling back to the async method if the
1652-
sync path returns `false` or throws. Similarly, `_final()` tries `endSync()`
1652+
sync path returns `false`. Similarly, `_final()` tries `endSync()`
16531653
before `end()`. When the sync path succeeds, the callback is deferred via
16541654
`queueMicrotask` to preserve the async resolution contract.
16551655

β€Žlib/internal/streams/iter/classic.jsβ€Ž

Lines changed: 16 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ const {
5757
const{
5858
toAsyncStreamable: kToAsyncStreamable,
5959
kValidatedSource,
60-
kSyncWriteAccepted,
6160
drainableProtocol,
6261
}=require('internal/streams/iter/types');
6362

@@ -765,41 +764,11 @@ function toWritable(writer) {
765764
consthasEndSync=hasEnd&&
766765
typeofwriter.endSync==='function';
767766
consthasFail=typeofwriter.fail==='function';
768-
consthasSyncWriteAccepted=
769-
typeofwriter[kSyncWriteAccepted]==='function';
770-
771-
functionsyncWriteAccepted(){
772-
returnhasSyncWriteAccepted&&writer[kSyncWriteAccepted]();
773-
}
774-
775-
functionfinishAfterSyncBackpressure(cb){
776-
letondrain;
777-
try{
778-
if(typeofwriter[drainableProtocol]==='function'){
779-
ondrain=writer[drainableProtocol]();
780-
}
781-
}catch(err){
782-
cb(err);
783-
return;
784-
}
785-
if(ondrain!==null&&ondrain!==undefined){
786-
PromisePrototypeThen(ondrain,(drained)=>{
787-
if(drained===false){
788-
cb(newERR_INVALID_STATE.TypeError('Stream closed by consumer'));
789-
return;
790-
}
791-
cb();
792-
},cb);
793-
return;
794-
}
795-
queueMicrotask(cb);
796-
}
797-
798767
// Try-sync-first pattern: attempt the synchronous method and fall back to the
799-
// async method if it returns false without accepting the data, or if it
800-
// throws. When the sync path succeeds, the callback is deferred via
801-
// queueMicrotask to preserve the async resolution contract that Writable
802-
// internals expect from _write/_writev/_final callbacks.
768+
// async method if it returns false (data not accepted synchronously).
769+
// When the sync path succeeds, the callback is deferred via queueMicrotask
770+
// to preserve the async resolution contract that Writable internals expect
771+
// from _write/_writev/_final callbacks.
803772

804773
function_write(chunk,encoding,cb){
805774
constbytes=typeofchunk==='string' ?
@@ -810,13 +779,10 @@ function toWritable(writer) {
810779
queueMicrotask(cb);
811780
return;
812781
}
813-
if(syncWriteAccepted()){
814-
// The chunk was accepted; false only signaled backpressure.
815-
finishAfterSyncBackpressure(cb);
816-
return;
817-
}
818-
}catch{
819-
// Sync path threw -- fall through to async.
782+
// WriteSync returned false: not accepted, fall through to async.
783+
}catch(err){
784+
cb(err);
785+
return;
820786
}
821787
}
822788
try{
@@ -839,13 +805,10 @@ function toWritable(writer) {
839805
queueMicrotask(cb);
840806
return;
841807
}
842-
if(syncWriteAccepted()){
843-
// The chunks were accepted; false only signaled backpressure.
844-
finishAfterSyncBackpressure(cb);
845-
return;
846-
}
847-
}catch{
848-
// Sync path threw -- fall through to async.
808+
// WritevSync returned false: not accepted, fall through to async.
809+
}catch(err){
810+
cb(err);
811+
return;
849812
}
850813
}
851814
try{
@@ -867,8 +830,10 @@ function toWritable(writer) {
867830
queueMicrotask(cb);
868831
return;
869832
}
870-
}catch{
871-
// Sync path threw -- fall through to async.
833+
// Result < 0: can't end synchronously, fall through to async.
834+
}catch(err){
835+
cb(err);
836+
return;
872837
}
873838
}
874839
try{

β€Žlib/internal/streams/iter/consumers.jsβ€Ž

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,7 @@ function merge(...args) {
480480
);
481481
}
482482

483+
letprimaryError;
483484
try{
484485
while(activeCount>0||ready.length>0){
485486
signal?.throwIfAborted();
@@ -500,22 +501,46 @@ function merge(...args) {
500501
});
501502
}
502503
}
504+
}catch(err){
505+
primaryError=err;
503506
}finally{
504-
// Clean up: return all iterators
505-
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
506-
if(iterator.return){
507-
try{
508-
awaititerator.return();
509-
}catch{
510-
// Ignore return errors
511-
}
512-
}
513-
});
507+
// Clean up: return all iterators. Cleanup errors are not
508+
// swallowed - a broken iterator.return() (e.g., failing to
509+
// release a resource) should be visible to the caller.
510+
awaitcleanupIterators(iterators,primaryError);
514511
}
515512
},
516513
};
517514
}
518515

516+
asyncfunctioncleanupIterators(iterators,primaryError){
517+
letcleanupError;
518+
awaitSafePromiseAllReturnVoid(iterators,async(iterator)=>{
519+
if(iterator.return){
520+
try{
521+
awaititerator.return();
522+
}catch(err){
523+
// Keep the first cleanup error encountered.
524+
cleanupError??=err;
525+
}
526+
}
527+
});
528+
if(cleanupError!==undefined){
529+
if(primaryError!==undefined){
530+
// Both a primary error and a cleanup error occurred.
531+
// Wrap in SuppressedError so neither is lost:
532+
// .error = primaryError, .suppressed = cleanupError.
533+
// eslint-disable-next-line no-restricted-syntax
534+
thrownewSuppressedError(primaryError,cleanupError);
535+
}
536+
// No primary error - the cleanup error is the only error.
537+
throwcleanupError;
538+
}
539+
if(primaryError!==undefined){
540+
throwprimaryError;
541+
}
542+
}
543+
519544
module.exports={
520545
array,
521546
arrayBuffer,

β€Žlib/internal/streams/iter/pull.jsβ€Ž

Lines changed: 5 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,6 @@ const {
6363
}=require('internal/streams/iter/utils');
6464

6565
const{
66-
drainableProtocol,
67-
kSyncWriteAcceptedOnFalse,
6866
kValidatedSource,
6967
kValidatedTransform,
7068
toAsyncStreamable,
@@ -863,18 +861,6 @@ async function* createAsyncPipeline(source, transforms, signal) {
863861
}
864862
}
865863

866-
/**
867-
* Check if a false sync write result means accepted backpressure.
868-
* @param {object} writer - The writer whose sync method returned.
869-
* @param {*} result - The return value from writeSync() or writevSync().
870-
* @returns {boolean}
871-
*/
872-
functionisAcceptedSyncWriteBackpressure(writer,result){
873-
returnresult===false&&
874-
writer[kSyncWriteAcceptedOnFalse]===true&&
875-
writer.desiredSize===0;
876-
}
877-
878864
// =============================================================================
879865
// Public API: pull() and pullSync()
880866
// =============================================================================
@@ -963,9 +949,7 @@ function pipeToSync(source, ...args) {
963949
break;
964950
}
965951
if(hasWritevSync&&batch.length>1){
966-
constresult=writer.writevSync(batch);
967-
if(result===false&&
968-
!isAcceptedSyncWriteBackpressure(writer,result)){
952+
if(writer.writevSync(batch)===false){
969953
break;
970954
}
971955
for(leti=0;i<batch.length;i++){
@@ -974,9 +958,7 @@ function pipeToSync(source, ...args) {
974958
}else{
975959
for(leti=0;i<batch.length;i++){
976960
constchunk=batch[i];
977-
constresult=writer.writeSync(chunk);
978-
if(result===false&&
979-
!isAcceptedSyncWriteBackpressure(writer,result)){
961+
if(writer.writeSync(chunk)===false){
980962
canContinue=false;
981963
break;
982964
}
@@ -1027,28 +1009,13 @@ async function pipeTo(source, ...args) {
10271009
consthasWritevSync=typeofwriter.writevSync==='function';
10281010
consthasEndSync=typeofwriter.endSync==='function';
10291011

1030-
functionwaitForSyncBackpressure(){
1031-
constondrain=writer[drainableProtocol];
1032-
returnondrain?.call(writer);
1033-
}
1034-
1035-
asyncfunctionwriteBatchAfterAcceptedBackpressure(batch,startIndex){
1036-
awaitwaitForSyncBackpressure();
1037-
awaitwriteBatchAsyncFallback(batch,startIndex);
1038-
}
1039-
10401012
// Async fallback for writeBatch when sync write fails partway through.
10411013
// Continues writing from batch[startIndex] using async write().
10421014
asyncfunctionwriteBatchAsyncFallback(batch,startIndex){
10431015
for(leti=startIndex;i<batch.length;i++){
10441016
constchunk=batch[i];
1045-
constresult=hasWriteSync&&writer.writeSync(chunk);
1046-
if(result){
1017+
if(hasWriteSync&&writer.writeSync(chunk)){
10471018
// Sync retry succeeded
1048-
}elseif(isAcceptedSyncWriteBackpressure(writer,result)){
1049-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1050-
awaitwaitForSyncBackpressure();
1051-
continue;
10521019
}else{
10531020
constresult=writer.write(
10541021
chunk,signal ? {__proto__: null, signal } : undefined);
@@ -1065,14 +1032,7 @@ async function pipeTo(source, ...args) {
10651032
// is required. Callers must check: const p = writeBatch(b); if (p) await p;
10661033
functionwriteBatch(batch){
10671034
if(hasWritev&&batch.length>1){
1068-
constresult=hasWritevSync&&writer.writevSync(batch);
1069-
if(!result){
1070-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1071-
for(leti=0;i<batch.length;i++){
1072-
totalBytes+=TypedArrayPrototypeGetByteLength(batch[i]);
1073-
}
1074-
returnwaitForSyncBackpressure();
1075-
}
1035+
if(!hasWritevSync||!writer.writevSync(batch)){
10761036
constopts=signal ? {__proto__: null, signal } : undefined;
10771037
constwritevResult=writer.writev(batch,opts);
10781038
if(writevResult===undefined){
@@ -1094,14 +1054,8 @@ async function pipeTo(source, ...args) {
10941054
}
10951055
for(leti=0;i<batch.length;i++){
10961056
constchunk=batch[i];
1097-
constresult=hasWriteSync&&writer.writeSync(chunk);
1098-
if(!result){
1099-
if(isAcceptedSyncWriteBackpressure(writer,result)){
1100-
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);
1101-
returnwriteBatchAfterAcceptedBackpressure(batch,i+1);
1102-
}
1057+
if(!hasWriteSync||!writer.writeSync(chunk)){
11031058
// Sync path failed at index i - fall back to async for the rest.
1104-
// Count bytes for chunks already written synchronously (0..i-1).
11051059
returnwriteBatchAsyncFallback(batch,i);
11061060
}
11071061
totalBytes+=TypedArrayPrototypeGetByteLength(chunk);

β€Žlib/internal/streams/iter/push.jsβ€Ž

Lines changed: 2 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@ const {
3232

3333
const{
3434
drainableProtocol,
35-
kSyncWriteAccepted,
36-
kSyncWriteAcceptedOnFalse,
3735
}=require('internal/streams/iter/types');
3836

3937
const{
@@ -364,19 +362,6 @@ class PushQueue {
364362
this.#pendingEnd =pending;
365363
}
366364

367-
/**
368-
* Force-enqueue chunks into the slots buffer, bypassing capacity checks.
369-
* Used by PushWriter.writeSync() for 'block' policy where the data is
370-
* accepted but false is returned as a backpressure signal.
371-
*/
372-
forceEnqueue(chunks){
373-
this.#slots.push(chunks);
374-
for(leti=0;i<chunks.length;i++){
375-
this.#bytesWritten +=TypedArrayPrototypeGetByteLength(chunks[i]);
376-
}
377-
this.#resolvePendingReads();
378-
}
379-
380365
/**
381366
* Wait for backpressure to clear (desiredSize > 0).
382367
* @returns {Promise<void>}
@@ -558,16 +543,11 @@ class PushQueue {
558543

559544
classPushWriter{
560545
#queue;
561-
#syncWriteAccepted =false;
562546

563547
constructor(queue){
564548
this.#queue =queue;
565549
}
566550

567-
[kSyncWriteAccepted](){
568-
returnthis.#syncWriteAccepted;
569-
}
570-
571551
[drainableProtocol](){
572552
constdesired=this.desiredSize;
573553
if(desired===null)returnnull;
@@ -579,10 +559,6 @@ class PushWriter {
579559
returnthis.#queue.desiredSize;
580560
}
581561

582-
get[kSyncWriteAcceptedOnFalse](){
583-
returnthis.#queue.backpressurePolicy==='block';
584-
}
585-
586562
write(chunk,options){
587563
if(!options?.signal&&this.#queue.canWriteSync()){
588564
constbytes=toUint8Array(chunk);
@@ -607,36 +583,16 @@ class PushWriter {
607583
}
608584

609585
writeSync(chunk){
610-
this.#syncWriteAccepted =false;
611586
constbytes=toUint8Array(chunk);
612-
constresult=this.#queue.writeSync([bytes]);
613-
if(!result&&this.#queue.backpressurePolicy==='block'&&
614-
this.#queue.desiredSize===0){
615-
// Block policy: force-enqueue and return false as backpressure signal.
616-
// Data IS accepted; false tells caller to slow down.
617-
this.#queue.forceEnqueue([bytes]);
618-
this.#syncWriteAccepted =true;
619-
returnfalse;
620-
}
621-
this.#syncWriteAccepted =result;
622-
returnresult;
587+
returnthis.#queue.writeSync([bytes]);
623588
}
624589

625590
writevSync(chunks){
626-
this.#syncWriteAccepted =false;
627591
if(!ArrayIsArray(chunks)){
628592
thrownewERR_INVALID_ARG_TYPE('chunks','Array',chunks);
629593
}
630594
constbytes=convertChunks(chunks);
631-
constresult=this.#queue.writeSync(bytes);
632-
if(!result&&this.#queue.backpressurePolicy==='block'&&
633-
this.#queue.desiredSize===0){
634-
this.#queue.forceEnqueue(bytes);
635-
this.#syncWriteAccepted =true;
636-
returnfalse;
637-
}
638-
this.#syncWriteAccepted =result;
639-
returnresult;
595+
returnthis.#queue.writeSync(bytes);
640596
}
641597

642598
end(options){

0 commit comments

Comments
Β (0)