crypto: guard against size_t overflow on experimental 32-bit arch

Refs: https://hackerone.com/reports/3655230
Signed-off-by: Filip Skokan <panva.ip@gmail.com>
PR-URL: #62626
Backport-PR-URL: #63563
Refs: https://hackerone.com/reports/3655230
Reviewed-By: Juan José Arboleda <soyjuanarbol@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com>

Author: panva

src/crypto/crypto_turboshake.cc

@@ -587,7 +587,6 @@ bool KangarooTwelveTraits::DeriveBits(Environment* env,
  ByteSource* out,
  CryptoJobMode mode) {
 CHECK_GT(params.output_length, 0);
-char* buf = MallocOpenSSL<char>(params.output_length);
 
 constuint8_t* input = reinterpret_cast<constuint8_t*>(params.data.data());
 size_t input_len = params.data.size();
@@ -596,6 +595,17 @@ bool KangarooTwelveTraits::DeriveBits(Environment* env,
 reinterpret_cast<constuint8_t*>(params.customization.data());
 size_t custom_len = params.customization.size();
 
+// Guard against size_t overflow in KangarooTwelve's s_len computation:
+// s_len = msg_len + custom_len + LengthEncode(custom_len).size()
+// LengthEncode produces at most sizeof(size_t) + 1 bytes.
+staticconstexprsize_tkMaxLengthEncodeSize = sizeof(size_t) + 1;
+if (input_len > SIZE_MAX - custom_len ||
+ input_len + custom_len > SIZE_MAX - kMaxLengthEncodeSize) {
+returnfalse;
+ }
+
+char* buf = MallocOpenSSL<char>(params.output_length);
+
 switch (params.variant) {
 case KangarooTwelveVariant::KT128:
 KT128(input,

test/pummel/test-webcrypto-kangarootwelve-32bit-overflow.js

@@ -0,0 +1,41 @@
+'use strict';
+constcommon=require('../common');
+
+if(!common.hasCrypto)
+common.skip('missing crypto');
+
+// KangarooTwelve: data + customization size_t overflow on 32-bit platforms.
+// On 32-bit, msg_len + custom_len + LengthEncode(custom_len).size() can wrap
+// size_t, causing an undersized allocation and heap buffer overflow.
+// This test verifies the guard rejects such inputs.
+// When kMaxLength < 2^32 two max-sized buffers can overflow a 32-bit size_t.
+
+const{ kMaxLength }=require('buffer');
+
+if(kMaxLength>=2**32)
+common.skip('only relevant when kMaxLength < 2^32');
+
+constassert=require('assert');
+const{ subtle }=globalThis.crypto;
+
+letdata,customization;
+try{
+data=newUint8Array(kMaxLength);
+customization=newUint8Array(kMaxLength);
+}catch{
+common.skip('insufficient memory to allocate test buffers');
+}
+
+(async()=>{
+awaitassert.rejects(
+subtle.digest(
+{name: 'KT128',outputLength: 256, customization },
+data),
+{name: 'OperationError'});
+
+awaitassert.rejects(
+subtle.digest(
+{name: 'KT256',outputLength: 512, customization },
+data),
+{name: 'OperationError'});
+})().then(common.mustCall());