From c6a5721586094139c17fb316c3744bde48ba962d Mon Sep 17 00:00:00 2001 From: Fedor Chelnokov Date: Tue, 8 Sep 2026 11:22:06 +0300 Subject: [PATCH 1/6] Reject vertex/triangle counts whose allocation size would overflow size_t --- lib/openctm.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/openctm.c b/lib/openctm.c index 8310a59..575d58f 100644 --- a/lib/openctm.c +++ b/lib/openctm.c @@ -1234,6 +1234,18 @@ CTMEXPORT void CTMCALL ctmLoadCustom(CTMcontext aContext, CTMreadfn aReadFn, self->mUVMapCount = _ctmStreamReadUINT(self); self->mAttribMapCount = _ctmStreamReadUINT(self); flags = _ctmStreamReadUINT(self); + + // Reject counts so large that a per-element allocation size (count * stride) + // would overflow size_t and wrap to an undersized buffer. The largest strides + // are 16 bytes per vertex and 12 per triangle; 32 is a safe bound for both. + if(self->mVertexCount > ((size_t) -1) / 32 || + self->mTriangleCount > ((size_t) -1) / 32) + { + _ctmClearMesh(self); + self->mError = CTM_BAD_FORMAT; + return; + } + _ctmStreamReadSTRING(self, &self->mFileComment); if(self->mError != CTM_NONE) { From 8e70fe6ba48dbf47e1e3c8248a94794d5013f30c Mon Sep 17 00:00:00 2001 From: Fedor Chelnokov Date: Tue, 8 Sep 2026 11:37:37 +0300 Subject: [PATCH 2/6] Free partial mesh on mVertices allocation failure Match the surrounding OOM error paths, which all call _ctmClearMesh before returning. --- lib/openctm.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/openctm.c b/lib/openctm.c index 575d58f..449ee0e 100644 --- a/lib/openctm.c +++ b/lib/openctm.c @@ -1257,6 +1257,7 @@ CTMEXPORT void CTMCALL ctmLoadCustom(CTMcontext aContext, CTMreadfn aReadFn, self->mVertices = (CTMfloat *) malloc(self->mVertexCount * sizeof(CTMfloat) * 3); if(!self->mVertices) { + _ctmClearMesh(self); self->mError = CTM_OUT_OF_MEMORY; return; } From ae76f6676cb12ed23df6201f00cb7258b11077d1 Mon Sep 17 00:00:00 2001 From: Fedor Chelnokov Date: Tue, 8 Sep 2026 11:49:29 +0300 Subject: [PATCH 3/6] Bound count guard by 32-bit UINT_MAX and compute float-map size in size_t The previous guard rejected counts above size_t max / 32, so on a 64-bit size_t it never fired. The mesh allocations in ctmLoadCustom multiply by a size_t sizeof and are safe there, but several count * stride products are computed in 32-bit CTMuint and wrap regardless of size_t width: - _ctmAllocateFloatMaps: size was CTMuint, so aChannels * 4 * mVertexCount (up to 16 bytes/vertex) truncated to 32 bits before malloc/memset, while the decoder then wrote mVertexCount * channels floats -> heap overflow. - stream.c packed int/float temp buffers: malloc(aCount * aSize * 4), all 32-bit operands. On 64-bit these sit behind multi-GB sibling allocations, but under Linux memory overcommit those can succeed, making the truncated map write reachable (not merely a large-allocation DoS). Bounding both counts by 0xFFFFFFFF / 32 keeps count * stride (stride <= 16) below 2^32 on every platform, closing the CTMuint products at the source; the wasm32 (32-bit size_t) overflow this PR targets stays closed. Also compute the float-map allocation size in size_t so it is correct independent of the guard. --- lib/openctm.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/openctm.c b/lib/openctm.c index 449ee0e..ab39ce8 100644 --- a/lib/openctm.c +++ b/lib/openctm.c @@ -1149,7 +1149,8 @@ static CTMuint _ctmAllocateFloatMaps(_CTMcontext * self, _CTMfloatmap ** aMapListPtr, CTMuint aCount, CTMuint aChannels) { _CTMfloatmap ** mapListPtr; - CTMuint i, size; + CTMuint i; + size_t size; mapListPtr = aMapListPtr; for(i = 0; i < aCount; ++ i) @@ -1235,11 +1236,11 @@ CTMEXPORT void CTMCALL ctmLoadCustom(CTMcontext aContext, CTMreadfn aReadFn, self->mAttribMapCount = _ctmStreamReadUINT(self); flags = _ctmStreamReadUINT(self); - // Reject counts so large that a per-element allocation size (count * stride) - // would overflow size_t and wrap to an undersized buffer. The largest strides - // are 16 bytes per vertex and 12 per triangle; 32 is a safe bound for both. - if(self->mVertexCount > ((size_t) -1) / 32 || - self->mTriangleCount > ((size_t) -1) / 32) + // Reject counts whose count * stride could wrap an allocation to an + // undersized buffer. Largest stride is 16 bytes; 32 is a safe bound. The + // 32-bit UINT_MAX bound also covers strides computed in 32-bit CTMuint. + if(self->mVertexCount > 0xFFFFFFFFu / 32 || + self->mTriangleCount > 0xFFFFFFFFu / 32) { _ctmClearMesh(self); self->mError = CTM_BAD_FORMAT; From 95445855ed7f358e11cf15b8ab5cb4b3d0b8fe49 Mon Sep 17 00:00:00 2001 From: Fedor Chelnokov Date: Tue, 8 Sep 2026 12:01:36 +0300 Subject: [PATCH 4/6] Tighten count guard to the exact 16-byte stride bound --- lib/openctm.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/openctm.c b/lib/openctm.c index ab39ce8..e5086e8 100644 --- a/lib/openctm.c +++ b/lib/openctm.c @@ -1237,10 +1237,10 @@ CTMEXPORT void CTMCALL ctmLoadCustom(CTMcontext aContext, CTMreadfn aReadFn, flags = _ctmStreamReadUINT(self); // Reject counts whose count * stride could wrap an allocation to an - // undersized buffer. Largest stride is 16 bytes; 32 is a safe bound. The - // 32-bit UINT_MAX bound also covers strides computed in 32-bit CTMuint. - if(self->mVertexCount > 0xFFFFFFFFu / 32 || - self->mTriangleCount > 0xFFFFFFFFu / 32) + // undersized buffer. The largest stride is 16 bytes (4 floats per vertex). + // The 32-bit UINT_MAX bound also covers products computed in 32-bit CTMuint. + if(self->mVertexCount > 0xFFFFFFFFu / 16 || + self->mTriangleCount > 0xFFFFFFFFu / 16) { _ctmClearMesh(self); self->mError = CTM_BAD_FORMAT; From de43e4d686f61c918c8fde297ff63c41ab7ea988 Mon Sep 17 00:00:00 2001 From: Fedor Chelnokov Date: Tue, 8 Sep 2026 12:01:36 +0300 Subject: [PATCH 5/6] Free MG1 temporary indices when the packed-int read fails --- lib/compressMG1.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/compressMG1.c b/lib/compressMG1.c index f70206f..68814d8 100644 --- a/lib/compressMG1.c +++ b/lib/compressMG1.c @@ -282,7 +282,10 @@ int _ctmUncompressMesh_MG1(_CTMcontext * self) return CTM_FALSE; } if(!_ctmStreamReadPackedInts(self, (CTMint *) indices, self->mTriangleCount, 3, CTM_FALSE)) + { + free(indices); return CTM_FALSE; + } // Restore indices _ctmRestoreIndices(self, indices); From 2fd8f34c668117d15725358a3eb8e65f55385a6c Mon Sep 17 00:00:00 2001 From: Fedor Chelnokov Date: Tue, 8 Sep 2026 12:12:32 +0300 Subject: [PATCH 6/6] Explain why the count guard is bounded by UINT_MAX and not SIZE_MAX --- lib/openctm.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/openctm.c b/lib/openctm.c index e5086e8..183313a 100644 --- a/lib/openctm.c +++ b/lib/openctm.c @@ -1238,7 +1238,8 @@ CTMEXPORT void CTMCALL ctmLoadCustom(CTMcontext aContext, CTMreadfn aReadFn, // Reject counts whose count * stride could wrap an allocation to an // undersized buffer. The largest stride is 16 bytes (4 floats per vertex). - // The 32-bit UINT_MAX bound also covers products computed in 32-bit CTMuint. + // UINT_MAX rather than SIZE_MAX because count * 3, count * 4 and the temp + // buffer sizes in stream.c are still computed in 32-bit CTMuint everywhere. if(self->mVertexCount > 0xFFFFFFFFu / 16 || self->mTriangleCount > 0xFFFFFFFFu / 16) {