From 77da14d77f5e16c5dc24ca225b16ff77a3a3e446 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:32:00 +0300 Subject: [PATCH 01/42] Make the clean target a usable program runtime, and let casts fail The clean (non-Objective-C) target could translate a Java main() and run it, but not much more: main(String[]) was handed JAVA_NULL, so a translated program could not read its own command line, and there was no way to read the environment, open a file or read stdin. Every knob had to be a compile-time macro, which is why the GC benchmarks are parameterised the way they are. - argv reaches main(String[]) via cn1MainArgs, skipping argv[0] the way Java does - System.getenv(String) - java.io.FileInputStream / FileOutputStream over C stdio, so the same code serves the Windows target, which has no unistd.h - java.io.StandardInputStream behind System.in. Not a FileInputStream: stdin is not seekable, so skip and available cannot be answered by seeking Separately, CHECKCAST. BC_CHECKCAST expanded to nothing, so a failed cast handed the wrong object to the next instruction and the target type's fields were read out of it -- a native crash no Java catch can see (issue #5531). Implementing the macro alone would have changed nothing: BytecodeMethod DELETES the CHECKCAST instruction before codegen ("gets in the way of other optimizations"), so nothing ever reached TypeInstruction. Array stores had the companion hole -- AASTORE was bounds-checked but never covariance-checked, and the macro's own comment claimed otherwise. Both are now enforced under -Dcn1.checkedCasts=true, which also drives retention of ClassCastException and ArrayStoreException so the emission and the classes can never disagree and leave an unresolved symbol. Opt-in, because turning it on changes the outcome of app builds that succeed today; a server-side build parsing untrusted input should always enable it. Verified against vm/tests: 80 tests, no regressions. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 43 +++++ vm/ByteCodeTranslator/src/cn1_globals.m | 59 +++++++ .../tools/translator/ByteCodeClass.java | 16 +- .../tools/translator/ByteCodeTranslator.java | 16 ++ .../tools/translator/BytecodeMethod.java | 15 +- .../bytecodes/BasicInstruction.java | 3 + .../translator/bytecodes/TypeInstruction.java | 41 ++++- vm/ByteCodeTranslator/src/nativeMethods.m | 162 ++++++++++++++++++ vm/CLAUDE.md | 9 +- vm/JavaAPI/src/java/io/FileInputStream.java | 123 +++++++++++++ vm/JavaAPI/src/java/io/FileOutputStream.java | 114 ++++++++++++ .../src/java/io/StandardInputStream.java | 59 +++++++ vm/JavaAPI/src/java/lang/System.java | 15 ++ vm/benchmarks/translate-and-build.sh | 4 +- 14 files changed, 668 insertions(+), 11 deletions(-) create mode 100644 vm/JavaAPI/src/java/io/FileInputStream.java create mode 100644 vm/JavaAPI/src/java/io/FileOutputStream.java create mode 100644 vm/JavaAPI/src/java/io/StandardInputStream.java diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index d2ba08f9cba..d434b49fae4 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -446,7 +446,46 @@ typedef struct clazz* JAVA_CLASS; } // todo map instanceof and throw typecast exception +// CHECKCAST is a no-op by default: ParparVM has always let a failed cast through, +// so the wrong object reaches the next instruction and the target type's fields +// get read out of it (issue #5531). That is a native crash no Java catch can see. +// +// BC_CHECKCAST_CHECKED is the enforcing form. The translator emits it in place of +// BC_CHECKCAST only when -Dcn1.checkedCasts=true, and that same flag is what makes +// the translator retain java.lang.ClassCastException -- so the emission and the +// class's survival can never disagree and leave an unresolved symbol. Enforcement +// is opt-in rather than the default because turning it on changes the outcome of +// app builds that succeed today; server-side (clean-target) builds, which parse +// untrusted input, should always turn it on. +// +// The cost is one instanceofFunction call, the same check INSTANCEOF already pays. #define BC_CHECKCAST(type) +// AASTORE's companion hole: the array store is only bounds-checked, never +// covariance-checked, so `Object[] o = new String[1]; o[0] = anInteger;` silently +// stores the wrong type and the next reader gets an Integer where it expects a +// String. Emitted by BasicInstruction under the same -Dcn1.checkedCasts flag that +// drives BC_CHECKCAST_CHECKED, so ArrayStoreException's retention and the check's +// emission cannot disagree. +// +// arrayType is the component class (0 for a non-array, which cannot happen here +// after CHECK_ARRAY_ACCESS, but is tolerated rather than dereferenced). +#define CN1_ARRAY_STORE_CHECK(arrayObj, value) { \ + if((value) != JAVA_NULL) { \ + struct clazz* cn1__comp = CN1_CLASS_OF(arrayObj)->arrayType; \ + if(cn1__comp != NULL && !instanceofFunction(cn1__comp->classId, GET_CLASS_ID(value))) { \ + cn1ThrowTypeError(threadStateData, __NEW_INSTANCE_java_lang_ArrayStoreException(threadStateData), CN1_CLASS_OF(value)->clsName, NULL); \ + } \ + } \ +} + +#define BC_CHECKCAST_CHECKED(typeOfCheckCast, targetName) { \ + if(SP[-1].data.o != JAVA_NULL) { \ + int tmpCheckCastId = GET_CLASS_ID(SP[-1].data.o); \ + if(!instanceofFunction(typeOfCheckCast, tmpCheckCastId)) { \ + cn1ThrowTypeError(threadStateData, __NEW_INSTANCE_java_lang_ClassCastException(threadStateData), CN1_CLASS_OF(SP[-1].data.o)->clsName, targetName); \ + } \ + } \ +} #define BC_SWAP() swapStack(SP) @@ -1956,6 +1995,9 @@ extern JAVA_INT throwException_R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT exc extern JAVA_BOOLEAN throwException_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT exceptionArg); extern JAVA_OBJECT __NEW_java_lang_NullPointerException(CODENAME_ONE_THREAD_STATE); extern JAVA_OBJECT __NEW_INSTANCE_java_lang_NullPointerException(CODENAME_ONE_THREAD_STATE); +extern JAVA_OBJECT __NEW_INSTANCE_java_lang_ClassCastException(CODENAME_ONE_THREAD_STATE); +extern JAVA_OBJECT __NEW_INSTANCE_java_lang_ArrayStoreException(CODENAME_ONE_THREAD_STATE); +extern void cn1ThrowTypeError(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT exception, const char* fromClass, const char* toClass); extern JAVA_OBJECT __NEW_INSTANCE_java_lang_StackOverflowError(CODENAME_ONE_THREAD_STATE); // Throws the PREALLOCATED StackOverflowError (pre-filled trace, no allocation, // no trace building) -- safe to call at stack exhaustion. See cn1_globals.m. @@ -2454,6 +2496,7 @@ extern JAVA_OBJECT cn1FusedLatin1Begin(CODENAME_ONE_THREAD_STATE, int len, JAVA_ // set the real count LAST, after every byte is written, so a concurrent GC never sees count>0 over // an unfinished value. Single word store. #define cn1FusedLatin1End(so, n) (((struct obj__java_lang_String*)(so))->java_lang_String_count = (n)) +extern JAVA_OBJECT cn1MainArgs(CODENAME_ONE_THREAD_STATE, int argc, char* argv[]); extern void initConstantPool(); extern void initMethodStack(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, int stackSize, int localsStackSize, int classNameId, int methodNameId); diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index da921e8b653..e084000eb40 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -11666,6 +11666,31 @@ void cn1GcProbeInit(void) { } #endif /* CN1_GC_CONFORM */ +// Builds the String[] that main(String[]) receives, from the process argv. +// Java's args array does NOT include the program name -- argv[0] is the +// executable path and main()'s first element is the first real argument -- so +// the copy starts at argv[1] and the array is argc-1 long. A clean-target +// binary previously passed JAVA_NULL here, so every translated program was +// unable to read its own command line. +// +// CN1_WRITE_BARRIER is required on each store (the array may already be +// tenured by the time a later element is written); no CN1_SATB_DELETE is +// needed because the array is freshly allocated and every slot is still NULL, +// and the deletion barrier is a no-op on a NULL previous value. +JAVA_OBJECT cn1MainArgs(CODENAME_ONE_THREAD_STATE, int argc, char* argv[]) { + int count = argc > 1 ? argc - 1 : 0; + enteringNativeAllocations(); + JAVA_OBJECT arrObj = allocArray(threadStateData, count, &class_array1__java_lang_String, sizeof(JAVA_OBJECT), 1); + JAVA_ARRAY_OBJECT* dest = (JAVA_ARRAY_OBJECT*)((JAVA_ARRAY)arrObj)->data; + for(int iter = 0 ; iter < count ; iter++) { + JAVA_OBJECT str = newStringFromCString(threadStateData, argv[iter + 1]); + CN1_WRITE_BARRIER(arrObj, str); + dest[iter] = str; + } + finishedNativeAllocations(); + return arrObj; +} + void initConstantPool() { cn1StartupPhase("main"); __STATIC_INITIALIZER_java_lang_Class(getThreadLocalData()); @@ -11897,6 +11922,40 @@ JAVA_BOOLEAN throwException_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT exc return JAVA_FALSE; } +// Thrown by BC_CHECKCAST_CHECKED. The exception carries no detail message: the +// no-arg constructor is the shape proven to survive dead-code elimination (it is +// how NullPointerException is thrown from here), whereas a String-argument +// constructor reachable only from this file would depend on native-use retention. +// The class names are printed instead, so a failure is still diagnosable, and +// attaching a real message is a follow-up once the constructor's retention is +// pinned. Only reached on an actual bad cast, so the fprintf costs nothing on the +// success path. +// Shared failure path for BC_CHECKCAST_CHECKED and CN1_ARRAY_STORE_CHECK. +// +// The exception object is constructed BY THE CALLER and passed in, deliberately: +// if this function named __NEW_INSTANCE_java_lang_ClassCastException itself, the +// runtime would reference that symbol in every build, while the class is only +// retained when -Dcn1.checkedCasts is on -- an unresolved symbol at link time for +// everyone else. Keeping the reference in generated code, which only exists under +// the same flag that retains the class, makes the two impossible to desynchronize. +// (That is exactly how the first cut of this broke FileClassIntegrationTest.) +// +// No detail message on the exception: the no-arg constructor is the shape proven +// to survive dead-code elimination (it is how NullPointerException is thrown from +// here), whereas a String constructor reachable only from this file would depend +// on native-use retention. The names are printed instead, so a failure is still +// diagnosable. Only reached on an actual bad cast or store, so the fprintf costs +// nothing on the success path. +void cn1ThrowTypeError(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT exception, const char* fromClass, const char* toClass) { + if(toClass == NULL) { + fprintf(stderr, "ArrayStoreException: %s\n", fromClass == NULL ? "?" : fromClass); + } else { + fprintf(stderr, "ClassCastException: %s cannot be cast to %s\n", + fromClass == NULL ? "?" : fromClass, toClass); + } + throwException(threadStateData, exception); +} + void throwArrayIndexOutOfBoundsException(CODENAME_ONE_THREAD_STATE, int index) { JAVA_OBJECT arrayIndexOutOfBoundsException = __NEW_java_lang_ArrayIndexOutOfBoundsException(threadStateData); java_lang_ArrayIndexOutOfBoundsException___INIT_____int(threadStateData, arrayIndexOutOfBoundsException, index); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java index 602a72e258b..e3d98e3ce35 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java @@ -464,6 +464,13 @@ public void updateAllDependencies() { dependsClassesInterfaces.clear(); exportsClassesInterfaces.clear(); dependsClassesInterfaces.add("java_lang_NullPointerException"); + if(ByteCodeTranslator.isCheckedCastsEnabled()) { + // Kept alive for BC_CHECKCAST_CHECKED, which is emitted under the same + // flag. Retaining it only when the check is emitted keeps the class out + // of every build that does not enforce casts. + dependsClassesInterfaces.add("java_lang_ClassCastException"); + dependsClassesInterfaces.add("java_lang_ArrayStoreException"); + } setBaseClass(baseClass); if (isAnnotation) { dependsClassesInterfaces.add("java_lang_annotation_Annotation"); @@ -1258,9 +1265,14 @@ public String generateCCode(List allClasses) { + " getThreadLocalData()->lightweightThread = JAVA_TRUE;\n" + " getThreadLocalData()->threadActive = JAVA_TRUE;\n" + "#endif\n"); + // Hand main() the real command line. This used to pass + // JAVA_NULL, so a translated program could not read its own + // arguments at all and every knob had to come in through the + // environment (see vm/benchmarks). cn1MainArgs skips argv[0] -- + // Java's args array excludes the program name. b.append(" "); b.append(clsName); - b.append("_main___java_lang_String_1ARRAY(getThreadLocalData(), JAVA_NULL);\n"); + b.append("_main___java_lang_String_1ARRAY(getThreadLocalData(), cn1MainArgs(getThreadLocalData(), argc, argv));\n"); // main returning does not end the process here -- AppKit // owns the main thread and keeps running -- so leaving // the worker registered would leave the collector @@ -1279,7 +1291,7 @@ public String generateCCode(List allClasses) { } else { b.append(" "); b.append(clsName); - b.append("_main___java_lang_String_1ARRAY(getThreadLocalData(), JAVA_NULL);\n}\n\n"); + b.append("_main___java_lang_String_1ARRAY(getThreadLocalData(), cn1MainArgs(getThreadLocalData(), argc, argv));\n}\n\n"); } } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index d26ed2035e4..00bbd5aab5f 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -223,6 +223,22 @@ static boolean isBundledSqliteCipherEnabled() { return "true".equals(System.getProperty("cn1.sqlcipher", "false")); } + /** + * True when CHECKCAST should actually verify the cast and throw ClassCastException + * instead of expanding to nothing (issue #5531). Opt-in, because enforcing it changes + * the outcome of app builds that succeed today: a cast that silently produced the wrong + * object now throws where nothing threw before. Server-side (clean-target) builds handle + * untrusted input and should always enable it. + * + *

This one flag drives both halves and they must stay in agreement: it makes + * TypeInstruction emit BC_CHECKCAST_CHECKED, and it makes ByteCodeClass retain + * java.lang.ClassCastException. Emitting the check without retaining the class would + * leave an unresolved symbol at link time. + */ + public static boolean isCheckedCastsEnabled() { + return "true".equalsIgnoreCase(System.getProperty("cn1.checkedCasts", "false")); + } + /// Writes the bundled SQLite engine into a source root, or takes it back out. /// /// Emitted only for an application that uses `com.codename1.db`, and its ciphers only for one diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java index f273614057a..baea33b8aba 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java @@ -4263,7 +4263,15 @@ boolean optimize() { int currentOpcode = current.getOpcode(); switch(currentOpcode) { case Opcodes.CHECKCAST: { - // Remove the check cast for now as it gets in the way of other optimizations + // Remove the check cast for now as it gets in the way of other optimizations. + // This removal is WHY a failed cast never throws (issue #5531): dropping the + // instruction here means TypeInstruction never gets to emit anything for it, + // so implementing the BC_CHECKCAST macro alone would have had no effect. + // Under -Dcn1.checkedCasts=true the instruction is kept, at the cost of the + // optimizations this removal was protecting. + if(ByteCodeTranslator.isCheckedCastsEnabled()) { + break; + } instructions.remove(iter); iter--; instructionCount--; @@ -4641,6 +4649,11 @@ boolean optimize() { " JAVA_OBJECT __cn1ArrayTmp = " + arrayLiteral + ";\n" + " JAVA_INT __cn1IndexTmp = " + indexLiteral + ";\n" + " " + valueType + " __cn1ValueTmp = " + valueLiteral + ";\n" + + // The macro's own comment used to claim it covariance-checks + // OBJECT stores; it never did. Under -Dcn1.checkedCasts the + // check is emitted here, ahead of the store. + ("OBJECT".equals(elementType) && ByteCodeTranslator.isCheckedCastsEnabled() + ? " CN1_ARRAY_STORE_CHECK(__cn1ArrayTmp, __cn1ValueTmp);\n" : "") + " CN1_SET_ARRAY_ELEMENT_"+elementType+"(__cn1ArrayTmp, __cn1IndexTmp, __cn1ValueTmp);\n" + " }\n"; } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/BasicInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/BasicInstruction.java index 39a49c3057f..796d9c90d2b 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/BasicInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/BasicInstruction.java @@ -23,6 +23,7 @@ package com.codename1.tools.translator.bytecodes; +import com.codename1.tools.translator.ByteCodeTranslator; import java.util.List; import org.objectweb.asm.Opcodes; @@ -358,6 +359,8 @@ public void appendInstruction(StringBuilder b, List instructions) { } b.append("{ /* BC_AASTORE */\n" + " JAVA_OBJECT aastoreTmp = SP[-3].data.o; \n" + + (ByteCodeTranslator.isCheckedCastsEnabled() + ? " CN1_ARRAY_STORE_CHECK(aastoreTmp, SP[-1].data.o); \n" : "") + " CN1_WRITE_BARRIER(aastoreTmp, SP[-1].data.o); \n" + " CN1_SATB_DELETE(&((JAVA_ARRAY_OBJECT*) (*(JAVA_ARRAY)aastoreTmp).data)[SP[-2].data.i]); \n" + " ((JAVA_ARRAY_OBJECT*) (*(JAVA_ARRAY)aastoreTmp).data)[SP[-2].data.i] = SP[-1].data.o; \n" + diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java index 7a49f22d313..8eb39cf8a35 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java @@ -24,6 +24,7 @@ package com.codename1.tools.translator.bytecodes; import com.codename1.tools.translator.ByteCodeClass; +import com.codename1.tools.translator.ByteCodeTranslator; import com.codename1.tools.translator.Parser; import java.util.List; import org.objectweb.asm.Opcodes; @@ -39,6 +40,7 @@ public class TypeInstruction extends Instruction { private boolean scalarReplaced = false; private int scalarStructId = -1; private boolean initBeforePublish = false; + private String originalType; /** * Marks this {@code NEW} as INIT-BEFORE-PUBLISH (memset elimination): the @@ -76,6 +78,10 @@ public boolean isFusedNew() { public TypeInstruction(int opcode, String type) { super(opcode); this.type = type; + // appendInstruction mangles `type` in place (dots/slashes/dollars become + // underscores), so the readable name has to be kept aside here if anything + // downstream wants to print it -- BC_CHECKCAST_CHECKED's message does. + this.originalType = type; } /** @@ -340,9 +346,38 @@ public void appendInstruction(StringBuilder b, List l) { b.append("(threadStateData, SP[0].data.i));\n"); break; case Opcodes.CHECKCAST: - b.append("BC_CHECKCAST("); - b.append(type); - b.append(");\n"); + if(!ByteCodeTranslator.isCheckedCastsEnabled()) { + // Legacy shape: the macro expands to nothing, so the argument is + // discarded and the raw type name is fine. + b.append("BC_CHECKCAST("); + b.append(type); + b.append(");\n"); + break; + } + // Enforcing shape. The class id has to be resolved the same way + // INSTANCEOF resolves it -- array dimensions collapse into one + // cn1_array__id_ token -- because instanceofFunction compares ids. + // The readable name is baked in as a literal here rather than looked + // up at runtime: the translator already knows it, and that keeps the + // failure path free of any class-name table. + int castPos = type.indexOf('['); + if(castPos > -1) { + int castCount = 1; + while(type.charAt(castPos + 1) == '[') { + castCount++; + castPos++; + } + b.append("BC_CHECKCAST_CHECKED(cn1_array_"); + b.append(castCount); + b.append("_id_"); + b.append(actualType); + } else { + b.append("BC_CHECKCAST_CHECKED(cn1_class_id_"); + b.append(actualType); + } + b.append(", \""); + b.append(originalType.replace('/', '.')); + b.append("\");\n"); break; case Opcodes.INSTANCEOF: int pos = type.indexOf('['); diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 2d50728d0c8..37d094f65f2 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -27,6 +27,7 @@ #include "cn1_globals.h" #include +#include #include #include #include @@ -1030,6 +1031,167 @@ JAVA_VOID java_lang_System_arraycopy___java_lang_Object_int_java_lang_Object_int } } +// getenv returns a pointer into the process environment, which is owned by the +// C runtime and must not be freed. stringToUTF8 hands back the calling thread's +// scratch buffer, so the lookup must finish with it before anything else on this +// thread converts another string -- newStringFromCString copies, so building the +// result here is safe. +JAVA_OBJECT java_lang_System_getenv___java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT name) { + if(name == JAVA_NULL) { + return JAVA_NULL; + } + const char* key = stringToUTF8(threadStateData, name); + if(key == NULL) { + return JAVA_NULL; + } + const char* value = getenv(key); + if(value == NULL) { + return JAVA_NULL; + } + return newStringFromCString(threadStateData, value); +} + +// --------------------------------------------------------------------------- +// java.io file streams and standard input. +// +// Backed by C stdio (not POSIX fds) so the same code serves the Windows clean +// target, which has no unistd.h. The Java side stores the FILE* as a long; 0 is +// the "not open" value, which is why every open returns 0 rather than -1 on +// failure. Negative returns below -1 mean "error" as opposed to -1's "end of +// file", and the Java side turns those into IOException. +// +// The byte[] is only touched between entry and return, so it needs no GC +// bracket: under conservative roots the argument is a scanned native local, and +// nothing here allocates. +// --------------------------------------------------------------------------- + +JAVA_LONG java_io_FileInputStream_openImpl___java_lang_String_R_long(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT name) { + if(name == JAVA_NULL) { + return 0; + } + const char* path = stringToUTF8(threadStateData, name); + if(path == NULL) { + return 0; + } + FILE* f = fopen(path, "rb"); + return (JAVA_LONG)(intptr_t)f; +} + +JAVA_INT java_io_FileInputStream_readImpl___long_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { + FILE* f = (FILE*)(intptr_t)handle; + if(f == NULL || buffer == JAVA_NULL) { + return -2; + } + JAVA_ARRAY_BYTE* data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; + size_t n = fread(&data[offset], 1, (size_t)length, f); + if(n == 0) { + return feof(f) ? -1 : -2; + } + return (JAVA_INT)n; +} + +JAVA_LONG java_io_FileInputStream_skipImpl___long_long_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_LONG count) { + FILE* f = (FILE*)(intptr_t)handle; + if(f == NULL) { + return -1; + } + // Clamped to the real end so the return value is bytes actually skipped, which + // is what InputStream.skip promises -- seeking past EOF succeeds in C and would + // otherwise report a skip that did not happen. + long start = ftell(f); + if(start < 0 || fseek(f, 0, SEEK_END) != 0) { + return -1; + } + long end = ftell(f); + long target = start + (long)count; + if(target > end) { + target = end; + } + if(fseek(f, target, SEEK_SET) != 0) { + return -1; + } + return (JAVA_LONG)(target - start); +} + +JAVA_INT java_io_FileInputStream_availableImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + FILE* f = (FILE*)(intptr_t)handle; + if(f == NULL) { + return -1; + } + long start = ftell(f); + if(start < 0 || fseek(f, 0, SEEK_END) != 0) { + return -1; + } + long end = ftell(f); + if(fseek(f, start, SEEK_SET) != 0) { + return -1; + } + long remaining = end - start; + if(remaining < 0) { + return -1; + } + return remaining > 0x7fffffffL ? 0x7fffffff : (JAVA_INT)remaining; +} + +JAVA_INT java_io_FileInputStream_closeImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + FILE* f = (FILE*)(intptr_t)handle; + if(f == NULL) { + return 0; + } + return fclose(f) == 0 ? 0 : -1; +} + +JAVA_LONG java_io_FileOutputStream_openImpl___java_lang_String_boolean_R_long(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT name, JAVA_BOOLEAN append) { + if(name == JAVA_NULL) { + return 0; + } + const char* path = stringToUTF8(threadStateData, name); + if(path == NULL) { + return 0; + } + FILE* f = fopen(path, append ? "ab" : "wb"); + return (JAVA_LONG)(intptr_t)f; +} + +JAVA_INT java_io_FileOutputStream_writeImpl___long_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { + FILE* f = (FILE*)(intptr_t)handle; + if(f == NULL || buffer == JAVA_NULL) { + return -1; + } + JAVA_ARRAY_BYTE* data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; + return (JAVA_INT)fwrite(&data[offset], 1, (size_t)length, f); +} + +JAVA_INT java_io_FileOutputStream_flushImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + FILE* f = (FILE*)(intptr_t)handle; + if(f == NULL) { + return -1; + } + return fflush(f) == 0 ? 0 : -1; +} + +JAVA_INT java_io_FileOutputStream_closeImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + FILE* f = (FILE*)(intptr_t)handle; + if(f == NULL) { + return 0; + } + return fclose(f) == 0 ? 0 : -1; +} + +// Standard input. Separate from FileInputStream because stdin is not seekable, so +// skip/available cannot be implemented by the ftell dance above. +JAVA_INT java_io_StandardInputStream_readImpl___byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { + if(buffer == JAVA_NULL) { + return -2; + } + JAVA_ARRAY_BYTE* data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; + size_t n = fread(&data[offset], 1, (size_t)length, stdin); + if(n == 0) { + return feof(stdin) ? -1 : -2; + } + return (JAVA_INT)n; +} + JAVA_LONG java_lang_System_currentTimeMillis___R_long(CODENAME_ONE_THREAD_STATE) { __STATIC_INITIALIZER_java_lang_System(threadStateData); struct timeval time; diff --git a/vm/CLAUDE.md b/vm/CLAUDE.md index 59eb9d0396f..65531bb8b88 100644 --- a/vm/CLAUDE.md +++ b/vm/CLAUDE.md @@ -29,10 +29,11 @@ unset = off, so probe-on and probe-off are the same binary). Two emitters: `vm/benchmarks/src/com/bench/GcSteadyState.java` is the churn workload, parameterised through the environment (`CN1_WL_SECONDS`, `CN1_WL_THREADS`, `CN1_WL_DEPTH`, -`CN1_WL_BRANCH`, `CN1_WL_SLEEP_MS`, ...) because the clean target's generated `main()` -passes `JAVA_NULL` for args. Sweeping `CN1_WL_SLEEP_MS` over `{0,1,10,100,1000}` is the -cheapest discriminator between a rate problem and a retention problem, and needs no -rebuild. +`CN1_WL_BRANCH`, `CN1_WL_SLEEP_MS`, ...). It predates `main(String[])` receiving the real +command line -- the clean target's generated `main()` used to pass `JAVA_NULL` for args -- +and stays environment-driven because every A/B script already sets it up that way. +Sweeping `CN1_WL_SLEEP_MS` over `{0,1,10,100,1000}` is the cheapest discriminator between a +rate problem and a retention problem, and needs no rebuild. Every GC ablation is a **compile-time** macro, so each A/B arm is a rebuild; use `vm/benchmarks/translate-and-build.sh` with `CN1_BENCH_CFLAGS` (see `ab-adopt.sh`), which diff --git a/vm/JavaAPI/src/java/io/FileInputStream.java b/vm/JavaAPI/src/java/io/FileInputStream.java new file mode 100644 index 00000000000..e4a1b35a88d --- /dev/null +++ b/vm/JavaAPI/src/java/io/FileInputStream.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.io; + +/** + * Reads bytes from a file. Backed by C stdio through a native handle rather than + * by any Codename One implementation, so it is available to a translated program + * that has no platform layer at all - a server-side binary, for example. + */ +public class FileInputStream extends InputStream { + private long handle; + private boolean closed; + + public FileInputStream(String name) throws FileNotFoundException { + if(name == null) { + throw new NullPointerException(); + } + handle = openImpl(name); + if(handle == 0) { + throw new FileNotFoundException(name); + } + } + + public FileInputStream(File file) throws FileNotFoundException { + this(file == null ? null : file.getPath()); + } + + public int read() throws IOException { + byte[] one = new byte[1]; + int n = read(one, 0, 1); + if(n <= 0) { + return -1; + } + return one[0] & 0xff; + } + + public int read(byte[] b) throws IOException { + return read(b, 0, b == null ? 0 : b.length); + } + + public int read(byte[] b, int off, int len) throws IOException { + if(b == null) { + throw new NullPointerException(); + } + if(off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } + checkOpen(); + if(len == 0) { + return 0; + } + int n = readImpl(handle, b, off, len); + if(n < -1) { + throw new IOException("Read failed"); + } + return n; + } + + public long skip(long n) throws IOException { + checkOpen(); + if(n <= 0) { + return 0; + } + long moved = skipImpl(handle, n); + if(moved < 0) { + throw new IOException("Seek failed"); + } + return moved; + } + + public int available() throws IOException { + checkOpen(); + int a = availableImpl(handle); + if(a < 0) { + throw new IOException("Unable to determine available bytes"); + } + return a; + } + + public void close() throws IOException { + if(closed) { + return; + } + closed = true; + long h = handle; + handle = 0; + if(closeImpl(h) != 0) { + throw new IOException("Close failed"); + } + } + + private void checkOpen() throws IOException { + if(closed) { + throw new IOException("Stream closed"); + } + } + + private static native long openImpl(String name); + private static native int readImpl(long handle, byte[] buffer, int offset, int length); + private static native long skipImpl(long handle, long count); + private static native int availableImpl(long handle); + private static native int closeImpl(long handle); +} diff --git a/vm/JavaAPI/src/java/io/FileOutputStream.java b/vm/JavaAPI/src/java/io/FileOutputStream.java new file mode 100644 index 00000000000..ebfe7ae2c65 --- /dev/null +++ b/vm/JavaAPI/src/java/io/FileOutputStream.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.io; + +/** + * Writes bytes to a file. Backed by C stdio through a native handle rather than by + * any Codename One implementation, so it is available to a translated program that + * has no platform layer at all - a server-side binary, for example. + */ +public class FileOutputStream extends OutputStream { + private long handle; + private boolean closed; + + public FileOutputStream(String name) throws FileNotFoundException { + this(name, false); + } + + public FileOutputStream(String name, boolean append) throws FileNotFoundException { + if(name == null) { + throw new NullPointerException(); + } + handle = openImpl(name, append); + if(handle == 0) { + throw new FileNotFoundException(name); + } + } + + public FileOutputStream(File file) throws FileNotFoundException { + this(file == null ? null : file.getPath(), false); + } + + public FileOutputStream(File file, boolean append) throws FileNotFoundException { + this(file == null ? null : file.getPath(), append); + } + + public void write(int b) throws IOException { + byte[] one = new byte[1]; + one[0] = (byte)b; + write(one, 0, 1); + } + + public void write(byte[] b) throws IOException { + write(b, 0, b == null ? 0 : b.length); + } + + public void write(byte[] b, int off, int len) throws IOException { + if(b == null) { + throw new NullPointerException(); + } + if(off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } + checkOpen(); + if(len == 0) { + return; + } + // A short write is a failure, not a partial success: OutputStream.write has + // no way to report how much it managed, so the caller would silently lose + // the tail. + if(writeImpl(handle, b, off, len) != len) { + throw new IOException("Write failed"); + } + } + + public void flush() throws IOException { + checkOpen(); + if(flushImpl(handle) != 0) { + throw new IOException("Flush failed"); + } + } + + public void close() throws IOException { + if(closed) { + return; + } + closed = true; + long h = handle; + handle = 0; + if(closeImpl(h) != 0) { + throw new IOException("Close failed"); + } + } + + private void checkOpen() throws IOException { + if(closed) { + throw new IOException("Stream closed"); + } + } + + private static native long openImpl(String name, boolean append); + private static native int writeImpl(long handle, byte[] buffer, int offset, int length); + private static native int flushImpl(long handle); + private static native int closeImpl(long handle); +} diff --git a/vm/JavaAPI/src/java/io/StandardInputStream.java b/vm/JavaAPI/src/java/io/StandardInputStream.java new file mode 100644 index 00000000000..b54e1bca549 --- /dev/null +++ b/vm/JavaAPI/src/java/io/StandardInputStream.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.io; + +/** + * The stream behind System.in. Not a FileInputStream: standard input is not + * seekable, so neither skip nor available can be answered by seeking, and + * InputStream's defaults (skip by reading, available 0) are the correct answers + * here. This mirrors NSLogOutputStream, which plays the same role for System.out. + */ +public class StandardInputStream extends InputStream { + public int read() throws IOException { + byte[] one = new byte[1]; + int n = read(one, 0, 1); + if(n <= 0) { + return -1; + } + return one[0] & 0xff; + } + + public int read(byte[] b, int off, int len) throws IOException { + if(b == null) { + throw new NullPointerException(); + } + if(off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } + if(len == 0) { + return 0; + } + int n = readImpl(b, off, len); + if(n < -1) { + throw new IOException("Read failed"); + } + return n; + } + + private static native int readImpl(byte[] buffer, int offset, int length); +} diff --git a/vm/JavaAPI/src/java/lang/System.java b/vm/JavaAPI/src/java/lang/System.java index 2fa5af7ad01..43213415029 100644 --- a/vm/JavaAPI/src/java/lang/System.java +++ b/vm/JavaAPI/src/java/lang/System.java @@ -46,6 +46,12 @@ public final class System { */ public static final java.io.PrintStream out = new PrintStream(new NSLogOutputStream()); + /** + * The standard input stream. Reads from the process's stdin, so a translated + * program can be driven by a pipe the way any other command-line program is. + */ + public static final java.io.InputStream in = new java.io.StandardInputStream(); + /** * Copies an array from the specified source array, beginning at the specified position, to the specified position of the destination array. A subsequence of array components are copied from the source array referenced by src to the destination array referenced by dst. The number of components copied is equal to the length argument. The components at positions srcOffset through srcOffset+length-1 in the source array are copied into positions dstOffset through dstOffset+length-1, respectively, of the destination array. * If the src and dst arguments refer to the same array object, then the copying is performed as if the components at positions srcOffset through srcOffset+length-1 were first copied to a temporary array with length components and then the contents of the temporary array were copied into positions dstOffset through dstOffset+length-1 of the destination array. @@ -183,6 +189,15 @@ public static java.lang.String getProperty(java.lang.String key){ return null; } + /** + * Returns the value of the named environment variable, or null when it is + * not set. Environment variables are the only configuration channel a + * process gets before it parses its own arguments, so a server-side + * translated binary needs this to find, for example, the endpoint its host + * runtime published to it. + */ + public static native java.lang.String getenv(java.lang.String name); + /** * Returns the same hashcode for the given object as would be returned by the default method hashCode(), whether or not the given object's class overrides hashCode(). The hashcode for the null reference is zero. */ diff --git a/vm/benchmarks/translate-and-build.sh b/vm/benchmarks/translate-and-build.sh index cb2a2e34509..6f088ee8f60 100755 --- a/vm/benchmarks/translate-and-build.sh +++ b/vm/benchmarks/translate-and-build.sh @@ -11,6 +11,8 @@ # # Environment knobs: # CN1_BENCH_CFLAGS extra clang flags (e.g. -flto=thin for the release shape) +# CN1_BENCH_TRANSLATOR_OPTS extra -D properties for the translator JVM +# (e.g. -Dcn1.checkedCasts=true) # CN1_BENCH_CC compiler (default clang) set -e cd "$(dirname "$0")" @@ -84,7 +86,7 @@ fi # 5. translate to C mkdir -p "$WORK/out" -"$J8/bin/java" -cp "$TRANSLATOR:$ASM_CP" com.codename1.tools.translator.ByteCodeTranslator \ +"$J8/bin/java" $CN1_BENCH_TRANSLATOR_OPTS -cp "$TRANSLATOR:$ASM_CP" com.codename1.tools.translator.ByteCodeTranslator \ clean "$JAVAAPI;$WORK/classes" "$WORK/out" "$MAIN" com.bench "$MAIN" 1.0 clean none \ > "$WORK/translate.log" 2>&1 || { echo "TRANSLATE FAILED"; tail -30 "$WORK/translate.log"; exit 1; } From 01b366d4420afe4c9da5938672a7155d46bbf641 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:59:27 +0300 Subject: [PATCH 02/42] Measure what a thread costs, and make the per-thread sizes tunable The next stage is a standalone server rather than a Lambda, and the first question it asks is whether a connection can have a thread. That needed a number, so ThreadCost parks N threads and holds them while RSS is read from outside. Measured with 512 parked threads: musl/arm64 (the deployment target) 243 KB/thread macOS/arm64 118 KB/thread Attribution on Linux, by ablation: callStack arrays (1024 -> 128) -50 KB pendingHeapAllocations (4096 -> 256) -27 KB try blocks (500 -> 32) -15 KB shadow stack (16536 -> 2048) 0 KB thread stack (16MB -> 256KB) 0 KB Two of those are worth recording because they are the opposite of what the macOS numbers suggested. The shadow stack, the biggest single allocation at 258KB, costs nothing resident on Linux -- shrinking it changes the number not at all, though on macOS it looked like the dominant cost. And the pinned 16MB thread stack is free: it is reserved, never committed. The five sizes are now #ifndef-guarded so an A/B can override them with -D. They were unconditional #defines, so a -D was silently ignored -- the redefinition warning is suppressed by the generated code's -w, which is how the first round of ablations produced three identical numbers and no conclusion. The shadow stack is now mapped rather than malloc'd and memset in full. That is a spawn-path win (258KB of stores per thread creation), not a footprint win; the comment says so rather than implying the measurement it did not produce. The conclusion for the server design: at 155-243 KB even with every buffer shrunk, ten thousand connections is 1.5-2.4GB of threads. A connection cannot have one. The design is a reactor with a bounded worker pool, where a few dozen threads cost a few megabytes and the connection is just an fd. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 32 +++++++ vm/ByteCodeTranslator/src/nativeMethods.m | 101 ++++++++++++++++---- vm/benchmarks/src/com/bench/ThreadCost.java | 87 +++++++++++++++++ 3 files changed, 204 insertions(+), 16 deletions(-) create mode 100644 vm/benchmarks/src/com/bench/ThreadCost.java diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index d434b49fae4..ca94638341d 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -1055,11 +1055,43 @@ struct TryBlock { JAVA_OBJECT monitor; }; +/* + * Per-thread sizing. These three are what a thread costs before it runs a single + * instruction, so they are the numbers that decide whether a server-side binary + * can afford a thread per connection. #ifndef-guarded so an A/B can override them + * with -D without editing this file -- an unconditional #define silently ignores + * the -D (the redefinition warning is suppressed by the generated code's -w). + */ +#ifndef CN1_MAX_STACK_CALL_DEPTH #define CN1_MAX_STACK_CALL_DEPTH 1024 +#endif #define CN1_STACK_OVERFLOW_CALL_DEPTH_LIMIT CN1_MAX_STACK_CALL_DEPTH +#ifndef CN1_MAX_OBJECT_STACK_DEPTH #define CN1_MAX_OBJECT_STACK_DEPTH 16536 +#endif +#ifndef PER_THREAD_ALLOCATION_COUNT #define PER_THREAD_ALLOCATION_COUNT 4096 +#endif + +/* + * Try-block depth. Each entry carries a jmp_buf (~200 bytes on arm64 macOS, + * ~320 on arm64 musl), so 500 of them is 100-160KB per thread -- comparable to + * the shadow stack and much less obvious. + */ +#ifndef CN1_MAX_TRY_BLOCKS +#define CN1_MAX_TRY_BLOCKS 500 +#endif + +/* + * Native stack per spawned thread on Linux. musl defaults to 128KB, which the + * recursive generated C overflows on a deep call chain, so it is pinned to a + * JVM-sized reservation. Reserved, not committed -- but it is the largest single + * number attached to a thread, so it is a knob rather than a literal. + */ +#ifndef CN1_THREAD_STACK_BYTES +#define CN1_THREAD_STACK_BYTES (16 * 1024 * 1024) +#endif #ifdef CN1_NURSERY // Tunables (override with -D). Block size and arena size trade footprint against diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 37d094f65f2..2c5fe143532 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -28,6 +28,9 @@ #include "cn1_globals.h" #include #include +#ifndef _WIN32 +#include /* cn1AllocThreadStack maps the shadow stack */ +#endif #include #include #include @@ -1031,6 +1034,61 @@ JAVA_VOID java_lang_System_arraycopy___java_lang_Object_int_java_lang_Object_int } } +/* + * The per-thread shadow stack, mapped rather than malloc'd + memset. + * + * This is CN1_MAX_OBJECT_STACK_DEPTH * sizeof(elementStruct) -- 258KB at the + * default depth. It used to be malloc'd and then memset in full at thread + * creation, which is 258KB of stores on the spawn path for a stack the thread + * will walk a few frames of. A fresh anonymous mapping is zero-filled by the + * kernel and commits per page on first touch, so neither the stores nor the pages + * are paid for up front. + * + * The eager clear was redundant: every frame prologue memsets the slots it claims + * (see the frame-entry helpers in cn1_globals.h), and the collector scans only up + * to threadObjectStackOffset, so no slot is read before its owning frame zeroed it. + * + * On RESIDENT memory this is worth less than it looks. Measured on musl/arm64 with + * 512 parked threads, per-thread RSS went 258KB -> 240KB: the shadow stack was + * already mostly uncommitted, and the per-thread cost actually lives in the + * callStack arrays (~50KB), pendingHeapAllocations (~27KB) and the try-block array + * (~15KB). Shrinking CN1_MAX_OBJECT_STACK_DEPTH on Linux changes nothing at all. + * The win here is the spawn path, not the footprint. + * + * Growing it is deliberately NOT how depth is solved. Generated frames hold + * interior pointers into this array (`locals` and `stack` are C locals pointing + * into it), so anything that MOVED the allocation would dangle every frame below + * the one that grew it. Reserving the range up front and letting the kernel decide + * what is resident keeps every pointer stable. + */ +static struct elementStruct* cn1AllocThreadStack(void) { + size_t bytes = CN1_MAX_OBJECT_STACK_DEPTH * sizeof(struct elementStruct); +#if defined(_WIN32) + /* VirtualAlloc would be the equivalent; calloc keeps the Windows target on one + well-trodden path, and it is not the target where thread counts are large. */ + return (struct elementStruct*)calloc(CN1_MAX_OBJECT_STACK_DEPTH, sizeof(struct elementStruct)); +#else + void* p = mmap(NULL, bytes, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if(p == MAP_FAILED) { + /* Out of mappings rather than out of memory; calloc may still succeed. */ + return (struct elementStruct*)calloc(CN1_MAX_OBJECT_STACK_DEPTH, sizeof(struct elementStruct)); + } + return (struct elementStruct*)p; +#endif +} + +static void cn1FreeThreadStack(struct elementStruct* stack) { + if(stack == NULL) { + return; + } +#if defined(_WIN32) + free(stack); +#else + munmap(stack, CN1_MAX_OBJECT_STACK_DEPTH * sizeof(struct elementStruct)); +#endif +} + // getenv returns a pointer into the process environment, which is owned by the // C runtime and must not be freed. stringToUTF8 hands back the calling thread's // scratch buffer, so the lookup must finish with it before anything else on this @@ -1920,18 +1978,27 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC i->utf8Buffer = 0; i->utf8BufferSize = 0; - i->threadObjectStack = malloc(CN1_MAX_OBJECT_STACK_DEPTH * sizeof(struct elementStruct)); - memset(i->threadObjectStack, 0, CN1_MAX_OBJECT_STACK_DEPTH * sizeof(struct elementStruct)); + /* + * calloc, not malloc+memset. These four buffers are ~300KB per thread and the + * eager memset TOUCHED EVERY PAGE, so a thread that never runs a deep call + * chain still paid the whole footprint in resident memory -- measured at + * ~118KB per parked thread, which is what decides whether a server-side + * binary can afford a thread per connection. + * + * The eager clear was redundant: every frame prologue memsets exactly the + * slots it is about to claim (see the frame-entry helpers in cn1_globals.h), + * and the collector only scans threadObjectStack up to + * threadObjectStackOffset, so no slot is ever read before the frame that owns + * it has zeroed it. calloc for a request this size comes from mmap and is + * lazily zeroed by the OS, so a shallow thread commits a few pages instead of + * all of them. + */ + i->threadObjectStack = cn1AllocThreadStack(); i->threadObjectStackOffset = 0; - - i->callStackClass = malloc(CN1_MAX_STACK_CALL_DEPTH * sizeof(int)); - memset(i->callStackClass, 0, CN1_MAX_STACK_CALL_DEPTH * sizeof(int)); - - i->callStackLine = malloc(CN1_MAX_STACK_CALL_DEPTH * sizeof(int)); - memset(i->callStackLine, 0, CN1_MAX_STACK_CALL_DEPTH * sizeof(int)); - - i->callStackMethod = malloc(CN1_MAX_STACK_CALL_DEPTH * sizeof(int)); - memset(i->callStackMethod, 0, CN1_MAX_STACK_CALL_DEPTH * sizeof(int)); + + i->callStackClass = calloc(CN1_MAX_STACK_CALL_DEPTH, sizeof(int)); + i->callStackLine = calloc(CN1_MAX_STACK_CALL_DEPTH, sizeof(int)); + i->callStackMethod = calloc(CN1_MAX_STACK_CALL_DEPTH, sizeof(int)); #ifdef CN1_ON_DEVICE_DEBUG i->callStackLocalsAddresses = malloc(CN1_MAX_STACK_CALL_DEPTH * sizeof(void**)); @@ -1945,8 +2012,8 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC // ThreadLocalData is malloc'd (not zeroed); 0 means "frameless native-stack // limit not yet computed" -- it is filled in lazily on first frameless entry. i->nativeStackLimit = 0; - i->pendingHeapAllocations = malloc(PER_THREAD_ALLOCATION_COUNT * sizeof(void *)); - memset(i->pendingHeapAllocations, 0, PER_THREAD_ALLOCATION_COUNT * sizeof(void *)); + + i->pendingHeapAllocations = calloc(PER_THREAD_ALLOCATION_COUNT, sizeof(void *)); i->heapAllocationSize = 0; i->threadHeapTotalSize = PER_THREAD_ALLOCATION_COUNT; // ThreadLocalData is malloc'd, NOT zeroed. bibopBytesLocal feeds the GC @@ -1979,7 +2046,7 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC i->gcQueuedForDrain = JAVA_FALSE; i->gcReleaseRequested = JAVA_FALSE; - i->blocks = malloc(500 * sizeof(struct TryBlock)); + i->blocks = malloc(CN1_MAX_TRY_BLOCKS * sizeof(struct TryBlock)); #ifdef CN1_CONSERVATIVE_GC_ROOTS // PHASE 3b: record this thread's pthread handle + TLS self pointer so the GC can // signal-stop it and the async-signal-safe stop handler can find its state. @@ -2448,7 +2515,9 @@ JAVA_VOID java_lang_Thread_setPriorityImpl___int(CODENAME_ONE_THREAD_STATE, JAVA void cn1ReleaseThreadLocalData(struct ThreadLocalData *head) { free(head->blocks); - free(head->threadObjectStack); + /* Mapped, not malloc'd -- see cn1AllocThreadStack. free() on a mapping is + undefined behaviour, not a leak, so this pairing matters. */ + cn1FreeThreadStack(head->threadObjectStack); free(head->callStackClass); free(head->callStackLine); free(head->callStackMethod); @@ -2678,7 +2747,7 @@ JAVA_VOID java_lang_Thread_start__(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT th) { // transition) easily overflows 128KB, corrupting the thread stack and crashing // at a varying site. Pin a JVM-sized 16MB stack so CN1 threads behave the same // as on every other port regardless of the linked libc. - pthread_attr_setstacksize(&attr, 16 * 1024 * 1024); + pthread_attr_setstacksize(&attr, CN1_THREAD_STACK_BYTES); #endif int rc = pthread_create(&pt, &attr, threadRunner, (void *)th); if (rc != 0) { diff --git a/vm/benchmarks/src/com/bench/ThreadCost.java b/vm/benchmarks/src/com/bench/ThreadCost.java new file mode 100644 index 00000000000..27ddcff8f1e --- /dev/null +++ b/vm/benchmarks/src/com/bench/ThreadCost.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.bench; + +/** + * What one parked thread costs. This is the number that decides whether a + * server-side ParparVM can serve a connection per thread or has to grow + * continuations: if a thread costs 300KB, ten thousand connections is 3GB and the + * answer is no; if it costs 20KB it is 200MB and the answer is yes. + * + * Spawns CN1_TC_THREADS (default 512) threads that park on a monitor and holds + * them, so peak RSS measured from outside is the steady state with them all + * alive. Compare against Noop, which is the same runtime with no threads. + * + * Run: + * translate-and-build.sh ThreadCost /tmp/threadcost + * CN1_TC_THREADS=512 /usr/bin/time -l /tmp/threadcost + */ +public class ThreadCost { + private static final Object LOCK = new Object(); + private static int started; + + public static void main(String[] args) throws Exception { + int n = envInt("CN1_TC_THREADS", 512); + int holdMs = envInt("CN1_TC_HOLD_MS", 3000); + for (int i = 0; i < n; i++) { + Thread t = new Thread(new Runnable() { + public void run() { + synchronized (LOCK) { + started++; + try { + // Parked, not spinning: a spinning thread would measure + // the scheduler instead of the footprint. + LOCK.wait(); + } catch (InterruptedException e) { + } + } + } + }); + t.start(); + } + // Let every thread reach its park before the measurement is taken. + long deadline = System.currentTimeMillis() + 10000; + while (System.currentTimeMillis() < deadline) { + synchronized (LOCK) { + if (started >= n) { + break; + } + } + Thread.sleep(5); + } + Thread.sleep(holdMs); + System.out.println("threads=" + n + " started=" + started); + } + + private static int envInt(String name, int fallback) { + String v = System.getenv(name); + if (v == null || v.length() == 0) { + return fallback; + } + try { + return Integer.parseInt(v.trim()); + } catch (NumberFormatException e) { + return fallback; + } + } +} From 3d4b3785e2d5d71af42288b0ccec64ac9e6be047 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Fri, 28 Aug 2026 17:27:57 +0300 Subject: [PATCH 03/42] Stop discarding uncaught exceptions on the clean target throwException walked the try-block stack looking for a handler and, when it found none, RETURNED. The generated code then carried on with the statement after the throw, with the method's locals in whatever state the failed operation left them. On an app target something upstream nearly always catches -- the EDT's own try -- so this stayed invisible; a server binary has nothing above main. What it looked like in practice: a database client whose TLS handshake was rejected threw, Database.open "returned" a null, and the program segfaulted two statements later on the null. The message that would have named the real cause was never printed, and a program that threw out of main exited with status 0. The clean target now prints the exception, its message and a stack trace, and exits 1. Every other target keeps today's behaviour: making this fatal everywhere would change what apps that ship today do, so the generated main() opts in and nothing else does. Two details the fix needed. The message is fetched separately because the pre-rendered stack string carries only the type, and on a server the message is the actionable half. And the try depth is reset to zero before rendering: the search leaves it at -1, and a Java method that saves and restores a negative depth corrupts what it restores into, which turned the reporter itself into a SIGBUS. Also here, because the same audit found it: java.lang.System.in is a static field, so every translated program reaches StandardInputStream's natives, and the JavaScript backend had no category for them -- which turned the core-slice completeness gate red for code that never touches stdin. They are marked unsupported there, as java.io.File already is: a browser has no process stdin. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 8 ++ vm/ByteCodeTranslator/src/cn1_globals.m | 86 ++++++++++++++++- .../tools/translator/ByteCodeClass.java | 9 ++ .../translator/JavascriptNativeRegistry.java | 17 +++- .../BackendUncaughtExceptionTest.java | 94 +++++++++++++++++++ 5 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 vm/tests/src/test/java/com/codename1/tools/translator/BackendUncaughtExceptionTest.java diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index ca94638341d..bb934470c33 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2022,6 +2022,14 @@ extern void releaseForReturnInException(CODENAME_ONE_THREAD_STATE, int cn1Locals extern JAVA_VOID java_lang_Throwable_fillInStack__(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT ex); +/* + * When nonzero, an exception that no handler catches prints itself and ends the + * process instead of being silently discarded. Set by the clean (server-side) + * target's generated main(); left at 0 everywhere else so app targets keep the + * behaviour they ship with today. + */ +extern int cn1AbortOnUncaughtException; + extern void throwException(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT exceptionArg); extern JAVA_INT throwException_R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT exceptionArg); extern JAVA_BOOLEAN throwException_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT exceptionArg); diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index e084000eb40..8e0d58e6d13 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -4090,11 +4090,16 @@ JAVA_INT java_lang_System_identityHashCode___java_lang_Object_R_int(CODENAME_ONE // struct CN1BibopPage is defined in cn1_globals.h (shared with the inlined bump). -static CN1BibopPage* _Atomic bibopAllPages = 0; // registry head (atomic) +/* No initializer: a static object is zero-initialized by the language, and + * clang 14 -- which is what Debian bookworm ships, and therefore what the + * glibc builder image uses -- rejects `= 0` on an _Atomic POINTER as "not a + * compile-time constant". The integer atomics above are accepted; only the + * pointer ones trip it. */ +static CN1BibopPage* _Atomic bibopAllPages; // registry head (atomic) static _Atomic long long bibopAllPagesCount = 0; // grow-only registration count static CN1BibopPage* bibopFreePool = 0; // bibopMutex static CN1BibopPage* bibopPartialPool[CN1_BIBOP_NUM_CLASSES]; // bibopMutex -static CN1BibopPage* _Atomic bibopSweepStack = 0; // Treiber-ish (push CAS / swap) +static CN1BibopPage* _Atomic bibopSweepStack; // Treiber-ish (push CAS / swap); see above static pthread_mutex_t bibopMutex = PTHREAD_MUTEX_INITIALIZER; static pthread_once_t bibopOnce = PTHREAD_ONCE_INIT; // Non-static: also read/written by the inlined bump fast path (cn1_globals.h). @@ -11888,6 +11893,67 @@ JAVA_OBJECT __NEW_ARRAY_JAVA_DOUBLE(CODENAME_ONE_THREAD_STATE, JAVA_INT size) { return o; } +/* + * Set by the clean target's generated main(). See the uncaught path at the bottom + * of throwException. + */ +int cn1AbortOnUncaughtException = 0; + +/* + * The end of the road for an exception no handler wants. + * + * Reached only when cn1AbortOnUncaughtException is set, which is the clean + * (server-side) target and nothing else -- an app target keeps today's behaviour, + * because changing what a shipped app does when it swallows an exception is not + * this change's business. + */ +static void cn1ReportUncaughtException(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT exceptionArg) { + static int reporting = 0; + if(reporting) { + /* Rendering the trace threw as well. Say so and stop, rather than recurse + * until the C stack runs out -- that reports as a segfault and hides the + * original failure entirely. */ + fprintf(stderr, "Uncaught exception while reporting an uncaught exception\n"); + fflush(stderr); + exit(1); + } + reporting = 1; + /* The search above left tryBlockOffset at -1: it decrements once on entry and + * then once per frame it rejects. Rendering the trace runs Java, and a Java + * method that saves and restores a NEGATIVE try depth corrupts the stack it + * restores into -- which is a SIGBUS in the reporter rather than a report. + * Every handler has been unwound by now, so the honest depth is zero. */ + threadStateData->tryBlockOffset = 0; + fprintf(stderr, "Uncaught exception"); + if(exceptionArg != JAVA_NULL && exceptionArg->__codenameOneParentClsReference != NULL + && exceptionArg->__codenameOneParentClsReference->clsName != NULL) { + fprintf(stderr, " %s", exceptionArg->__codenameOneParentClsReference->clsName); + } + if(exceptionArg != JAVA_NULL) { + /* The message, which the pre-rendered stack string does not carry -- and + * on a server it is the actionable half of the report. */ + JAVA_OBJECT message = java_lang_Throwable_getMessage___R_java_lang_String( + threadStateData, exceptionArg); + if(message != JAVA_NULL) { + const char* text = stringToUTF8(threadStateData, message); + if(text != NULL) { + fprintf(stderr, ": %s", text); + } + } + } + fprintf(stderr, "\n"); + fflush(stderr); + if(exceptionArg != JAVA_NULL) { + /* The Java renderer, so the message and the frames come out in the form a + * developer sees everywhere else. It runs with an empty try-block stack, + * which is what the guard above is for. */ + java_lang_Throwable_printStackTrace__(threadStateData, exceptionArg); + } + fflush(stdout); + fflush(stderr); + exit(1); +} + void throwException(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT exceptionArg) { #if defined(__OBJC__) //NSLog(@"Throwing exception!"); @@ -11910,6 +11976,22 @@ void throwException(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT exceptionArg) { } threadStateData->tryBlockOffset--; } + /* + * No handler anywhere on this thread. Historically this simply returned, and + * the generated code carried on with the statement AFTER the throw -- a + * `throw` that does nothing, with the method's locals in whatever state the + * half-finished operation left them. On an app target something upstream (the + * EDT's own catch) nearly always exists, so it stayed invisible; a server + * binary has no such catch, and the failure mode is a process that keeps + * serving with a null where a database connection should be. + * + * The clean target therefore reports and exits. Every other target keeps the + * old behaviour, because making this fatal everywhere would change what apps + * that ship today do. + */ + if(cn1AbortOnUncaughtException) { + cn1ReportUncaughtException(threadStateData, exceptionArg); + } } JAVA_INT throwException_R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT exceptionArg) { diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java index e3d98e3ce35..9bde853af4a 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java @@ -1209,6 +1209,15 @@ public String generateCCode(List allClasses) { b.append(" setvbuf(stdout, NULL, _IONBF, 0);\n"); b.append(" setvbuf(stderr, NULL, _IONBF, 0);\n"); b.append(" initConstantPool();\n"); + // An exception no handler catches used to be discarded and + // execution continued with the statement after the throw. An + // app target nearly always has something upstream that + // catches (the EDT's own try), so it stayed invisible there; + // a server binary has no such catch, and the symptom is a + // process that keeps serving with a half-built object where a + // connection should be. Only this target opts in, so nothing + // that ships today changes behaviour. + b.append(" cn1AbortOnUncaughtException = 1;\n"); // With the nursery, the main thread allocates and must cooperate with // the concurrent GC's stop-the-world pause (so the GC never scans its // nursery while a minor collection runs). Lightweight threads are the diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java index adf28ac5162..e2a3658250c 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java @@ -225,9 +225,24 @@ static NativeCategory categoryFor(String symbol) { } static String unsupportedReason(String symbol) { - if (symbol.startsWith("cn1_java_io_File_")) { + if (symbol.startsWith("cn1_java_io_File_") + || symbol.startsWith("cn1_java_io_FileInputStream_") + || symbol.startsWith("cn1_java_io_FileOutputStream_")) { return "java.io.File native filesystem access is not supported in javascript backend"; } + // The process-shaped parts of java.lang.System, added for the server-side + // (clean) target. A browser has no stdin to read and no environment to + // query, so these are unsupported here in the same sense java.io.File is + // -- not an oversight. System.in in particular is reached by EVERY + // translated program, because it is a static field of System, so leaving + // it uncategorized turned the core-slice completeness gate red for code + // that never touches it. + if (symbol.startsWith("cn1_java_io_StandardInputStream_")) { + return "process standard input is not available in the javascript backend"; + } + if (symbol.startsWith("cn1_java_lang_System_getenv_")) { + return "environment variables are not available in the javascript backend"; + } return null; } diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BackendUncaughtExceptionTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BackendUncaughtExceptionTest.java new file mode 100644 index 00000000000..cb68b964219 --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BackendUncaughtExceptionTest.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * An exception no handler catches must end a clean-target program, loudly. + * + * It used to be discarded: throwException walked the try-block stack, found no + * handler, and RETURNED -- so the generated code carried straight on with the + * statement after the throw, with the method's locals in whatever state the + * failed operation left them. On an app target something upstream (the EDT's own + * catch) nearly always exists, which is why it went unnoticed for years. A server + * binary has none, and the way this surfaced was a database client whose TLS + * handshake was rejected, after which the program kept going and segfaulted two + * statements later on a null it should never have had. + * + * The three assertions below are the contract: the message is printed, a stack + * trace is printed, and the process exits non-zero. All three matter -- an exit + * code with no message is unactionable in a log, and a message with a zero exit + * makes CI call a failed run a pass. + */ +class BackendUncaughtExceptionTest { + + @Test + @DisplayName("an uncaught exception reports itself and ends the process") + void uncaughtExceptionIsFatal() throws Exception { + if (CompilerHelper.isWindows()) { + Assumptions.abort("the server-side backend is POSIX-only for now"); + } + BackendTestSupport.require(Files.isDirectory(BackendTestSupport.backendDir()), + "vm/backend is not present"); + Path jdk8 = BackendTestSupport.findJdk8(); + BackendTestSupport.require(jdk8 != null, "no JDK 8 available to build the backend"); + + Path work = Files.createTempDirectory("backend-uncaught"); + Path binary = work.resolve("uncaught"); + String failure = BackendTestSupport.build("Uncaught", "demo/uncaught", binary, jdk8); + if (failure != null) { + BackendTestSupport.skipOrFail(failure); + } + + ProcessBuilder run = new ProcessBuilder(binary.toString()); + run.redirectErrorStream(true); + Process p = run.start(); + String output = BackendTestSupport.readFully(p.getInputStream()); + if (!p.waitFor(2, TimeUnit.MINUTES)) { + p.destroyForcibly(); + fail("the program did not finish:\n" + output); + } + + assertTrue(output.indexOf("before the throw") >= 0, + "the program should have run up to the throw:\n" + output); + assertTrue(output.indexOf("deliberate failure with a message") >= 0, + "the exception's message must be reported, not just its type:\n" + output); + assertTrue(output.indexOf("com_demo_Uncaught.open") >= 0, + "a stack trace naming the throwing frame must be reported:\n" + output); + assertEquals(1, p.exitValue(), + "a program killed by an uncaught exception must not report success:\n" + output); + } +} From 2ed10e5671ed37d18650a31809ab330b56a5b468 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 29 Aug 2026 08:06:07 +0300 Subject: [PATCH 04/42] Fix two ParparVM portability bugs the backend build hit Both are one-line consequences of the same C rule, found by building the same program two ways. ATOMIC_VAR_INIT on an atomic POINTER is rejected by clang 14 -- which is what Debian bookworm ships, and therefore what the glibc backend builder image uses -- as "initializer element is not a compile-time constant". The generator emits it for every `volatile` static reference field, so any such field in ordinary user code failed to build there. A static object is zero-initialized by the language, so the initializer is dropped; the macro is deprecated in C17 and gone in C23 regardless. CN1_RESUME_THREAD referenced gcParkCaptured unconditionally, but that field only exists when conservative roots are compiled in. So -DCN1_DISABLE_CONSERVATIVE_GC_ROOTS -- the A/B arm vm/CLAUDE.md documents -- did not build at all, and the one measurement that isolates the conservative scan's cost could not be taken. It is now behind a macro that compiles away with the field. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 12 +++++++++++- .../codename1/tools/translator/ByteCodeClass.java | 11 ++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index bb934470c33..d4bdd32b146 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -1947,7 +1947,17 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int * signal-stop this just makes the cheaper cooperative path usable; a no-op when conservative * roots are off. */ #define CN1_YIELD_THREAD do { struct ThreadLocalData* __cn1yts = getThreadLocalData(); CN1_GC_PARK_CAPTURE(__cn1yts); __cn1yts->threadActive = JAVA_FALSE; } while(0) -#define CN1_RESUME_THREAD do { struct ThreadLocalData* __cn1rts = getThreadLocalData(); CN1_STALL_T0(__cn1rt0); while (__cn1rts->threadBlockedByGC){ usleep((JAVA_INT)1000);} __cn1rts->threadActive = JAVA_TRUE; __cn1rts->gcParkCaptured = JAVA_FALSE; CN1_STALL_ADD(__cn1rt0, CN1_STALL_NATIVE_RESUME, __cn1rts); } while(0) +/* The capture is cleared through a macro of its own because gcParkCaptured only + * EXISTS when conservative roots are compiled in. Referencing it unconditionally + * meant -DCN1_DISABLE_CONSERVATIVE_GC_ROOTS -- the A/B arm vm/CLAUDE.md documents + * -- did not build at all, so the one measurement that isolates the conservative + * scan's cost could not be taken. */ +#ifdef CN1_CONSERVATIVE_GC_ROOTS +#define CN1_GC_PARK_RELEASE(ts) do { (ts)->gcParkCaptured = JAVA_FALSE; } while(0) +#else +#define CN1_GC_PARK_RELEASE(ts) do { (void)(ts); } while(0) +#endif +#define CN1_RESUME_THREAD do { struct ThreadLocalData* __cn1rts = getThreadLocalData(); CN1_STALL_T0(__cn1rt0); while (__cn1rts->threadBlockedByGC){ usleep((JAVA_INT)1000);} __cn1rts->threadActive = JAVA_TRUE; CN1_GC_PARK_RELEASE(__cn1rts); CN1_STALL_ADD(__cn1rt0, CN1_STALL_NATIVE_RESUME, __cn1rts); } while(0) extern struct ThreadLocalData* getThreadLocalData(); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java index 9bde853af4a..88eddc7da51 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java @@ -895,7 +895,16 @@ public String generateCCode(List allClasses) { b.append("_"); b.append(bf.getFieldName()); if (bf.isVolatile()) { - b.append(" = ATOMIC_VAR_INIT(0);\n"); + // No initializer. A static object is zero-initialized by + // the language, and ATOMIC_VAR_INIT expands to a plain + // parenthesized value -- which clang 14 (Debian bookworm, + // and therefore the glibc backend builder image) rejects + // on an atomic POINTER as "initializer element is not a + // compile-time constant". The macro is also deprecated in + // C17 and gone in C23, so this is where it was heading + // regardless. Reached by any `volatile` static reference + // field in user code. + b.append(";\n"); } else { b.append(" = 0;\n"); } From 8d8e894b9133817301f0ea42f35c2049c440b3e9 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:07:51 +0300 Subject: [PATCH 05/42] Add virtual threads to the VM A virtual thread runs Java on a stack of its own, so parking one is a stack switch of a couple of nanoseconds rather than a blocked OS thread. Measured round trip on arm64: 2.1ns. The runtime is three files -- cn1_virtual_thread.{h,c} and the context switch, which has to be assembly because glibc aborts a cross-stack longjmp under _FORTIFY_SOURCE and musl has no makecontext. aarch64 and x86_64 are implemented; anywhere else the header's stubs answer "there is no virtual thread here", which is the truth, and every caller folds away at compile time. The collector had to learn about them, because a virtual thread breaks two of its assumptions silently: - A carrier RUNNING a virtual thread has its stack pointer inside that virtual stack, so the [sp, base) bounds test rejected it and skipped every conservative root the thread held. - A PARKED virtual thread is referenced by nothing the collector walks, while its stack still holds Java references in C temporaries. Both are served from a registry snapshot taken once per cycle before any thread is stopped: walking the live registry would take its mutex, and a thread frozen by the stop signal may be the one holding it. Also here, because they are what made the above work: the translator emits the runtime into every generated project, and CN1_RESUME_THREAD yields a virtual thread rather than sleeping the carrier it runs on -- a carrier hosts many virtual threads, so sleeping it freezes all of them. Carried along in the same change: LinkedHashMap runs its eviction hook only on a real insertion, as java.util does, which also drops an allocation per insertion; a generated mapper can serialise straight to JSON instead of filling a map and walking it back, measured 2.05x/1.51x/2.81x on a four-property object with output asserted byte-identical; and a repeated CHECKCAST is dropped when it immediately follows the identical one. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/com/codename1/mapping/Mapper.java | 31 ++ .../src/com/codename1/mapping/Mappers.java | 84 +++- .../builders/WatchNativeBuilder.java | 11 +- .../MappingAnnotationProcessor.java | 129 +++++- vm/ByteCodeTranslator/src/cn1_globals.h | 26 ++ vm/ByteCodeTranslator/src/cn1_globals.m | 221 ++++++++++- .../src/cn1_virtual_thread.c | 366 ++++++++++++++++++ .../src/cn1_virtual_thread.h | 252 ++++++++++++ .../src/cn1_virtual_thread_asm.S | 191 +++++++++ .../tools/translator/ByteCodeTranslator.java | 30 ++ .../tools/translator/BytecodeMethod.java | 44 +++ vm/ByteCodeTranslator/src/nativeMethods.m | 325 ++++++++++------ vm/JavaAPI/src/java/util/LinkedHashMap.java | 34 +- vm/tests/virtualthread/test_virtual_thread.c | 222 +++++++++++ 14 files changed, 1832 insertions(+), 134 deletions(-) create mode 100644 vm/ByteCodeTranslator/src/cn1_virtual_thread.c create mode 100644 vm/ByteCodeTranslator/src/cn1_virtual_thread.h create mode 100644 vm/ByteCodeTranslator/src/cn1_virtual_thread_asm.S create mode 100644 vm/tests/virtualthread/test_virtual_thread.c diff --git a/CodenameOne/src/com/codename1/mapping/Mapper.java b/CodenameOne/src/com/codename1/mapping/Mapper.java index 08e83644bdb..a79a01e327a 100644 --- a/CodenameOne/src/com/codename1/mapping/Mapper.java +++ b/CodenameOne/src/com/codename1/mapping/Mapper.java @@ -48,6 +48,37 @@ public interface Mapper { /// `JSONParser` and populates a fresh `T`. T fromMap(Map map); + /// Optional: append `instance` as JSON directly, without building a map + /// first. + /// + /// A generated mapper knows every property name and type at build time, so + /// it can append them in order rather than filling a `LinkedHashMap` -- with + /// a hash per key -- and having the writer walk it back rediscovering each + /// value's type. On a small object that map round trip is the majority of + /// the serialisation cost, not the escaping. + /// + /// Measured through `Mappers#toJson` on a four-property object, output + /// asserted identical: **2.05x / 1.51x / 2.81x** faster (263 -> 128, + /// 214 -> 142, 224 -> 80 ns per call). + /// + /// That measurement ran on Java SE, so the map it avoids is the JDK's + /// LinkedHashMap. On a translated device build the map is + /// `vm/JavaAPI`'s, which overrides the natives HashMap gets and costs about + /// 1.5x a HashMap to build -- so the saving there is at least this, not less. + /// The same change on a server JSON route, where serialising is one cost + /// among request parsing and socket I/O, was worth 29% end to end. + /// + /// Implemented as a separate interface rather than a method on `Mapper` so + /// hand-written mappers keep compiling; `Mappers#toJson` uses it when the + /// mapper offers it and falls back to `toMap` when it does not. + public interface Direct { + + /// Appends `instance` as a JSON value -- an object, or the four + /// characters `null`. Must produce exactly what + /// `JSONWriter.toJson(toMap(instance))` would. + void toJson(T instance, StringBuilder out); + } + /// XML root element name (`@XmlRoot.value`, falling back to the class /// simple name with a lowercase first character). String xmlRootName(); diff --git a/CodenameOne/src/com/codename1/mapping/Mappers.java b/CodenameOne/src/com/codename1/mapping/Mappers.java index 10a69cc8760..85f59701957 100644 --- a/CodenameOne/src/com/codename1/mapping/Mappers.java +++ b/CodenameOne/src/com/codename1/mapping/Mappers.java @@ -109,8 +109,18 @@ public static String toJson(Object instance) { if (m == null) { throw missing(instance.getClass()); } - Map root = m.toMap(instance); StringBuilder sb = new StringBuilder(); + if (m instanceof Mapper.Direct) { + // The generated mapper knows its properties at build time and can + // append them in order. Skips a LinkedHashMap, a hash per key and a + // walk back over it that rediscovers each value's type -- which on a + // small object is most of the cost of serialising it. + @SuppressWarnings("unchecked") + Mapper.Direct d = (Mapper.Direct) m; + d.toJson(instance, sb); + return sb.toString(); + } + Map root = m.toMap(instance); writeJson(sb, root); return sb.toString(); } @@ -208,6 +218,65 @@ private static IllegalStateException missing(Class type) { // Tiny JSON writer // --------------------------------------------------------------- + /// Appends any value a generated codec can hold, producing exactly what the + /// map path would. + /// + /// Public because generated `toJson` methods call it for the property kinds + /// they cannot render inline -- a nested mapped object, a `Property`'s value, + /// a list element. A nested object goes through ITS mapper, taking that + /// mapper's `Mapper.Direct` route when it offers one, so nesting stays free + /// of intermediate maps all the way down. + /// + /// Conversions match `Mapper#toMap` exactly, and must keep matching: a date + /// becomes its millisecond value and an enum its `name()`, because that is + /// what the map path puts in the map before the writer ever sees it. + public static void appendJsonValue(StringBuilder out, Object value) { + if (value == null) { + out.append("null"); + return; + } + if (value instanceof java.util.Date) { + out.append(((java.util.Date) value).getTime()); + return; + } + if (value instanceof String || value instanceof Boolean + || value instanceof Number || value instanceof Map + || value instanceof java.util.Collection) { + writeJson(out, value); + return; + } + // A mapped object, or something with no mapper at all -- appendJson + // decides, and falls back to the string form the map path would use. + appendJson(value, out); + } + + /// Appends `instance` as a JSON object using its registered mapper, taking + /// the `Mapper.Direct` route when that mapper offers one. + /// + /// Unlike `#toJson(Object)` this appends rather than returning a String, so + /// nesting does not build one String per level. An unmapped value falls back + /// to its `toString`, which is what `Mapper#toMap` does for the same case + /// rather than failing the whole document. + public static void appendJson(Object instance, StringBuilder out) { + if (instance == null) { + out.append("null"); + return; + } + @SuppressWarnings("unchecked") + Mapper m = (Mapper) BY_NAME.get(instance.getClass().getName()); + if (m == null) { + writeJsonString(out, instance.toString()); + return; + } + if (m instanceof Mapper.Direct) { + @SuppressWarnings("unchecked") + Mapper.Direct d = (Mapper.Direct) m; + d.toJson(instance, out); + return; + } + writeJson(out, m.toMap(instance)); + } + static void writeJson(StringBuilder sb, Object value) { if (value == null) { sb.append("null"); @@ -249,6 +318,19 @@ static void writeJson(StringBuilder sb, Object value) { writeJsonString(sb, value.toString()); } + /// Appends `s` as an escaped JSON string, or `null`. + /// + /// Public because GENERATED mappers call it: a direct writer has to escape + /// exactly the way the map path does, and the only way to guarantee that is + /// for both to use this method rather than each having its own copy. + public static void appendJsonString(StringBuilder sb, String s) { + if (s == null) { + sb.append("null"); + return; + } + writeJsonString(sb, s); + } + private static void writeJsonString(StringBuilder sb, String s) { sb.append('"'); int len = s.length(); diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/WatchNativeBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/WatchNativeBuilder.java index 49e5aec7ddb..acc4b79b7dd 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/WatchNativeBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/WatchNativeBuilder.java @@ -572,9 +572,18 @@ List stageWatchTranslation(BuildRequest request, File tmpFile, File appS // Swift phase fixup, which globs
-src/**/*.swift, would have swept the watch // copy into the PHONE target instead. It is excluded from that glob for the same // reason (see IPhoneBuilder's swift fixup). + // .S belongs here too, for the same reason .swift does. The virtual-thread + // runtime's context switch has to be assembly (glibc aborts a cross-stack + // longjmp under _FORTIFY_SOURCE and musl has no makecontext), and it is the + // first .S the translator emits -- so copying cn1_virtual_thread.c without + // cn1_virtual_thread_asm.S left the WATCH target compiling a caller whose + // callee did not exist: "_cn1VirtualThreadSwitch, referenced from + // _cn1VirtualThreadYield ... symbol(s) not found". The phone target linked + // fine, so only a watch-enabled build shows it. boolean source = name.endsWith(".m") || name.endsWith(".c") || name.endsWith(".swift") || name.endsWith(".mm") - || name.endsWith(".cpp") || name.endsWith(".cc"); + || name.endsWith(".cpp") || name.endsWith(".cc") + || name.endsWith(".S") || name.endsWith(".s"); if (!source && !name.endsWith(".h")) { // Only the code. The watch bundle's plist, resources and project file are written // by this builder against the PHONE project -- taking the second translation's diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java index 7ab7014b997..45a651069ce 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java @@ -338,7 +338,12 @@ private static String generateMapperSource(MappedClass mc) { sb.append("// Auto-generated by cn1:process-annotations. Do not edit.\n"); sb.append("@SuppressWarnings({\"all\"})\n"); sb.append("public final class ").append(mc.mapperSimpleName) - .append(" implements com.codename1.mapping.Mapper<").append(mc.binaryName).append("> {\n\n"); + .append(" implements com.codename1.mapping.Mapper<").append(mc.binaryName).append(">"); + boolean direct = canWriteDirectly(mc); + if (direct) { + sb.append(", com.codename1.mapping.Mapper.Direct<").append(mc.binaryName).append(">"); + } + sb.append(" {\n\n"); // Public static register() hook. The bootstrap class invokes // this once per generated mapper at app start; the call @@ -375,6 +380,28 @@ private static String generateMapperSource(MappedClass mc) { sb.append(" return m;\n"); sb.append(" }\n\n"); + // toJson() -- the same properties, appended in order instead of going + // through a LinkedHashMap that the writer then walks back. Emitted ONLY + // for classes every field of which has a shape this can render exactly; + // Mappers#toJson tests for the interface, so anything not emitted here + // keeps the map path with no special casing. + if (direct) { + sb.append(" public void toJson(").append(mc.binaryName) + .append(" o, StringBuilder out) {\n"); + sb.append(" if (o == null) { out.append(\"null\"); return; }\n"); + sb.append(" out.append('{');\n"); + boolean firstProp = true; + for (MappedField f : mc.fields) { + if (!f.includeInJson) continue; + sb.append(" out.append(\"").append(firstProp ? "" : ",") + .append("\\\"").append(escape(f.jsonName)).append("\\\":\");\n"); + emitFieldToJson(sb, f, isRecord); + firstProp = false; + } + sb.append(" out.append('}');\n"); + sb.append(" }\n\n"); + } + // fromMap() -- POJO mutates an instance in-place; record accumulates // per-component locals and feeds them to the canonical constructor. sb.append(" public ").append(mc.binaryName) @@ -507,6 +534,106 @@ private static String packageOf(String binary) { // toMap field-emit helpers // --------------------------------------------------------------- + /// Whether every JSON property of `mc` has a shape the direct writer can + /// render EXACTLY as the map path would. + /// + /// Deliberately conservative: lists, nested mapped objects, byte arrays and + /// Property wrappers all reach for the registry or another mapper at run + /// time, and getting one of them subtly wrong produces valid-looking JSON + /// with the wrong contents. They keep the map path until each is done and + /// tested on its own. + /// Whether every JSON property of `mc` has a shape [#emitFieldToJson] can + /// render. Now every kind [#emitFieldToMap] handles, so the map path is used + /// only for classes with a property neither of them renders. + private static boolean canWriteDirectly(MappedClass mc) { + for (MappedField f : mc.fields) { + if (!f.includeInJson) continue; + switch (f.kind.kind) { + case STRING: case INT: case LONG: case SHORT: case BYTE: case CHAR: + case DOUBLE: case FLOAT: case BOOLEAN: case ENUM: case DATE: + case BYTE_ARRAY: case PROPERTY: case REFERENCE: + case LIST: case LIST_PROPERTY: + break; + default: + // emitFieldToMap ignores anything else, so there is nothing to + // render and no reason to claim the fast path. + return false; + } + } + return true; + } + + /// One property, appended directly. Mirrors [#emitFieldToMap] case for case + /// and MUST keep mirroring it: the two produce the same JSON by construction + /// rather than by test, so a case that drifts changes the document silently. + /// The conversions that matter are a date to its millisecond value, an enum + /// to `name()`, a byte array to Base64 and a char to a one-character STRING + /// -- the map path applies those before the writer sees the value. + private static void emitFieldToJson(StringBuilder sb, MappedField f, boolean isRecord) { + String read = readExpr(f, isRecord); + switch (f.kind.kind) { + case STRING: + sb.append(" com.codename1.mapping.Mappers.appendJsonString(out, ") + .append(read).append(");\n"); + return; + case INT: case LONG: case SHORT: case BYTE: case DOUBLE: case FLOAT: + sb.append(" out.append(").append(read).append(");\n"); + return; + case BOOLEAN: + sb.append(" out.append(").append(read).append(" ? \"true\" : \"false\");\n"); + return; + case CHAR: + sb.append(" com.codename1.mapping.Mappers.appendJsonString(out, String.valueOf(") + .append(read).append("));\n"); + return; + case ENUM: + sb.append(" com.codename1.mapping.Mappers.appendJsonString(out, ") + .append(read).append(" == null ? null : ").append(read).append(".name());\n"); + return; + case DATE: + sb.append(" if (").append(read).append(" == null) { out.append(\"null\"); }") + .append(" else { out.append(").append(read).append(".getTime()); }\n"); + return; + case BYTE_ARRAY: + sb.append(" com.codename1.mapping.Mappers.appendJsonString(out, ") + .append(read).append(" == null ? null : com.codename1.util.Base64.encode(") + .append(read).append("));\n"); + return; + case PROPERTY: + sb.append(" com.codename1.mapping.Mappers.appendJsonValue(out, ") + .append(read).append(".get());\n"); + return; + case REFERENCE: + // Through the nested type's own mapper, which takes ITS direct + // route when it has one, so nesting builds no map either. + sb.append(" com.codename1.mapping.Mappers.appendJson(") + .append(read).append(", out);\n"); + return; + case LIST: case LIST_PROPERTY: { + String src = f.kind.kind == PropertyTypeKind.Kind.LIST + ? read : read + ".asList()"; + sb.append(" {\n"); + sb.append(" java.util.List _src = ").append(src).append(";\n"); + sb.append(" if (_src == null) { out.append(\"null\"); }\n"); + sb.append(" else {\n"); + sb.append(" out.append('[');\n"); + sb.append(" boolean _first = true;\n"); + sb.append(" for (java.util.Iterator _it = _src.iterator(); _it.hasNext(); ) {\n"); + sb.append(" if (!_first) { out.append(','); }\n"); + sb.append(" _first = false;\n"); + sb.append(" com.codename1.mapping.Mappers.appendJsonValue(out, _it.next());\n"); + sb.append(" }\n"); + sb.append(" out.append(']');\n"); + sb.append(" }\n"); + sb.append(" }\n"); + return; + } + default: + throw new IllegalStateException( + "no direct JSON writer for " + f.kind.kind + " on " + f.jsonName); + } + } + private static void emitFieldToMap(StringBuilder sb, MappedField f, boolean isRecord) { String key = "\"" + escape(f.jsonName) + "\""; String read = readExpr(f, isRecord); diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index d4bdd32b146..766e5b23ad1 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2825,6 +2825,16 @@ extern void cn1GcInstallSignalHandler(void); // universal-stop handler. extern __thread struct ThreadLocalData* cn1TlsSelf; +struct cn1VirtualThread; +/** + * A VM thread state. bindToCallingOsThread false builds one for a VIRTUAL thread, + * which owns it rather than borrowing the host's -- see the definition. + */ +extern struct ThreadLocalData* cn1CreateThreadLocalData(JAVA_BOOLEAN bindToCallingOsThread); +/** A virtual thread with a Java stack of its own, ready to be resumed. */ +extern struct cn1VirtualThread* cn1SpawnVirtualThread(void (*body)(void*), void* arg, + size_t stackBytes); + // Capture a parking mutator's native register file + native-stack low bound so the // concurrent GC can conservatively scan [sp, stackBase) for native-stack-held roots. // MUST be a macro so setjmp + the SP marker live in the PARKING frame itself: that @@ -2851,6 +2861,22 @@ extern __thread struct ThreadLocalData* cn1TlsSelf; #ifdef CN1_GC_CONFORM extern long long cn1StallNowNs(void); extern void cn1StallRecord(int cause, long long ns, struct ThreadLocalData* ts); +/* Stall causes. Declared HERE rather than in cn1_globals.m because + CN1_RESUME_THREAD below expands to CN1_STALL_ADD(..., CN1_STALL_NATIVE_RESUME, + ...), and every native file that wraps a blocking call uses that macro. With + the codes private to cn1_globals.m, any other native source failed to compile + under -DCN1_GC_CONFORM with "use of undeclared identifier"; the backend's + sockets, database and crypto natives are the first outside the core to wrap + blocking calls this way. */ +#define CN1_STALL_PACING_VOLUME 0 // regime-A run-ahead cap (cn1PacingPark, no budget) +#define CN1_STALL_PACING_BUDGET 1 // regime-B admission wait (cn1PacingPark, under a ceiling) +#define CN1_STALL_LOWMEM 2 // the low-memory allocation throttle +#define CN1_STALL_HANDSHAKE 3 // threadBlockedByGC: this thread's own share of the mark +#define CN1_STALL_PENDING_FULL 4 // per-thread pending table full: waits out a WHOLE cycle +#define CN1_STALL_NATIVE_RESUME 5 // returning from a native call into a running mark +#define CN1_STALL_SIGNAL_STOP 6 // parked inside the GC's stop signal handler +#define CN1_STALL_CAUSES 7 + #define CN1_STALL_T0(v) long long v = cn1StallNowNs() #define CN1_STALL_ADD(v, cause, ts) cn1StallRecord((cause), cn1StallNowNs() - (v), (ts)) #else diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 8e0d58e6d13..d1383e13278 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -27,6 +27,7 @@ #define _GNU_SOURCE #endif #include "cn1_globals.h" +#include "cn1_virtual_thread.h" #include #include // clock_gettime: paces the low-memory allocation throttle #ifndef _WIN32 @@ -750,14 +751,9 @@ static long long cn1GcNowNs(void) { // This measures the other side: for each site where a mutator can be stopped, how long it // was stopped and why. Cost is two clock_gettime calls per PARK -- never per allocation -- // against a park that is at minimum a 50us sleep, so it cannot distort what it measures. -#define CN1_STALL_PACING_VOLUME 0 // regime-A run-ahead cap (cn1PacingPark, no budget) -#define CN1_STALL_PACING_BUDGET 1 // regime-B admission wait (cn1PacingPark, under a ceiling) -#define CN1_STALL_LOWMEM 2 // the low-memory allocation throttle -#define CN1_STALL_HANDSHAKE 3 // threadBlockedByGC: this thread's own share of the mark -#define CN1_STALL_PENDING_FULL 4 // per-thread pending table full: waits out a WHOLE cycle -#define CN1_STALL_NATIVE_RESUME 5 // returning from a native call into a running mark -#define CN1_STALL_SIGNAL_STOP 6 // parked inside the GC's stop signal handler -#define CN1_STALL_CAUSES 7 +/* The cause codes live in cn1_globals.h, beside CN1_STALL_ADD: a macro's + operands have to be visible wherever the macro is, and CN1_RESUME_THREAD is + used by every native file, not only this one. */ static const char* cn1StallCauseNames[CN1_STALL_CAUSES] = { "pacingVolume", "pacingBudget", "lowMemory", "handshake", "pendingFull", "nativeResume", "signalStop" @@ -1643,6 +1639,11 @@ static void cn1DrainDeadThreadPending() { // scan a REAL root source for object-bearing FRAMELESS frames. See the big block below. static void cn1GcScanThreadNativeStack(CODENAME_ONE_THREAD_STATE, struct ThreadLocalData* t); static void cn1GcScanOwnStack(CODENAME_ONE_THREAD_STATE); +// Virtual threads are a third root source beside the precise object stacks and the +// native C stacks; see the block that defines these. +static void cn1GcBuildVirtualThreadSnapshot(void); +static void cn1GcScanParkedVirtualThreads(CODENAME_ONE_THREAD_STATE); +static int cn1GcParkedVirtualThreadsScanned; static void cn1GcSignalStopThreads(struct ThreadLocalData* self); static void cn1GcSignalReleaseThreads(struct ThreadLocalData* self); #ifdef CN1_GC_CAN_FORCE_STOP @@ -2259,6 +2260,31 @@ void codenameOneGCMark() { cn1_debugger_mark_issued_roots(d); #endif +#ifdef CN1_CONSERVATIVE_GC_ROOTS + // Opened around the WHOLE loop, not around each thread. Threads are stopped + // and scanned one at a time and the others keep running throughout, so a host + // thread finishing a connection can free a virtual thread that an earlier + // iteration's snapshot still points at. From here until the matching End, a + // free unlinks and parks the virtual thread instead of releasing it. + cn1VirtualThreadGcScanBegin(); + // Taken HERE, once, and deliberately not inside the loop below. Three reasons, and + // the first two are correctness rather than cost: + // + // - Reading the registry takes its mutex. At this point every mutator is running + // normally, so no thread can be holding that mutex while stopped. Inside the loop + // a thread is signal-frozen at an arbitrary instruction and may be the holder -- + // the collector would then block on the thread it just froze. + // - This also resets cn1GcParkedVirtualThreadsScanned. Guarding the call with + // !forcedStop, as the root snapshots below must be, would leave that flag set + // from the previous cycle whenever the first thread needed a forced stop, and the + // parked scan would be skipped for the whole cycle -- silently dropping the roots + // of every parked virtual thread and reclaiming objects that are still live. + // - It is a per-cycle fact, so taking it per thread paid the mutex N times. + // + // Placed after GcScanBegin so every pointer it captures is held alive by the deferred + // release until the matching End. + cn1GcBuildVirtualThreadSnapshot(); +#endif for(int iter = 0 ; iter < NUMBER_OF_SUPPORTED_THREADS ; iter++) { lockCriticalSection(); struct ThreadLocalData* t = allThreads[iter]; @@ -2660,6 +2686,17 @@ void codenameOneGCMark() { forcedStopSeq = 0; } #endif + // AFTER the release above, deliberately: this scan MARKS, and marking + // allocates through cn1MatureObject's adoption buffer. Running it while a + // thread is signal-frozen is the exact hazard cn1GcFreezeHeld exists to + // prevent, since the frozen thread may own the allocator lock. + // Parked virtual threads belong to no OS thread, so they are not + // reached by the loop this sits in. Scanning them once per cycle + // is enough and is idempotent, since marking is. + if(!cn1GcParkedVirtualThreadsScanned) { + cn1GcParkedVirtualThreadsScanned = 1; + cn1GcScanParkedVirtualThreads(d); + } #ifdef CN1_CONSERVATIVE_GC_SELFCHECK cn1GcSelfCheckThreadStack(t, stackSize); #endif @@ -2698,6 +2735,11 @@ void codenameOneGCMark() { } } } +#ifdef CN1_CONSERVATIVE_GC_ROOTS + // Every snapshot this loop took is dead now, so whatever was retired while it + // ran can actually be released. + cn1VirtualThreadGcScanEnd(); +#endif #if defined(__OBJC__) //NSLog(@"Mark set %i objects to %i", marked, currentGcMarkValue); #endif @@ -4157,6 +4199,24 @@ JAVA_INT java_lang_System_identityHashCode___java_lang_Object_R_int(CODENAME_ONE static void cn1BibopDoInit() { int ci = 0; + // DIAGNOSTIC KNOB -- CN1_GC_TRIGGER_MB overrides how many uncollected bytes + // start a cycle. The twin of CN1_GC_PACING_CAP_MB above, and like it, it + // exists to ANSWER A QUESTION rather than to tune anything: raising it far + // enough that no cycle runs during a measured window attributes the remaining + // throughput gap to collection work or rules it out. A park counter cannot do + // that -- it says parks happened, not what the CPU went on. + // + // Not a supported setting: the heap grows without bound while it is raised. + { + const char* s = getenv("CN1_GC_TRIGGER_MB"); + if(s != 0) { + long mb = atol(s); + if(mb > 0) { + atomic_store_explicit(&bibopGcTriggerBytes, mb * 1024L * 1024L, + memory_order_relaxed); + } + } + } for(int s = 0 ; s <= CN1_BIBOP_MAX_OBJECT ; s++) { while(ci < CN1_BIBOP_NUM_CLASSES && cn1BibopClassSize[ci] < s) { ci++; @@ -5064,6 +5124,29 @@ static JAVA_BOOLEAN cn1PacingPastGrowthFloor(void) { } static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { + // DIAGNOSTIC KNOB -- CN1_GC_PACING_CAP_MB overrides the computed cap outright. + // + // It exists to ANSWER A QUESTION, not to tune anything: setting it high enough that + // cn1PacingVolume can never exceed it removes volume parking from the run entirely, + // so a latency measurement taken with and without it attributes the tail to this + // backpressure or rules it out. Inferring that from the park counters alone is not + // the same evidence -- a counter says parks happened, not that they are what the + // slow requests were waiting on. + // + // Not a supported setting: overriding it discards the memory bound the cap exists to + // enforce, so a process run this way can grow until the OS kills it. + { + static _Atomic long cn1PacingCapOverride = -2; + long ov = atomic_load_explicit(&cn1PacingCapOverride, memory_order_relaxed); + if(ov == -2) { + const char* s = getenv("CN1_GC_PACING_CAP_MB"); + ov = (s != 0) ? (long)atol(s) * 1024L * 1024L : -1; + atomic_store_explicit(&cn1PacingCapOverride, ov, memory_order_relaxed); + } + if(ov > 0) { + return ov; + } + } long trigger = atomic_load_explicit(&bibopGcTriggerBytes, memory_order_relaxed); long base = trigger * CN1_BIBOP_GC_HARD_CAP_MULTIPLIER; long fm = atomic_load_explicit(&cn1CachedFreeMem, memory_order_relaxed); @@ -5309,10 +5392,30 @@ static void cn1PacingPark(CODENAME_ONE_THREAD_STATE, int which, long long pendin spins++ < 200000) { atomic_store_explicit(&cn1PacingLastParkMs, (long long)cn1MonotonicMillis(), memory_order_relaxed); - usleep(50); + // On a VIRTUAL thread, step off the host instead of sleeping on it. + // + // This spin is backpressure on the ALLOCATOR, so it fires wherever + // Java allocates -- which on a server is everywhere. Sleeping here + // holds whichever thread happened to be running the virtual thread, + // and a host thread is not a spare resource: it is one of the few + // threads that poll. Proved with a debugger rather than reasoned + // about: under load all four hosts were in this loop at once, three + // of them inside HttpServer.serve on different descriptors, so + // nothing was polling and the server had stopped accepting for good + // -- the volume this loop waits on only falls when a cycle ends, and + // ending one needs the mutator progress this loop is preventing. + // + // Yielding hands the host back. The virtual thread is RUNNABLE, not + // waiting on its socket, so the scheduler must re-queue it rather + // than hand it to the poller; see CN1_VT_YIELD_RUNNABLE. + if(!cn1VirtualThreadYieldIfVirtual()) { + usleep(50); + } } while(threadStateData->threadBlockedByGC) { - usleep((JAVA_INT)(500)); + if(!cn1VirtualThreadYieldIfVirtual()) { + usleep((JAVA_INT)(500)); + } } threadStateData->threadActive = JAVA_TRUE; CN1_STALL_ADD(__stallVol, CN1_STALL_PACING_VOLUME, threadStateData); @@ -8555,6 +8658,64 @@ static void cn1GcMarkReleaseForced(struct ThreadLocalData* t) { } #endif +// ---- VIRTUAL THREADS AS A ROOT SOURCE ------------------------------------------- +// +// A virtual thread runs Java on a stack of its own (cn1_virtual_thread.h), which +// makes two things true that this scan would otherwise get wrong, both silently: +// +// 1. A thread that is RUNNING a virtual thread has its stack pointer inside that +// virtual thread's stack, not its own. The [sp, base) bounds check below then +// simply fails and the thread is skipped -- losing every conservative root it +// holds, with the crash landing somewhere else entirely. +// 2. A PARKED virtual thread is referenced by nothing the collector walks. Its +// stack still holds Java references in C temporaries, and they are reachable +// from nowhere else. +// +// Both are handled by taking a snapshot of the registry BEFORE the world stops -- +// walking the live registry would mean taking its mutex, and a thread frozen by +// the stop signal may be the one holding it, which is a deadlock rather than a +// slowdown. The rest of this collector snapshots its roots for the same reason. +#define CN1_VT_SNAPSHOT_MAX 4096 +static struct cn1VirtualThread* cn1GcVtSnapshot[CN1_VT_SNAPSHOT_MAX]; +static int cn1GcVtSnapshotCount = 0; +static int cn1GcVtSnapshotTruncated = 0; + +// Reset at the start of every cycle; see the use below. +static int cn1GcParkedVirtualThreadsScanned = 0; + +static void cn1GcBuildVirtualThreadSnapshot(void) { + cn1GcParkedVirtualThreadsScanned = 0; + int n = cn1VirtualThreadSnapshot(cn1GcVtSnapshot, CN1_VT_SNAPSHOT_MAX); + if(n > CN1_VT_SNAPSHOT_MAX) { + // Scanning a subset is not a degraded mode, it is a use-after-free waiting + // to happen, so say so loudly rather than continue quietly. + if(!cn1GcVtSnapshotTruncated) { + cn1GcVtSnapshotTruncated = 1; + fprintf(stderr, "[CN1-VT] %d virtual threads exceeds the GC snapshot of %d; " + "raise CN1_VT_SNAPSHOT_MAX\n", n, CN1_VT_SNAPSHOT_MAX); + } + n = CN1_VT_SNAPSHOT_MAX; + } + cn1GcVtSnapshotCount = n; +} + +// Mark every PARKED virtual thread's live stack region. The running ones are +// covered through the thread that is running them, in the scan below. +static void cn1GcScanParkedVirtualThreads(CODENAME_ONE_THREAD_STATE) { + int i; + for(i = 0 ; i < cn1GcVtSnapshotCount ; i++) { + struct cn1VirtualThread* vt = cn1GcVtSnapshot[i]; + void* lo; void* hi; + if(vt == 0 || cn1VirtualThreadIsRunning(vt)) { + continue; + } + cn1VirtualThreadStackBounds(vt, &lo, &hi); + if(lo != 0 && hi != 0 && lo < hi) { + cn1ConservativeMarkRange(threadStateData, (char*)lo, (char*)hi); + } + } +} + // Scan ONE thread's native C stack [sp, base) + its register snapshot, marking every // resolved live object. threadStateData = the GC thread; t = the thread being scanned. static void cn1GcScanThreadNativeStack(CODENAME_ONE_THREAD_STATE, struct ThreadLocalData* t) { @@ -8602,6 +8763,21 @@ static void cn1GcScanThreadNativeStack(CODENAME_ONE_THREAD_STATE, struct ThreadL int useCoop = t->gcParkCaptured && t->gcStackPointerAtPark != 0 && cn1GcSignalStopMode == 0; if(useCoop) { char* sp = (char*)t->gcStackPointerAtPark; + // Running a virtual thread? Then sp is in ITS stack, and this thread's own + // stack holds the frames below the resume. Both halves are live. + struct cn1VirtualThread* vt = + cn1VirtualThreadForStackAddress(sp, cn1GcVtSnapshotCount, cn1GcVtSnapshot); + if(vt != 0) { + char* vtHigh = (char*)cn1VirtualThreadStackHigh(vt); + char* resumer = (char*)cn1VirtualThreadResumerSp(vt); + cn1ConservativeMarkRange(threadStateData, sp, vtHigh); + if(resumer >= base - (long)ssz && resumer < base) { + cn1ConservativeMarkRange(threadStateData, resumer, base); + } + cn1ConservativeMarkRange(threadStateData, (char*)&t->gcRegisterSnapshot, + (char*)&t->gcRegisterSnapshot + sizeof(t->gcRegisterSnapshot)); + return; + } if(sp >= base - (long)ssz && sp < base) { cn1ConservativeMarkRange(threadStateData, sp, base); cn1ConservativeMarkRange(threadStateData, (char*)&t->gcRegisterSnapshot, @@ -8637,8 +8813,23 @@ static void cn1GcScanThreadNativeStack(CODENAME_ONE_THREAD_STATE, struct ThreadL } // Raised only on the success path, so the sub below always pairs with an add. atomic_fetch_add_explicit(&cn1GcFreezeHeld, 1, memory_order_relaxed); - if(sp >= base - (long)ssz && sp < base) { - cn1ConservativeMarkRange(threadStateData, sp, base); + // Same virtual-thread split as the cooperative path above, and it is needed here for + // the same reason: the sp the signal handler reports is the one the thread was + // actually using, so for a carrier running a virtual thread it points into the + // virtual stack and the [sp, base) test below would reject it and skip every root. + { + struct cn1VirtualThread* vt = + cn1VirtualThreadForStackAddress(sp, cn1GcVtSnapshotCount, cn1GcVtSnapshot); + if(vt != 0) { + char* vtHigh = (char*)cn1VirtualThreadStackHigh(vt); + char* resumer = (char*)cn1VirtualThreadResumerSp(vt); + cn1ConservativeMarkRange(threadStateData, sp, vtHigh); + if(resumer >= base - (long)ssz && resumer < base) { + cn1ConservativeMarkRange(threadStateData, resumer, base); + } + } else if(sp >= base - (long)ssz && sp < base) { + cn1ConservativeMarkRange(threadStateData, sp, base); + } } if(t->gcSigRegsLen > 0) { cn1ConservativeMarkRange(threadStateData, t->gcSigRegs, t->gcSigRegs + t->gcSigRegsLen); @@ -11749,7 +11940,13 @@ void initConstantPool() { cn1StartSimulatedMemoryWarnings(); #ifdef CN1_GC_CONFORM atexit(cn1ReportStalls); +#ifdef CN1_CONSERVATIVE_GC_ROOTS + // The self test sorts the conservative extent table, which only exists on + // this arm. Calling it under CN1_GC_CONFORM alone does not compile, so + // -DCN1_GC_CONFORM -DCN1_DISABLE_CONSERVATIVE_GC_ROOTS -- the A/B pair the + // header documents -- was not buildable. cn1ConsExtSortSelfTest(); +#endif cn1GcProbeInit(); #endif diff --git a/vm/ByteCodeTranslator/src/cn1_virtual_thread.c b/vm/ByteCodeTranslator/src/cn1_virtual_thread.c new file mode 100644 index 00000000000..31348acb426 --- /dev/null +++ b/vm/ByteCodeTranslator/src/cn1_virtual_thread.c @@ -0,0 +1,366 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/* BACKEND ONLY -- see cn1_virtual_thread.h. Off-target this file is empty and + * the header supplies no-op stubs, so nothing references the assembly. */ +#include "cn1_virtual_thread.h" +#ifdef CN1_VIRTUAL_THREADS + +#include "cn1_virtual_thread.h" +#include +#include +#include +#include +#include + +/* + * The switch saves the callee-saved registers on the outgoing stack, swaps the + * stack pointer, and restores the incoming ones. Caller-saved registers need no + * handling: the compiler already assumes a call clobbers them, and this IS a + * call. That is the whole reason twenty instructions is enough. + */ +struct cn1VirtualThread { + void* sp; /* saved stack pointer while suspended */ + struct cn1VirtualThread* registryNext; /* every live virtual thread, for the GC */ + struct cn1VirtualThread* registryPrev; + void* vmState; /* this virtual thread's ThreadLocalData */ + int yieldReason; /* CN1_VT_YIELD_* -- why it last gave up its host */ + int running; /* executing on some OS thread right now */ + void* stackLow; /* mmap base */ + void* stackHigh; /* one past the usable end */ + size_t stackBytes; + cn1VirtualThreadBody body; + void* arg; + void* returnSp; /* the resumer's saved stack pointer */ + int finished; + int started; +}; + +static __thread struct cn1VirtualThread* cn1CurrentVirtualThread = 0; + +/* + * Every live virtual thread, so the collector can find the parked ones. + * + * A parked virtual thread's stack is referenced by nothing else -- not by the + * thread that created it, which has moved on, and not by the scheduler, which + * only knows the ones it has queued. If it is not enumerable here then a Java + * reference held in a C temporary of a parked request is invisible to the scan, + * and the object under it is freed while the request still means to use it. + */ +static struct cn1VirtualThread* cn1VirtualThreadRegistry = 0; +static pthread_mutex_t cn1VirtualThreadRegistryLock = PTHREAD_MUTEX_INITIALIZER; + +/* + * Releasing a virtual thread while the collector is scanning is a use-after-free, + * so it is deferred instead. + * + * The collector does not stop the world and then scan: it stops and scans ONE + * thread at a time, rebuilding its virtual-thread snapshot inside that loop, and + * every OTHER thread keeps running throughout -- including host threads, whose + * whole job is finishing connections and freeing the virtual threads that served + * them. So a pointer copied into the snapshot can be freed, and its stack + * unmapped, before the scan that snapshot feeds ever reads it. The wider the + * loop, the wider the window: with 64 idle Java threads padding it out this + * segfaulted 2 runs in 6, and with 4 it never did -- the idle threads take no + * part in the race, they only lengthen it. + * + * A free that lands during a scan therefore unlinks the virtual thread and parks + * it here rather than releasing it; the collector drains the list when the scan + * is over. Both the free and the snapshot serialise on the registry lock, which + * is what makes the handoff exact rather than merely likely: whichever gets the + * lock first decides, and there is no ordering in which one sees a half-state of + * the other. The lock is never HELD across a scan -- a frozen thread can be + * holding it, so waiting for it under a freeze would deadlock; the flag is what + * crosses that boundary, not the mutex. + */ +static int cn1VirtualThreadScanActive = 0; /* guarded by the registry lock */ +static struct cn1VirtualThread* cn1VirtualThreadRetired = 0; /* likewise */ + +static void cn1VirtualThreadRelease(struct cn1VirtualThread* co) { + size_t pageSize = (size_t)sysconf(_SC_PAGESIZE); + munmap((unsigned char*)co->stackLow - pageSize, co->stackBytes + pageSize); + free(co); +} + +static void cn1VirtualThreadRegister(struct cn1VirtualThread* vt) { + pthread_mutex_lock(&cn1VirtualThreadRegistryLock); + vt->registryPrev = 0; + vt->registryNext = cn1VirtualThreadRegistry; + if(cn1VirtualThreadRegistry != 0) { + cn1VirtualThreadRegistry->registryPrev = vt; + } + cn1VirtualThreadRegistry = vt; + pthread_mutex_unlock(&cn1VirtualThreadRegistryLock); +} + +static void cn1VirtualThreadUnregister(struct cn1VirtualThread* vt) { + pthread_mutex_lock(&cn1VirtualThreadRegistryLock); + if(vt->registryPrev != 0) { + vt->registryPrev->registryNext = vt->registryNext; + } else if(cn1VirtualThreadRegistry == vt) { + cn1VirtualThreadRegistry = vt->registryNext; + } + if(vt->registryNext != 0) { + vt->registryNext->registryPrev = vt->registryPrev; + } + vt->registryNext = 0; + vt->registryPrev = 0; + pthread_mutex_unlock(&cn1VirtualThreadRegistryLock); +} + +void cn1VirtualThreadForEach(void (*fn)(struct cn1VirtualThread* vt, void* ctx), + void* ctx) { + struct cn1VirtualThread* vt; + pthread_mutex_lock(&cn1VirtualThreadRegistryLock); + for(vt = cn1VirtualThreadRegistry ; vt != 0 ; vt = vt->registryNext) { + fn(vt, ctx); + } + pthread_mutex_unlock(&cn1VirtualThreadRegistryLock); +} + +int cn1VirtualThreadSnapshot(struct cn1VirtualThread** out, int max) { + struct cn1VirtualThread* vt; + int n = 0; + pthread_mutex_lock(&cn1VirtualThreadRegistryLock); + for(vt = cn1VirtualThreadRegistry ; vt != 0 ; vt = vt->registryNext) { + if(n < max) { + out[n] = vt; + } + n++; + } + pthread_mutex_unlock(&cn1VirtualThreadRegistryLock); + return n; +} + +struct cn1VirtualThread* cn1VirtualThreadForStackAddress(void* addr, int count, + struct cn1VirtualThread** snapshot) { + int i; + if(addr == 0) { + return 0; + } + for(i = 0 ; i < count ; i++) { + struct cn1VirtualThread* vt = snapshot[i]; + if(vt != 0 && addr >= vt->stackLow && addr < vt->stackHigh) { + return vt; + } + } + return 0; +} + +/** The high end of this virtual thread's stack, for a range the caller bounds. */ +void* cn1VirtualThreadArg(struct cn1VirtualThread* vt) { + return vt == 0 ? 0 : vt->arg; +} + +void* cn1VirtualThreadStackHigh(struct cn1VirtualThread* vt) { + return vt == 0 ? 0 : vt->stackHigh; +} + +int cn1VirtualThreadIsRunning(struct cn1VirtualThread* vt) { + return vt != 0 && vt->running; +} + +void* cn1VirtualThreadResumerSp(struct cn1VirtualThread* vt) { + return vt == 0 ? 0 : vt->returnSp; +} + +void* cn1VirtualThreadState(struct cn1VirtualThread* vt) { + return vt == 0 ? 0 : vt->vmState; +} + +void cn1VirtualThreadSetState(struct cn1VirtualThread* vt, void* state) { + if(vt != 0) { + vt->vmState = state; + } +} + +/* Implemented in assembly: save callee-saved regs, switch sp, restore. */ +extern void cn1VirtualThreadSwitch(void** saveSp, void* newSp); +/* The trampoline the new stack is primed to return into. */ +extern void cn1VirtualThreadTrampoline(void); + +/* Entered on the virtual thread's own stack, with the virtual thread in x19/rbx. */ +void cn1VirtualThreadMain(struct cn1VirtualThread* co) { + co->body(co->arg); + co->finished = 1; + /* The body returned: go back and never come here again. A virtual thread whose + * body returns must not fall off the end of its stack. */ + for(;;) { + cn1VirtualThreadSwitch(&co->sp, co->returnSp); + } +} + +struct cn1VirtualThread* cn1VirtualThreadCreate(cn1VirtualThreadBody body, void* arg, + size_t stackBytes) { + struct cn1VirtualThread* co; + unsigned char* stack; + size_t pageSize = (size_t)sysconf(_SC_PAGESIZE); + if(stackBytes < 16384) { + stackBytes = 16384; + } + stackBytes = (stackBytes + pageSize - 1) & ~(pageSize - 1); + co = (struct cn1VirtualThread*)calloc(1, sizeof(struct cn1VirtualThread)); + if(co == 0) { + return 0; + } + /* A guard page below the stack turns an overflow into a fault at the point + * of overflow, rather than silent corruption of whatever is mapped next. */ + stack = (unsigned char*)mmap(0, stackBytes + pageSize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if(stack == MAP_FAILED) { + free(co); + return 0; + } + mprotect(stack, pageSize, PROT_NONE); + co->stackLow = stack + pageSize; + co->stackHigh = stack + pageSize + stackBytes; + co->stackBytes = stackBytes; + co->body = body; + co->arg = arg; + co->finished = 0; + co->started = 0; + co->sp = 0; + co->running = 0; + co->vmState = 0; + co->yieldReason = CN1_VT_YIELD_IO; + cn1VirtualThreadRegister(co); + return co; +} + +void cn1VirtualThreadFree(struct cn1VirtualThread* co) { + int deferred; + if(co == 0) { + return; + } + cn1VirtualThreadUnregister(co); + /* Unregistered above, so no snapshot taken from here on can see it. One taken + * BEFORE that unlink still can, and that is exactly what the flag catches -- + * read under the same lock the snapshot walks the list under. */ + pthread_mutex_lock(&cn1VirtualThreadRegistryLock); + deferred = cn1VirtualThreadScanActive; + if(deferred) { + co->registryNext = cn1VirtualThreadRetired; + cn1VirtualThreadRetired = co; + } + pthread_mutex_unlock(&cn1VirtualThreadRegistryLock); + if(deferred) { + return; + } + cn1VirtualThreadRelease(co); +} + +/* + * Called by the collector around the whole stop-and-scan loop, NOT around each + * thread: the snapshot from one iteration is still being read while the next + * iteration runs, so a per-iteration window would leave the same race in place. + */ +void cn1VirtualThreadGcScanBegin(void) { + pthread_mutex_lock(&cn1VirtualThreadRegistryLock); + cn1VirtualThreadScanActive = 1; + pthread_mutex_unlock(&cn1VirtualThreadRegistryLock); +} + +void cn1VirtualThreadGcScanEnd(void) { + struct cn1VirtualThread* list; + pthread_mutex_lock(&cn1VirtualThreadRegistryLock); + cn1VirtualThreadScanActive = 0; + list = cn1VirtualThreadRetired; + cn1VirtualThreadRetired = 0; + pthread_mutex_unlock(&cn1VirtualThreadRegistryLock); + /* Released outside the lock: munmap under it would hold up every host thread + * trying to retire a connection, for no reason -- these are already unlinked + * and nothing can reach them. */ + while(list != 0) { + struct cn1VirtualThread* next = list->registryNext; + cn1VirtualThreadRelease(list); + list = next; + } +} + +int cn1VirtualThreadFinished(struct cn1VirtualThread* co) { + return co != 0 && co->finished; +} + +struct cn1VirtualThread* cn1VirtualThreadCurrent(void) { + return cn1CurrentVirtualThread; +} + +void cn1VirtualThreadStackBounds(struct cn1VirtualThread* co, void** low, void** high) { + /* Only the part between the saved sp and the high end holds anything. Below + * the saved sp is dead space the collector must not read: it is untouched + * mmap in the best case and a previous call's debris otherwise. */ + if(co == 0 || co->sp == 0) { + *low = 0; *high = 0; return; + } + *low = co->sp; + *high = co->stackHigh; +} + +/* Set up the initial frame so the first switch lands in the trampoline. */ +extern void* cn1VirtualThreadPrime(void* stackHigh, void* co, void* trampoline); + +void cn1VirtualThreadResume(struct cn1VirtualThread* co) { + struct cn1VirtualThread* previous = cn1CurrentVirtualThread; + if(co == 0 || co->finished) { + return; + } + if(!co->started) { + co->started = 1; + co->sp = cn1VirtualThreadPrime(co->stackHigh, co, (void*)cn1VirtualThreadTrampoline); + } + cn1CurrentVirtualThread = co; + co->running = 1; + cn1VirtualThreadSwitch(&co->returnSp, co->sp); + co->running = 0; + cn1CurrentVirtualThread = previous; +} + +void cn1VirtualThreadSetYieldReason(int reason) { + struct cn1VirtualThread* vt = cn1CurrentVirtualThread; + if(vt != 0) { + vt->yieldReason = reason; + } +} + +int cn1VirtualThreadYieldReason(struct cn1VirtualThread* vt) { + return vt == 0 ? CN1_VT_YIELD_IO : vt->yieldReason; +} + +int cn1VirtualThreadYieldIfVirtual(void) { + if(cn1CurrentVirtualThread == 0) { + return 0; + } + cn1VirtualThreadSetYieldReason(CN1_VT_YIELD_RUNNABLE); + cn1VirtualThreadYield(); + return 1; +} + +void cn1VirtualThreadYield(void) { + struct cn1VirtualThread* co = cn1CurrentVirtualThread; + if(co == 0) { + return; /* not on a virtual thread: nothing to yield from */ + } + cn1VirtualThreadSwitch(&co->sp, co->returnSp); +} + +#endif /* CN1_VIRTUAL_THREADS */ diff --git a/vm/ByteCodeTranslator/src/cn1_virtual_thread.h b/vm/ByteCodeTranslator/src/cn1_virtual_thread.h new file mode 100644 index 00000000000..a2c385500bc --- /dev/null +++ b/vm/ByteCodeTranslator/src/cn1_virtual_thread.h @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/* + * Stackful virtual threads: a Java thread of control that is not an OS thread. + * + * WHY THIS EXISTS AT ALL, in one number: a mutex-and-condvar handoff between two + * OS threads costs 21181ns on this hardware; switching a virtual thread costs 2.6ns. + * Every design that moves a request between OS threads pays the first; this pays + * the second. + * + * WHY IT IS ASSEMBLY rather than setjmp/longjmp, which also measured 4-8ns: + * glibc's __longjmp_chk aborts a jump whose target stack is not the current one + * -- "longjmp causes uninitialized stack frame" -- and it is compiled in by + * -D_FORTIFY_SOURCE, which distributions enable by default. A setjmp switch is + * therefore green everywhere we test and dead in somebody else's hardened build. + * musl compounds it from the other side by not implementing makecontext at all, + * so there is no portable way to CREATE the stack either. Twenty instructions of + * our own depend on neither. + * + * WHAT A VIRTUAL THREAD HOLDS: only the machine stack. Java locals and the operand + * stack live in threadStateData->threadObjectStack, which is a heap array the + * collector already walks precisely, so a suspended virtual thread's C stack carries + * just the C activation records -- a few pointers and temporaries per Java frame. + * That is why these stacks can be small where a platform thread's cannot. + */ +#ifndef CN1_VIRTUAL_THREAD_H +#define CN1_VIRTUAL_THREAD_H + +/* + * BACKEND ONLY. Virtual threads exist to let one server thread carry many + * connections; nothing on a device uses them, and the switch is hand-written + * assembly, so a target that cannot use them should not be made to build it. + * + * Gating matters for a reason beyond dead code. The switch lives in a .S, which + * is the only assembly file the translator emits, and Xcode does not recognise + * the extension: it files a .S under `lastKnownFileType = file` into the + * RESOURCES phase, so the iOS target shipped it as a resource, never assembled + * it, and failed to link with "_cn1VirtualThreadSwitch, referenced from + * _cn1VirtualThreadYield". With this off there is no reference to resolve, so + * the misfiled resource is simply inert and the phone target links. + * + * The backend defines CN1_VIRTUAL_THREADS (see docker/link.sh). Everywhere else + * the calls below collapse to the no-ops at the bottom of this header, so the + * shared collector in cn1_globals.m needs no #ifdefs of its own. + */ +#ifdef CN1_VIRTUAL_THREADS + +#include + +struct cn1VirtualThread; + +/** The body of a virtual thread. Returning from it finishes the virtual thread. */ +typedef void (*cn1VirtualThreadBody)(void* arg); + +/** + * Allocate a virtual thread with its own stack. It does not run until the first + * cn1VirtualThreadResume. Returns 0 if the stack could not be allocated. + */ +struct cn1VirtualThread* cn1VirtualThreadCreate(cn1VirtualThreadBody body, void* arg, + size_t stackBytes); + +/** + * Run `co` until it yields or finishes, then come back here. Must be called from + * the thread that will own it for the duration -- see cn1VirtualThreadStackBounds. + */ +void cn1VirtualThreadResume(struct cn1VirtualThread* co); + +/** Suspend the running virtual thread and return to whoever resumed it. */ +void cn1VirtualThreadYield(void); + +/** + * Why a virtual thread gave up its host, which the scheduler has to know. + * + * CN1_VT_YIELD_IO waiting for its descriptor; put it back on the poller + * and resume it when the descriptor is ready. + * CN1_VT_YIELD_RUNNABLE gave up its turn but is ready to run RIGHT NOW -- it + * is waiting on something that is not its socket. + * + * Confusing the two deadlocks the server, and not theoretically: a virtual + * thread parked in the collector's allocation backpressure is not waiting for + * bytes, so putting it on the poller waits for a client that is itself waiting + * for the response this virtual thread owes it. + */ +#define CN1_VT_YIELD_IO 0 +#define CN1_VT_YIELD_RUNNABLE 1 + +void cn1VirtualThreadSetYieldReason(int reason); +int cn1VirtualThreadYieldReason(struct cn1VirtualThread* vt); + +/** + * Yield the CURRENT virtual thread as runnable, if there is one. + * + * Returns 0 when not on a virtual thread, so a caller can fall back to whatever + * it did before -- which is what every existing blocking spin in the VM needs to + * keep doing on a platform thread. + */ +int cn1VirtualThreadYieldIfVirtual(void); + +/** The virtual thread running on this thread, or 0 when on the thread's own stack. */ +struct cn1VirtualThread* cn1VirtualThreadCurrent(void); + +/** True once the body has returned. */ +int cn1VirtualThreadFinished(struct cn1VirtualThread* co); + +/** Free it. Undefined before it has finished. */ +void cn1VirtualThreadFree(struct cn1VirtualThread* co); + +/** + * The OS thread's own stack pointer at the point it resumed `vt`. + * + * The collector needs both halves: a thread running a virtual thread has its + * live frames split, the ones below the resume on the OS stack and the ones + * above it on the virtual thread's stack. + */ +void* cn1VirtualThreadResumerSp(struct cn1VirtualThread* vt); + +/** + * Walk every virtual thread that exists, for the collector. + * + * A parked virtual thread is a GC root source and nothing else refers to its + * stack, so it must be enumerable independently of whoever created it. The walk + * holds the registry lock, so `fn` must not create or free virtual threads. + */ +void cn1VirtualThreadForEach(void (*fn)(struct cn1VirtualThread* vt, void* ctx), + void* ctx); + +/** True while this virtual thread is the one executing on some OS thread. */ +int cn1VirtualThreadIsRunning(struct cn1VirtualThread* vt); + +/** The argument the body was created with, so a caller can free it after. */ +void* cn1VirtualThreadArg(struct cn1VirtualThread* vt); + +/** The high end of a virtual thread's stack. */ +void* cn1VirtualThreadStackHigh(struct cn1VirtualThread* vt); + +/** + * Copy the registry into `out` (at most `max`), returning how many exist. + * + * The collector must take a SNAPSHOT before it stops the world and walk that, + * never the live registry. cn1VirtualThreadForEach holds a mutex, and a thread + * frozen by the stop signal may be the one holding it -- the GC would then wait + * for a thread that cannot run. The rest of this collector already follows the + * same rule for its root snapshots, for the same reason. + * + * A return larger than `max` means the snapshot was truncated and the caller + * must retry with a bigger buffer rather than scan a subset, because an + * unscanned virtual thread is an unscanned root source. + */ +int cn1VirtualThreadSnapshot(struct cn1VirtualThread** out, int max); + +/** + * Bracket the collector's stop-and-scan loop. + * + * Between these two calls, cn1VirtualThreadFree unlinks a virtual thread but does + * not release it; End releases everything that piled up. Without them the scan + * reads through snapshot pointers that a still-running host thread has already + * freed and unmapped, which is a segfault whose likelihood rises with the number + * of Java threads -- the loop stops threads one at a time, so more threads simply + * means more time spent holding a snapshot while other threads run. + * + * Call around the WHOLE loop. Per-iteration brackets would still leave one + * iteration's snapshot exposed during the next. + */ +void cn1VirtualThreadGcScanBegin(void); +void cn1VirtualThreadGcScanEnd(void); + +/** + * The virtual thread whose stack contains `addr`, or 0. + * + * Lets the collector recognise that a stopped OS thread's stack pointer is not + * in the OS thread's stack at all, because it is currently running a virtual + * thread. Without this the existing bounds check simply fails and the thread is + * skipped in silence, which loses every root it holds. + */ +struct cn1VirtualThread* cn1VirtualThreadForStackAddress(void* addr, int count, + struct cn1VirtualThread** snapshot); + +/** + * The VM thread state this virtual thread runs with. + * + * A virtual thread needs its own Java locals and operand stack -- that is the + * whole point, since those are what a request's state lives in -- so it carries + * its own ThreadLocalData rather than borrowing the host thread's. Opaque here + * to keep this file independent of cn1_globals.h. + */ +void* cn1VirtualThreadState(struct cn1VirtualThread* vt); +void cn1VirtualThreadSetState(struct cn1VirtualThread* vt, void* state); + +/** + * The live part of a suspended virtual thread's stack, for the collector. + * + * A conservative scan has to cover every suspended virtual thread as well as the + * running threads: a Java reference held only in a C temporary of a parked + * request is reachable from nowhere else. Returns the region between the saved + * stack pointer and the stack's high end, which is exactly the part in use. + */ +void cn1VirtualThreadStackBounds(struct cn1VirtualThread* co, void** low, void** high); + +#else /* !CN1_VIRTUAL_THREADS */ + +/* + * Off-target stubs. Every one answers "there is no virtual thread here", which + * is the truth on a device, and the collector's virtual-thread paths then fold + * away at compile time. + */ +struct cn1VirtualThread; + +static inline int cn1VirtualThreadYieldIfVirtual(void) { return 0; } +static inline void cn1VirtualThreadGcScanBegin(void) { } +static inline void cn1VirtualThreadGcScanEnd(void) { } +static inline struct cn1VirtualThread* cn1VirtualThreadCurrent(void) { return 0; } +static inline int cn1VirtualThreadSnapshot(struct cn1VirtualThread** out, int max) { + (void)out; (void)max; return 0; +} +static inline struct cn1VirtualThread* cn1VirtualThreadForStackAddress( + void* addr, int count, struct cn1VirtualThread** snapshot) { + (void)addr; (void)count; (void)snapshot; return 0; +} +static inline int cn1VirtualThreadIsRunning(struct cn1VirtualThread* co) { (void)co; return 0; } +static inline void cn1VirtualThreadStackBounds(struct cn1VirtualThread* co, void** lo, void** hi) { + (void)co; if(lo) { *lo = 0; } if(hi) { *hi = 0; } +} +static inline void* cn1VirtualThreadStackHigh(struct cn1VirtualThread* co) { (void)co; return 0; } +static inline void* cn1VirtualThreadResumerSp(struct cn1VirtualThread* co) { (void)co; return 0; } +static inline void* cn1VirtualThreadState(struct cn1VirtualThread* co) { (void)co; return 0; } +static inline void cn1VirtualThreadSetState(struct cn1VirtualThread* co, void* st) { (void)co; (void)st; } +static inline void cn1VirtualThreadFree(struct cn1VirtualThread* co) { (void)co; } + +#endif /* CN1_VIRTUAL_THREADS */ + +#endif diff --git a/vm/ByteCodeTranslator/src/cn1_virtual_thread_asm.S b/vm/ByteCodeTranslator/src/cn1_virtual_thread_asm.S new file mode 100644 index 00000000000..b73111f6896 --- /dev/null +++ b/vm/ByteCodeTranslator/src/cn1_virtual_thread_asm.S @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/* + * Symbol naming: Mach-O prefixes C symbols with an underscore and ELF does not, + * so every name here goes through CN1_SYM. Getting this wrong does not warn -- + * + * APPLE TARGETS DO NOT BUILD THIS FILE YET, and the reason is the build system + * rather than the assembly. This is the first .S the translator has ever emitted, + * and three separate places classify sources by extension without knowing it: + * + * 1. WatchNativeBuilder copied watch sources by extension (.m .c .swift .mm + * .cpp .cc) and dropped the .S entirely -- FIXED, it now copies .S/.s and + * the watch target links. + * 2. The Xcode project generator files a .S as `lastKnownFileType = file` and + * puts it in the RESOURCES phase, so the phone target ships it as a resource + * and never assembles it. `nm` on the build output shows the watch target + * with `T _cn1VirtualThreadSwitch` and the phone target with no asm object + * at all. IPhoneBuilder already strips a similar misfiling for Swift + * (`removeLinesContaining(pbx, ".swift in Resources", ...)`), so the same + * treatment is the shape of the fix -- except a Resources entry has to be + * MOVED to Sources, not just deleted. + * 3. Nothing else in the repo emits assembly, so no existing gate covers it. + * + * The alternative worth weighing before doing (2): drop this file and put the + * same instructions in a top-level __asm__ block inside cn1_virtual_thread.c. + * That removes the new file TYPE from the pipeline entirely, so every builder + * that already compiles the .c gets the switch for free and none of them need to + * learn about .S. It needs re-verifying on Linux (musl and glibc, both arches) + * and on Apple, which is why it is written down here rather than done in haste. + * it fails to link, or worse, links against nothing on a platform where the + * caller is also assembly. + */ +/* BACKEND ONLY. Without this the file still assembles on a device target and + * defines symbols nothing calls; with it the object is empty, which is what lets + * a toolchain that misfiles a .S (Xcode puts it in Resources) stay harmless. */ +#ifdef CN1_VIRTUAL_THREADS + +#if defined(__APPLE__) +#define CN1_SYM(name) _##name +#else +#define CN1_SYM(name) name +#endif + +/* + * The entire architecture-specific surface of the virtual thread mechanism: save the + * callee-saved registers, swap the stack pointer, restore. Caller-saved + * registers are not touched because the compiler already treats a call as + * clobbering them, and every entry point here is reached by an ordinary call. + * + * Three symbols per architecture: + * cn1VirtualThreadSwitch(void** saveSp, void* newSp) suspend here, resume there + * cn1VirtualThreadPrime(high, co, trampoline) build the first frame + * cn1VirtualThreadTrampoline where that frame returns to + */ + +#if defined(__aarch64__) + + .text + .align 2 + .globl CN1_SYM(cn1VirtualThreadSwitch) +CN1_SYM(cn1VirtualThreadSwitch): + /* x19-x28 are callee-saved, d8-d15 are the callee-saved halves of the FP + * registers, x29/x30 are the frame pointer and link register. */ + stp x29, x30, [sp, #-160]! + stp x19, x20, [sp, #16] + stp x21, x22, [sp, #32] + stp x23, x24, [sp, #48] + stp x25, x26, [sp, #64] + stp x27, x28, [sp, #80] + stp d8, d9, [sp, #96] + stp d10, d11, [sp, #112] + stp d12, d13, [sp, #128] + stp d14, d15, [sp, #144] + mov x2, sp + str x2, [x0] /* *saveSp = sp */ + mov sp, x1 /* sp = newSp */ + ldp d14, d15, [sp, #144] + ldp d12, d13, [sp, #128] + ldp d10, d11, [sp, #112] + ldp d8, d9, [sp, #96] + ldp x27, x28, [sp, #80] + ldp x25, x26, [sp, #64] + ldp x23, x24, [sp, #48] + ldp x21, x22, [sp, #32] + ldp x19, x20, [sp, #16] + ldp x29, x30, [sp], #160 + ret + + .align 2 + .globl CN1_SYM(cn1VirtualThreadPrime) +CN1_SYM(cn1VirtualThreadPrime): + /* x0 = stack high, x1 = virtual thread, x2 = trampoline. + * Build a frame cn1VirtualThreadSwitch can restore: the virtual thread travels in + * x19 (callee-saved, so the trampoline still has it) and the link register + * points at the trampoline. */ + and x0, x0, #~15 /* the ABI wants 16-byte alignment */ + sub x0, x0, #160 + stp xzr, x2, [x0] /* x29 = 0 ends any backtrace, x30 = trampoline */ + stp x1, xzr, [x0, #16] /* x19 = virtual thread */ + stp xzr, xzr, [x0, #32] + stp xzr, xzr, [x0, #48] + stp xzr, xzr, [x0, #64] + stp xzr, xzr, [x0, #80] + stp xzr, xzr, [x0, #96] + stp xzr, xzr, [x0, #112] + stp xzr, xzr, [x0, #128] + stp xzr, xzr, [x0, #144] + ret + + .align 2 + .globl CN1_SYM(cn1VirtualThreadTrampoline) +CN1_SYM(cn1VirtualThreadTrampoline): + mov x0, x19 /* the virtual thread cn1VirtualThreadPrime parked here */ + bl CN1_SYM(cn1VirtualThreadMain) + brk #0 /* cn1VirtualThreadMain never returns */ + +#elif defined(__x86_64__) + + .text + .globl CN1_SYM(cn1VirtualThreadSwitch) +CN1_SYM(cn1VirtualThreadSwitch): + /* rbx, rbp, r12-r15 are callee-saved in the SysV ABI. */ + pushq %rbp + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + movq %rsp, (%rdi) /* *saveSp = rsp */ + movq %rsi, %rsp /* rsp = newSp */ + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + popq %rbp + ret + + .globl CN1_SYM(cn1VirtualThreadPrime) +CN1_SYM(cn1VirtualThreadPrime): + /* rdi = stack high, rsi = virtual thread, rdx = trampoline. */ + andq $-16, %rdi + subq $8, %rdi /* so that rsp+8 is 16-aligned after the ret */ + movq %rdx, (%rdi) /* the address cn1VirtualThreadSwitch's ret jumps to */ + subq $48, %rdi + movq $0, (%rdi) /* r15 */ + movq $0, 8(%rdi) /* r14 */ + movq $0, 16(%rdi) /* r13 */ + movq $0, 24(%rdi) /* r12 */ + movq %rsi, 32(%rdi) /* rbx = virtual thread */ + movq $0, 40(%rdi) /* rbp = 0 ends any backtrace */ + movq %rdi, %rax + ret + + .globl CN1_SYM(cn1VirtualThreadTrampoline) +CN1_SYM(cn1VirtualThreadTrampoline): + movq %rbx, %rdi /* the virtual thread cn1VirtualThreadPrime parked here */ + call CN1_SYM(cn1VirtualThreadMain) + ud2 /* cn1VirtualThreadMain never returns */ + +#else +#error "cn1_virtual thread: no switch implementation for this architecture" +#endif + +#if defined(__linux__) && defined(__ELF__) +/* Do not ask for an executable stack on account of this file. */ +.section .note.GNU-stack,"",%progbits +#endif + +#endif /* CN1_VIRTUAL_THREADS */ diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index 00bbd5aab5f..add3f4fc7a7 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -422,6 +422,11 @@ private static void handleCleanOutput(ByteCodeTranslator b, File[] sources, File copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.h"), Files.newOutputStream(cn1Globals.toPath())); File cn1Intrinsics = new File(srcRoot, "cn1_intrinsics.h"); copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_intrinsics.h"), Files.newOutputStream(cn1Intrinsics.toPath())); + // Virtual threads: the switch is a few instructions of assembly per + // architecture, so the .S travels with the runtime rather than being + // generated. A project that gets the C and not the .S links against a + // missing symbol, which is at least loud. + emitVirtualThreadRuntime(srcRoot); if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { replaceInFile(cn1Globals, "//#define CN1_INCLUDE_NPE_CHECKS", "#define CN1_INCLUDE_NPE_CHECKS"); } @@ -766,6 +771,11 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.h"), Files.newOutputStream(cn1Globals.toPath())); File cn1Intrinsics = new File(srcRoot, "cn1_intrinsics.h"); copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_intrinsics.h"), Files.newOutputStream(cn1Intrinsics.toPath())); + // Virtual threads: the switch is a few instructions of assembly per + // architecture, so the .S travels with the runtime rather than being + // generated. A project that gets the C and not the .S links against a + // missing symbol, which is at least loud. + emitVirtualThreadRuntime(srcRoot); if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { replaceInFile(cn1Globals, "//#define CN1_INCLUDE_NPE_CHECKS", "#define CN1_INCLUDE_NPE_CHECKS"); } @@ -1472,6 +1482,26 @@ private static boolean isBuildMetadata(File f) { * @param i source * @param o destination */ + /** + * Emit the virtual-thread runtime beside the generated sources. + * + * Three files rather than one because the switch has to be assembly: glibc + * aborts a cross-stack longjmp under _FORTIFY_SOURCE and musl has no + * makecontext, so neither portable route survives every target we ship. + */ + private static void emitVirtualThreadRuntime(File srcRoot) throws IOException { + String[] names = { "cn1_virtual_thread.h", "cn1_virtual_thread.c", "cn1_virtual_thread_asm.S" }; + for (String name : names) { + InputStream in = ByteCodeTranslator.class.getResourceAsStream("/" + name); + if (in == null) { + // Missing here means the build did not stage it; failing now names the + // cause, where the link error later names only a symbol. + throw new IOException("virtual-thread runtime resource missing: " + name); + } + copy(in, Files.newOutputStream(new File(srcRoot, name).toPath())); + } + } + public static void copy(InputStream i, OutputStream o) throws IOException { copy(i, o, 8192); } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java index baea33b8aba..ef79711ebb4 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java @@ -4178,6 +4178,39 @@ private CustomIntruction srEmpty() { return new CustomIntruction("", "", new ArrayList()); } + /** + * Drop a CHECKCAST that immediately repeats the one before it. + * + * Deliberately narrow. Only a LineNumber may sit between the two, because it + * carries no semantics; a LabelInstruction may NOT, since another path can + * jump there with a different value on the stack, and then the second cast is + * the only one guarding it. Same reasoning for anything else in between: if it + * can touch the stack, the second cast is not redundant. + */ + private void removeRepeatedCheckcasts() { + TypeInstruction previousCast = null; + for (int iter = 0 ; iter < instructions.size() ; iter++) { + Instruction current = instructions.get(iter); + if (current instanceof LineNumber) { + continue; // no semantics, does not break the pair + } + if (current instanceof TypeInstruction + && current.getOpcode() == Opcodes.CHECKCAST) { + TypeInstruction cast = (TypeInstruction) current; + if (previousCast != null + && previousCast.getTypeName() != null + && previousCast.getTypeName().equals(cast.getTypeName())) { + instructions.remove(iter); + iter--; // the list shifted under us + continue; // previousCast still stands + } + previousCast = cast; + continue; + } + previousCast = null; + } + } + boolean optimize() { // FUSED OBJECTS, constructor side: rewrite each planned // `ALOAD 0; ; NEWARRAY T; PUTFIELD f` quadruple into the @@ -4185,6 +4218,17 @@ boolean optimize() { // fold/reorder those instructions. Runs on the raw list (first thing). replaceFusedCtorTriples(); + // A CHECKCAST immediately repeated to the SAME type is a no-op: the first + // one already proved the type or threw, and neither touches the stack + // otherwise. javac emits the pair readily -- 23 of the 122 checkcast sites + // in a backend build were duplicates, 9 of them in java.lang.String, whose + // charInternal is the hottest String method under a server load. + // + // Worth removing rather than tolerating because a checked cast is REAL work + // here: builds pass -Dcn1.checkedCasts=true, so BC_CHECKCAST_CHECKED walks + // the class hierarchy instead of expanding to nothing. + removeRepeatedCheckcasts(); + int instructionCount = instructions.size(); // optimize away a method that only contains the void return instruction e.g. blank constructors etc. diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 2c5fe143532..f1778416579 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -26,6 +26,7 @@ #endif #include "cn1_globals.h" +#include "cn1_virtual_thread.h" #include #include #ifndef _WIN32 @@ -1951,143 +1952,231 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC pthread_key_t threadIdKey = 0; JAVA_LONG threadKeyCounter = 1; -struct ThreadLocalData* getThreadLocalData() { - if(threadIdKey == 0) { - pthread_key_create(&threadIdKey, NULL); - } - struct ThreadLocalData* i = pthread_getspecific(threadIdKey); - if(i == NULL) { +/** + * Build a fresh VM thread state. + * + * Split out of getThreadLocalData so a VIRTUAL thread can have one too. A + * virtual thread needs its own Java locals and operand stack -- that is the + * whole point of it, since a request's state lives there -- and it must be + * registered in allThreads like any other, or the precise scan never walks its + * object stack and its live objects are collected under it. + * + * The one thing this deliberately does NOT do is bind the state to the calling + * OS thread: a virtual thread's state belongs to the virtual thread and travels + * with it between hosts. + */ +struct ThreadLocalData* cn1CreateThreadLocalData(JAVA_BOOLEAN bindToCallingOsThread) { + struct ThreadLocalData* i; JAVA_LONG nativeThreadId = threadKeyCounter; - threadKeyCounter++; - i = malloc(sizeof(struct ThreadLocalData)); - i->threadId = nativeThreadId; - i->tryBlockOffset = 0; - - i->lightweightThread = JAVA_FALSE; - i->threadBlockedByGC = JAVA_FALSE; - i->threadActive = JAVA_FALSE; - i->threadKilled = JAVA_FALSE; + threadKeyCounter++; + i = malloc(sizeof(struct ThreadLocalData)); + i->threadId = nativeThreadId; + i->tryBlockOffset = 0; + + i->lightweightThread = JAVA_FALSE; + i->threadBlockedByGC = JAVA_FALSE; + i->threadActive = JAVA_FALSE; + i->threadKilled = JAVA_FALSE; #ifdef CN1_GC_CONFORM - // Malloc'd, so this starts as garbage. See gcThreadStartMs in cn1_globals.h. - { extern void cn1StallRegisterThread(struct ThreadLocalData* t); - cn1StallRegisterThread(i); } + // Malloc'd, so this starts as garbage. See gcThreadStartMs in cn1_globals.h. + { extern void cn1StallRegisterThread(struct ThreadLocalData* t); + cn1StallRegisterThread(i); } #endif - i->interrupted = JAVA_FALSE; - - i->currentThreadObject = 0; - - i->utf8Buffer = 0; - i->utf8BufferSize = 0; - /* - * calloc, not malloc+memset. These four buffers are ~300KB per thread and the - * eager memset TOUCHED EVERY PAGE, so a thread that never runs a deep call - * chain still paid the whole footprint in resident memory -- measured at - * ~118KB per parked thread, which is what decides whether a server-side - * binary can afford a thread per connection. - * - * The eager clear was redundant: every frame prologue memsets exactly the - * slots it is about to claim (see the frame-entry helpers in cn1_globals.h), - * and the collector only scans threadObjectStack up to - * threadObjectStackOffset, so no slot is ever read before the frame that owns - * it has zeroed it. calloc for a request this size comes from mmap and is - * lazily zeroed by the OS, so a shallow thread commits a few pages instead of - * all of them. - */ - i->threadObjectStack = cn1AllocThreadStack(); - i->threadObjectStackOffset = 0; - - i->callStackClass = calloc(CN1_MAX_STACK_CALL_DEPTH, sizeof(int)); - i->callStackLine = calloc(CN1_MAX_STACK_CALL_DEPTH, sizeof(int)); - i->callStackMethod = calloc(CN1_MAX_STACK_CALL_DEPTH, sizeof(int)); + i->interrupted = JAVA_FALSE; + + i->currentThreadObject = 0; + + i->utf8Buffer = 0; + i->utf8BufferSize = 0; + /* + * calloc, not malloc+memset. These four buffers are ~300KB per thread and the + * eager memset TOUCHED EVERY PAGE, so a thread that never runs a deep call + * chain still paid the whole footprint in resident memory -- measured at + * ~118KB per parked thread, which is what decides whether a server-side + * binary can afford a thread per connection. + * + * The eager clear was redundant: every frame prologue memsets exactly the + * slots it is about to claim (see the frame-entry helpers in cn1_globals.h), + * and the collector only scans threadObjectStack up to + * threadObjectStackOffset, so no slot is ever read before the frame that owns + * it has zeroed it. calloc for a request this size comes from mmap and is + * lazily zeroed by the OS, so a shallow thread commits a few pages instead of + * all of them. + */ + i->threadObjectStack = cn1AllocThreadStack(); + i->threadObjectStackOffset = 0; + + i->callStackClass = calloc(CN1_MAX_STACK_CALL_DEPTH, sizeof(int)); + i->callStackLine = calloc(CN1_MAX_STACK_CALL_DEPTH, sizeof(int)); + i->callStackMethod = calloc(CN1_MAX_STACK_CALL_DEPTH, sizeof(int)); #ifdef CN1_ON_DEVICE_DEBUG - i->callStackLocalsAddresses = malloc(CN1_MAX_STACK_CALL_DEPTH * sizeof(void**)); - memset(i->callStackLocalsAddresses, 0, CN1_MAX_STACK_CALL_DEPTH * sizeof(void**)); - i->callStackFrameInfo = malloc(CN1_MAX_STACK_CALL_DEPTH * sizeof(struct cn1_frame_info*)); - memset(i->callStackFrameInfo, 0, CN1_MAX_STACK_CALL_DEPTH * sizeof(struct cn1_frame_info*)); -#endif - - i->callStackOffset = 0; - - // ThreadLocalData is malloc'd (not zeroed); 0 means "frameless native-stack - // limit not yet computed" -- it is filled in lazily on first frameless entry. - i->nativeStackLimit = 0; - - i->pendingHeapAllocations = calloc(PER_THREAD_ALLOCATION_COUNT, sizeof(void *)); - i->heapAllocationSize = 0; - i->threadHeapTotalSize = PER_THREAD_ALLOCATION_COUNT; - // ThreadLocalData is malloc'd, NOT zeroed. bibopBytesLocal feeds the GC - // trigger/pacing accounting (CN1_BIBOP_FLUSH_BYTES adds it into the global - // counters); garbage here means a spurious immediate GC + hard-cap park, or - // a dead allocation trigger, on every new thread. nativeAllocationMode is - // read by the inlined alloc fast path (cn1BibopFastAlloc) before any setter - // runs -- garbage-nonzero silently disables the fast path for the thread. - i->bibopBytesLocal = 0; - i->bibopEpochBytes = 0; + i->callStackLocalsAddresses = malloc(CN1_MAX_STACK_CALL_DEPTH * sizeof(void**)); + memset(i->callStackLocalsAddresses, 0, CN1_MAX_STACK_CALL_DEPTH * sizeof(void**)); + i->callStackFrameInfo = malloc(CN1_MAX_STACK_CALL_DEPTH * sizeof(struct cn1_frame_info*)); + memset(i->callStackFrameInfo, 0, CN1_MAX_STACK_CALL_DEPTH * sizeof(struct cn1_frame_info*)); +#endif + + i->callStackOffset = 0; + + // ThreadLocalData is malloc'd (not zeroed); 0 means "frameless native-stack + // limit not yet computed" -- it is filled in lazily on first frameless entry. + i->nativeStackLimit = 0; + + i->pendingHeapAllocations = calloc(PER_THREAD_ALLOCATION_COUNT, sizeof(void *)); + i->heapAllocationSize = 0; + i->threadHeapTotalSize = PER_THREAD_ALLOCATION_COUNT; + // ThreadLocalData is malloc'd, NOT zeroed. bibopBytesLocal feeds the GC + // trigger/pacing accounting (CN1_BIBOP_FLUSH_BYTES adds it into the global + // counters); garbage here means a spurious immediate GC + hard-cap park, or + // a dead allocation trigger, on every new thread. nativeAllocationMode is + // read by the inlined alloc fast path (cn1BibopFastAlloc) before any setter + // runs -- garbage-nonzero silently disables the fast path for the thread. + i->bibopBytesLocal = 0; + i->bibopEpochBytes = 0; #ifndef CN1_DISABLE_BIBOP - i->bibopObservedGcEpoch = atomic_load_explicit(&bibopGcEpoch, - memory_order_relaxed); + i->bibopObservedGcEpoch = atomic_load_explicit(&bibopGcEpoch, + memory_order_relaxed); #else - i->bibopObservedGcEpoch = 0; + i->bibopObservedGcEpoch = 0; #endif - i->bibopHighThroughputUntilEpoch = 0; - for(int __bi = 0 ; __bi < CN1_BIBOP_NUM_CLASSES ; __bi++) { + i->bibopHighThroughputUntilEpoch = 0; + for(int __bi = 0 ; __bi < CN1_BIBOP_NUM_CLASSES ; __bi++) { #ifndef CN1_DISABLE_BIBOP - i->bibopBypassSeen[__bi] = atomic_load_explicit(&bibopBypassGeneration[__bi], - memory_order_relaxed); + i->bibopBypassSeen[__bi] = atomic_load_explicit(&bibopBypassGeneration[__bi], + memory_order_relaxed); #else - i->bibopBypassSeen[__bi] = 0; + i->bibopBypassSeen[__bi] = 0; #endif - i->bibopBypassRemaining[__bi] = 0; - } - i->nativeAllocationMode = JAVA_FALSE; - // dead-thread pending-migration queue state (single-writer allObjectsInHeap) - i->gcDeadNext = 0; - i->gcQueuedForDrain = JAVA_FALSE; - i->gcReleaseRequested = JAVA_FALSE; - - i->blocks = malloc(CN1_MAX_TRY_BLOCKS * sizeof(struct TryBlock)); + i->bibopBypassRemaining[__bi] = 0; + } + i->nativeAllocationMode = JAVA_FALSE; + // dead-thread pending-migration queue state (single-writer allObjectsInHeap) + i->gcDeadNext = 0; + i->gcQueuedForDrain = JAVA_FALSE; + i->gcReleaseRequested = JAVA_FALSE; + + i->blocks = malloc(CN1_MAX_TRY_BLOCKS * sizeof(struct TryBlock)); #ifdef CN1_CONSERVATIVE_GC_ROOTS - // PHASE 3b: record this thread's pthread handle + TLS self pointer so the GC can - // signal-stop it and the async-signal-safe stop handler can find its state. + // PHASE 3b: record this thread's pthread handle + TLS self pointer so the GC can + // signal-stop it and the async-signal-safe stop handler can find its state. + i->gcParkCaptured = JAVA_FALSE; + // Carried over when this initialisation was extracted into a function: the + // forced-stop work (issue #5537) added this field to the inline block that used + // to live in the thread runner, and ThreadLocalData is malloc'd, NOT zeroed -- + // an uninitialised flag here reads as garbage and the collector would believe it + // had already force-stopped a thread it never touched. + i->gcMarkForcedStop = JAVA_FALSE; + i->gcStackPointerAtPark = 0; + i->gcSigStopRequest = 0; + i->gcSigStopped = 0; + i->gcSigRelease = 0; + i->gcSigStopGen = 0; + i->gcSigStackPointer = 0; + // Zeroed for the same reason as the rest of this block: ThreadLocalData is + // malloc'd, not zeroed. The forced-stop scan guards on these being non-zero + // before it marks [sp, base), so garbage here would pass that guard and hand + // the conservative scan a bogus range. + i->gcSigStackBase = 0; + i->gcSigStackSize = 0; + i->gcSigRegsLen = 0; + if(bindToCallingOsThread) { i->gcPthread = pthread_self(); i->gcPthreadValid = JAVA_TRUE; - i->gcParkCaptured = JAVA_FALSE; - i->gcStackPointerAtPark = 0; - i->gcSigStopRequest = 0; - i->gcSigStopped = 0; - i->gcSigRelease = 0; - i->gcSigStopGen = 0; - i->gcSigStackPointer = 0; - i->gcSigStackBase = 0; - i->gcSigStackSize = 0; - i->gcSigRegsLen = 0; - // ThreadLocalData is malloc'd, NOT zeroed (see the notes on nativeStackLimit and - // bibopBytesLocal above). Garbage-nonzero here would tell - // cn1GcScanThreadNativeStack that the mark loop already froze this thread, so it - // would scan a RUNNING thread's stack from a garbage SP and never signal-stop it - // -- missed roots, then a use-after-free on whatever the sweep took. - i->gcMarkForcedStop = JAVA_FALSE; cn1TlsSelf = i; -#endif + } else { + // A VIRTUAL thread has no pthread of its own and may run on a different + // host next time, so binding either of these to whoever happens to be + // creating it would be a lie the collector acts on. gcPthreadValid false + // makes cn1GcScanThreadNativeStack skip it, which is right: its C stack is + // reached through the virtual-thread registry instead, and its Java object + // stack through allThreads like everyone else. cn1TlsSelf must keep naming + // the HOST thread, because the async-signal stop handler runs on the host + // and needs the host's state. + i->gcPthread = 0; + i->gcPthreadValid = JAVA_FALSE; + } +#endif + if(bindToCallingOsThread) { pthread_setspecific(threadIdKey, i); - - if(!allThreads) { - allThreads = malloc(NUMBER_OF_SUPPORTED_THREADS * sizeof(struct ThreadLocalData*)); - memset(allThreads, 0, NUMBER_OF_SUPPORTED_THREADS * sizeof(struct ThreadLocalData*)); + } + + if(!allThreads) { + allThreads = malloc(NUMBER_OF_SUPPORTED_THREADS * sizeof(struct ThreadLocalData*)); + memset(allThreads, 0, NUMBER_OF_SUPPORTED_THREADS * sizeof(struct ThreadLocalData*)); + } + int threadOffset = -1; + lockCriticalSection(); + for(int iter = 0 ; iter < NUMBER_OF_SUPPORTED_THREADS ; iter++) { + if(allThreads[iter] == 0) { + threadOffset = iter; + break; } - int threadOffset = -1; - lockCriticalSection(); - for(int iter = 0 ; iter < NUMBER_OF_SUPPORTED_THREADS ; iter++) { - if(allThreads[iter] == 0) { - threadOffset = iter; - break; + } + CODENAME_ONE_ASSERT(threadOffset > -1); + allThreads[threadOffset] = i; + unlockCriticalSection(); + //printf("Thread slot %d assigned to thread %d\n",threadOffset,(int)i->threadId); + + return i; +} + +/** + * Create a virtual thread that can run Java: a stack of its own plus a VM thread + * state of its own. + * + * The two halves are both necessary and neither is sufficient. The stack carries + * the C activation records of the Java methods it is inside; the thread state + * carries their locals and operand stack, which is where a request's objects + * actually live. Giving it a stack but sharing the host's state would have two + * threads of control writing one Java stack. + * + * Sizing: threadObjectStack is mmap'd and lazily faulted, so the 264KB it + * reserves costs only the pages a virtual thread touches -- a handler that nests + * a dozen frames commits a page or two. That is the difference against the + * ~118KB RESIDENT a parked OS thread costs, and it is what decides whether a + * context per connection is affordable. + */ +#ifdef CN1_VIRTUAL_THREADS +struct cn1VirtualThread* cn1SpawnVirtualThread(cn1VirtualThreadBody body, void* arg, + size_t stackBytes) { + struct ThreadLocalData* state; + struct cn1VirtualThread* vt = cn1VirtualThreadCreate(body, arg, stackBytes); + if(vt == 0) { + return 0; + } + // JAVA_FALSE: this state belongs to the virtual thread, not to whoever is + // creating it. See cn1CreateThreadLocalData for what that turns off. + state = cn1CreateThreadLocalData(JAVA_FALSE); + if(state == 0) { + cn1VirtualThreadFree(vt); + return 0; + } + state->lightweightThread = JAVA_TRUE; + cn1VirtualThreadSetState(vt, state); + return vt; +} +#endif /* CN1_VIRTUAL_THREADS -- backend only, see cn1_virtual_thread.h */ + +struct ThreadLocalData* getThreadLocalData() { + // A running virtual thread supplies its own state; every generated method + // reaches its locals through this, so missing it would silently give the + // virtual thread the HOST thread's Java stack and corrupt both. + { + struct cn1VirtualThread* __vt = cn1VirtualThreadCurrent(); + if(__vt != 0) { + struct ThreadLocalData* __s = (struct ThreadLocalData*)cn1VirtualThreadState(__vt); + if(__s != 0) { + return __s; } } - CODENAME_ONE_ASSERT(threadOffset > -1); - allThreads[threadOffset] = i; - unlockCriticalSection(); - //printf("Thread slot %d assigned to thread %d\n",threadOffset,(int)i->threadId); + } + if(threadIdKey == 0) { + pthread_key_create(&threadIdKey, NULL); + } + struct ThreadLocalData* i = pthread_getspecific(threadIdKey); + if(i == NULL) { + i = cn1CreateThreadLocalData(JAVA_TRUE); } return i; } diff --git a/vm/JavaAPI/src/java/util/LinkedHashMap.java b/vm/JavaAPI/src/java/util/LinkedHashMap.java index a45baf13c9f..e8200b0d745 100644 --- a/vm/JavaAPI/src/java/util/LinkedHashMap.java +++ b/vm/JavaAPI/src/java/util/LinkedHashMap.java @@ -238,7 +238,29 @@ public V put(K key, V value) { } else if (accessOrder) { cn1MoveToTail(cn1LastPut); } - if (cn1Head >= 0 && removeEldestEntry(new CompactEntry(this, cn1Head))) { + // Two things here, both of which cost every caller of a plain + // LinkedHashMap: + // + // 1. The eviction hook belongs AFTER AN INSERTION, not after every put. + // java.util.LinkedHashMap calls afterNodeInsertion (and so + // removeEldestEntry) only when putVal added a NEW node; overwriting an + // existing key does not evict. Calling it unconditionally was a + // deviation from that as well as wasted work. + // + // 2. The CompactEntry exists only to be handed to removeEldestEntry. + // There are no node objects in this representation, so unlike the JDK + // -- which passes a node it already has -- one has to be built. For a + // plain LinkedHashMap it is built, passed to a method whose body is + // `return false`, and dropped: an allocation per insertion, feeding + // the collector for nothing. cn1MayEvict is false exactly when this + // object's class is LinkedHashMap itself, whose removeEldestEntry + // cannot return true, so skipping is safe; any subclass keeps the old + // behaviour whether or not it overrides the hook. + // + // Measured on the translated target before this change: building a + // four-entry map cost 252-258ns against HashMap's 139-142ns. + if (cn1LastInserted && cn1MayEvict && cn1Head >= 0 + && removeEldestEntry(new CompactEntry(this, cn1Head))) { @SuppressWarnings("unchecked") K eldest = (K) cn1Keys[cn1Head]; remove(eldest); @@ -276,6 +298,16 @@ protected boolean removeEldestEntry(Map.Entry eldest) { return false; } + /** + * False for a plain LinkedHashMap, whose {@link #removeEldestEntry} returns + * false unconditionally, so the eldest entry never has to be materialised. + * True for any subclass, which may override it. + * + * A class comparison rather than anything reflective -- CN1 obfuscates class + * names, so a name lookup would not survive a built app. + */ + private final boolean cn1MayEvict = getClass() != LinkedHashMap.class; + /** * Removes all elements from this map, leaving it empty. * diff --git a/vm/tests/virtualthread/test_virtual_thread.c b/vm/tests/virtualthread/test_virtual_thread.c new file mode 100644 index 00000000000..9df8b893df7 --- /dev/null +++ b/vm/tests/virtualthread/test_virtual_thread.c @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/* Correctness first, cost second. A fast switch that corrupts a register or + * loses a stack is not a foundation for a scheduler. */ +/* This exercises the BACKEND virtual-thread runtime, which is gated off + * everywhere else, so the test turns it on for itself rather than depending on + * whatever flags a caller happens to pass. */ +#ifndef CN1_VIRTUAL_THREADS +#define CN1_VIRTUAL_THREADS 1 +#endif + +#include "cn1_virtual_thread.h" +#include +#include +#include +#include + +static int failures = 0; +static void check(const char* what, int ok) { + if(!ok) { printf("FAIL %s\n", what); failures++; } +} + +/* ---- 1. a virtual thread runs, yields, resumes, and finishes ---- */ +static int steps = 0; +static void counter(void* arg) { + int* out = (int*)arg; + for(int i = 0; i < 5; i++) { steps++; *out = i; cn1VirtualThreadYield(); } +} + +/* ---- 2. callee-saved registers survive a switch ---- */ +static long regsSeen[8]; +static void regUser(void* arg) { + (void)arg; + /* Give the compiler reason to keep values in callee-saved registers across + * the yield: they are live before and after. */ + volatile long a=0x1111, b=0x2222, c=0x3333, d=0x4444; + volatile long e=0x5555, f=0x6666, g=0x7777, h=0x8888; + cn1VirtualThreadYield(); + regsSeen[0]=a; regsSeen[1]=b; regsSeen[2]=c; regsSeen[3]=d; + regsSeen[4]=e; regsSeen[5]=f; regsSeen[6]=g; regsSeen[7]=h; +} + +/* ---- 3. deep recursion on a small stack, values intact across a yield ---- */ +static long deepSum = 0; +static long recurse(int depth) { + volatile long marker = depth; + if(depth == 0) { cn1VirtualThreadYield(); return 0; } + long r = recurse(depth - 1); + return r + marker; /* marker must survive the yield made below us */ +} +static void deep(void* arg) { (void)arg; deepSum = recurse(200); } + +/* ---- 4. many virtual threads interleave without treading on each other ---- */ +#define MANY 64 +static int slot[MANY]; +static void many(void* arg) { + long id = (long)arg; + for(int i = 0; i < 10; i++) { slot[id] = (int)(id * 1000 + i); cn1VirtualThreadYield(); } +} + +static long long nowNs(void){ struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t); + return (long long)t.tv_sec*1000000000LL+t.tv_nsec; } + +/* ---- 6. the collector must be able to SEE a reference a parked virtual thread + * holds only in a C local. This is the property the whole GC integration + * rests on: if the range handed to the scan does not cover it, the object + * is freed while a parked request still means to use it, and the crash + * lands nowhere near the cause. ---- */ +static volatile void* hiddenRef = 0; +static void holder(void* arg) { + /* `mine` exists only here, in a C local, on this virtual thread's stack */ + void* volatile mine = arg; + cn1VirtualThreadYield(); + /* still ours after the park */ + hiddenRef = mine; +} + +static int rangeContains(struct cn1VirtualThread* vt, void* needle) { + void *lo, *hi; char** w; + cn1VirtualThreadStackBounds(vt, &lo, &hi); + if(lo == 0 || hi == 0) return 0; + for(w = (char**)lo ; (void*)w < hi ; w++) { + if(*w == (char*)needle) return 1; + } + return 0; +} + +/* ---- 7. the registry must enumerate every live virtual thread ---- */ +static int registrySeen = 0; +static void countOne(struct cn1VirtualThread* vt, void* ctx) { + (void)vt; (void)ctx; registrySeen++; +} + +int main(void) { + /* 1 */ + int seen = -1; + struct cn1VirtualThread* co = cn1VirtualThreadCreate(counter, &seen, 64*1024); + check("create", co != 0); + for(int i = 0; i < 5; i++) { + cn1VirtualThreadResume(co); + check("yield value", seen == i); + } + cn1VirtualThreadResume(co); + check("finishes", cn1VirtualThreadFinished(co)); + check("ran every step", steps == 5); + cn1VirtualThreadResume(co); /* resuming a finished one is a no-op */ + check("resume after finish is safe", cn1VirtualThreadFinished(co)); + cn1VirtualThreadFree(co); + + /* 2 */ + memset(regsSeen, 0, sizeof(regsSeen)); + co = cn1VirtualThreadCreate(regUser, 0, 64*1024); + cn1VirtualThreadResume(co); /* runs to the yield */ + { volatile long clobber[8]; /* stomp the registers in between */ + for(int i=0;i<8;i++) clobber[i]=0xDEAD0000L+i; + (void)clobber; } + cn1VirtualThreadResume(co); /* must still see its own values */ + check("callee-saved registers survive", + regsSeen[0]==0x1111 && regsSeen[1]==0x2222 && regsSeen[2]==0x3333 && + regsSeen[3]==0x4444 && regsSeen[4]==0x5555 && regsSeen[5]==0x6666 && + regsSeen[6]==0x7777 && regsSeen[7]==0x8888); + cn1VirtualThreadFree(co); + + /* 3 */ + co = cn1VirtualThreadCreate(deep, 0, 256*1024); + cn1VirtualThreadResume(co); + cn1VirtualThreadResume(co); + check("deep stack intact across yield", deepSum == 200L*201L/2); + cn1VirtualThreadFree(co); + + /* 4 */ + struct cn1VirtualThread* cs[MANY]; + for(long i = 0; i < MANY; i++) cs[i] = cn1VirtualThreadCreate(many, (void*)i, 32*1024); + for(int round = 0; round < 10; round++) + for(int i = 0; i < MANY; i++) cn1VirtualThreadResume(cs[i]); + int ok = 1; + for(long i = 0; i < MANY; i++) if(slot[i] != (int)(i*1000+9)) ok = 0; + check("64 virtual threads stayed independent", ok); + + /* 5 stack bounds must be inside the virtual thread's own stack */ + { void *lo, *hi; cn1VirtualThreadStackBounds(cs[0], &lo, &hi); + check("stack bounds sane", lo != 0 && hi != 0 && lo < hi); } + for(long i = 0; i < MANY; i++) { cn1VirtualThreadResume(cs[i]); cn1VirtualThreadFree(cs[i]); } + + /* cost */ + struct cn1VirtualThread* fast = cn1VirtualThreadCreate(counter, &seen, 64*1024); + const long N = 500000; + long long t0 = nowNs(); + for(long i = 0; i < N; i++) cn1VirtualThreadResume(fast); + long long t1 = nowNs(); + printf("switch cost %.1f ns (round trip, %ld resumes)\n", (double)(t1-t0)/N, N); + cn1VirtualThreadFree(fast); + + /* 6: a parked virtual thread's C local must be inside the scanned range */ + { int marker; + struct cn1VirtualThread* h = cn1VirtualThreadCreate(holder, &marker, 64*1024); + cn1VirtualThreadResume(h); /* runs to the yield, now parked */ + check("parked stack range covers a C local", + rangeContains(h, &marker)); + check("parked virtual thread is not running", !cn1VirtualThreadIsRunning(h)); + cn1VirtualThreadResume(h); + check("resumed and kept its value", hiddenRef == (void*)&marker); + cn1VirtualThreadFree(h); } + + /* 7: registry membership tracks create and free */ + { struct cn1VirtualThread* a = cn1VirtualThreadCreate(counter, &seen, 32*1024); + struct cn1VirtualThread* b = cn1VirtualThreadCreate(counter, &seen, 32*1024); + registrySeen = 0; cn1VirtualThreadForEach(countOne, 0); + check("registry sees both", registrySeen == 2); + cn1VirtualThreadFree(a); + registrySeen = 0; cn1VirtualThreadForEach(countOne, 0); + check("registry sees one after free", registrySeen == 1); + cn1VirtualThreadFree(b); + registrySeen = 0; cn1VirtualThreadForEach(countOne, 0); + check("registry empty after both freed", registrySeen == 0); } + + /* 8: a free that lands during a GC scan defers the release + * + * The collector copies raw virtual-thread pointers into a snapshot and reads + * through them while other threads are still running and still freeing. So a + * free between Begin and End must leave the memory readable -- if it unmaps, + * the read below faults and this test dies rather than reporting, which is + * exactly the production failure. It is left OUT of the registry immediately + * either way, so no later snapshot picks it up. */ + { struct cn1VirtualThread* v = cn1VirtualThreadCreate(counter, &seen, 32*1024); + void* lo; void* hi; + volatile unsigned char* probe; + cn1VirtualThreadResume(v); /* start it so sp is set */ + cn1VirtualThreadStackBounds(v, &lo, &hi); + probe = (volatile unsigned char*)lo; + cn1VirtualThreadGcScanBegin(); + cn1VirtualThreadFree(v); + registrySeen = 0; cn1VirtualThreadForEach(countOne, 0); + check("freed during a scan leaves the registry at once", registrySeen == 0); + check("freed during a scan is still readable", (*probe | 1) != 0); + cn1VirtualThreadGcScanEnd(); + /* released for real now; nothing may read it again */ } + + printf(failures ? "FAILURES: %d\n" : "ALL VIRTUAL THREAD TESTS PASSED\n", failures); + return failures ? 1 : 0; +} From 7f85de951ae00681286905e2ec03b4a806062d4b Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 31 Aug 2026 20:04:01 +0300 Subject: [PATCH 06/42] Yield the virtual thread instead of sleeping its carrier CN1_RESUME_THREAD waited out a collection with usleep(1000). Two things make that expensive on the backend and neither is visible at the call site. It sleeps the CARRIER, and a carrier hosts many virtual threads: hostCount is min(workers, cores), so on a two-core pin sixty four connections share two carriers. One carrier sleeping a millisecond freezes about thirty two connections that were ready to run, which is the shape of a server whose median is healthy and whose tail is not. And it is a sleep-poll, so the wait is quantised to the sleep interval however briefly the flag was actually held. The measured worst case was 1923us: two iterations of a 1ms sleep waiting for something that had long since cleared. The pacing park already yielded here; this site did not, and it is the hottest of the four -- once per syscall return, 204105 times in a twenty second run against 9 for the handshake. Platform threads still sleep, having nothing to yield to, and off the backend the stub answers "not virtual" so the macro folds back to exactly the old loop. This shortens the wait; it does not remove it. The thread is still held until the collector has drained the whole worklist reachable from its roots rather than merely captured them, which is a separate question and a larger one. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 766e5b23ad1..687f08491bc 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -37,6 +37,11 @@ #include #include #include +/* For CN1_RESUME_THREAD, which yields a virtual thread rather than sleeping the + carrier it runs on. Off the backend every entry point here is a static inline + stub answering "there is no virtual thread", so the macro folds back to the + plain sleep and every other platform is byte-for-byte unchanged. */ +#include "cn1_virtual_thread.h" #include // Darwin's setjmp/longjmp SAVE and RESTORE the caller's signal mask -- a sigprocmask @@ -1957,7 +1962,15 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int #else #define CN1_GC_PARK_RELEASE(ts) do { (void)(ts); } while(0) #endif -#define CN1_RESUME_THREAD do { struct ThreadLocalData* __cn1rts = getThreadLocalData(); CN1_STALL_T0(__cn1rt0); while (__cn1rts->threadBlockedByGC){ usleep((JAVA_INT)1000);} __cn1rts->threadActive = JAVA_TRUE; CN1_GC_PARK_RELEASE(__cn1rts); CN1_STALL_ADD(__cn1rt0, CN1_STALL_NATIVE_RESUME, __cn1rts); } while(0) +/* Sleeping here sleeps the CARRIER, and a carrier hosts many virtual threads -- + * hostCount is min(workers, cores), so on a 2-core pin 64 connections share 2 + * carriers. One carrier sleeping a millisecond therefore freezes ~32 connections + * that were ready to run, which is why p50 stays good while p99 does not. Yield + * instead when this is a virtual thread: the carrier goes and serves the others, + * and the collector gets its safepoint just the same. The pacing park already + * did this; this site, the hottest of the four (once per syscall return), did + * not. Platform threads still sleep -- there is nothing to yield to. */ +#define CN1_RESUME_THREAD do { struct ThreadLocalData* __cn1rts = getThreadLocalData(); CN1_STALL_T0(__cn1rt0); while (__cn1rts->threadBlockedByGC){ if(!cn1VirtualThreadYieldIfVirtual()) { usleep((JAVA_INT)1000); } } __cn1rts->threadActive = JAVA_TRUE; CN1_GC_PARK_RELEASE(__cn1rts); CN1_STALL_ADD(__cn1rt0, CN1_STALL_NATIVE_RESUME, __cn1rts); } while(0) extern struct ThreadLocalData* getThreadLocalData(); From 7cfea66af8604c58bee921faa9b6edeae69a3045 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 1 Sep 2026 03:54:53 +0300 Subject: [PATCH 07/42] Keep the virtual-thread API out of the conservative-roots block cn1SpawnVirtualThread and cn1CreateThreadLocalData were declared inside #ifdef CN1_CONSERVATIVE_GC_ROOTS. Neither has anything to do with how the collector finds its roots, and burying them there broke -DCN1_DISABLE_CONSERVATIVE_GC_ROOTS -- the precise threadObjectStack arm that vm/CLAUDE.md documents -- with an undeclared cn1SpawnVirtualThread in the backend's native sources. C being what it is, the implicit declaration then also produced an int-to-pointer conversion, so the failure named the wrong thing. Found while measuring that arm rather than by building it, which is the point: nothing builds it. The default build is unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 687f08491bc..506c4a82ee4 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2821,6 +2821,21 @@ extern JAVA_BOOLEAN removeObjectFromHeapCollection(CODENAME_ONE_THREAD_STATE, JA extern void codenameOneGCMark(); extern void codenameOneGCSweep(); +/* Thread-state and virtual-thread construction. Declared OUTSIDE the + conservative-roots block: neither depends on how the collector finds its roots, + and burying them there broke -DCN1_DISABLE_CONSERVATIVE_GC_ROOTS -- the precise + threadObjectStack arm vm/CLAUDE.md documents -- with an undeclared + cn1SpawnVirtualThread in the backend's native sources. */ +struct cn1VirtualThread; +/** + * A VM thread state. bindToCallingOsThread false builds one for a VIRTUAL thread, + * which owns it rather than borrowing the host's -- see the definition. + */ +extern struct ThreadLocalData* cn1CreateThreadLocalData(JAVA_BOOLEAN bindToCallingOsThread); +/** A virtual thread with a Java stack of its own, ready to be resumed. */ +extern struct cn1VirtualThread* cn1SpawnVirtualThread(void (*body)(void*), void* arg, + size_t stackBytes); + #ifdef CN1_CONSERVATIVE_GC_ROOTS // PHASE 3b production conservative-root API. cn1ConservativeResolve maps an // arbitrary machine word to the base of the live heap object it points into @@ -2838,16 +2853,6 @@ extern void cn1GcInstallSignalHandler(void); // universal-stop handler. extern __thread struct ThreadLocalData* cn1TlsSelf; -struct cn1VirtualThread; -/** - * A VM thread state. bindToCallingOsThread false builds one for a VIRTUAL thread, - * which owns it rather than borrowing the host's -- see the definition. - */ -extern struct ThreadLocalData* cn1CreateThreadLocalData(JAVA_BOOLEAN bindToCallingOsThread); -/** A virtual thread with a Java stack of its own, ready to be resumed. */ -extern struct cn1VirtualThread* cn1SpawnVirtualThread(void (*body)(void*), void* arg, - size_t stackBytes); - // Capture a parking mutator's native register file + native-stack low bound so the // concurrent GC can conservatively scan [sp, stackBase) for native-stack-held roots. // MUST be a macro so setjmp + the SP marker live in the PARKING frame itself: that From 67a61b0392027ad5e2302e7a5a65ad72e9919d9e Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:08:04 +0300 Subject: [PATCH 08/42] Capture errno before the GC safepoint in the Linux socket read CN1_RESUME_THREAD is a safepoint: it can park the thread on a timed wait while a collection runs, and that overwrites errno. Reading errno after it recorded the WAIT's outcome rather than the read's, so lastError handed Java an error belonging to something else entirely. Captured at the syscall instead. The do/while EINTR retry idiom elsewhere is already safe -- it reads errno before the resume. Co-Authored-By: Claude Opus 5 (1M context) --- Ports/LinuxPort/nativeSources/cn1_linux_socket.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Ports/LinuxPort/nativeSources/cn1_linux_socket.c b/Ports/LinuxPort/nativeSources/cn1_linux_socket.c index 30ca4935d2b..679aa7873ab 100644 --- a/Ports/LinuxPort/nativeSources/cn1_linux_socket.c +++ b/Ports/LinuxPort/nativeSources/cn1_linux_socket.c @@ -108,6 +108,7 @@ JAVA_INT com_codename1_impl_linux_LinuxNative_socketRead___long_byte_1ARRAY_int_ CN1Socket* s = (CN1Socket*) (intptr_t) socket; char* data; ssize_t n; + int readErrno; if (!s || s->fd < 0 || buffer == JAVA_NULL || length <= 0) { return -1; } @@ -119,6 +120,11 @@ JAVA_INT com_codename1_impl_linux_LinuxNative_socketRead___long_byte_1ARRAY_int_ * syscall. Every other CN1 port's blocking I/O does the same. */ CN1_YIELD_THREAD; n = read(s->fd, data + offset, (size_t) length); + /* Captured before CN1_RESUME_THREAD. The resume is a GC safepoint and can park + * this thread on a timed wait, which overwrites errno -- so lastError below + * reported the WAIT's outcome rather than the read's, handing Java a misleading + * error for a failure that had nothing to do with it. */ + readErrno = errno; CN1_RESUME_THREAD; /* Keep the buffer array object reachable across the parked read: only `data` (an * interior pointer) is used, so the optimizer may drop `buffer` and the concurrent GC @@ -126,7 +132,7 @@ JAVA_INT com_codename1_impl_linux_LinuxNative_socketRead___long_byte_1ARRAY_int_ * Windows port where this manifested on the cn1ss WebSocket reader). Force liveness. */ CN1_SOCKET_KEEP_ALIVE(buffer); if (n <= 0) { - s->lastError = n < 0 ? errno : 0; + s->lastError = n < 0 ? readErrno : 0; if (n == 0) { s->connected = 0; } From 15bd6976f3226d36be4e0189d8f70dcd0bd40689 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:12:40 +0300 Subject: [PATCH 09/42] Stop waiting on a thread the GC cannot stop The mark phase signals every thread and spins until it answers, so it can scan the thread's native stack conservatively. A thread that never answers is not scanned either way -- the caller returns 0 and reads nothing -- so the wait buys literally nothing, and one such thread cost 267ms of a 280ms mark, every cycle. Count consecutive timeouts per thread and skip a thread that has failed three of them, re-probing every 64th attempt so one that becomes responsive is picked back up, and clearing the count the moment it answers. The forced-stop escalation (issue #5537) must NOT be throttled this way, so the implementation takes a maySkip flag and the escalation passes 0. It retries every CN1_GC_SAFEPOINT_WAIT_MAX_US precisely to ride out a transient or descheduled handler; skipping those retries would leave the collector waiting on threadActive for tens of seconds, turning a recoverable timeout into exactly the whole-VM pause the escalation exists to prevent. Measured on the server workload: stackMs 269 -> 0.20. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 6 ++++ vm/ByteCodeTranslator/src/cn1_globals.m | 39 ++++++++++++++++++++--- vm/ByteCodeTranslator/src/nativeMethods.m | 1 + 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 506c4a82ee4..689ec90e32c 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -1368,6 +1368,12 @@ struct ThreadLocalData { volatile sig_atomic_t gcSigStopped; // handler publishes the gen it parked for volatile sig_atomic_t gcSigRelease; // GC publishes highest released gen (monotonic) volatile sig_atomic_t gcSigStopGen; // generation counter (GC thread writes only) + /* Consecutive cn1GcSignalStopOne timeouts for this thread. A thread that never + answers the stop signal is not scanned either way -- the caller returns without + reading its stack -- so signalling it at all, and then waiting, buys nothing. + One such thread cost 267ms of a 280ms mark, every cycle. Stop attempting it once + it has proved unresponsive, and clear this the moment it answers. */ + int gcStopFailures; void* volatile gcSigStackPointer; // SP captured inside the signal handler // [sp,base) high bound and stack size, resolved BEFORE a forced freeze and reused // while it is held. cn1GcStackBase must not be called under one: it is two plain diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index d1383e13278..403c966a892 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -8539,9 +8539,24 @@ void cn1GcInstallSignalHandler(void) { // Signal-stop one thread, returning its captured SP (or 0 on failure/timeout). The // resolver snapshot MUST already be built (we do not realloc after the thread freezes). -static char* cn1GcSignalStopOne(struct ThreadLocalData* t) { +/* maySkip: whether this caller tolerates the unresponsive-thread throttle below. The + per-cycle native-stack scan does -- a thread it cannot stop is one it does not scan + either way. The FORCED-STOP ESCALATION does NOT: it retries every + CN1_GC_SAFEPOINT_WAIT_MAX_US precisely to ride out a transient or descheduled + handler, and throttling those retries would leave the collector waiting on + threadActive for tens of seconds, turning a recoverable timeout into the whole-VM + pause the escalation exists to prevent. */ +static char* cn1GcSignalStopOneImpl(struct ThreadLocalData* t, int maySkip) { #if !defined(_WIN32) if(!t->gcPthreadValid) return 0; + // SKIP a thread that has proved unresponsive rather than waiting on it again. + // Re-probe every 64th attempt so one that becomes responsive is picked back up. + if(maySkip && t->gcStopFailures >= 3) { + if(t->gcStopFailures < 1000000000) { t->gcStopFailures++; } + if((t->gcStopFailures & 63) != 0) { + return 0; + } + } // Next generation for this thread (only the GC thread writes it). gcSigRelease // is MONOTONIC and never reset -- see the handler's generation handshake. int gen = (int)t->gcSigStopGen + 1; @@ -8558,6 +8573,9 @@ void cn1GcInstallSignalHandler(void) { if((spins & 1023) == 0) usleep(50); } if((int)t->gcSigStopped != gen) { + // Counted only for the caller that can act on it: an escalation timeout says the + // thread was busy for 250ms, not that it never answers. + if(maySkip && t->gcStopFailures < 1000000) { t->gcStopFailures++; } // Abandon: the signal may still be pending, and the handler may ALREADY be // past its request gate about to park. PRE-RELEASE the generation so that // park (whenever it happens) exits immediately instead of spinning forever @@ -8566,12 +8584,23 @@ void cn1GcInstallSignalHandler(void) { t->gcSigStopRequest = 0; return 0; } + t->gcStopFailures = 0; // answered: stop skipping it return (char*)t->gcSigStackPointer; #else return 0; #endif } +/* Per-cycle native-stack scan: may skip a thread that has proved unresponsive. */ +static char* cn1GcSignalStopOne(struct ThreadLocalData* t) { + return cn1GcSignalStopOneImpl(t, 1); +} + +/* Forced-stop escalation: never skips -- see the note on the impl. */ +static char* cn1GcSignalStopOneForEscalation(struct ThreadLocalData* t) { + return cn1GcSignalStopOneImpl(t, 0); +} + static void cn1GcSignalReleaseOne(struct ThreadLocalData* t) { #if !defined(_WIN32) t->gcSigRelease = t->gcSigStopGen; // monotonic: frees this AND any older park @@ -8619,9 +8648,11 @@ static JAVA_BOOLEAN cn1GcMarkForceStopUncooperative(struct ThreadLocalData* t) { // decline). Sized well above one thread's plausible adoption count per cycle. cn1GcAdoptReserve(16384); #endif - if(cn1GcSignalStopOne(t) == 0) { - // Timed out. cn1GcSignalStopOne has already pre-released the generation, so - // nothing is left stranded. + if(cn1GcSignalStopOneForEscalation(t) == 0) { + // Timed out. cn1GcSignalStopOneImpl has already pre-released the generation, + // so nothing is left stranded. This caller does NOT skip on repeated timeouts + // -- see the maySkip note on the impl -- so the CN1_GC_SAFEPOINT_WAIT_MAX_US + // (250ms) retry loop above keeps signalling until the thread answers. return JAVA_FALSE; } #ifdef CN1_NURSERY diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index f1778416579..5dbd94ee6a2 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -2077,6 +2077,7 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC // malloc'd, not zeroed. The forced-stop scan guards on these being non-zero // before it marks [sp, base), so garbage here would pass that guard and hand // the conservative scan a bogus range. + i->gcStopFailures = 0; i->gcSigStackBase = 0; i->gcSigStackSize = 0; i->gcSigRegsLen = 0; From 6e6a18a8c122a0b5cfa042acf75d40f2d5a4a330 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:12:40 +0300 Subject: [PATCH 10/42] Turn virtual threads on wherever the switch exists, and file .S as assembly Two halves of one bug. Virtual threads were gated on a build flag that only the server build set, and the flag was justified by an Xcode misfiling it was working around: Xcode has no mapping for the .S extension, so an unrecognised one becomes `lastKnownFileType = file` and lands the file in the RESOURCES phase, where it is copied into the bundle and never assembled. The iOS target then failed to link naming _cn1VirtualThreadSwitch, whose source was sitting right there in the project. Gating the feature off made the misfiled resource inert, so the phone target linked and the misfiling stayed hidden. Fix the misfiling instead: .S maps to sourcecode.asm.asm (preprocessed, which the capability gate in the file needs) and .s to sourcecode.asm, and both route into the Sources phase rather than Resources. Every future assembly file gets this too. That removes the reason for the flag, so the gate becomes a capability test: on anywhere the switch is written for -- aarch64 and x86_64, excluding Windows, whose calling convention needs its own prologue -- virtual threads are on. There is no separate "server build" of the VM; a flag would only mean the feature is off in every build nobody remembered to set it in. Elsewhere the header's no-op stubs answer "there is no virtual thread here", which is true, so the collector needs no #ifdefs and every call folds away. CN1_DISABLE_VIRTUAL_THREADS forces that path. The predicate is repeated verbatim in the .S, which is preprocessed assembly and cannot include the header -- the two must stay identical or the link breaks on the switch symbol. Also excludes LinkedHashMap from the copyright gate: it is Apache Harmony source and keeps its Apache-2.0 notice. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/copyright-header-exclusions.txt | 1 + .../src/cn1_virtual_thread.c | 5 +-- .../src/cn1_virtual_thread.h | 35 +++++++++++-------- .../src/cn1_virtual_thread_asm.S | 13 +++++-- .../tools/translator/ByteCodeTranslator.java | 16 +++++++-- 5 files changed, 48 insertions(+), 22 deletions(-) diff --git a/scripts/copyright-header-exclusions.txt b/scripts/copyright-header-exclusions.txt index 35fa46dd25f..3924e0e5a11 100644 --- a/scripts/copyright-header-exclusions.txt +++ b/scripts/copyright-header-exclusions.txt @@ -28,3 +28,4 @@ vm/ByteCodeTranslator/src/cn1_sqlite3.h | SQLite3 Multiple Ciphers public header vm/ByteCodeTranslator/src/cn1_sqlite3_amalgamation.h | SQLite3 Multiple Ciphers amalgamation, upstream MIT notice over public-domain SQLite Ports/JavaScriptPort/src/main/webapp/js/sqlite3mc.js | SQLite3 Multiple Ciphers WebAssembly loader, Emscripten generated, MIT over public-domain SQLite Ports/JavaScriptPort/src/main/webapp/js/sqlite3-opfs-async-proxy.js | SQLite3 Multiple Ciphers OPFS proxy worker, MIT over public-domain SQLite +vm/JavaAPI/src/java/util/LinkedHashMap.java | Apache Harmony source retaining its original Apache-2.0 notice diff --git a/vm/ByteCodeTranslator/src/cn1_virtual_thread.c b/vm/ByteCodeTranslator/src/cn1_virtual_thread.c index 31348acb426..b67b9bdfbe0 100644 --- a/vm/ByteCodeTranslator/src/cn1_virtual_thread.c +++ b/vm/ByteCodeTranslator/src/cn1_virtual_thread.c @@ -21,8 +21,9 @@ * need additional information or have any questions. */ -/* BACKEND ONLY -- see cn1_virtual_thread.h. Off-target this file is empty and - * the header supplies no-op stubs, so nothing references the assembly. */ +/* See cn1_virtual_thread.h for the capability gate. On a target the switch is not + * written for, this file is empty and the header supplies no-op stubs, so nothing + * references the assembly. */ #include "cn1_virtual_thread.h" #ifdef CN1_VIRTUAL_THREADS diff --git a/vm/ByteCodeTranslator/src/cn1_virtual_thread.h b/vm/ByteCodeTranslator/src/cn1_virtual_thread.h index a2c385500bc..b59baa8b2f4 100644 --- a/vm/ByteCodeTranslator/src/cn1_virtual_thread.h +++ b/vm/ByteCodeTranslator/src/cn1_virtual_thread.h @@ -48,22 +48,27 @@ #define CN1_VIRTUAL_THREAD_H /* - * BACKEND ONLY. Virtual threads exist to let one server thread carry many - * connections; nothing on a device uses them, and the switch is hand-written - * assembly, so a target that cannot use them should not be made to build it. - * - * Gating matters for a reason beyond dead code. The switch lives in a .S, which - * is the only assembly file the translator emits, and Xcode does not recognise - * the extension: it files a .S under `lastKnownFileType = file` into the - * RESOURCES phase, so the iOS target shipped it as a resource, never assembled - * it, and failed to link with "_cn1VirtualThreadSwitch, referenced from - * _cn1VirtualThreadYield". With this off there is no reference to resolve, so - * the misfiled resource is simply inert and the phone target links. - * - * The backend defines CN1_VIRTUAL_THREADS (see docker/link.sh). Everywhere else - * the calls below collapse to the no-ops at the bottom of this header, so the - * shared collector in cn1_globals.m needs no #ifdefs of its own. + * Virtual threads are on wherever they CAN be, which is anywhere the hand-written + * context switch has an implementation. That is deliberately a capability test and + * not a build flag: there is no separate "server build" of the VM, so a flag would + * only mean the feature is off in every build nobody remembered to set it in. + * + * The switch has to be assembly -- glibc aborts a cross-stack longjmp under + * _FORTIFY_SOURCE and musl has no makecontext -- and it is written for aarch64 and + * x86_64. Anywhere else, and on Windows (whose calling convention needs its own + * prologue and whose stack has a guard page the switch would have to poke), the + * declarations below collapse to the no-ops at the bottom of this header. Those + * report "there is no virtual thread here", which is true, so the shared collector + * in cn1_globals.m needs no #ifdefs of its own and every call folds away. + * + * CN1_DISABLE_VIRTUAL_THREADS forces the no-op path on a target that would + * otherwise qualify. */ +#if !defined(CN1_VIRTUAL_THREADS) && !defined(CN1_DISABLE_VIRTUAL_THREADS) \ + && !defined(_WIN32) && (defined(__aarch64__) || defined(__x86_64__)) +#define CN1_VIRTUAL_THREADS 1 +#endif + #ifdef CN1_VIRTUAL_THREADS #include diff --git a/vm/ByteCodeTranslator/src/cn1_virtual_thread_asm.S b/vm/ByteCodeTranslator/src/cn1_virtual_thread_asm.S index b73111f6896..0a08d05de2d 100644 --- a/vm/ByteCodeTranslator/src/cn1_virtual_thread_asm.S +++ b/vm/ByteCodeTranslator/src/cn1_virtual_thread_asm.S @@ -51,9 +51,16 @@ * it fails to link, or worse, links against nothing on a platform where the * caller is also assembly. */ -/* BACKEND ONLY. Without this the file still assembles on a device target and - * defines symbols nothing calls; with it the object is empty, which is what lets - * a toolchain that misfiles a .S (Xcode puts it in Resources) stay harmless. */ +/* This is preprocessed assembly, so it cannot include cn1_virtual_thread.h -- the + * header's C declarations would not assemble. The predicate is therefore repeated + * here and MUST stay identical to the one in that header: if the two disagree the C + * side calls a switch this file did not define, and the link fails naming + * _cn1VirtualThreadSwitch. */ +#if !defined(CN1_VIRTUAL_THREADS) && !defined(CN1_DISABLE_VIRTUAL_THREADS) \ + && !defined(_WIN32) && (defined(__aarch64__) || defined(__x86_64__)) +#define CN1_VIRTUAL_THREADS 1 +#endif + #ifdef CN1_VIRTUAL_THREADS #if defined(__APPLE__) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index add3f4fc7a7..d4992b75e7a 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -941,7 +941,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File } } else { fileListEntry.append("; path = \""); - if(file.endsWith(".m") || file.endsWith(".c") || file.endsWith(".cpp") || file.endsWith(".mm") || file.endsWith(".h") || + if(file.endsWith(".m") || file.endsWith(".S") || file.endsWith(".s") || file.endsWith(".c") || file.endsWith(".cpp") || file.endsWith(".mm") || file.endsWith(".h") || file.endsWith(".swift") || file.endsWith(".bundle") || file.endsWith(".xcdatamodeld") || file.endsWith(".hh") || file.endsWith(".hpp") || file.endsWith(".xib") || file.endsWith(".metal")) { fileListEntry.append(file); @@ -977,7 +977,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File .append(" };\n"); } - if(file.endsWith(".m") || file.endsWith(".c") || file.endsWith(".cpp") || file.endsWith(".hh") || file.endsWith(".hpp") || + if(file.endsWith(".m") || file.endsWith(".S") || file.endsWith(".s") || file.endsWith(".c") || file.endsWith(".cpp") || file.endsWith(".hh") || file.endsWith(".hpp") || file.endsWith(".swift") || file.endsWith(".mm") || file.endsWith(".h") || file.endsWith(".bundle") || file.endsWith(".xcdatamodeld") || file.endsWith(".xib") || file.endsWith(".metal")) { @@ -1370,6 +1370,18 @@ private static String getFileType(String s) { if(s.endsWith(".m") || s.endsWith(".c")) { return "sourcecode.c.objc"; } + // Assembly. Xcode has no default mapping for .S/.s, and an unrecognised + // extension becomes `lastKnownFileType = file`, which lands the file in the + // RESOURCES phase: it ships into the bundle and is never assembled, so the + // link fails naming a symbol whose source is sitting right there in the + // project. .S is preprocessed before assembling (the capability gate in + // cn1_virtual_thread_asm.S needs that); .s is not. + if(s.endsWith(".S")) { + return "sourcecode.asm.asm"; + } + if(s.endsWith(".s")) { + return "sourcecode.asm"; + } if(s.endsWith(".xcassets")) { return "folder.assetcatalog"; } From fe581d5813ae4dddecb66a5de14bc8a46fd1a3a9 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:47:57 +0300 Subject: [PATCH 11/42] Assemble the .S the generated projects have been shipping unassembled Turning virtual threads on by capability rather than by a flag nobody set made three latent bugs reachable at once, all the same shape: the context switch was copied into the generated project and never assembled, so the C half linked against a symbol whose source was sitting in the same directory. - CMake globbed *.S only for the LINUX app type, and only when embedding resources -- the condition belonged to the resource blob, which used to be the only .S there is. Now any .S present drives both the ASM language and the glob, on every cmake target. - The WINDOWS app type is also cross-built with clang on a POSIX host, where _WIN32 is undefined, the switch is live, and MSVC's inability to assemble GNU syntax is irrelevant. That is a question about the compiler, and CMake can only answer it after project() has enabled C, so it is asked there rather than guessed from the app type. Under MSVC the variable stays unset and expands to nothing. - Xcode has no mapping for .S at all, so it became `lastKnownFileType = file` and landed in the RESOURCES phase, shipped into the bundle and never built. sourcecode.asm is the identifier for both spellings: Xcode's own StandardFileTypes.xcspec lists it as `Extensions = (s)` with `GccDialectName = assembler-with-cpp`, which is the preprocessing the file's capability gate needs. The neighbouring sourcecode.asm.asm is for .asm. Tests. BackendUncaughtExceptionTest needed a support class that does not exist here, and only ever reached the fix through a server binary; replaced by UncaughtExceptionIntegrationTest, which builds a clean-target program directly and asserts the whole contract -- message, stack frame, non-zero exit, and that execution stops AT the throw rather than carrying on, which is the half the other three can all pass without. test_virtual_thread.c was built by nothing. A hand-written context switch with no enforced coverage could break in any commit and stay green, so VirtualThreadRuntimeTest drives it from the suite, compiled out of the SAME staged classpath resources a generated project receives -- which also asserts those three files are present and agree with each other. The iOS project test now asserts the assembly is typed as assembly, IS in the Sources phase and is NOT in Resources. All three: the type alone does not prove the phase, and the phase alone does not prove it assembles. The generator's own source set is what caught the last of it. Two copies of replaceLibraryWithExecutableTarget matched the add_library line by its full argument LIST -- the shared one in CleanTargetIntegrationTest and a private duplicate at the bottom of FileClassIntegrationTest. Adding the assembly glob made both stop matching, so those tests built a library and then failed running an executable nothing had asked for. The shared one now matches the CALL and asserts the substitution happened; the duplicate is gone, and FileClassIntegration uses the shared one like the other twenty-two callers already did. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/com/codename1/mapping/Mapper.java | 2 - vm/ByteCodeTranslator/src/cn1_globals.h | 13 +- .../tools/translator/ByteCodeTranslator.java | 79 +++++++--- vm/ByteCodeTranslator/src/nativeMethods.m | 2 +- .../BackendUncaughtExceptionTest.java | 94 ------------ .../BytecodeInstructionIntegrationTest.java | 40 +++++ .../CleanTargetIntegrationTest.java | 16 +- .../translator/FileClassIntegrationTest.java | 38 +++-- .../UncaughtExceptionIntegrationTest.java | 142 ++++++++++++++++++ .../translator/VirtualThreadRuntimeTest.java | 128 ++++++++++++++++ vm/tests/virtualthread/test_virtual_thread.c | 6 +- 11 files changed, 420 insertions(+), 140 deletions(-) delete mode 100644 vm/tests/src/test/java/com/codename1/tools/translator/BackendUncaughtExceptionTest.java create mode 100644 vm/tests/src/test/java/com/codename1/tools/translator/UncaughtExceptionIntegrationTest.java create mode 100644 vm/tests/src/test/java/com/codename1/tools/translator/VirtualThreadRuntimeTest.java diff --git a/CodenameOne/src/com/codename1/mapping/Mapper.java b/CodenameOne/src/com/codename1/mapping/Mapper.java index a79a01e327a..4355f8eeedc 100644 --- a/CodenameOne/src/com/codename1/mapping/Mapper.java +++ b/CodenameOne/src/com/codename1/mapping/Mapper.java @@ -65,8 +65,6 @@ public interface Mapper { /// LinkedHashMap. On a translated device build the map is /// `vm/JavaAPI`'s, which overrides the natives HashMap gets and costs about /// 1.5x a HashMap to build -- so the saving there is at least this, not less. - /// The same change on a server JSON route, where serialising is one cost - /// among request parsing and socket I/O, was worth 29% end to end. /// /// Implemented as a separate interface rather than a method on `Mapper` so /// hand-written mappers keep compiling; `Mappers#toJson` uses it when the diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 689ec90e32c..babea79befb 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -38,9 +38,9 @@ #include #include /* For CN1_RESUME_THREAD, which yields a virtual thread rather than sleeping the - carrier it runs on. Off the backend every entry point here is a static inline - stub answering "there is no virtual thread", so the macro folds back to the - plain sleep and every other platform is byte-for-byte unchanged. */ + carrier it runs on. Where the switch is not implemented, every entry point here + is a static inline stub answering "there is no virtual thread", so the macro + folds back to the plain sleep and that platform is byte-for-byte unchanged. */ #include "cn1_virtual_thread.h" #include @@ -2831,7 +2831,7 @@ extern void codenameOneGCSweep(); conservative-roots block: neither depends on how the collector finds its roots, and burying them there broke -DCN1_DISABLE_CONSERVATIVE_GC_ROOTS -- the precise threadObjectStack arm vm/CLAUDE.md documents -- with an undeclared - cn1SpawnVirtualThread in the backend's native sources. */ + cn1SpawnVirtualThread in any native source that spawns one. */ struct cn1VirtualThread; /** * A VM thread state. bindToCallingOsThread false builds one for a VIRTUAL thread, @@ -2889,9 +2889,8 @@ extern void cn1StallRecord(int cause, long long ns, struct ThreadLocalData* ts); CN1_RESUME_THREAD below expands to CN1_STALL_ADD(..., CN1_STALL_NATIVE_RESUME, ...), and every native file that wraps a blocking call uses that macro. With the codes private to cn1_globals.m, any other native source failed to compile - under -DCN1_GC_CONFORM with "use of undeclared identifier"; the backend's - sockets, database and crypto natives are the first outside the core to wrap - blocking calls this way. */ + under -DCN1_GC_CONFORM with "use of undeclared identifier", which is every port + whose sockets, database or crypto natives wrap a blocking call this way. */ #define CN1_STALL_PACING_VOLUME 0 // regime-A run-ahead cap (cn1PacingPark, no budget) #define CN1_STALL_PACING_BUDGET 1 // regime-B admission wait (cn1PacingPark, under a ceiling) #define CN1_STALL_LOWMEM 2 // the low-memory allocation throttle diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index d4992b75e7a..cd53abd0133 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -1075,14 +1075,31 @@ private static void writeCmakeProject(File projectRoot, File srcRoot, String app // generated .S that .incbin's the resource blobs (ASM language). boolean embedResources = (windows && new File(srcRoot, "cn1_resources.rc").isFile()) || (linux && new File(srcRoot, "cn1_resources_data.S").isFile()); + // Assembly is driven by what is actually THERE, not by which feature put it + // there. The resource .S used to be the only one, so the ASM language and its + // glob were gated on embedResources; the virtual-thread context switch is a + // second .S, and under that gate the clean target compiled its C half and + // failed to link on _cn1VirtualThreadSwitch with the source sitting in the + // same directory. + boolean hasAsm = false; + String[] rootFiles = srcRoot.list(); + if (rootFiles != null) { + for (String f : rootFiles) { + if (f.endsWith(".S") || f.endsWith(".s")) { + hasAsm = true; + break; + } + } + } if (windows) { writer.append("project(").append(appName).append(embedResources ? " LANGUAGES C CXX RC)\n" : " LANGUAGES C CXX)\n"); } else if (linux) { - writer.append("project(").append(appName).append(embedResources + writer.append("project(").append(appName).append(hasAsm ? " LANGUAGES C ASM)\n" : " LANGUAGES C)\n"); } else { - writer.append("project(").append(appName).append(" LANGUAGES C)\n"); + writer.append("project(").append(appName).append(hasAsm + ? " LANGUAGES C ASM)\n" : " LANGUAGES C)\n"); } // C11 for (cn1_globals.h) and _Static_assert (Win32 shim); // supported by clang/clang-cl, gcc and Xcode's clang alike. @@ -1103,12 +1120,13 @@ private static void writeCmakeProject(File projectRoot, File srcRoot, String app writer.append("file(GLOB TRANSLATOR_SOURCES \"${CN1_APP_SOURCE_ROOT}/*.c\")\n"); writer.append("file(GLOB TRANSLATOR_HEADERS \"${CN1_APP_SOURCE_ROOT}/*.h\")\n"); if (linux) { - // The Linux executable is pure C (GTK/Cairo/Pango/GdkPixbuf are C - // libraries). The generated resource .S (.incbin of each classpath - // resource) is added when present so getResourceAsStream can read - // the blobs straight out of the ELF .rodata. + // The Linux executable is otherwise pure C (GTK/Cairo/Pango/GdkPixbuf + // are C libraries). Two things can put a .S beside it: the generated + // resource blob (.incbin of each classpath resource, so + // getResourceAsStream reads straight out of the ELF .rodata) and the + // virtual-thread context switch. Both are picked up by presence. String asmGlob = ""; - if (embedResources) { + if (hasAsm) { writer.append("file(GLOB TRANSLATOR_ASM_SOURCES \"${CN1_APP_SOURCE_ROOT}/*.S\")\n"); asmGlob = " ${TRANSLATOR_ASM_SOURCES}"; } @@ -1119,13 +1137,26 @@ private static void writeCmakeProject(File projectRoot, File srcRoot, String app } else if (windows) { // The port's nativeSources contribute the C++ DirectWrite layer. writer.append("file(GLOB TRANSLATOR_CXX_SOURCES \"${CN1_APP_SOURCE_ROOT}/*.cpp\")\n"); + // Assembly is a COMPILER question here, not an app-type one. MSVC cannot + // assemble GNU syntax, but the Windows app type is also cross-built with + // clang on a POSIX host, where _WIN32 is undefined, the virtual-thread + // switch is live, and the link fails without it. CMake knows which one it + // got only after project() has enabled C, so ask it there rather than + // guessing from the app type. Under MSVC the variable stays unset and + // expands to nothing. + if (hasAsm) { + writer.append("if(NOT MSVC)\n"); + writer.append(" enable_language(ASM)\n"); + writer.append(" file(GLOB TRANSLATOR_ASM_SOURCES \"${CN1_APP_SOURCE_ROOT}/*.S\")\n"); + writer.append("endif()\n"); + } if (embedResources) { // The resource script compiles to a .res linked into the exe, // putting the app's classpath resources in the PE resource section. writer.append("file(GLOB TRANSLATOR_RC_SOURCES \"${CN1_APP_SOURCE_ROOT}/*.rc\")\n"); - writer.append("add_executable(${PROJECT_NAME} ${TRANSLATOR_SOURCES} ${TRANSLATOR_CXX_SOURCES} ${TRANSLATOR_RC_SOURCES} ${TRANSLATOR_HEADERS})\n"); + writer.append("add_executable(${PROJECT_NAME} ${TRANSLATOR_SOURCES} ${TRANSLATOR_CXX_SOURCES} ${TRANSLATOR_ASM_SOURCES} ${TRANSLATOR_RC_SOURCES} ${TRANSLATOR_HEADERS})\n"); } else { - writer.append("add_executable(${PROJECT_NAME} ${TRANSLATOR_SOURCES} ${TRANSLATOR_CXX_SOURCES} ${TRANSLATOR_HEADERS})\n"); + writer.append("add_executable(${PROJECT_NAME} ${TRANSLATOR_SOURCES} ${TRANSLATOR_CXX_SOURCES} ${TRANSLATOR_ASM_SOURCES} ${TRANSLATOR_HEADERS})\n"); } writer.append("target_include_directories(${PROJECT_NAME} PUBLIC ${CN1_APP_SOURCE_ROOT})\n"); // Math lives in the CRT under MSVC (no separate libm to link); every @@ -1197,7 +1228,13 @@ private static void writeCmakeProject(File projectRoot, File srcRoot, String app writer.append(" target_link_libraries(${PROJECT_NAME} m)\n"); writer.append("endif()\n"); } else { - writer.append("add_library(${PROJECT_NAME} ${TRANSLATOR_SOURCES} ${TRANSLATOR_HEADERS})\n"); + String asmGlob = ""; + if (hasAsm) { + writer.append("file(GLOB TRANSLATOR_ASM_SOURCES \"${CN1_APP_SOURCE_ROOT}/*.S\")\n"); + asmGlob = " ${TRANSLATOR_ASM_SOURCES}"; + } + writer.append("add_library(${PROJECT_NAME} ${TRANSLATOR_SOURCES}") + .append(asmGlob).append(" ${TRANSLATOR_HEADERS})\n"); writer.append("target_include_directories(${PROJECT_NAME} PUBLIC ${CN1_APP_SOURCE_ROOT})\n"); } @@ -1370,16 +1407,18 @@ private static String getFileType(String s) { if(s.endsWith(".m") || s.endsWith(".c")) { return "sourcecode.c.objc"; } - // Assembly. Xcode has no default mapping for .S/.s, and an unrecognised - // extension becomes `lastKnownFileType = file`, which lands the file in the - // RESOURCES phase: it ships into the bundle and is never assembled, so the - // link fails naming a symbol whose source is sitting right there in the - // project. .S is preprocessed before assembling (the capability gate in - // cn1_virtual_thread_asm.S needs that); .s is not. - if(s.endsWith(".S")) { - return "sourcecode.asm.asm"; - } - if(s.endsWith(".s")) { + // Assembly. An extension Xcode does not recognise becomes + // `lastKnownFileType = file`, which lands the file in the RESOURCES phase: it + // ships into the bundle and is never assembled, so the link fails naming a + // symbol whose source is sitting right there in the project. + // + // sourcecode.asm is the identifier to use for BOTH spellings. Xcode's + // StandardFileTypes.xcspec lists it as `Extensions = (s)` with + // `GccDialectName = assembler-with-cpp`, so it runs the preprocessor -- which + // the capability gate in cn1_virtual_thread_asm.S needs. .S is not in any + // Extensions list of its own, and the neighbouring sourcecode.asm.asm is for + // .asm, not for it. + if(s.endsWith(".S") || s.endsWith(".s")) { return "sourcecode.asm"; } if(s.endsWith(".xcassets")) { diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 5dbd94ee6a2..fc36514cc61 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -2157,7 +2157,7 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC cn1VirtualThreadSetState(vt, state); return vt; } -#endif /* CN1_VIRTUAL_THREADS -- backend only, see cn1_virtual_thread.h */ +#endif /* CN1_VIRTUAL_THREADS -- see the capability gate in cn1_virtual_thread.h */ struct ThreadLocalData* getThreadLocalData() { // A running virtual thread supplies its own state; every generated method diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BackendUncaughtExceptionTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BackendUncaughtExceptionTest.java deleted file mode 100644 index cb68b964219..00000000000 --- a/vm/tests/src/test/java/com/codename1/tools/translator/BackendUncaughtExceptionTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ -package com.codename1.tools.translator; - -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * An exception no handler catches must end a clean-target program, loudly. - * - * It used to be discarded: throwException walked the try-block stack, found no - * handler, and RETURNED -- so the generated code carried straight on with the - * statement after the throw, with the method's locals in whatever state the - * failed operation left them. On an app target something upstream (the EDT's own - * catch) nearly always exists, which is why it went unnoticed for years. A server - * binary has none, and the way this surfaced was a database client whose TLS - * handshake was rejected, after which the program kept going and segfaulted two - * statements later on a null it should never have had. - * - * The three assertions below are the contract: the message is printed, a stack - * trace is printed, and the process exits non-zero. All three matter -- an exit - * code with no message is unactionable in a log, and a message with a zero exit - * makes CI call a failed run a pass. - */ -class BackendUncaughtExceptionTest { - - @Test - @DisplayName("an uncaught exception reports itself and ends the process") - void uncaughtExceptionIsFatal() throws Exception { - if (CompilerHelper.isWindows()) { - Assumptions.abort("the server-side backend is POSIX-only for now"); - } - BackendTestSupport.require(Files.isDirectory(BackendTestSupport.backendDir()), - "vm/backend is not present"); - Path jdk8 = BackendTestSupport.findJdk8(); - BackendTestSupport.require(jdk8 != null, "no JDK 8 available to build the backend"); - - Path work = Files.createTempDirectory("backend-uncaught"); - Path binary = work.resolve("uncaught"); - String failure = BackendTestSupport.build("Uncaught", "demo/uncaught", binary, jdk8); - if (failure != null) { - BackendTestSupport.skipOrFail(failure); - } - - ProcessBuilder run = new ProcessBuilder(binary.toString()); - run.redirectErrorStream(true); - Process p = run.start(); - String output = BackendTestSupport.readFully(p.getInputStream()); - if (!p.waitFor(2, TimeUnit.MINUTES)) { - p.destroyForcibly(); - fail("the program did not finish:\n" + output); - } - - assertTrue(output.indexOf("before the throw") >= 0, - "the program should have run up to the throw:\n" + output); - assertTrue(output.indexOf("deliberate failure with a message") >= 0, - "the exception's message must be reported, not just its type:\n" + output); - assertTrue(output.indexOf("com_demo_Uncaught.open") >= 0, - "a stack trace naming the throwing frame must be reported:\n" + output); - assertEquals(1, p.exitValue(), - "a program killed by an uncaught exception must not report success:\n" + output); - } -} diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java index 2a1aa78d52a..0e88b50b8aa 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java @@ -1069,6 +1069,23 @@ void handleIosOutputGeneratesProjectStructure(CompilerHelper.CompilerConfig conf assertTrue(pbxproj.contains("CoreText.framework"), "iOS projects must link CoreText for IOSNative bundled font registration"); + // The assembly file must be typed AND filed as a source. An extension + // Xcode does not recognise gets `lastKnownFileType = file` and lands in + // the Resources phase, where it is copied into the bundle and never + // assembled -- a green build that fails to link on a symbol whose source + // is right there in the project. Assert both halves: the type alone does + // not prove the phase, and the phase alone does not prove it assembles. + assertTrue(Files.exists(srcRoot.resolve("cn1_virtual_thread_asm.S")), + "the virtual-thread switch must travel with the generated sources"); + String asmReference = fileReferenceLine(pbxproj, "cn1_virtual_thread_asm.S"); + assertTrue(asmReference.contains("lastKnownFileType = sourcecode.asm"), + "cn1_virtual_thread_asm.S must be typed as assembly, not left as `file`:\n" + + asmReference); + assertTrue(buildPhase(pbxproj, "PBXSourcesBuildPhase").contains("cn1_virtual_thread_asm.S"), + "cn1_virtual_thread_asm.S must be in the Sources build phase"); + assertFalse(buildPhase(pbxproj, "PBXResourcesBuildPhase").contains("cn1_virtual_thread_asm.S"), + "cn1_virtual_thread_asm.S must not be shipped as a resource"); + // Verify bundle copied assertTrue(Files.exists(srcRoot.resolve("test.bundle"))); assertTrue(Files.exists(srcRoot.resolve("test.bundle/info.txt"))); @@ -1426,4 +1443,27 @@ void testArithmeticExpressionCoverage() { // or mock if possible. But here we can check basic behavior. } + + /** + * The text of one pbxproj section, so a "contains" question can be asked of the + * SOURCES phase rather than of the whole file, where every path appears at least + * once as a file reference and the answer is always yes. + */ + private static String buildPhase(String pbxproj, String isa) { + int at = pbxproj.indexOf("isa = " + isa); + assertTrue(at >= 0, "the generated project has no " + isa); + int end = pbxproj.indexOf("};", at); + assertTrue(end >= 0, "unterminated " + isa + " in the generated project"); + return pbxproj.substring(at, end); + } + + /** The PBXFileReference line naming this file, for a failure message that shows the real type. */ + private static String fileReferenceLine(String pbxproj, String fileName) { + for (String line : pbxproj.split("\n")) { + if (line.contains("PBXFileReference") && line.contains(fileName)) { + return line.trim(); + } + } + return "(no PBXFileReference names " + fileName + ")"; + } } diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetIntegrationTest.java index 522586513a1..08777a6f050 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetIntegrationTest.java @@ -1714,10 +1714,18 @@ static void replaceLibraryWithExecutableTarget(Path cmakeLists, String sourceDir String linkLine = CompilerHelper.isWindows() ? "" : "\ntarget_link_libraries(${PROJECT_NAME} m)"; - String replacement = content.replace( - "add_library(${PROJECT_NAME} ${TRANSLATOR_SOURCES} ${TRANSLATOR_HEADERS})", - "add_executable(${PROJECT_NAME} ${TRANSLATOR_SOURCES} ${TRANSLATOR_HEADERS})" + linkLine - ); + // Match the CALL, not the whole argument list. Spelling the arguments out here + // means any source set the generator adds (the assembly glob was the one that + // caught this) silently stops matching, and every clean-target test then builds + // a LIBRARY and fails looking for an executable that was never asked for. + int at = content.indexOf("add_library(${PROJECT_NAME}"); + assertTrue(at >= 0, "the generated CMakeLists no longer declares add_library(${PROJECT_NAME}...):\n" + content); + int end = content.indexOf(')', at); + assertTrue(end >= 0, "unterminated add_library() in the generated CMakeLists:\n" + content); + String replacement = content.substring(0, at) + + "add_executable(" + content.substring(at + "add_library(".length(), end + 1) + + linkLine + + content.substring(end + 1); Files.write(cmakeLists, replacement.getBytes(StandardCharsets.UTF_8)); } diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java index eb0b3709d54..7403fd63cab 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ package com.codename1.tools.translator; import org.junit.jupiter.params.ParameterizedTest; @@ -70,7 +92,13 @@ public void testFileClassMethods(CompilerHelper.CompilerConfig config) throws Ex assertTrue(Files.exists(cmakeLists), "Translator should emit a CMake project"); Path srcRoot = distDir.resolve("FileTestApp-src"); - replaceLibraryWithExecutableTarget(cmakeLists, srcRoot.getFileName().toString()); + // The SHARED helper, not a private copy. The copy that used to live at the + // bottom of this file matched the add_library line by its full argument list, + // so the moment the generator gained an assembly glob it silently stopped + // matching -- and this test built a library, then failed to run an executable + // that was never asked for. + CleanTargetIntegrationTest.replaceLibraryWithExecutableTarget( + cmakeLists, srcRoot.getFileName().toString()); Path buildDir = distDir.resolve("build"); Files.createDirectories(buildDir); @@ -116,12 +144,4 @@ private String fileTestAppSource() { "}"; } - private void replaceLibraryWithExecutableTarget(Path cmakeLists, String sourceDirName) throws IOException { - String content = new String(Files.readAllBytes(cmakeLists), StandardCharsets.UTF_8); - String replacement = content.replace( - "add_library(${PROJECT_NAME} ${TRANSLATOR_SOURCES} ${TRANSLATOR_HEADERS})", - "add_executable(${PROJECT_NAME} ${TRANSLATOR_SOURCES} ${TRANSLATOR_HEADERS})\ntarget_link_libraries(${PROJECT_NAME} m)" - ); - Files.write(cmakeLists, replacement.getBytes(StandardCharsets.UTF_8)); - } } diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/UncaughtExceptionIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/UncaughtExceptionIntegrationTest.java new file mode 100644 index 00000000000..3aaa8788486 --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/UncaughtExceptionIntegrationTest.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.params.ParameterizedTest; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * An exception no handler catches must end a clean-target program, loudly. + * + * It used to be discarded: throwException walked the try-block stack, found no + * handler, and RETURNED -- so the generated code carried straight on with the + * statement after the throw, with the method's locals in whatever state the + * failed operation left them. On an app target something upstream (the EDT's own + * catch) nearly always exists, which is why it went unnoticed for years. A clean + * target has none, and the way this surfaced was a client whose TLS handshake was + * rejected, after which the program kept going and segfaulted two statements + * later on a null it should never have had. + * + * The four assertions below are the contract: execution stops AT the throw, the + * message is printed, a stack trace naming the throwing frame is printed, and the + * process exits non-zero. All four matter -- an exit code with no message is + * unactionable in a log, a message with a zero exit makes CI call a failed run a + * pass, and if execution continues past the throw the other three can all hold + * while the bug is still there. + */ +class UncaughtExceptionIntegrationTest { + + @ParameterizedTest + @org.junit.jupiter.params.provider.MethodSource("com.codename1.tools.translator.BytecodeInstructionIntegrationTest#provideCompilerConfigs") + void uncaughtExceptionIsFatal(CompilerHelper.CompilerConfig config) throws Exception { + Parser.cleanup(); + + Path sourceDir = Files.createTempDirectory("uncaught-sources"); + Path classesDir = Files.createTempDirectory("uncaught-classes"); + Path javaApiDir = Files.createTempDirectory("uncaught-java-api"); + Files.write(sourceDir.resolve("UncaughtApp.java"), + uncaughtSource().getBytes(StandardCharsets.UTF_8)); + + JavascriptTargetIntegrationTest.compileAgainstJavaApi(config, sourceDir, classesDir, javaApiDir); + + Path outputDir = Files.createTempDirectory("uncaught-output"); + CleanTargetIntegrationTest.runTranslator(classesDir, outputDir, "UncaughtApp"); + + Path distDir = outputDir.resolve("dist"); + Path cmakeLists = distDir.resolve("CMakeLists.txt"); + assertTrue(Files.exists(cmakeLists), "Translator should emit a CMake project"); + CleanTargetIntegrationTest.replaceLibraryWithExecutableTarget(cmakeLists, "UncaughtApp-src"); + + Path buildDir = distDir.resolve("build"); + Files.createDirectories(buildDir); + List configure = new java.util.ArrayList<>(Arrays.asList( + "cmake", "-S", distDir.toString(), "-B", buildDir.toString(), + "-DCMAKE_BUILD_TYPE=Release")); + configure.addAll(CompilerHelper.cmakeToolchainArgs()); + CleanTargetIntegrationTest.runCommand(configure, distDir); + CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), distDir); + + // Deliberately NOT runCommand: that asserts a zero exit, and a zero exit is + // precisely the failure this test exists to catch. + Path executable = buildDir.resolve(CompilerHelper.executableName("UncaughtApp")); + ProcessBuilder run = new ProcessBuilder(executable.toString()); + run.redirectErrorStream(true); + Process p = run.start(); + String output = new String(readFully(p), StandardCharsets.UTF_8); + if (!p.waitFor(2, TimeUnit.MINUTES)) { + p.destroyForcibly(); + fail("the program did not finish:\n" + output); + } + + assertTrue(output.contains("UNCAUGHT_BEFORE"), + "the program should have run up to the throw:\n" + output); + assertTrue(output.contains("deliberate failure with a message"), + "the exception's message must be reported, not just its type:\n" + output); + assertTrue(output.contains("UncaughtApp.open"), + "a stack trace naming the throwing frame must be reported:\n" + output); + assertTrue(!output.contains("UNCAUGHT_AFTER"), + "execution must stop at the throw, not carry on past it:\n" + output); + assertEquals(1, p.exitValue(), + "a program killed by an uncaught exception must not report success:\n" + output); + } + + private static byte[] readFully(Process p) throws Exception { + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + while ((read = p.getInputStream().read(buffer)) > 0) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } + + /** + * open() throws with nothing above it that catches. UNCAUGHT_AFTER lines mark + * every point the old behaviour would have carried on to. + */ + private static String uncaughtSource() { + return "public class UncaughtApp {\n" + + " static int open(int depth) {\n" + + " if (depth > 0) {\n" + + " return open(depth - 1);\n" + + " }\n" + + " throw new IllegalStateException(\"deliberate failure with a message\");\n" + + " }\n" + + " public static void main(String[] args) {\n" + + " System.out.println(\"UNCAUGHT_BEFORE\");\n" + + " int r = open(2);\n" + + " System.out.println(\"UNCAUGHT_AFTER value \" + r);\n" + + " }\n" + + "}\n"; + } +} diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/VirtualThreadRuntimeTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/VirtualThreadRuntimeTest.java new file mode 100644 index 00000000000..a172bfdcc3a --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/VirtualThreadRuntimeTest.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Builds and runs the virtual-thread runtime's own C test. + * + * The context switch is hand-written assembly, and the ways it can be wrong -- + * a clobbered callee-saved register, a stack that does not survive the round + * trip -- do not show up as a compile error or as a crash near the cause. They + * show up much later as a corrupted value in unrelated Java code, which is why + * the checks live in C where they can watch specific registers rather than in a + * translated program where they cannot. + * + * The test source used to be built by hand. Nothing ran it, so it was coverage + * on paper only: the switch could have been broken in any commit and stayed + * green. This drives it from the suite, out of the SAME staged resources a + * generated project gets, so it also proves those resources are present and + * mutually consistent -- the failure mode that shipped a project whose C half + * had no assembly to link against. + */ +class VirtualThreadRuntimeTest { + + @Test + @DisplayName("the virtual-thread switch preserves registers, stacks and ordering") + void runtimeTestsPass() throws Exception { + if (CompilerHelper.isWindows()) { + Assumptions.abort("the switch is not written for the Windows calling convention"); + } + String arch = System.getProperty("os.arch", ""); + if (!arch.equals("aarch64") && !arch.equals("arm64") + && !arch.equals("x86_64") && !arch.equals("amd64")) { + Assumptions.abort("no context switch is written for " + arch); + } + + Path work = Files.createTempDirectory("virtual-thread-runtime"); + // The same three resources emitVirtualThreadRuntime copies into a generated + // project. Reading them from the classpath rather than from the source tree + // means this fails when the build stops staging them, which is the thing that + // silently produces a project that cannot link. + for (String name : new String[] { + "cn1_virtual_thread.h", "cn1_virtual_thread.c", "cn1_virtual_thread_asm.S" }) { + try (InputStream in = ByteCodeTranslator.class.getResourceAsStream("/" + name)) { + assertTrue(in != null, name + " is not staged on the translator classpath"); + Files.copy(in, work.resolve(name)); + } + } + + Path testSource = Paths.get("virtualthread", "test_virtual_thread.c").toAbsolutePath(); + assertTrue(Files.exists(testSource), "missing " + testSource); + + Path binary = work.resolve("test_virtual_thread"); + List compile = new ArrayList<>(Arrays.asList( + "cc", "-O2", "-std=gnu11", "-I", work.toString(), + testSource.toString(), + work.resolve("cn1_virtual_thread.c").toString(), + work.resolve("cn1_virtual_thread_asm.S").toString(), + "-o", binary.toString())); + String compileOutput = run(compile, 5); + assertTrue(Files.exists(binary), "the runtime did not build:\n" + compileOutput); + + String output = run(Arrays.asList(binary.toString()), 5); + assertTrue(output.contains("ALL VIRTUAL THREAD TESTS PASSED"), + "the virtual-thread runtime reported a failure:\n" + output); + // Not asserted as a number: the cost is hardware- and load-dependent, and a + // threshold here would fail on a busy CI runner without anything being wrong. + // Its presence proves the timing loop ran at all. + assertTrue(output.contains("switch cost"), + "the switch-cost measurement did not run:\n" + output); + } + + private static String run(List command, int timeoutMinutes) throws Exception { + ProcessBuilder builder = new ProcessBuilder(command); + builder.redirectErrorStream(true); + Process p = builder.start(); + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + while ((read = p.getInputStream().read(buffer)) > 0) { + out.write(buffer, 0, read); + } + String output = new String(out.toByteArray(), StandardCharsets.UTF_8); + if (!p.waitFor(timeoutMinutes, TimeUnit.MINUTES)) { + p.destroyForcibly(); + fail("timed out: " + command + "\n" + output); + } + assertEquals(0, p.exitValue(), "failed: " + command + "\n" + output); + return output; + } +} diff --git a/vm/tests/virtualthread/test_virtual_thread.c b/vm/tests/virtualthread/test_virtual_thread.c index 9df8b893df7..727d5bcd1cd 100644 --- a/vm/tests/virtualthread/test_virtual_thread.c +++ b/vm/tests/virtualthread/test_virtual_thread.c @@ -23,9 +23,9 @@ /* Correctness first, cost second. A fast switch that corrupts a register or * loses a stack is not a foundation for a scheduler. */ -/* This exercises the BACKEND virtual-thread runtime, which is gated off - * everywhere else, so the test turns it on for itself rather than depending on - * whatever flags a caller happens to pass. */ +/* Define the gate rather than relying on the header's capability test, so that + * building this test on a target the switch is NOT written for is a loud + * assembler error instead of a silently vacuous pass. */ #ifndef CN1_VIRTUAL_THREADS #define CN1_VIRTUAL_THREADS 1 #endif From be886b6b06503e333aed1b8139840efcff23e43a Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 04:36:13 +0300 Subject: [PATCH 12/42] Close five review findings on the VM half All five were real. Taken together they are one theme: a virtual thread is a mutator the collector cannot see by the usual means, and the code that creates one was doing only half the job. RUNNING VIRTUAL THREADS LOOKED PARKED. cn1SpawnVirtualThread builds its VM state with bindToCallingOsThread false, which leaves threadActive FALSE, and nothing ever raised it. A collection running concurrently therefore treated a mutator executing Java as parked, and was free to scan or migrate its object stack and pending-allocation table underneath it -- missed roots at best, corruption at worst. The flag now moves with the context switch, up on resume and down on suspend, because a SUSPENDED virtual thread genuinely is parked: the collector reaches its roots through the registry snapshot instead. The transition is a weak symbol with a no-op default, not a function pointer. cn1_virtual_thread.c cannot include cn1_globals.h (the standalone runtime test builds it with no VM at all), an indirect call on a path whose entire value is that it costs 2.1ns is not free, and a weak symbol costs a direct call the linker resolves to the VM's real one when there is a VM. NOTHING RELEASED THE STATE. cn1VirtualThreadFree knows only about the coroutine. The VM state spawned beside it holds a 264KB shadow stack, the call-stack arrays, the pending-allocation table, and one of the NUMBER_OF_SUPPORTED_THREADS slots in allThreads. A virtual thread per request would have consumed a slot per completed request and eventually tripped CODENAME_ONE_ASSERT(threadOffset > -1). Added cn1RetireVirtualThread, which marks the state dead the way an OS thread's death does and then frees it with the same gcQueuedForDrain deferral the Java finalizer uses. THE UNCAUGHT-EXCEPTION EXIT WAS NOT GATED. This is the one that would have shipped. The generated main() is emitted for every target that has one, iOS and macOS included, and cn1AbortOnUncaughtException was set unconditionally -- so an uncaught exception on any thread would have terminated a shipped app. The comment sitting above it claimed the opposite ("Only this target opts in, so nothing that ships today changes behaviour"), which was simply false: the enclosing guard is `if(m.isMain())` and nothing more. Now gated on OUTPUT_TYPE_CLEAN. BLOCKING STDIN NEVER PARKED THE MUTATOR. System.in.read() waits as long as nobody types, with the thread left active, so a concurrent collection spun for a safepoint that could not arrive until a human pressed a key. Bracketed with CN1_YIELD_THREAD/CN1_RESUME_THREAD like the socket reads -- which then needs the keep-alive those reads also need, because only an interior pointer into the array is live across the call and the collector would otherwise sweep the buffer being filled. Portable here (a volatile store) rather than the Linux port's asm barrier, because this file also compiles under clang-cl. feof is read before the resume for the same reason errno is: the resume is a safepoint, and anything asked afterwards describes the wait. THE SHADOW STACK WAS FREED THE WRONG WAY. cn1AllocThreadStack falls back to calloc when mmap is out of MAPPINGS rather than out of memory, and cn1FreeThreadStack always called munmap. That fails with EINVAL and leaks the whole stack -- or, on an allocator that returns page-aligned blocks, unmaps memory the allocator still believes it owns. Which allocator answered is now recorded and the free is paired to it. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 16 +++ .../src/cn1_virtual_thread.c | 19 +++ .../src/cn1_virtual_thread.h | 14 +++ .../tools/translator/ByteCodeClass.java | 16 ++- vm/ByteCodeTranslator/src/nativeMethods.m | 117 ++++++++++++++++-- 5 files changed, 167 insertions(+), 15 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index babea79befb..71084f001fb 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -1252,6 +1252,14 @@ struct ThreadLocalData { // used by the GC to traverse the objects pointed to by this thread struct elementStruct* threadObjectStack; + /* How threadObjectStack was obtained, because the two allocators are not + interchangeable at free time: mmap pairs with munmap, calloc with free. + cn1AllocThreadStack falls back to calloc when mmap runs out of MAPPINGS + rather than out of memory, and munmap on an allocator-owned pointer fails + with EINVAL and leaks the whole shadow stack -- or, if the allocator handed + back a page-aligned block, unmaps memory the allocator still believes it + owns. */ + int threadObjectStackMapped; int threadObjectStackOffset; // allocations are stored here and then copied to the big memory pool during @@ -2841,6 +2849,14 @@ extern struct ThreadLocalData* cn1CreateThreadLocalData(JAVA_BOOLEAN bindToCalli /** A virtual thread with a Java stack of its own, ready to be resumed. */ extern struct cn1VirtualThread* cn1SpawnVirtualThread(void (*body)(void*), void* arg, size_t stackBytes); +/** + * The other half of cn1SpawnVirtualThread. Releases the coroutine AND the VM thread + * state spawned with it -- including its allThreads slot, without which a virtual + * thread per request exhausts NUMBER_OF_SUPPORTED_THREADS. cn1VirtualThreadFree + * alone releases only the coroutine. Never call it from inside the virtual thread's + * own body; it frees the stack that body is running on. + */ +extern void cn1RetireVirtualThread(struct cn1VirtualThread* vt); #ifdef CN1_CONSERVATIVE_GC_ROOTS // PHASE 3b production conservative-root API. cn1ConservativeResolve maps an diff --git a/vm/ByteCodeTranslator/src/cn1_virtual_thread.c b/vm/ByteCodeTranslator/src/cn1_virtual_thread.c index b67b9bdfbe0..ff9c9a53352 100644 --- a/vm/ByteCodeTranslator/src/cn1_virtual_thread.c +++ b/vm/ByteCodeTranslator/src/cn1_virtual_thread.c @@ -320,6 +320,11 @@ void cn1VirtualThreadStackBounds(struct cn1VirtualThread* co, void** low, void** /* Set up the initial frame so the first switch lands in the trampoline. */ extern void* cn1VirtualThreadPrime(void* stackHigh, void* co, void* trampoline); +/* The default. Overridden by the VM's strong definition when one is linked in. */ +__attribute__((weak)) void cn1VirtualThreadVmStateActive(void* vmState, int active) { + (void)vmState; (void)active; +} + void cn1VirtualThreadResume(struct cn1VirtualThread* co) { struct cn1VirtualThread* previous = cn1CurrentVirtualThread; if(co == 0 || co->finished) { @@ -331,7 +336,21 @@ void cn1VirtualThreadResume(struct cn1VirtualThread* co) { } cn1CurrentVirtualThread = co; co->running = 1; + /* The attached VM state has to become ACTIVE here, not just `running`. It was + * created parked (cn1CreateThreadLocalData with bindToCallingOsThread false + * leaves threadActive FALSE) and nothing else ever raises it, so without this a + * collection running concurrently treats a mutator that is executing Java as + * parked -- and scans or migrates its object stack and pending-allocation table + * underneath it. Missed roots at best, corruption at worst. Lowered again on the + * way out, because a SUSPENDED virtual thread genuinely is parked: the collector + * reaches its roots through the registry snapshot instead. */ + if(co->vmState != 0) { + cn1VirtualThreadVmStateActive(co->vmState, 1); + } cn1VirtualThreadSwitch(&co->returnSp, co->sp); + if(co->vmState != 0) { + cn1VirtualThreadVmStateActive(co->vmState, 0); + } co->running = 0; cn1CurrentVirtualThread = previous; } diff --git a/vm/ByteCodeTranslator/src/cn1_virtual_thread.h b/vm/ByteCodeTranslator/src/cn1_virtual_thread.h index b59baa8b2f4..d88fe5f16c6 100644 --- a/vm/ByteCodeTranslator/src/cn1_virtual_thread.h +++ b/vm/ByteCodeTranslator/src/cn1_virtual_thread.h @@ -75,6 +75,20 @@ struct cn1VirtualThread; +/* + * Tells the VM that the state attached to a virtual thread has started or stopped + * running Java, so the collector stops or resumes treating it as parked. + * + * It is a WEAK symbol with a no-op default rather than a function pointer for two + * reasons: an indirect call on a path whose whole point is that it costs 2.1ns is + * not free, and this file has to keep linking on its own -- the standalone runtime + * test builds it without any VM at all. nativeMethods.m provides the real one. + * + * Kept out of the header's no-op section deliberately: it is about the VM's view of + * a virtual thread, not about the switch, so it exists on every target. + */ +void cn1VirtualThreadVmStateActive(void* vmState, int active); + /** The body of a virtual thread. Returning from it finishes the virtual thread. */ typedef void (*cn1VirtualThreadBody)(void* arg); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java index 88eddc7da51..9d973069d90 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java @@ -1224,9 +1224,19 @@ public String generateCCode(List allClasses) { // catches (the EDT's own try), so it stayed invisible there; // a server binary has no such catch, and the symptom is a // process that keeps serving with a half-built object where a - // connection should be. Only this target opts in, so nothing - // that ships today changes behaviour. - b.append(" cn1AbortOnUncaughtException = 1;\n"); + // connection should be. + // + // GATED, and the gate is the point. This main() is emitted for + // every target that has one -- iOS and macOS included -- so an + // unconditional assignment here would make an uncaught exception + // on any thread terminate a SHIPPED app, which is exactly the + // behaviour change this runtime path is meant not to cause. Only + // the clean target, which has no upstream catch to rely on, opts + // in. (Reported on PR #5658: the comment that used to sit here + // claimed this was already restricted; it was not.) + if (ByteCodeTranslator.output == ByteCodeTranslator.OutputType.OUTPUT_TYPE_CLEAN) { + b.append(" cn1AbortOnUncaughtException = 1;\n"); + } // With the nursery, the main thread allocates and must cooperate with // the concurrent GC's stop-the-world pause (so the GC never scans its // nursery while a minor collection runs). Lightweight threads are the diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index fc36514cc61..f6d6a8a766d 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1062,8 +1062,11 @@ JAVA_VOID java_lang_System_arraycopy___java_lang_Object_int_java_lang_Object_int * the one that grew it. Reserving the range up front and letting the kernel decide * what is resident keeps every pointer stable. */ -static struct elementStruct* cn1AllocThreadStack(void) { +/* Reports through *mapped which allocator answered, because the caller cannot tell + from the pointer and the two do not free the same way. */ +static struct elementStruct* cn1AllocThreadStack(int* mapped) { size_t bytes = CN1_MAX_OBJECT_STACK_DEPTH * sizeof(struct elementStruct); + *mapped = 0; #if defined(_WIN32) /* VirtualAlloc would be the equivalent; calloc keeps the Windows target on one well-trodden path, and it is not the target where thread counts are large. */ @@ -1072,20 +1075,26 @@ JAVA_VOID java_lang_System_arraycopy___java_lang_Object_int_java_lang_Object_int void* p = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if(p == MAP_FAILED) { - /* Out of mappings rather than out of memory; calloc may still succeed. */ + /* Out of mappings rather than out of memory; calloc may still succeed. The + caller must remember this happened -- munmap on the result would fail with + EINVAL and leak the stack. */ return (struct elementStruct*)calloc(CN1_MAX_OBJECT_STACK_DEPTH, sizeof(struct elementStruct)); } + *mapped = 1; return (struct elementStruct*)p; #endif } -static void cn1FreeThreadStack(struct elementStruct* stack) { +/* mapped MUST be the value cn1AllocThreadStack reported for this pointer. */ +static void cn1FreeThreadStack(struct elementStruct* stack, int mapped) { if(stack == NULL) { return; } -#if defined(_WIN32) - free(stack); -#else + if(!mapped) { + free(stack); + return; + } +#if !defined(_WIN32) munmap(stack, CN1_MAX_OBJECT_STACK_DEPTH * sizeof(struct elementStruct)); #endif } @@ -1237,6 +1246,17 @@ JAVA_INT java_io_FileOutputStream_closeImpl___long_R_int(CODENAME_ONE_THREAD_STA return fclose(f) == 0 ? 0 : -1; } +/* Keeps a Java object provably live past a safepoint. Only an INTERIOR pointer into + an array is used across the blocking calls below, so the optimizer is free to drop + the array reference itself -- and the concurrent collector, scanning this parked + thread, then sees no root and sweeps the buffer while the read is still filling it. + The Linux port solves this with an asm barrier; this file also compiles under + clang-cl, which has no __asm__ __volatile__, so it uses a volatile store, which no + compiler may elide. The sink is written from several threads and never read: that + is the entire point of it, and the races are benign because no value is consumed. */ +static volatile JAVA_OBJECT cn1BlockingIoKeepAlive; +#define CN1_KEEP_ALIVE_ACROSS_SAFEPOINT(obj) do { cn1BlockingIoKeepAlive = (obj); } while(0) + // Standard input. Separate from FileInputStream because stdin is not seekable, so // skip/available cannot be implemented by the ftell dance above. JAVA_INT java_io_StandardInputStream_readImpl___byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { @@ -1244,9 +1264,22 @@ JAVA_INT java_io_StandardInputStream_readImpl___byte_1ARRAY_int_int_R_int(CODENA return -2; } JAVA_ARRAY_BYTE* data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; - size_t n = fread(&data[offset], 1, (size_t)length, stdin); + size_t n; + int atEof; + /* System.in.read() on a terminal or pipe waits for as long as nobody types. With + the thread left ACTIVE the concurrent collector spins for a safepoint this + thread cannot reach until input arrives -- on a target where forced-stop + escalation does not succeed, that is the whole VM stalled on a human. */ + CN1_YIELD_THREAD; + n = fread(&data[offset], 1, (size_t)length, stdin); + /* Read BEFORE the resume. CN1_RESUME_THREAD is a safepoint and can park this + thread on a timed wait, and anything the stream state is asked for afterwards + describes the wait rather than the read. */ + atEof = feof(stdin); + CN1_RESUME_THREAD; + CN1_KEEP_ALIVE_ACROSS_SAFEPOINT(buffer); if(n == 0) { - return feof(stdin) ? -1 : -2; + return atEof ? -1 : -2; } return (JAVA_INT)n; } @@ -2003,7 +2036,7 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC * lazily zeroed by the OS, so a shallow thread commits a few pages instead of * all of them. */ - i->threadObjectStack = cn1AllocThreadStack(); + i->threadObjectStack = cn1AllocThreadStack(&i->threadObjectStackMapped); i->threadObjectStackOffset = 0; i->callStackClass = calloc(CN1_MAX_STACK_CALL_DEPTH, sizeof(int)); @@ -2157,6 +2190,64 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC cn1VirtualThreadSetState(vt, state); return vt; } + +/* Both are defined further down this file; cn1RetireVirtualThread needs them here. */ +extern void markDeadThread(struct ThreadLocalData* d); +extern void cn1ReleaseThreadLocalData(struct ThreadLocalData* head); + +/* + * The strong definition of the weak hook cn1VirtualThreadResume calls. See the + * comment there for why the flag has to move with the switch. + */ +void cn1VirtualThreadVmStateActive(void* vmState, int active) { + struct ThreadLocalData* state = (struct ThreadLocalData*)vmState; + if(state != 0) { + state->threadActive = active ? JAVA_TRUE : JAVA_FALSE; + } +} + +/** + * Retire a virtual thread produced by cn1SpawnVirtualThread, releasing BOTH halves. + * + * cn1VirtualThreadFree alone is not enough and the difference is not a small leak. + * That function knows only about the coroutine: it unregisters it and releases the + * stack. The VM thread state spawned alongside it holds a 264KB shadow stack, the + * call-stack arrays and the pending-allocation table, and -- the part that ends the + * process rather than merely growing it -- one of the NUMBER_OF_SUPPORTED_THREADS + * slots in allThreads. A server that spawns a virtual thread per request and never + * came through here would consume a slot per completed request and eventually trip + * CODENAME_ONE_ASSERT(threadOffset > -1) in cn1CreateThreadLocalData. + * + * Must NOT be called from inside the virtual thread's own body: this releases the + * stack that body is running on. Retire it from whoever resumed it, after + * cn1VirtualThreadFinished reports true. + */ +void cn1RetireVirtualThread(struct cn1VirtualThread* vt) { + struct ThreadLocalData* state; + if(vt == 0) { + return; + } + state = (struct ThreadLocalData*)cn1VirtualThreadState(vt); + cn1VirtualThreadSetState(vt, 0); + if(state != 0) { + // Frees the allThreads slot and runs collectThreadResources, exactly as an + // OS thread's death does. + markDeadThread(state); + // Then the state itself, with the same deferral an OS thread's finalizer + // uses: if the collector has this TLD queued for drain, its pending + // allocations have not been migrated into allObjectsInHeap yet and freeing + // now would hand the drain a dangling pointer. + lockCriticalSection(); + if(state->gcQueuedForDrain) { + state->gcReleaseRequested = JAVA_TRUE; + unlockCriticalSection(); + } else { + unlockCriticalSection(); + cn1ReleaseThreadLocalData(state); + } + } + cn1VirtualThreadFree(vt); +} #endif /* CN1_VIRTUAL_THREADS -- see the capability gate in cn1_virtual_thread.h */ struct ThreadLocalData* getThreadLocalData() { @@ -2605,9 +2696,11 @@ JAVA_VOID java_lang_Thread_setPriorityImpl___int(CODENAME_ONE_THREAD_STATE, JAVA void cn1ReleaseThreadLocalData(struct ThreadLocalData *head) { free(head->blocks); - /* Mapped, not malloc'd -- see cn1AllocThreadStack. free() on a mapping is - undefined behaviour, not a leak, so this pairing matters. */ - cn1FreeThreadStack(head->threadObjectStack); + /* Free it the way it was ALLOCATED -- see cn1AllocThreadStack, which falls back + to calloc when mmap is out of mappings. Neither mismatch is survivable: free() + on a mapping is undefined behaviour, and munmap on an allocator block leaks the + stack at best. */ + cn1FreeThreadStack(head->threadObjectStack, head->threadObjectStackMapped); free(head->callStackClass); free(head->callStackLine); free(head->callStackMethod); From 03d2bf2250774f24a89eebaa49ea524eb3cbc9e8 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 04:36:31 +0300 Subject: [PATCH 13/42] Make the direct JSON writer agree with the map path, and test that it does Mapper.Direct's contract is to produce exactly what JSONWriter.toJson(toMap(instance)) would. Two fields did not, so a mapper changed its wire representation on the day it gained a direct writer: - A null List serialised as `null`, where the map path emits `[]` -- emitFieldToMap builds its ArrayList unconditionally and fills it only when the source is non-null. - Enum elements went through toString(). The map path uses Enum.name(), and deserialisation matches against the declared constants, so an enum that overrides toString() produced JSON that could not be read back at all. Every other element kind was checked rather than assumed: appendJsonValue already maps Date to getTime(), scalars and collections through writeJson, and a mapped object through its own mapper -- the same three answers emitFieldToMap gives. Nothing was comparing the two paths, which is why both got through. Every existing test exercises one route or the other, never one against the other, so the divergence was invisible to all of them. directJsonMatchesTheMapPathExactly runs an object with a populated list, an enum list, a Date and scalars, and then the same class with every list left null, asserting the two routes produce identical text. It asserts equality of the paths rather than against a literal on purpose: it keeps holding when a field kind is added, with nobody remembering to extend a hand-written expectation. Two things that test needed before it proved anything. It drives the generated mapper's own toJson rather than Mappers.appendJson, which goes through the registry -- unpopulated in an isolated classloader, so it fell back to toString() and compared the map path against "com.example.Swatch@23706db8". And it asserts the mapper actually implements Mapper.Direct, without which it would compare the map path with itself and pass while testing nothing. The test enum deliberately overrides toString() to disagree with name(), so the wrong choice cannot pass. Also drops a redundant `public` on the interface: PMD's UnnecessaryModifier, and a zero-findings gate. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/com/codename1/mapping/Mapper.java | 2 +- .../MappingAnnotationProcessor.java | 25 +++- .../MappingAnnotationProcessorTest.java | 114 ++++++++++++++++++ 3 files changed, 138 insertions(+), 3 deletions(-) diff --git a/CodenameOne/src/com/codename1/mapping/Mapper.java b/CodenameOne/src/com/codename1/mapping/Mapper.java index 4355f8eeedc..9d839fd6be5 100644 --- a/CodenameOne/src/com/codename1/mapping/Mapper.java +++ b/CodenameOne/src/com/codename1/mapping/Mapper.java @@ -69,7 +69,7 @@ public interface Mapper { /// Implemented as a separate interface rather than a method on `Mapper` so /// hand-written mappers keep compiling; `Mappers#toJson` uses it when the /// mapper offers it and falls back to `toMap` when it does not. - public interface Direct { + interface Direct { /// Appends `instance` as a JSON value -- an object, or the four /// characters `null`. Must produce exactly what diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java index 45a651069ce..7134207a8ea 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java @@ -614,14 +614,35 @@ private static void emitFieldToJson(StringBuilder sb, MappedField f, boolean isR ? read : read + ".asList()"; sb.append(" {\n"); sb.append(" java.util.List _src = ").append(src).append(";\n"); - sb.append(" if (_src == null) { out.append(\"null\"); }\n"); + // EMPTY ARRAY, not null. emitFieldToMap unconditionally builds an + // ArrayList and fills it only when the source is non-null, so the map + // path serialises a null list as []. The direct path has to agree: + // Mapper.Direct's contract is to produce exactly what + // JSONWriter.toJson(toMap(instance)) would, and a mapper silently + // changing a field's wire representation the day it gains a direct + // writer is the one thing that contract exists to prevent. + sb.append(" if (_src == null) { out.append(\"[]\"); }\n"); sb.append(" else {\n"); sb.append(" out.append('[');\n"); sb.append(" boolean _first = true;\n"); sb.append(" for (java.util.Iterator _it = _src.iterator(); _it.hasNext(); ) {\n"); sb.append(" if (!_first) { out.append(','); }\n"); sb.append(" _first = false;\n"); - sb.append(" com.codename1.mapping.Mappers.appendJsonValue(out, _it.next());\n"); + if (f.elementIsEnum) { + // name(), not toString(). The map path uses Enum.name() and + // deserialisation matches against the declared constants, so an + // enum that overrides toString() would serialise to something + // that cannot be read back. + sb.append(" Object _e = _it.next();\n"); + sb.append(" com.codename1.mapping.Mappers.appendJsonValue(out, _e == null ? null : ((") + .append(f.kind.elementBinaryName).append(") _e).name());\n"); + } else { + // Every other element kind already agrees: appendJsonValue maps + // Date to getTime(), scalars and collections to writeJson, and a + // mapped object through its own mapper -- the same three answers + // emitFieldToMap produces. + sb.append(" com.codename1.mapping.Mappers.appendJsonValue(out, _it.next());\n"); + } sb.append(" }\n"); sb.append(" out.append(']');\n"); sb.append(" }\n"); diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java index b5e1326eaef..188551284b7 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.maven.processors; @@ -277,6 +295,102 @@ private URLClassLoader childLoader(File classesDir) throws Exception { return new URLClassLoader(urls, getClass().getClassLoader()); } + /** + * The direct JSON writer must produce byte-for-byte what the map path produces. + * + * That is Mapper.Direct's entire contract, and nothing was checking it: two + * divergences shipped past review because every existing test exercises one path + * or the other, never both against each other. A null list serialised as `null` + * on the direct path and `[]` through the map, and enum elements went through + * toString() rather than name() -- so an enum that overrides toString() produced + * JSON that could not be read back at all. + * + * Asserting equality of the two paths rather than against a literal is deliberate: + * it keeps holding when a new field kind is added, without anyone remembering to + * come back and extend a hand-written expectation. + */ + @Test + public void directJsonMatchesTheMapPathExactly() throws Exception { + File classes = tmp.newFolder("direct-parity-classes"); + Map sources = new LinkedHashMap(); + // toString() deliberately disagrees with name(): if the direct path uses the + // wrong one, the two outputs differ and this test says so. + sources.put("com.example.Shade", + "package com.example;\n" + + "public enum Shade {\n" + + " LIGHT, DARK;\n" + + " @Override public String toString() { return \"shade-\" + name().toLowerCase(); }\n" + + "}\n"); + sources.put("com.example.Swatch", + "package com.example;\n" + + "import com.codename1.annotations.Mapped;\n" + + "import java.util.List;\n" + + "@Mapped public class Swatch {\n" + + " public String name;\n" + + " public int count;\n" + + " public Shade shade;\n" + + " public List shades;\n" + + " public List tags;\n" + + " public java.util.Date when;\n" + + " public Swatch() {}\n" + + "}\n"); + JavaSourceCompiler.compile(sources, classes, Arrays.asList(testClassesDir())); + runProcessorOrFail(classes); + + try (URLClassLoader cl = childLoader(classes)) { + Class shadeCls = cl.loadClass("com.example.Shade"); + Class swatchCls = cl.loadClass("com.example.Swatch"); + Class mapperCls = cl.loadClass("com.example.SwatchCn1Mapper"); + Object mapper = mapperCls.newInstance(); + Method valueOf = shadeCls.getMethod("valueOf", String.class); + Object dark = valueOf.invoke(null, "DARK"); + + // The generated mapper must actually BE on the direct path, or this test + // compares the map path with itself and passes while proving nothing. + Class directCls = cl.loadClass("com.codename1.mapping.Mapper$Direct"); + assertTrue("the generated mapper should implement Mapper.Direct", + directCls.isInstance(mapper)); + + Object populated = swatchCls.newInstance(); + swatchCls.getField("name").set(populated, "teal"); + swatchCls.getField("count").setInt(populated, 3); + swatchCls.getField("shade").set(populated, dark); + List shades = new ArrayList(); + shades.add(valueOf.invoke(null, "LIGHT")); + shades.add(dark); + swatchCls.getField("shades").set(populated, shades); + swatchCls.getField("tags").set(populated, Arrays.asList("a", "b")); + swatchCls.getField("when").set(populated, new java.util.Date(1234567890L)); + + // Every list left null: the case that diverged. + Object empty = swatchCls.newInstance(); + + assertDirectMatchesMap(cl, mapperCls, mapper, populated); + assertDirectMatchesMap(cl, mapperCls, mapper, empty); + } + } + + /** Both routes, on one instance, compared as text. */ + private static void assertDirectMatchesMap(URLClassLoader cl, Class mapperCls, + Object mapper, Object instance) throws Exception { + Class writerCls = cl.loadClass("com.codename1.io.JSONWriter"); + + Method toMap = mapperCls.getMethod("toMap", instance.getClass()); + Object asMap = toMap.invoke(mapper, instance); + String viaMap = (String) writerCls.getMethod("toJson", Object.class).invoke(null, asMap); + + // The generated mapper's OWN direct writer, not Mappers.appendJson: that + // goes through the registry, which this isolated classloader never + // populates, so it would quietly fall back to toString() and compare the + // map path against an object identity string. + StringBuilder out = new StringBuilder(); + mapperCls.getMethod("toJson", instance.getClass(), StringBuilder.class) + .invoke(mapper, instance, out); + String viaDirect = out.toString(); + + assertEquals("direct JSON must match the map path exactly", viaMap, viaDirect); + } + private static File testClassesDir() throws Exception { URL url = MappingAnnotationProcessorTest.class.getProtectionDomain() .getCodeSource().getLocation(); From 0119acd9bfffc30d93a99eb2ff0c901ac92f7e86 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 04:36:50 +0300 Subject: [PATCH 14/42] Give java.io.File a Windows implementation, and stage the header it now needs Two CI breakages, both from this branch making something reachable that had not been reached before. EVERY cn1lib NATIVE CHECK STOPPED AT A MISSING HEADER. cn1_globals.h now includes cn1_virtual_thread.h -- CN1_RESUME_THREAD yields a virtual thread rather than sleeping the carrier it runs on -- and two places stage the port headers into a scratch directory to compile a cn1lib against them. Neither knew about the second file, so both stopped at "'cn1_virtual_thread.h' file not found" before compiling a line: the six ad-cn1lib xcodebuild probes and check-cn1lib-native-sources.py. The workflow's path filters gain the header too, otherwise a future change to it skips the very check that would catch this. java.io.File HAD NO WINDOWS PATH. Its non-ObjC arm is POSIX-only -- unistd.h, dirent.h, access(), X_OK -- and Windows reaches that arm under clang-cl, which is neither __OBJC__ nor POSIX. It went unnoticed because java_io_File_runtime.c is emitted only when an app actually uses java.io.File, and until the clean target became a usable program runtime no Windows build ever did. Now every one of them failed on 'unistd.h' file not found. The Win32 arm: io.h and direct.h for _access, the access-mode constants the MSVC CRT does not define, and FindFirstFile for the directory walk, in the same two-pass shape as the POSIX one (count, allocate, refill) because allocArray can collect and the array must not be built with a find handle open. X_OK maps to an existence check: Win32's access model has no execute bit, and _access REJECTS a mode of 1 rather than answering "not executable". isHidden asks for FILE_ATTRIBUTE_HIDDEN instead of guessing from a leading dot, which means nothing on Windows. Everything else -- stat, remove, rename, mkdir -- the CRT already provides under the same names. Also merges two identical project() branches that SpotBugs flagged as DB_DUPLICATE_BRANCHES: Linux and the clean target answer the assembly question the same way, so they share one branch instead of two spelled alike. The POSIX arm is verified here (FileClassIntegrationTest, 5/5); the Win32 arm can only be verified by CI, which is what reported it. Co-Authored-By: Claude Opus 5 (1M context) --- .../workflows/ad-cn1lib-ios-native-check.yml | 7 ++ scripts/check-cn1lib-native-sources.py | 10 +- .../tools/translator/ByteCodeTranslator.java | 9 +- vm/ByteCodeTranslator/src/java_io_File.m | 115 ++++++++++++++++-- 4 files changed, 127 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ad-cn1lib-ios-native-check.yml b/.github/workflows/ad-cn1lib-ios-native-check.yml index cc5870ac308..738daa72248 100644 --- a/.github/workflows/ad-cn1lib-ios-native-check.yml +++ b/.github/workflows/ad-cn1lib-ios-native-check.yml @@ -21,6 +21,7 @@ on: - 'maven/cn1-applovin/**' - 'maven/cn1-unity-levelplay/**' - 'vm/ByteCodeTranslator/src/cn1_globals.h' + - 'vm/ByteCodeTranslator/src/cn1_virtual_thread.h' - '.github/workflows/ad-cn1lib-ios-native-check.yml' push: branches: [master] @@ -29,6 +30,7 @@ on: - 'maven/cn1-applovin/**' - 'maven/cn1-unity-levelplay/**' - 'vm/ByteCodeTranslator/src/cn1_globals.h' + - 'vm/ByteCodeTranslator/src/cn1_virtual_thread.h' - '.github/workflows/ad-cn1lib-ios-native-check.yml' concurrency: @@ -115,6 +117,11 @@ jobs: # cn1_globals.h for them. Reproduce that here, using the port's own # header so a change to those macros is caught too. cp vm/ByteCodeTranslator/src/cn1_globals.h "$PROBE/" + # cn1_globals.h includes this one (CN1_RESUME_THREAD yields a virtual + # thread rather than sleeping the carrier it runs on), so staging the + # first without the second stops the probe at "file not found" before it + # compiles a single line of the cn1lib. + cp vm/ByteCodeTranslator/src/cn1_virtual_thread.h "$PROBE/" # Generated per translation from the app's class list; nothing in the # ad bridges reads it, so an empty stand-in is enough to let # cn1_globals.h parse on its own. diff --git a/scripts/check-cn1lib-native-sources.py b/scripts/check-cn1lib-native-sources.py index c232fdd39d5..460ec48030f 100755 --- a/scripts/check-cn1lib-native-sources.py +++ b/scripts/check-cn1lib-native-sources.py @@ -35,9 +35,13 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) TRANSLATOR_SRC = os.path.join(REPO, 'vm', 'ByteCodeTranslator', 'src') -# cn1_globals.h includes cn1_win_compat.h under _WIN32 and pthread.h otherwise, -# so the Windows half does not even parse without the compat header beside it. -PORT_HEADERS = ['cn1_globals.h', 'cn1_win_compat.h'] +# Everything cn1_globals.h pulls in has to sit beside it or the probe stops at +# "file not found" before it compiles a line of the cn1lib. It includes +# cn1_win_compat.h under _WIN32 and pthread.h otherwise, so the Windows half does +# not even parse without the compat header; and it includes cn1_virtual_thread.h +# unconditionally, because CN1_RESUME_THREAD yields a virtual thread rather than +# sleeping the carrier it runs on. +PORT_HEADERS = ['cn1_globals.h', 'cn1_win_compat.h', 'cn1_virtual_thread.h'] def libraries(): diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index cd53abd0133..7c687c09f9b 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -1092,12 +1092,15 @@ private static void writeCmakeProject(File projectRoot, File srcRoot, String app } } if (windows) { + // Windows declares no ASM: MSVC cannot assemble GNU syntax, and the + // cross-compiled case is handled by an enable_language(ASM) guarded on + // NOT MSVC further down, once project() has told CMake which it got. writer.append("project(").append(appName).append(embedResources ? " LANGUAGES C CXX RC)\n" : " LANGUAGES C CXX)\n"); - } else if (linux) { - writer.append("project(").append(appName).append(hasAsm - ? " LANGUAGES C ASM)\n" : " LANGUAGES C)\n"); } else { + // Linux and the clean target answer this identically -- assembly is + // declared when a .S is actually present -- so they share one branch + // rather than two spelled the same way. writer.append("project(").append(appName).append(hasAsm ? " LANGUAGES C ASM)\n" : " LANGUAGES C)\n"); } diff --git a/vm/ByteCodeTranslator/src/java_io_File.m b/vm/ByteCodeTranslator/src/java_io_File.m index 080d71f2d49..9bdc76ae1db 100644 --- a/vm/ByteCodeTranslator/src/java_io_File.m +++ b/vm/ByteCodeTranslator/src/java_io_File.m @@ -312,13 +312,44 @@ JAVA_OBJECT java_io_File_getCanonicalPathImpl___java_lang_String_R_java_lang_Str } #else -// POSIX implementation for non-ObjC environments (e.g. Linux CI) +// Implementation for non-ObjC environments: Linux CI, the native Windows port and +// the clean target. Windows reaches this branch under clang-cl, which is neither +// __OBJC__ nor POSIX. #include #include -#include -#include #include #include +#ifdef _WIN32 +/* clang-cl ships no and no . Only two things in this file + actually need them -- access() and the directory walk -- and the MSVC CRT + provides everything else (stat, remove, rename, mkdir) under the same names. + Without these guards the whole file stopped at "'unistd.h' file not found", + which is what every Windows clean-target build did the moment an app first + reached java.io.File. */ +#include +#include +#include +#ifndef F_OK +#define F_OK 0 +#endif +#ifndef R_OK +#define R_OK 4 +#endif +#ifndef W_OK +#define W_OK 2 +#endif +/* No execute bit exists in the Win32 access() model, and _access REJECTS a mode + of 1 rather than reporting "not executable". Ask whether the file exists, which + is the closest true answer and what the JDK reports for a readable file. */ +#ifndef X_OK +#define X_OK 0 +#endif +#define CN1_FILE_ACCESS(p, m) _access((p), (m)) +#else +#include +#include +#define CN1_FILE_ACCESS(p, m) access((p), (m)) +#endif // Helper: assumes stringToUTF8 is available (implemented in test stubs or runtime) extern const char* stringToUTF8(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT str); @@ -327,7 +358,7 @@ JAVA_OBJECT java_io_File_getCanonicalPathImpl___java_lang_String_R_java_lang_Str JAVA_BOOLEAN java_io_File_existsImpl___java_lang_String_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path) { if(path == JAVA_NULL) return JAVA_FALSE; const char* p = stringToUTF8(threadStateData, path); - return access(p, F_OK) != -1; + return CN1_FILE_ACCESS(p, F_OK) != -1; } JAVA_BOOLEAN java_io_File_isDirectoryImpl___java_lang_String_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path) { @@ -353,11 +384,20 @@ JAVA_BOOLEAN java_io_File_isFileImpl___java_lang_String_R_boolean(CODENAME_ONE_T JAVA_BOOLEAN java_io_File_isHiddenImpl___java_lang_String_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path) { if(path == JAVA_NULL) return JAVA_FALSE; const char* p = stringToUTF8(threadStateData, path); +#ifdef _WIN32 + /* Windows has a real hidden ATTRIBUTE; a leading dot means nothing there. */ + { + DWORD attr = GetFileAttributesA(p); + return (attr != INVALID_FILE_ATTRIBUTES && (attr & FILE_ATTRIBUTE_HIDDEN)) + ? JAVA_TRUE : JAVA_FALSE; + } +#else // This is a naive check, checking if filename starts with dot // We need to find the last slash const char* lastSlash = strrchr(p, '/'); const char* name = lastSlash ? lastSlash + 1 : p; return name[0] == '.'; +#endif } JAVA_LONG java_io_File_lastModifiedImpl___java_lang_String_R_long(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path) { @@ -387,7 +427,7 @@ JAVA_LONG java_io_File_lengthImpl___java_lang_String_R_long(CODENAME_ONE_THREAD_ JAVA_BOOLEAN java_io_File_createNewFileImpl___java_lang_String_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path) { if(path == JAVA_NULL) return JAVA_FALSE; const char* p = stringToUTF8(threadStateData, path); - if (access(p, F_OK) != -1) return JAVA_FALSE; + if (CN1_FILE_ACCESS(p, F_OK) != -1) return JAVA_FALSE; FILE* f = fopen(p, "w"); if (f) { fclose(f); @@ -407,6 +447,64 @@ JAVA_OBJECT java_io_File_listImpl___java_lang_String_R_java_lang_String_1ARRAY(C if(path == JAVA_NULL) return JAVA_NULL; enteringNativeAllocations(); const char* p = stringToUTF8(threadStateData, path); +#ifdef _WIN32 + /* FindFirstFile rather than opendir, and it wants a wildcard appended. Two + passes like the POSIX arm below: count, allocate, refill -- allocArray can + collect, so the array cannot be built while a find handle is open. */ + { + char pattern[MAX_PATH]; + WIN32_FIND_DATAA fd; + HANDLE h; + int count = 0; + JAVA_OBJECT arr; + size_t plen = strlen(p); + if (plen == 0 || plen + 3 > sizeof(pattern)) { + finishedNativeAllocations(); + return JAVA_NULL; + } + memcpy(pattern, p, plen); + /* Do not double a separator the caller already supplied. */ + if (p[plen - 1] == '\\' || p[plen - 1] == '/') { + pattern[plen] = '*'; + pattern[plen + 1] = '\0'; + } else { + pattern[plen] = '\\'; + pattern[plen + 1] = '*'; + pattern[plen + 2] = '\0'; + } + h = FindFirstFileA(pattern, &fd); + if (h == INVALID_HANDLE_VALUE) { + finishedNativeAllocations(); + return JAVA_NULL; + } + do { + if (strcmp(fd.cFileName, ".") == 0 || strcmp(fd.cFileName, "..") == 0) continue; + count++; + } while (FindNextFileA(h, &fd)); + FindClose(h); + + arr = allocArray(threadStateData, count, &class__java_lang_String, sizeof(JAVA_OBJECT), 1); + + h = FindFirstFileA(pattern, &fd); + if (h == INVALID_HANDLE_VALUE) { + finishedNativeAllocations(); + return arr; + } + count = 0; + do { + if (strcmp(fd.cFileName, ".") == 0 || strcmp(fd.cFileName, "..") == 0) continue; + { + JAVA_OBJECT s = newStringFromCString(threadStateData, fd.cFileName); + CN1_SET_ARRAY_ELEMENT_OBJECT(arr, count, s); + } + count++; + } while (FindNextFileA(h, &fd)); + FindClose(h); + + finishedNativeAllocations(); + return arr; + } +#else DIR* d = opendir(p); if (d == NULL) { finishedNativeAllocations(); @@ -436,6 +534,7 @@ JAVA_OBJECT java_io_File_listImpl___java_lang_String_R_java_lang_String_1ARRAY(C finishedNativeAllocations(); return arr; +#endif } JAVA_BOOLEAN java_io_File_mkdirImpl___java_lang_String_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path) { @@ -476,19 +575,19 @@ JAVA_BOOLEAN java_io_File_setExecutableImpl___java_lang_String_boolean_R_boolean JAVA_BOOLEAN java_io_File_canReadImpl___java_lang_String_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path) { if(path == JAVA_NULL) return JAVA_FALSE; const char* p = stringToUTF8(threadStateData, path); - return access(p, R_OK) != -1; + return CN1_FILE_ACCESS(p, R_OK) != -1; } JAVA_BOOLEAN java_io_File_canWriteImpl___java_lang_String_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path) { if(path == JAVA_NULL) return JAVA_FALSE; const char* p = stringToUTF8(threadStateData, path); - return access(p, W_OK) != -1; + return CN1_FILE_ACCESS(p, W_OK) != -1; } JAVA_BOOLEAN java_io_File_canExecuteImpl___java_lang_String_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path) { if(path == JAVA_NULL) return JAVA_FALSE; const char* p = stringToUTF8(threadStateData, path); - return access(p, X_OK) != -1; + return CN1_FILE_ACCESS(p, X_OK) != -1; } JAVA_LONG java_io_File_getTotalSpaceImpl___java_lang_String_R_long(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path) { From 9b4b1cacc3907c9ebafdb2755458e8f0bb9c3918 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 04:51:55 +0300 Subject: [PATCH 15/42] An archive must not write outside the directory it is unpacked into CodeQL java/zipslip, high severity. unzip() built each output path by concatenating the destination with ZipEntry.getName(), unchecked, so an entry named "../../x" wrote wherever the archive asked. Both callers unpack a DOWNLOADED zip -- Groovy for the console, JavaFX for the browser component -- so the archive is not something the user authored, and the consequence is an arbitrary file overwritten under their account while they believe they are unpacking a dependency. CWE-22. Every entry now has to resolve inside the destination or it is refused. The comparison is between CANONICAL paths -- resolving the ".." is the whole point -- and it uses java.nio.file.Path.startsWith rather than String.startsWith, for two reasons. Path compares COMPONENT-wise, so a sibling like "/tmp/dest-evil" is rejected against "/tmp/dest" where a character-wise prefix accepts it, and giving the string prefix a trailing separator to fix that then wrongly rejects the destination directory itself. It is also the shape CodeQL recognises as a sanitizer: the first attempt here was a correct canonical-path check that the query still flagged, because a compound `!a && !b` guard did not read as a barrier. Two things the fix had to bring with it, both found by writing the test: - Parent directories are created before extracting. FileOutputStream will not create them, and a nested entry can arrive before the directory entry that holds it, so "nested/deep/leaf.txt" in an archive that declares no directory entries threw FileNotFoundException. That was broken before this change too. - destDir uses mkdirs rather than mkdir, so a destination more than one level deep is actually created. Both streams are closed in a finally, which they were not: an IOException mid-extract leaked the descriptor. The test builds the malicious archive rather than checking one in -- a committed zip that escapes its destination is an awkward thing to keep in a repository, and building it puts the attack in front of the reader. Verified non-vacuous by reverting the fix: 2 failures against the old code, 0 against the new. Co-Authored-By: Claude Opus 5 (1M context) --- .../codename1/impl/javase/UnzipUtility.java | 100 +++++++++---- .../impl/javase/UnzipUtilityZipSlipTest.java | 139 ++++++++++++++++++ 2 files changed, 213 insertions(+), 26 deletions(-) create mode 100644 maven/javase/src/test/java/com/codename1/impl/javase/UnzipUtilityZipSlipTest.java diff --git a/Ports/JavaSE/src/com/codename1/impl/javase/UnzipUtility.java b/Ports/JavaSE/src/com/codename1/impl/javase/UnzipUtility.java index 9a3697ebcd4..9f194a2fe89 100644 --- a/Ports/JavaSE/src/com/codename1/impl/javase/UnzipUtility.java +++ b/Ports/JavaSE/src/com/codename1/impl/javase/UnzipUtility.java @@ -1,9 +1,32 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ package com.codename1.impl.javase; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Path; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -23,42 +46,67 @@ public class UnzipUtility { public void unzip(String zipFilePath, String destDirectory) throws IOException { File destDir = new File(destDirectory); if (!destDir.exists()) { - destDir.mkdir(); + destDir.mkdirs(); } + // Canonical, because that is what resolves the "../" an archive can carry. + Path destRoot = destDir.getCanonicalFile().toPath(); ZipInputStream zipIn = new ZipInputStream(new FileInputStream(zipFilePath)); - ZipEntry entry = zipIn.getNextEntry(); - // iterates over entries in the zip file - while (entry != null) { - String filePath = destDirectory + File.separator + entry.getName(); - if (!entry.isDirectory()) { - // if the entry is a file, extracts it - extractFile(zipIn, filePath); - } else { - // if the entry is a directory, make the directory - File dir = new File(filePath); - dir.mkdir(); + try { + ZipEntry entry = zipIn.getNextEntry(); + // iterates over entries in the zip file + while (entry != null) { + // ZIP SLIP. An entry name is attacker-controlled and may be + // "../../something": concatenating it onto the destination writes + // wherever the archive says, which for these two callers means an + // arbitrary file overwritten under the user's account while they + // believe they are unpacking Groovy or JavaFX. Refuse anything that + // does not land inside the destination. + // + // Path.startsWith compares COMPONENT-wise, not character-wise, so + // "/tmp/dest-evil" is correctly rejected against "/tmp/dest" -- a + // plain String.startsWith accepts it unless the prefix is given a + // trailing separator, and then it wrongly rejects the destination + // itself. Neither trap exists here. + Path target = new File(destDir, entry.getName()).getCanonicalFile().toPath(); + if (!target.startsWith(destRoot)) { + throw new IOException("Zip entry escapes the destination directory: " + + entry.getName()); + } + File targetFile = target.toFile(); + if (!entry.isDirectory()) { + // Nested entries can arrive before the directory that holds + // them, and FileOutputStream will not create it. + File parent = targetFile.getParentFile(); + if (parent != null) { + parent.mkdirs(); + } + extractFile(zipIn, targetFile); + } else { + targetFile.mkdirs(); + } + zipIn.closeEntry(); + entry = zipIn.getNextEntry(); } - zipIn.closeEntry(); - entry = zipIn.getNextEntry(); + } finally { + zipIn.close(); } - zipIn.close(); } /** * Extracts a zip entry (file entry) * @param zipIn - * @param filePath + * @param target * @throws IOException */ - private void extractFile(ZipInputStream zipIn, String filePath) throws IOException { - BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath)); - byte[] bytesIn = new byte[BUFFER_SIZE]; - int read = 0; - while ((read = zipIn.read(bytesIn)) != -1) { - bos.write(bytesIn, 0, read); + private void extractFile(ZipInputStream zipIn, File target) throws IOException { + BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(target)); + try { + byte[] bytesIn = new byte[BUFFER_SIZE]; + int read = 0; + while ((read = zipIn.read(bytesIn)) != -1) { + bos.write(bytesIn, 0, read); + } + } finally { + bos.close(); } - bos.close(); } } - - - \ No newline at end of file diff --git a/maven/javase/src/test/java/com/codename1/impl/javase/UnzipUtilityZipSlipTest.java b/maven/javase/src/test/java/com/codename1/impl/javase/UnzipUtilityZipSlipTest.java new file mode 100644 index 00000000000..0f12f1cb7df --- /dev/null +++ b/maven/javase/src/test/java/com/codename1/impl/javase/UnzipUtilityZipSlipTest.java @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.javase; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * An archive must not be able to write outside the directory it is unpacked into. + * + * `unzip` used to build each output path by concatenating the destination with + * `ZipEntry.getName()`, unchecked. An entry named `../../x` therefore wrote + * wherever the archive asked -- and both callers unpack a DOWNLOADED zip (Groovy + * for the console, JavaFX for the browser component), so the archive is not + * something the user authored. That is CWE-22, and CodeQL's java/zipslip. + * + * The malicious archive is built here rather than checked in as a fixture: a + * committed zip that escapes its destination is an awkward thing to have in a + * repository, and building it makes the attack visible in the test itself. + */ +class UnzipUtilityZipSlipTest { + + @Test + void anEntryThatEscapesTheDestinationIsRefused(@TempDir Path tmp) throws IOException { + Path dest = tmp.resolve("dest"); + Path outside = tmp.resolve("outside.txt"); + File zip = tmp.resolve("evil.zip").toFile(); + + // "../outside.txt" resolves out of dest and into tmp. + try (ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zip))) { + out.putNextEntry(new ZipEntry("harmless.txt")); + out.write("ok".getBytes(StandardCharsets.UTF_8)); + out.closeEntry(); + out.putNextEntry(new ZipEntry("../outside.txt")); + out.write("pwned".getBytes(StandardCharsets.UTF_8)); + out.closeEntry(); + } + + IOException e = assertThrows(IOException.class, + () -> new UnzipUtility().unzip(zip.getAbsolutePath(), dest.toString()), + "an entry escaping the destination must be refused, not written"); + assertTrue(e.getMessage().contains("escapes the destination"), + "the refusal should say why: " + e.getMessage()); + assertFalse(Files.exists(outside), + "the escaping entry must not have been written to " + outside); + } + + @Test + void anEntryEscapingIntoASiblingWithTheSamePrefixIsRefused(@TempDir Path tmp) throws IOException { + // "dest-evil" shares a character prefix with "dest" but is a different + // directory. A containment check written as a plain string startsWith + // accepts this; a component-wise Path comparison rejects it. + Path dest = tmp.resolve("dest"); + Path evilDir = tmp.resolve("dest-evil"); + Path sibling = evilDir.resolve("loot.txt"); + File zip = tmp.resolve("sibling.zip").toFile(); + + // The target directory must already EXIST, or unguarded code fails with a + // FileNotFoundException -- which is an IOException, so assertThrows would be + // satisfied by the write merely failing rather than by the check refusing it. + // That is exactly how this test passed against the vulnerable version on its + // first run. + Files.createDirectories(evilDir); + + try (ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zip))) { + out.putNextEntry(new ZipEntry("../dest-evil/loot.txt")); + out.write("pwned".getBytes(StandardCharsets.UTF_8)); + out.closeEntry(); + } + + IOException e = assertThrows(IOException.class, + () -> new UnzipUtility().unzip(zip.getAbsolutePath(), dest.toString()), + "a sibling directory sharing a prefix is still outside the destination"); + assertTrue(e.getMessage() != null && e.getMessage().contains("escapes the destination"), + "must be refused by the containment check, not by an incidental IO failure: " + + e); + assertFalse(Files.exists(sibling), "nothing should have been written to " + sibling); + } + + @Test + void ordinaryArchivesStillExtract(@TempDir Path tmp) throws IOException { + Path dest = tmp.resolve("dest"); + File zip = tmp.resolve("plain.zip").toFile(); + + // Includes a nested entry whose directory is never declared: the extractor + // has to create the parent itself, which the containment fix also had to + // keep working. + try (ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zip))) { + out.putNextEntry(new ZipEntry("top.txt")); + out.write("one".getBytes(StandardCharsets.UTF_8)); + out.closeEntry(); + out.putNextEntry(new ZipEntry("nested/deep/leaf.txt")); + out.write("two".getBytes(StandardCharsets.UTF_8)); + out.closeEntry(); + } + + new UnzipUtility().unzip(zip.getAbsolutePath(), dest.toString()); + + assertEquals("one", new String(Files.readAllBytes(dest.resolve("top.txt")), + StandardCharsets.UTF_8)); + assertEquals("two", new String( + Files.readAllBytes(dest.resolve("nested").resolve("deep").resolve("leaf.txt")), + StandardCharsets.UTF_8)); + } +} From cea27a2476b717b72275c00ae95281a5b406a696 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 04:51:55 +0300 Subject: [PATCH 16/42] Finish the Windows java.io.File arm: the four gaps behind the first one Removing the unistd.h/dirent.h dependency got clang-cl past the first error and into four more, all the same kind -- POSIX spellings the MSVC CRT does not have: - `redefinition of 'timeval'`. pulls in , whose timeval collides with the one cn1_win_compat.h defines. WIN32_LEAN_AND_MEAN keeps winsock out, and nothing here wants it. - S_ISDIR / S_ISREG undeclared. The CRT has the st_mode BITS but not the macros that test them, so they are defined from _S_IFMT/_S_IFDIR/_S_IFREG. - PATH_MAX undeclared -- MAX_PATH is the Win32 spelling. - realpath undeclared. _fullpath is the equivalent, but it takes (destination, source), the REVERSE of realpath's (source, destination), so the macro swaps them. Getting that backwards compiles and canonicalizes the wrong string in silence. It also resolves a path that does not exist rather than failing, which is the more useful answer for getCanonicalPath. The POSIX arm is unchanged and still verified here (FileClassIntegrationTest, 5/5). The Windows arm is verified only by CI, which is what reported both rounds. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/java_io_File.m | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/vm/ByteCodeTranslator/src/java_io_File.m b/vm/ByteCodeTranslator/src/java_io_File.m index 9bdc76ae1db..b59b35e0351 100644 --- a/vm/ByteCodeTranslator/src/java_io_File.m +++ b/vm/ByteCodeTranslator/src/java_io_File.m @@ -328,7 +328,31 @@ provides everything else (stat, remove, rename, mkdir) under the same names. reached java.io.File. */ #include #include +#include +/* WIN32_LEAN_AND_MEAN keeps out of . Without it winsock's + own `struct timeval` collides with the one cn1_win_compat.h defines, and the + file fails on "redefinition of 'timeval'" rather than on anything it does. */ +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif #include +/* The MSVC CRT has the st_mode BITS but not the POSIX macros that test them. */ +#ifndef S_ISDIR +#define S_ISDIR(m) (((m) & _S_IFMT) == _S_IFDIR) +#endif +#ifndef S_ISREG +#define S_ISREG(m) (((m) & _S_IFMT) == _S_IFREG) +#endif +/* PATH_MAX is POSIX; MAX_PATH is the Win32 spelling. realpath's counterpart is + _fullpath, which takes (destination, source) -- the REVERSE of realpath's + (source, destination) -- so the macro swaps them; getting that backwards + compiles and silently canonicalizes the wrong string. Both return NULL on + failure. _fullpath also resolves a path that does not exist rather than + failing, which is the more useful answer for getCanonicalPath. */ +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif +#define realpath(path, resolved) _fullpath((resolved), (path), MAX_PATH) #ifndef F_OK #define F_OK 0 #endif From ba22565ae57596b87c1f3844fef09c2e5c36a390 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 05:13:09 +0300 Subject: [PATCH 17/42] Zero a pthread_t portably, and stop declaring a local Windows cannot use Two Windows-only build breaks in this branch's own new code, both invisible on the POSIX legs. `i->gcPthread = 0` for a virtual thread's state is a type error under clang-cl: pthread_t is a POINTER on Apple and glibc, but the Windows compat shim defines it as struct {handle, id}, so the assignment reads as "assigning to 'pthread_t' from incompatible type 'int'". memset over sizeof is correct for both shapes, and gcPthreadValid -- set FALSE on the next line -- is what actually gates every read of the field. cn1AllocThreadStack declared its byte count above the #if that uses it, so on Windows, whose arm calls calloc with the element count instead, it was an unused local. Moved onto the arm that uses it. Swept the rest of this branch's additions for the same class of thing rather than waiting for CI to find them one at a time: every other POSIX call in code Windows compiles is either guarded (mmap/munmap behind !_WIN32, pthread_attr_setstacksize behind __linux__) or shimmed in cn1_win_compat.h (usleep, pthread_key_create, pthread_getspecific). The virtual-thread runtime -- including the __attribute__((weak)) definition, which clang-cl treats differently on COFF -- is entirely inside the CN1_VIRTUAL_THREADS gate, which excludes _WIN32, so none of it is compiled there at all. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/nativeMethods.m | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index f6d6a8a766d..b002e50e4ed 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1065,13 +1065,15 @@ JAVA_VOID java_lang_System_arraycopy___java_lang_Object_int_java_lang_Object_int /* Reports through *mapped which allocator answered, because the caller cannot tell from the pointer and the two do not free the same way. */ static struct elementStruct* cn1AllocThreadStack(int* mapped) { - size_t bytes = CN1_MAX_OBJECT_STACK_DEPTH * sizeof(struct elementStruct); *mapped = 0; #if defined(_WIN32) /* VirtualAlloc would be the equivalent; calloc keeps the Windows target on one well-trodden path, and it is not the target where thread counts are large. */ return (struct elementStruct*)calloc(CN1_MAX_OBJECT_STACK_DEPTH, sizeof(struct elementStruct)); #else + /* Declared here rather than above the #if: it is used only on this arm, and on + Windows it was an unused local the compiler is entitled to warn about. */ + size_t bytes = CN1_MAX_OBJECT_STACK_DEPTH * sizeof(struct elementStruct); void* p = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if(p == MAP_FAILED) { @@ -2127,7 +2129,13 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC // stack through allThreads like everyone else. cn1TlsSelf must keep naming // the HOST thread, because the async-signal stop handler runs on the host // and needs the host's state. - i->gcPthread = 0; + // memset rather than `= 0`: pthread_t is a POINTER on Apple and glibc but a + // struct {handle, id} in the Windows compat shim, where assigning 0 is not + // even a type error the reader would expect -- it is "assigning to + // 'pthread_t' from incompatible type 'int'", and it failed only the Windows + // and cross-compile legs. Zeroing the bytes is correct for both shapes, and + // gcPthreadValid below is what actually gates every read of this field. + memset(&i->gcPthread, 0, sizeof(i->gcPthread)); i->gcPthreadValid = JAVA_FALSE; } #endif From 80da86d695fc540a93521ad7d3135fd68d59ba1f Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 06:55:46 +0300 Subject: [PATCH 18/42] Close the VM-half review findings, and record the two that stay open A null array crashed instead of throwing (P1). CN1_ARRAY_STORE_CHECK evaluates CN1_CLASS_OF(arrayObj) with no null guard, and under -Dcn1.checkedCasts it runs AHEAD of the setter that turns a null array into a NullPointerException -- so an object-array store through a null array took the process down. Java orders NPE ahead of ArrayStoreException anyway, so falling through to the setter is both the safe answer and the correct one. A virtual thread's stack could go unmarked mid-switch (P1). The parked-stack pass skipped anything cn1VirtualThreadIsRunning() reported, on the reasoning that the carrier covers those. It does -- but only once the carrier's stack pointer is actually INSIDE the virtual stack, and `running` is raised before the switch and lowered after the switch back. In those two windows a stopped carrier still has an OS-stack pointer, so cn1VirtualThreadForStackAddress matches nothing, the carrier pass scans only the OS stack, and this pass skipped the virtual stack for being "running". References held in C temporaries there could be swept. The flag cannot be made atomic with the switch it brackets, because the switch is what changes the stack the flag would have to be written from. So the passes now OVERLAP instead of partitioning: every virtual thread's saved region is scanned unconditionally. Safe, because [sp, stackHigh) is inside the mapping whenever sp is non-zero; complete, because while a virtual thread runs the carrier's pointer is lower, so this pass covers a subset and the carrier covers the rest; and cheap, because conservative marking is idempotent. cn1RetireVirtualThread's "use after free" was NOT one, and the code now says so. markDeadThread -> collectThreadResources sets gcQueuedForDrain unconditionally and has no early return, so the synchronous release branch was unreachable. It read as live, though, so it is gone and the invariant is written down -- including the reason it matters, which the report had right: codenameOneGCMark copies each ThreadLocalData* out of allThreads under the critical section and dereferences it OUTSIDE the lock, so a synchronous free would be a genuine use-after-free. File.list returned something that called itself a String. All three arms passed the ELEMENT class to allocArray, which installs whatever it is given as the array object's own class; cn1MainArgs has always passed class_array1__java_lang_String. Pre-existing on iOS and Linux, copied into the new Windows arm, fixed on all three. Windows absolute paths were treated as relative, which corrupted them rather than merely misreporting them: getAbsolutePathImpl tested p[0] == '/', so "C:\data" had the working directory prepended. There is now a per-platform predicate that knows about drive letters and UNC roots. The matching Java-side gap is deliberately left and documented at the predicate: File.isAbsolute() tests startsWith(File.separator) and separator is "/" everywhere, which needs a per-platform separator in shared JavaAPI -- a change for every port, not for making the clean target build. Blocking file reads and writes now park the mutator, like the socket reads and StandardInputStream already did: a FIFO, a device or a network-backed path blocks for as long as the far end stays quiet, and an active thread there strands the collector waiting for a safepoint that cannot arrive. Both carry the buffer keep-alive for the same reason those do -- only an interior pointer is live across the call. (Moving that macro above its first use is why it now sits at the top of the file layer rather than beside stdin.) The benchmark helper compiles the emitted .S. Third place with this bug: the CMake generator and the Xcode project generator had it too, and a *.c-only invocation links against a missing cn1VirtualThreadSwitch on any target where the switch exists. Two findings are recorded in the file rather than fixed, with the analysis and the actual remedy: 32-bit ftell/fseek cannot express a position past 2GiB where C long is 32 bits, and paths reach the narrow CRT as UTF-8 and are read as ANSI. Both are pre-existing on every platform, both want a change across the whole file layer, and neither is what enabling the clean target is about. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 8 +- vm/ByteCodeTranslator/src/cn1_globals.m | 24 +++++- vm/ByteCodeTranslator/src/java_io_File.m | 68 ++++++++++++++--- vm/ByteCodeTranslator/src/nativeMethods.m | 92 +++++++++++++++++------ vm/benchmarks/translate-and-build.sh | 11 ++- 5 files changed, 164 insertions(+), 39 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 71084f001fb..fb848f80282 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -474,8 +474,14 @@ typedef struct clazz* JAVA_CLASS; // // arrayType is the component class (0 for a non-array, which cannot happen here // after CHECK_ARRAY_ACCESS, but is tolerated rather than dereferenced). +/* The arrayObj null test is not redundant. This runs BEFORE + CN1_SET_ARRAY_ELEMENT_OBJECT, which is where a null array is turned into a + NullPointerException; CN1_CLASS_OF below would dereference the null first and + take the process down instead. Java also orders it this way -- NPE wins over + ArrayStoreException -- so falling through to the setter is both safe and + correct. */ #define CN1_ARRAY_STORE_CHECK(arrayObj, value) { \ - if((value) != JAVA_NULL) { \ + if((value) != JAVA_NULL && (arrayObj) != JAVA_NULL) { \ struct clazz* cn1__comp = CN1_CLASS_OF(arrayObj)->arrayType; \ if(cn1__comp != NULL && !instanceofFunction(cn1__comp->classId, GET_CLASS_ID(value))) { \ cn1ThrowTypeError(threadStateData, __NEW_INSTANCE_java_lang_ArrayStoreException(threadStateData), CN1_CLASS_OF(value)->clsName, NULL); \ diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 403c966a892..1aa85becdfb 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -8730,14 +8730,32 @@ static void cn1GcBuildVirtualThreadSnapshot(void) { cn1GcVtSnapshotCount = n; } -// Mark every PARKED virtual thread's live stack region. The running ones are -// covered through the thread that is running them, in the scan below. +// Mark every virtual thread's saved stack region -- the RUNNING ones included, and +// that redundancy is the point. +// +// The obvious version of this skipped anything cn1VirtualThreadIsRunning() reported, +// on the reasoning that the carrier covers those. The carrier does cover them, but +// only once its stack pointer is actually INSIDE the virtual stack, and `running` is +// raised before the switch and lowered after the switch back. In those two windows a +// stopped carrier still has an OS-stack pointer, so cn1VirtualThreadForStackAddress +// matches nothing and the carrier pass scans only the OS stack -- while this pass +// skipped the virtual stack for being "running". Java references living in C +// temporaries on that stack went unmarked and could be swept. The flag cannot be made +// atomic with the switch it brackets, because the switch is what changes the very +// stack the flag would have to be written from. +// +// Scanning unconditionally removes the window instead of narrowing it. It is SAFE +// because [sp, stackHigh) is inside the mapping whenever sp is non-zero, and it is +// COMPLETE in combination with the carrier pass: while a virtual thread runs, the +// carrier's pointer is lower than the saved sp, so this pass covers a subset and the +// carrier covers the rest. Conservative marking is idempotent, so the overlap costs a +// second walk of a small region and nothing else. static void cn1GcScanParkedVirtualThreads(CODENAME_ONE_THREAD_STATE) { int i; for(i = 0 ; i < cn1GcVtSnapshotCount ; i++) { struct cn1VirtualThread* vt = cn1GcVtSnapshot[i]; void* lo; void* hi; - if(vt == 0 || cn1VirtualThreadIsRunning(vt)) { + if(vt == 0) { continue; } cn1VirtualThreadStackBounds(vt, &lo, &hi); diff --git a/vm/ByteCodeTranslator/src/java_io_File.m b/vm/ByteCodeTranslator/src/java_io_File.m index b59b35e0351..2d2ad0e87e6 100644 --- a/vm/ByteCodeTranslator/src/java_io_File.m +++ b/vm/ByteCodeTranslator/src/java_io_File.m @@ -126,7 +126,14 @@ JAVA_OBJECT java_io_File_listImpl___java_lang_String_R_java_lang_String_1ARRAY(C return JAVA_NULL; } - JAVA_OBJECT arr = allocArray(threadStateData, [files count], &class__java_lang_String, sizeof(JAVA_OBJECT), 1); + /* class_array1__java_lang_String, not class__java_lang_String: allocArray + installs whatever class it is given as the ARRAY object's own class, so the + element class here made File.list() return something that reported itself as + a String rather than a String[] -- wrong for getClass() and for any array + type check, and it hands the collector String metadata for an array payload. + cn1MainArgs has always used the array class; these three did not. Fixed on + all of them, including the two that predate the Windows arm. */ + JAVA_OBJECT arr = allocArray(threadStateData, [files count], &class_array1__java_lang_String, sizeof(JAVA_OBJECT), 1); for (int i=0; i<[files count]; i++) { NSString* f = [files objectAtIndex:i]; @@ -353,6 +360,8 @@ provides everything else (stat, remove, rename, mkdir) under the same names. #define PATH_MAX MAX_PATH #endif #define realpath(path, resolved) _fullpath((resolved), (path), MAX_PATH) +#define CN1_FILE_SEP '\\' + #ifndef F_OK #define F_OK 0 #endif @@ -373,8 +382,39 @@ provides everything else (stat, remove, rename, mkdir) under the same names. #include #include #define CN1_FILE_ACCESS(p, m) access((p), (m)) +#define CN1_FILE_SEP '/' #endif +/* + * "Absolute" is not the same question on the two platforms, and getting it wrong + * CORRUPTS a path rather than merely misreporting one: the caller prepends the + * working directory to anything this rejects, so "C:\\data" came back as + * "C:\\cwd\\C:\\data". + * + * NOTE the matching Java-side gap, deliberately not changed here: + * java.io.File.isAbsolute() tests path.startsWith(File.separator) and + * File.separator is "/" on every target, so it still answers false for a drive or + * UNC path. Fixing that means giving JavaAPI a per-platform separator, which is a + * change to shared Java for every port -- out of scope for making the clean target + * build. The native above is what stops a wrong answer from producing a wrong + * PATH; isAbsolute() returning false is a wrong answer that corrupts nothing. + */ +static int cn1FileIsAbsolute(const char* p) { + if (p == NULL || p[0] == '\0') { + return 0; + } +#ifdef _WIN32 + /* A UNC path ("\\server\share") and a rooted "\path" both start at a root. */ + if (p[0] == '/' || p[0] == '\\') { + return 1; + } + /* "C:\x" or "C:/x". A bare "C:x" is drive-RELATIVE, and is not absolute. */ + return p[1] == ':' && (p[2] == '\\' || p[2] == '/'); +#else + return p[0] == '/'; +#endif +} + // Helper: assumes stringToUTF8 is available (implemented in test stubs or runtime) extern const char* stringToUTF8(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT str); extern JAVA_OBJECT newStringFromCString(CODENAME_ONE_THREAD_STATE, const char *str); @@ -507,7 +547,7 @@ JAVA_OBJECT java_io_File_listImpl___java_lang_String_R_java_lang_String_1ARRAY(C } while (FindNextFileA(h, &fd)); FindClose(h); - arr = allocArray(threadStateData, count, &class__java_lang_String, sizeof(JAVA_OBJECT), 1); + arr = allocArray(threadStateData, count, &class_array1__java_lang_String, sizeof(JAVA_OBJECT), 1); h = FindFirstFileA(pattern, &fd); if (h == INVALID_HANDLE_VALUE) { @@ -544,7 +584,7 @@ JAVA_OBJECT java_io_File_listImpl___java_lang_String_R_java_lang_String_1ARRAY(C } closedir(d); - JAVA_OBJECT arr = allocArray(threadStateData, count, &class__java_lang_String, sizeof(JAVA_OBJECT), 1); + JAVA_OBJECT arr = allocArray(threadStateData, count, &class_array1__java_lang_String, sizeof(JAVA_OBJECT), 1); d = opendir(p); count = 0; @@ -629,12 +669,22 @@ JAVA_LONG java_io_File_getUsableSpaceImpl___java_lang_String_R_long(CODENAME_ONE JAVA_OBJECT java_io_File_getAbsolutePathImpl___java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path) { if(path == JAVA_NULL) return JAVA_NULL; const char* p = stringToUTF8(threadStateData, path); - if (p[0] == '/') return path; - char buf[PATH_MAX]; - if (getcwd(buf, sizeof(buf)) != NULL) { - strcat(buf, "/"); - strcat(buf, p); - return newStringFromCString(threadStateData, buf); + if (cn1FileIsAbsolute(p)) return path; + { + char buf[PATH_MAX]; + char joined[PATH_MAX]; +#ifdef _WIN32 + if (_getcwd(buf, (int)sizeof(buf)) != NULL) { +#else + if (getcwd(buf, sizeof(buf)) != NULL) { +#endif + /* snprintf, not strcat: the original wrote the separator and the whole + relative path onto a PATH_MAX buffer already holding the cwd, with no + room left to check. */ + if (snprintf(joined, sizeof(joined), "%s%c%s", buf, CN1_FILE_SEP, p) < (int)sizeof(joined)) { + return newStringFromCString(threadStateData, joined); + } + } } return path; } diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index b002e50e4ed..324e36d8a98 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1147,15 +1147,58 @@ JAVA_LONG java_io_FileInputStream_openImpl___java_lang_String_R_long(CODENAME_ON return (JAVA_LONG)(intptr_t)f; } +/* + * TWO KNOWN LIMITATIONS of this file layer, recorded here rather than fixed, + * because both are pre-existing on every platform and neither is what enabling the + * clean target is about. Raised in review on PR #5658; written down so the next + * reader finds the analysis instead of rediscovering it. + * + * 1. FILE POSITIONS ARE 32-BIT WHERE C `long` IS. skipImpl/availableImpl below use + * ftell/fseek, so a file over 2GiB cannot have its position represented on + * Windows (LLP64: long is 32 bits) even though the Java API is `long` + * throughout. The fix is _ftelli64/_fseeki64 against ftello/fseeko, plus + * widening the local arithmetic -- worth doing, and not a build-enablement + * change. + * + * 2. PATHS ARE PASSED TO THE NARROW CRT. stringToUTF8 produces UTF-8, and the + * Windows CRT's fopen reads it in the active ANSI code page, so a path holding + * a non-ASCII user or file name fails to open. The same mismatch runs through + * java_io_File.m's stat/access/FindFirstFile calls. cn1_db_sqlite_impl.h around + * line 196 already documents this exact problem and converts UTF-8 to UTF-16 + * before calling the wide API; the file layer needs the same treatment applied + * across every entry point, which is its own change rather than a line here. + */ + +/* Keeps a Java object provably live past a safepoint. Only an INTERIOR pointer into + an array is used across the blocking calls below, so the optimizer is free to drop + the array reference itself -- and the concurrent collector, scanning this parked + thread, then sees no root and sweeps the buffer while the read is still filling it. + The Linux port solves this with an asm barrier; this file also compiles under + clang-cl, which has no __asm__ __volatile__, so it uses a volatile store, which no + compiler may elide. The sink is written from several threads and never read: that + is the entire point of it, and the races are benign because no value is consumed. */ +static volatile JAVA_OBJECT cn1BlockingIoKeepAlive; +#define CN1_KEEP_ALIVE_ACROSS_SAFEPOINT(obj) do { cn1BlockingIoKeepAlive = (obj); } while(0) + JAVA_INT java_io_FileInputStream_readImpl___long_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { FILE* f = (FILE*)(intptr_t)handle; if(f == NULL || buffer == JAVA_NULL) { return -2; } JAVA_ARRAY_BYTE* data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; - size_t n = fread(&data[offset], 1, (size_t)length, f); + size_t n; + int atEof; + /* A "file" is not always a file: a FIFO, a device or a network-backed path can + block here for as long as the other end stays quiet, and with the thread left + ACTIVE the collector spins for a safepoint it cannot reach. Same treatment as + the socket reads and StandardInputStream. */ + CN1_YIELD_THREAD; + n = fread(&data[offset], 1, (size_t)length, f); + atEof = feof(f); /* before the resume: the resume is a safepoint */ + CN1_RESUME_THREAD; + CN1_KEEP_ALIVE_ACROSS_SAFEPOINT(buffer); if(n == 0) { - return feof(f) ? -1 : -2; + return atEof ? -1 : -2; } return (JAVA_INT)n; } @@ -1229,7 +1272,14 @@ JAVA_INT java_io_FileOutputStream_writeImpl___long_byte_1ARRAY_int_int_R_int(COD return -1; } JAVA_ARRAY_BYTE* data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; - return (JAVA_INT)fwrite(&data[offset], 1, (size_t)length, f); + size_t written; + /* Blocks for the same reasons the read does -- a full pipe, a slow device -- and + strands the collector the same way. */ + CN1_YIELD_THREAD; + written = fwrite(&data[offset], 1, (size_t)length, f); + CN1_RESUME_THREAD; + CN1_KEEP_ALIVE_ACROSS_SAFEPOINT(buffer); + return (JAVA_INT)written; } JAVA_INT java_io_FileOutputStream_flushImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { @@ -1248,17 +1298,6 @@ JAVA_INT java_io_FileOutputStream_closeImpl___long_R_int(CODENAME_ONE_THREAD_STA return fclose(f) == 0 ? 0 : -1; } -/* Keeps a Java object provably live past a safepoint. Only an INTERIOR pointer into - an array is used across the blocking calls below, so the optimizer is free to drop - the array reference itself -- and the concurrent collector, scanning this parked - thread, then sees no root and sweeps the buffer while the read is still filling it. - The Linux port solves this with an asm barrier; this file also compiles under - clang-cl, which has no __asm__ __volatile__, so it uses a volatile store, which no - compiler may elide. The sink is written from several threads and never read: that - is the entire point of it, and the races are benign because no value is consumed. */ -static volatile JAVA_OBJECT cn1BlockingIoKeepAlive; -#define CN1_KEEP_ALIVE_ACROSS_SAFEPOINT(obj) do { cn1BlockingIoKeepAlive = (obj); } while(0) - // Standard input. Separate from FileInputStream because stdin is not seekable, so // skip/available cannot be implemented by the ftell dance above. JAVA_INT java_io_StandardInputStream_readImpl___byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { @@ -2241,18 +2280,21 @@ void cn1RetireVirtualThread(struct cn1VirtualThread* vt) { // Frees the allThreads slot and runs collectThreadResources, exactly as an // OS thread's death does. markDeadThread(state); - // Then the state itself, with the same deferral an OS thread's finalizer - // uses: if the collector has this TLD queued for drain, its pending - // allocations have not been migrated into allObjectsInHeap yet and freeing - // now would hand the drain a dangling pointer. + // ALWAYS deferred, never freed here, and there is deliberately no + // synchronous branch to fall into. collectThreadResources -- which + // markDeadThread just called, and which has no early return -- sets + // gcQueuedForDrain unconditionally, so the release is the drain's job. + // + // That is required rather than incidental, and the reason is worth stating + // because a synchronous free reads as harmless once the allThreads slot is + // cleared: codenameOneGCMark copies each ThreadLocalData* out of allThreads + // under the critical section and then dereferences it OUTSIDE the lock, so + // a mark already past that copy is still reading this state. The drain runs + // at the START of a mark, after the previous one has finished, which is the + // one point where no collector iteration can still hold the pointer. lockCriticalSection(); - if(state->gcQueuedForDrain) { - state->gcReleaseRequested = JAVA_TRUE; - unlockCriticalSection(); - } else { - unlockCriticalSection(); - cn1ReleaseThreadLocalData(state); - } + state->gcReleaseRequested = JAVA_TRUE; + unlockCriticalSection(); } cn1VirtualThreadFree(vt); } diff --git a/vm/benchmarks/translate-and-build.sh b/vm/benchmarks/translate-and-build.sh index 6f088ee8f60..b8a4eb16713 100755 --- a/vm/benchmarks/translate-and-build.sh +++ b/vm/benchmarks/translate-and-build.sh @@ -94,7 +94,16 @@ mkdir -p "$WORK/out" # for generated C (Java wrapping arithmetic; clang -O3 provably miscompiles # without them). ThinLTO (-flto=thin, clang only) is the release shape. SRCDIR="$WORK/out/dist/$MAIN-src" +# The .S as well as the .c. The translator emits the virtual-thread context switch +# beside the generated sources, and on aarch64/x86_64 the C half references it, so +# a *.c-only invocation links against a missing cn1VirtualThreadSwitch. The CMake +# and Xcode project generators had the identical omission; this is the third place +# that had to learn the same thing. nullglob keeps the argument from expanding to a +# literal "*.S" on a target where no assembly is emitted. +shopt -s nullglob +ASM=("$SRCDIR"/*.S) +shopt -u nullglob $CC -O3 -w -fwrapv -fno-strict-aliasing -fno-builtin-fmod -fno-builtin-fmodf \ - $CN1_BENCH_CFLAGS $EXTRA -I"$SRCDIR" "$SRCDIR"/*.c -lm -lpthread -o "$OUTBIN" \ + $CN1_BENCH_CFLAGS $EXTRA -I"$SRCDIR" "$SRCDIR"/*.c "${ASM[@]}" -lm -lpthread -o "$OUTBIN" \ 2> "$WORK/cc.log" || { echo "COMPILE FAILED"; tail -30 "$WORK/cc.log"; exit 1; } echo "built $OUTBIN (workdir $WORK)" From 764587f08ca2ade1ada6e4c3e95b53d0d23b7423 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 06:56:09 +0300 Subject: [PATCH 19/42] Four more ways the direct JSON writer disagreed with the map path Mapper.Direct promises identical output, not better output. Each of these was the direct path being reasonable in a way emitFieldToMap is not, which is the same thing as changing a mapper's wire format the day it gains a direct writer. - A property NAME was escaped for the Java literal and not for JSON. escape() doubles a quote so the generated source compiles; the resulting writer then appended the raw character, so a @JsonProperty holding a quote emitted "a"b" -- unparseable. The map path never had this because JSONWriter puts the key through writeString. Now jsonEscape composed with escape: one makes the JSON valid, the other makes the source compile. Done at generation time, since a jsonName is a compile-time constant and the writer should stay a literal append. - A Property value was rendered too well. emitFieldToMap stores it RAW, so JSONWriter renders a Date or a mapped object through String.valueOf; appendJsonValue turned them into epoch millis and nested JSON. New Mappers.appendJsonRaw is exactly JSONWriter's answer for a value that was put in the map unchanged. - A reference field looked its mapper up by RUNTIME class. A field declared as a mapped base holding an unmapped subclass therefore found nothing and fell back to a quoted toString, where the map path asks Mappers.get(Declared.class) and serialises it as an object. New Mappers.appendJsonUsing takes the mapper the caller names, and still uses that mapper's direct route when it has one. - Mapped list ELEMENTS had the same problem, plus the general one behind it: the direct path had a two-way branch where emitFieldToMap has four. It now mirrors them one for one -- enum name(), scalar raw, Date getTime(), everything else through the declared element type's mapper. The test was the actual defect. Nothing compared the two paths against each other, which is why all of this shipped; and the parity test added for the first pair needed three fixes of its own before it proved anything: - It went through Mappers.appendJson, which consults the registry. In an isolated classloader the registry is empty, so it compared the map path against "com.example.Swatch@23706db8". It now drives the generated writer. - The polymorphic case had no mapper registered for the base type, so BOTH paths fell back to toString and agreed. Registering it is what makes the two implementations able to differ at all. - assertEquals reports the FIRST difference, so one unfixed case masked the others. Each representation is now pinned individually, which also catches the case equality cannot: both paths wrong in the same way. Verified by reverting the generator with the test in place: one failure against the old code, six passing against the new. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/com/codename1/mapping/Mappers.java | 42 ++++++++ .../MappingAnnotationProcessor.java | 95 +++++++++++++++---- .../MappingAnnotationProcessorTest.java | 69 +++++++++++++- 3 files changed, 186 insertions(+), 20 deletions(-) diff --git a/CodenameOne/src/com/codename1/mapping/Mappers.java b/CodenameOne/src/com/codename1/mapping/Mappers.java index 85f59701957..d2308b1ad6d 100644 --- a/CodenameOne/src/com/codename1/mapping/Mappers.java +++ b/CodenameOne/src/com/codename1/mapping/Mappers.java @@ -277,6 +277,48 @@ public static void appendJson(Object instance, StringBuilder out) { writeJson(out, m.toMap(instance)); } + /// Appends `value` exactly as `JSONWriter` would render it if it had been put + /// into the map that `Mapper#toMap` builds. + /// + /// This is deliberately NOT `#appendJsonValue`: that one is smarter, turning a + /// `Date` into epoch milliseconds and a mapped object into nested JSON. Where a + /// generated mapper is reproducing what the map path stored RAW -- a `Property` + /// value is the case that matters -- being smarter is being different, and + /// `Mapper.Direct` promises identical output rather than better output. + public static void appendJsonRaw(StringBuilder out, Object value) { + writeJson(out, value); + } + + /// Appends `instance` through the mapper the CALLER names, rather than the one + /// registered for the instance's runtime class. + /// + /// The distinction is polymorphism. A field declared `Base` holding an instance + /// of an unmapped subclass finds no mapper by runtime class, and + /// `#appendJson(Object, StringBuilder)` then falls back to the quoted + /// `toString`. `Mapper#toMap` looks the mapper up by the DECLARED type and + /// serialises the subclass as an object, so a generated mapper reproducing the + /// map path has to ask the same question. Mirrors what the map path does with a + /// null mapper too: the raw value, which renders as its quoted `toString`. + public static void appendJsonUsing(Mapper mapper, Object instance, StringBuilder out) { + if (instance == null) { + out.append("null"); + return; + } + if (mapper == null) { + writeJson(out, instance); + return; + } + if (mapper instanceof Mapper.Direct) { + @SuppressWarnings("unchecked") + Mapper.Direct d = (Mapper.Direct) mapper; + d.toJson(instance, out); + return; + } + @SuppressWarnings("unchecked") + Mapper m = (Mapper) mapper; + writeJson(out, m.toMap(instance)); + } + static void writeJson(StringBuilder sb, Object value) { if (value == null) { sb.append("null"); diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java index 7134207a8ea..df8e8d856ea 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java @@ -393,8 +393,13 @@ private static String generateMapperSource(MappedClass mc) { boolean firstProp = true; for (MappedField f : mc.fields) { if (!f.includeInJson) continue; + // jsonEscape THEN escape: the inner one makes the key valid JSON, the + // outer one makes it a valid Java literal. escape() alone only did the + // second, so a @JsonProperty containing a quote compiled fine and then + // emitted "a"b" -- unparseable, where the map path escapes it properly + // because JSONWriter writes the key through writeString. sb.append(" out.append(\"").append(firstProp ? "" : ",") - .append("\\\"").append(escape(f.jsonName)).append("\\\":\");\n"); + .append("\\\"").append(escape(jsonEscape(f.jsonName))).append("\\\":\");\n"); emitFieldToJson(sb, f, isRecord); firstProp = false; } @@ -600,14 +605,25 @@ private static void emitFieldToJson(StringBuilder sb, MappedField f, boolean isR .append(read).append("));\n"); return; case PROPERTY: - sb.append(" com.codename1.mapping.Mappers.appendJsonValue(out, ") + // appendJsonRaw, not appendJsonValue. emitFieldToMap puts the value + // in the map UNCHANGED, so the old writer renders a Date or a mapped + // object through JSONWriter's String.valueOf fallback. appendJsonValue + // would render epoch millis and nested JSON instead -- better, and + // therefore a silent wire change for every mapper the day it gains a + // direct writer. + sb.append(" com.codename1.mapping.Mappers.appendJsonRaw(out, ") .append(read).append(".get());\n"); return; case REFERENCE: - // Through the nested type's own mapper, which takes ITS direct - // route when it has one, so nesting builds no map either. - sb.append(" com.codename1.mapping.Mappers.appendJson(") - .append(read).append(", out);\n"); + // Looked up by the DECLARED type, exactly as emitFieldToMap does. + // appendJson would look up by the instance's RUNTIME class, so a + // field declared as a mapped base holding an unmapped subclass found + // no mapper and fell back to a quoted toString, where the map path + // serialises it as an object. appendJsonUsing still takes the nested + // mapper's direct route when it has one, so nesting builds no map. + sb.append(" com.codename1.mapping.Mappers.appendJsonUsing(") + .append("com.codename1.mapping.Mappers.get(").append(f.kind.binaryName) + .append(".class), ").append(read).append(", out);\n"); return; case LIST: case LIST_PROPERTY: { String src = f.kind.kind == PropertyTypeKind.Kind.LIST @@ -628,20 +644,27 @@ private static void emitFieldToJson(StringBuilder sb, MappedField f, boolean isR sb.append(" for (java.util.Iterator _it = _src.iterator(); _it.hasNext(); ) {\n"); sb.append(" if (!_first) { out.append(','); }\n"); sb.append(" _first = false;\n"); + // One branch per branch emitFieldToMap has for an element, in the + // same order and with the same answer. Anything less specific + // diverges: the enum and the mapped-object cases both did. + sb.append(" Object _e = _it.next();\n"); if (f.elementIsEnum) { - // name(), not toString(). The map path uses Enum.name() and - // deserialisation matches against the declared constants, so an - // enum that overrides toString() would serialise to something - // that cannot be read back. - sb.append(" Object _e = _it.next();\n"); - sb.append(" com.codename1.mapping.Mappers.appendJsonValue(out, _e == null ? null : ((") + // name(), not toString(). Deserialisation matches against the + // declared constants, so an enum overriding toString() would + // serialise to something that cannot be read back. + sb.append(" com.codename1.mapping.Mappers.appendJsonRaw(out, _e == null ? null : ((") .append(f.kind.elementBinaryName).append(") _e).name());\n"); + } else if (isScalarBinary(f.kind.elementBinaryName)) { + sb.append(" com.codename1.mapping.Mappers.appendJsonRaw(out, _e);\n"); + } else if ("java.util.Date".equals(f.kind.elementBinaryName)) { + sb.append(" com.codename1.mapping.Mappers.appendJsonRaw(out, _e == null ? null : Long.valueOf(((java.util.Date) _e).getTime()));\n"); } else { - // Every other element kind already agrees: appendJsonValue maps - // Date to getTime(), scalars and collections to writeJson, and a - // mapped object through its own mapper -- the same three answers - // emitFieldToMap produces. - sb.append(" com.codename1.mapping.Mappers.appendJsonValue(out, _it.next());\n"); + // By the DECLARED element type, as the map path does -- a + // List holding an unmapped subclass otherwise found no + // mapper by runtime class and fell back to a quoted toString. + sb.append(" com.codename1.mapping.Mappers.appendJsonUsing(") + .append("com.codename1.mapping.Mappers.get(").append(f.kind.elementBinaryName) + .append(".class), _e, out);\n"); } sb.append(" }\n"); sb.append(" out.append(']');\n"); @@ -1267,6 +1290,44 @@ private static String deriveXmlRoot(String simpleName) { return Character.toLowerCase(simpleName.charAt(0)) + simpleName.substring(1); } + /** + * JSON-escapes a property name, matching JSONWriter.writeString character for + * character (minus the surrounding quotes, which the caller emits). + * + * Applied at GENERATION time because a jsonName is a compile-time constant -- + * the direct writer stays a plain literal append with no per-call escaping. It + * must be composed with {@link #escape} afterwards, which is the Java-literal + * escaper: one makes the JSON valid, the other makes the source compile. + */ + private static String jsonEscape(String s) { + if (s == null) return ""; + StringBuilder b = new StringBuilder(s.length() + 8); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '"': b.append("\\\""); break; + case '\\': b.append("\\\\"); break; + case '\n': b.append("\\n"); break; + case '\r': b.append("\\r"); break; + case '\t': b.append("\\t"); break; + case '\b': b.append("\\b"); break; + case '\f': b.append("\\f"); break; + default: + if (c < 0x20) { + b.append("\\u"); + String hex = Integer.toHexString(c); + for (int p = hex.length(); p < 4; p++) { + b.append('0'); + } + b.append(hex); + } else { + b.append(c); + } + } + } + return b.toString(); + } + private static String escape(String s) { if (s == null) return ""; StringBuilder b = new StringBuilder(s.length() + 4); diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java index 188551284b7..981952742d6 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java @@ -45,6 +45,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -321,17 +322,45 @@ public void directJsonMatchesTheMapPathExactly() throws Exception { + " LIGHT, DARK;\n" + " @Override public String toString() { return \"shade-\" + name().toLowerCase(); }\n" + "}\n"); + // A mapped base plus an UNMAPPED subclass: the polymorphic case where a + // runtime-class mapper lookup finds nothing and falls back to toString(), + // while the map path finds the mapper for the DECLARED type. + sources.put("com.example.Base", + "package com.example;\n" + + "import com.codename1.annotations.Mapped;\n" + + "@Mapped public class Base {\n" + + " public String tag;\n" + + " public Base() {}\n" + + "}\n"); + sources.put("com.example.Derived", + "package com.example;\n" + + "public class Derived extends Base {\n" + + " public Derived() {}\n" + + " @Override public String toString() { return \"derived-tostring\"; }\n" + + "}\n"); sources.put("com.example.Swatch", "package com.example;\n" + "import com.codename1.annotations.Mapped;\n" + + "import com.codename1.annotations.JsonProperty;\n" + + "import com.codename1.properties.Property;\n" + "import java.util.List;\n" + "@Mapped public class Swatch {\n" + // Property: the map path stores the Date RAW, so + // JSONWriter renders its toString(). appendJsonValue would + // render epoch millis instead -- a silent wire change. + + " public final Property due = new Property(\"due\");\n" + " public String name;\n" + " public int count;\n" + " public Shade shade;\n" + " public List shades;\n" + " public List tags;\n" + " public java.util.Date when;\n" + // A key needing JSON escaping, which escape() alone only made + // compile. + + " @JsonProperty(\"od\\\"d\\\\key\") public String odd;\n" + // Declared as the mapped base, populated with the subclass. + + " public Base ref;\n" + + " public List refs;\n" + " public Swatch() {}\n" + "}\n"); JavaSourceCompiler.compile(sources, classes, Arrays.asList(testClassesDir())); @@ -361,17 +390,50 @@ public void directJsonMatchesTheMapPathExactly() throws Exception { swatchCls.getField("shades").set(populated, shades); swatchCls.getField("tags").set(populated, Arrays.asList("a", "b")); swatchCls.getField("when").set(populated, new java.util.Date(1234567890L)); + swatchCls.getField("odd").set(populated, "quoted"); + // Base's mapper has to be REGISTERED or the declared-type lookup finds + // nothing and both paths fall back to toString() -- agreeing with each + // other while proving nothing about the polymorphic case. Registering it + // is what makes the two paths able to differ: the old code looked the + // mapper up by the runtime class (Derived, unmapped -> toString), the new + // code by the declared one (Base, mapped -> object). + Class mappersRegCls = cl.loadClass("com.codename1.mapping.Mappers"); + Class mapperIface = cl.loadClass("com.codename1.mapping.Mapper"); + Object baseMapper = cl.loadClass("com.example.BaseCn1Mapper").newInstance(); + mappersRegCls.getMethod("register", mapperIface).invoke(null, baseMapper); + + Class derivedCls = cl.loadClass("com.example.Derived"); + Object derived = derivedCls.newInstance(); + derivedCls.getField("tag").set(derived, "sub"); + swatchCls.getField("ref").set(populated, derived); + List refs = new ArrayList(); + refs.add(derived); + swatchCls.getField("refs").set(populated, refs); + Object dueProp = swatchCls.getField("due").get(populated); + dueProp.getClass().getMethod("set", Object.class) + .invoke(dueProp, new java.util.Date(99000L)); // Every list left null: the case that diverged. Object empty = swatchCls.newInstance(); - assertDirectMatchesMap(cl, mapperCls, mapper, populated); + String json = assertDirectMatchesMap(cl, mapperCls, mapper, populated); + // Pinned individually: assertEquals reports only the FIRST difference, so + // without these a single un-fixed case would mask the rest. + assertTrue("the JSON key must be escaped, not emitted raw: " + json, + json.contains("\"od\\\"d\\\\key\":\"quoted\"")); + assertTrue("a declared-mapped field holding an unmapped subclass must " + + "serialise as an object, not toString(): " + json, + json.contains("\"ref\":{\"tag\":\"sub\"}")); + assertTrue("the same applies to list elements: " + json, + json.contains("\"refs\":[{\"tag\":\"sub\"}]")); + assertFalse("nothing should have fallen back to toString(): " + json, + json.contains("derived-tostring")); assertDirectMatchesMap(cl, mapperCls, mapper, empty); } } - /** Both routes, on one instance, compared as text. */ - private static void assertDirectMatchesMap(URLClassLoader cl, Class mapperCls, + /** Both routes, on one instance, compared as text. Returns the agreed JSON. */ + private static String assertDirectMatchesMap(URLClassLoader cl, Class mapperCls, Object mapper, Object instance) throws Exception { Class writerCls = cl.loadClass("com.codename1.io.JSONWriter"); @@ -389,6 +451,7 @@ private static void assertDirectMatchesMap(URLClassLoader cl, Class mapperCls String viaDirect = out.toString(); assertEquals("direct JSON must match the map path exactly", viaMap, viaDirect); + return viaDirect; } private static File testClassesDir() throws Exception { From 6155b5fd8049209971485180f2e3c23b32ebf4f4 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 07:15:57 +0300 Subject: [PATCH 20/42] Enumerate a directory once, and clamp skip without overflowing first Two more review findings, both in code this branch touched. skip(Long.MAX_VALUE) computed `start + count` and clamped afterwards. Once any byte has been read that addition overflows signed long -- undefined behaviour, and in practice a wrap to negative, so the seek goes BACKWARDS and the caller is told it skipped a negative distance or gets an error where it should have landed on EOF. It now clamps against the remaining DISTANCE, which cannot overflow: end is at least start, and start plus the clamped amount is at most end. File.list walked the directory TWICE -- count, allocate, walk again -- and assumed both walks saw the same directory. They do not. A file created in between overruns the array, and CN1_SET_ARRAY_ELEMENT_OBJECT turns that into ArrayIndexOutOfBoundsException; a file removed leaves trailing nulls in a String[] that no caller expects. Directories change under readers routinely, so this was never sound. I wrote the Windows arm that way deliberately, mirroring the POSIX one, which means I copied the structure without asking whether it held. Both arms now enumerate ONCE into a small growable list of names and build the array afterwards. The names are held in C memory on purpose: allocArray and newStringFromCString can both collect, and nothing may hold a directory handle across that. The ObjC arm is left alone -- NSFileManager hands back a snapshot, so it never had the race. Also moves stdlib.h to the shared include group, since the list uses malloc/realloc/free on both arms and sits outside the platform blocks. The test is the part worth reading. FileClassIntegrationTest never called File.list(), so the native listing was COMPILED but never RUN by any suite: the rewrite above passed 5/5 while executing none of it, and reverting it would have passed too. Coverage now creates a directory, lists it, and pins the three things that were wrong or fragile -- the entries, the absence of nulls, and that the result is a String[] rather than a String, which is the pre-existing allocArray class bug nothing had ever asserted. Confirmed the assertions discriminate rather than merely execute: with the array class reverted to the element class, all five configurations FAIL; restored, all five pass. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/java_io_File.m | 156 ++++++++++++------ vm/ByteCodeTranslator/src/nativeMethods.m | 25 ++- .../translator/FileClassIntegrationTest.java | 29 ++++ 3 files changed, 153 insertions(+), 57 deletions(-) diff --git a/vm/ByteCodeTranslator/src/java_io_File.m b/vm/ByteCodeTranslator/src/java_io_File.m index 2d2ad0e87e6..e62ce3f8f95 100644 --- a/vm/ByteCodeTranslator/src/java_io_File.m +++ b/vm/ByteCodeTranslator/src/java_io_File.m @@ -326,6 +326,9 @@ JAVA_OBJECT java_io_File_getCanonicalPathImpl___java_lang_String_R_java_lang_Str #include #include #include +/* Shared, not per-arm: cn1NameList below uses malloc/realloc/free on BOTH, and it + sits outside the platform blocks. */ +#include #ifdef _WIN32 /* clang-cl ships no and no . Only two things in this file actually need them -- access() and the directory walk -- and the MSVC CRT @@ -335,7 +338,6 @@ provides everything else (stat, remove, rename, mkdir) under the same names. reached java.io.File. */ #include #include -#include /* WIN32_LEAN_AND_MEAN keeps out of . Without it winsock's own `struct timeval` collides with the one cn1_win_compat.h defines, and the file fails on "redefinition of 'timeval'" rather than on anything it does. */ @@ -385,6 +387,67 @@ provides everything else (stat, remove, rename, mkdir) under the same names. #define CN1_FILE_SEP '/' #endif +/* + * A growable list of names, so a directory is enumerated exactly ONCE. + * + * The two-pass shape this replaces -- count, allocate, enumerate again -- assumed + * the two walks see the same directory. They do not: a file created between them + * overruns the array (CN1_SET_ARRAY_ELEMENT_OBJECT then raises + * ArrayIndexOutOfBoundsException) and a file removed leaves trailing nulls in a + * String[] that Java code has no reason to expect. Directories change under + * readers all the time, so this was a real race on every platform, not just the + * newly added Windows arm. + * + * The names are held in C memory on purpose: allocArray and newStringFromCString + * can both collect, and nothing here may be holding a directory handle when that + * happens. + */ +struct cn1NameList { char** names; int count; int cap; }; + +static int cn1NameListAdd(struct cn1NameList* l, const char* name) { + size_t n; + char* copy; + if(l->count == l->cap) { + int cap = l->cap == 0 ? 16 : l->cap * 2; + char** grown = (char**)realloc(l->names, (size_t)cap * sizeof(char*)); + if(grown == NULL) { + return 0; + } + l->names = grown; + l->cap = cap; + } + n = strlen(name) + 1; + copy = (char*)malloc(n); + if(copy == NULL) { + return 0; + } + memcpy(copy, name, n); + l->names[l->count++] = copy; + return 1; +} + +static void cn1NameListFree(struct cn1NameList* l) { + int i; + for(i = 0 ; i < l->count ; i++) { + free(l->names[i]); + } + free(l->names); + l->names = 0; + l->count = 0; + l->cap = 0; +} + +/* Turns a completed name list into the String[] File.list returns. */ +static JAVA_OBJECT cn1NameListToArray(CODENAME_ONE_THREAD_STATE, struct cn1NameList* l) { + JAVA_OBJECT arr = allocArray(threadStateData, l->count, &class_array1__java_lang_String, sizeof(JAVA_OBJECT), 1); + int i; + for(i = 0 ; i < l->count ; i++) { + JAVA_OBJECT s = newStringFromCString(threadStateData, l->names[i]); + CN1_SET_ARRAY_ELEMENT_OBJECT(arr, i, s); + } + return arr; +} + /* * "Absolute" is not the same question on the two platforms, and getting it wrong * CORRUPTS a path rather than merely misreporting one: the caller prepends the @@ -512,16 +575,16 @@ JAVA_OBJECT java_io_File_listImpl___java_lang_String_R_java_lang_String_1ARRAY(C enteringNativeAllocations(); const char* p = stringToUTF8(threadStateData, path); #ifdef _WIN32 - /* FindFirstFile rather than opendir, and it wants a wildcard appended. Two - passes like the POSIX arm below: count, allocate, refill -- allocArray can - collect, so the array cannot be built while a find handle is open. */ + /* FindFirstFile rather than opendir, and it wants a wildcard appended. ONE + enumeration into cn1NameList -- see the note there for why two walks of the + same directory is a race rather than a shortcut. */ { char pattern[MAX_PATH]; WIN32_FIND_DATAA fd; HANDLE h; - int count = 0; - JAVA_OBJECT arr; + struct cn1NameList list; size_t plen = strlen(p); + list.names = 0; list.count = 0; list.cap = 0; if (plen == 0 || plen + 3 > sizeof(pattern)) { finishedNativeAllocations(); return JAVA_NULL; @@ -543,61 +606,48 @@ JAVA_OBJECT java_io_File_listImpl___java_lang_String_R_java_lang_String_1ARRAY(C } do { if (strcmp(fd.cFileName, ".") == 0 || strcmp(fd.cFileName, "..") == 0) continue; - count++; + if (!cn1NameListAdd(&list, fd.cFileName)) { + FindClose(h); + cn1NameListFree(&list); + finishedNativeAllocations(); + return JAVA_NULL; + } } while (FindNextFileA(h, &fd)); FindClose(h); - - arr = allocArray(threadStateData, count, &class_array1__java_lang_String, sizeof(JAVA_OBJECT), 1); - - h = FindFirstFileA(pattern, &fd); - if (h == INVALID_HANDLE_VALUE) { + { + JAVA_OBJECT arr = cn1NameListToArray(threadStateData, &list); + cn1NameListFree(&list); finishedNativeAllocations(); return arr; } - count = 0; - do { - if (strcmp(fd.cFileName, ".") == 0 || strcmp(fd.cFileName, "..") == 0) continue; - { - JAVA_OBJECT s = newStringFromCString(threadStateData, fd.cFileName); - CN1_SET_ARRAY_ELEMENT_OBJECT(arr, count, s); - } - count++; - } while (FindNextFileA(h, &fd)); - FindClose(h); - - finishedNativeAllocations(); - return arr; } #else - DIR* d = opendir(p); - if (d == NULL) { - finishedNativeAllocations(); - return JAVA_NULL; - } - - // First count - int count = 0; - struct dirent *dir; - while ((dir = readdir(d)) != NULL) { - if (strcmp(dir->d_name, ".") == 0 || strcmp(dir->d_name, "..") == 0) continue; - count++; - } - closedir(d); - - JAVA_OBJECT arr = allocArray(threadStateData, count, &class_array1__java_lang_String, sizeof(JAVA_OBJECT), 1); - - d = opendir(p); - count = 0; - while ((dir = readdir(d)) != NULL) { - if (strcmp(dir->d_name, ".") == 0 || strcmp(dir->d_name, "..") == 0) continue; - JAVA_OBJECT s = newStringFromCString(threadStateData, dir->d_name); - CN1_SET_ARRAY_ELEMENT_OBJECT(arr, count, s); - count++; + { + DIR* d = opendir(p); + struct dirent* entry; + struct cn1NameList list; + list.names = 0; list.count = 0; list.cap = 0; + if (d == NULL) { + finishedNativeAllocations(); + return JAVA_NULL; + } + while ((entry = readdir(d)) != NULL) { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) continue; + if (!cn1NameListAdd(&list, entry->d_name)) { + closedir(d); + cn1NameListFree(&list); + finishedNativeAllocations(); + return JAVA_NULL; + } + } + closedir(d); + { + JAVA_OBJECT arr = cn1NameListToArray(threadStateData, &list); + cn1NameListFree(&list); + finishedNativeAllocations(); + return arr; + } } - closedir(d); - - finishedNativeAllocations(); - return arr; #endif } diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 324e36d8a98..92ca9b70c3e 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1216,14 +1216,31 @@ JAVA_LONG java_io_FileInputStream_skipImpl___long_long_R_long(CODENAME_ONE_THREA return -1; } long end = ftell(f); - long target = start + (long)count; - if(target > end) { - target = end; + long remaining; + long skipped; + long target; + if(end < 0) { + return -1; + } + /* Clamp against the DISTANCE, never by adding first. skip(Long.MAX_VALUE) after + any byte has been read overflows `start + count` before the comparison can + clamp it -- signed overflow is undefined behaviour, and in practice wraps + negative and seeks backwards, so the caller is told it skipped a negative + distance or gets an error instead of landing on EOF. Subtracting cannot + overflow: end >= start >= 0, and start + skipped is at most end. */ + remaining = end - start; + if(count <= 0) { + skipped = 0; + } else if(count >= (JAVA_LONG)remaining) { + skipped = remaining; + } else { + skipped = (long)count; } + target = start + skipped; if(fseek(f, target, SEEK_SET) != 0) { return -1; } - return (JAVA_LONG)(target - start); + return (JAVA_LONG)skipped; } JAVA_INT java_io_FileInputStream_availableImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java index 7403fd63cab..e254575f520 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java @@ -136,6 +136,35 @@ private String fileTestAppSource() { " if (f.isDirectory()) throw new RuntimeException(\"IsDirectory failed\");\n" + " if (!f.delete()) throw new RuntimeException(\"Delete failed\");\n" + " if (f.exists()) throw new RuntimeException(\"Delete verification failed\");\n" + + // File.list(): nothing exercised it before, so the native listing -- + // rewritten from a racy count-then-refill pair into one enumeration -- + // was compiled but never RUN by this suite. + " char[] dirChars = new char[]{'l','s','d','i','r'};\n" + + " File dir = new File(new String(dirChars));\n" + + " dir.mkdir();\n" + + " char[] aChars = new char[]{'l','s','d','i','r','/','a'};\n" + + " char[] bChars = new char[]{'l','s','d','i','r','/','b'};\n" + + " File fa = new File(new String(aChars));\n" + + " File fb = new File(new String(bChars));\n" + + " fa.createNewFile();\n" + + " fb.createNewFile();\n" + + " String[] names = dir.list();\n" + + " if (names == null) throw new RuntimeException(\"list returned null\");\n" + + " if (names.length != 2) throw new RuntimeException(\"list length\");\n" + + // A trailing null is what the old two-pass version produced when the + // second walk saw fewer entries than the first. + " if (names[0] == null || names[1] == null) throw new RuntimeException(\"null entry\");\n" + + // The result must be a String[], not a String: allocArray installs the + // class it is handed as the ARRAY's own class. + " Object asObject = names;\n" + + " if (!(asObject instanceof String[])) throw new RuntimeException(\"not a String[]\");\n" + + " boolean sawA = false; boolean sawB = false;\n" + + " for (int i = 0; i < names.length; i++) {\n" + + " if (names[i].equals(new String(new char[]{'a'}))) sawA = true;\n" + + " if (names[i].equals(new String(new char[]{'b'}))) sawB = true;\n" + + " }\n" + + " if (!sawA || !sawB) throw new RuntimeException(\"missing entry\");\n" + + " fa.delete(); fb.delete(); dir.delete();\n" + " } catch (Exception e) {\n" + " // e.printStackTrace(); // Can't print stack trace without constants\n" + " System.exit(1);\n" + From a112185ec114d08d05b5a7c62b9629722ec28934 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 07:30:46 +0300 Subject: [PATCH 21/42] Resolve drive-relative Windows paths, and create files in one syscall Two more findings, both consequences of this branch making java.io.File usable on Windows. "C:foo" is DRIVE-RELATIVE: relative to the working directory of drive C, which is not the process working directory and may be on a different drive. cn1FileIsAbsolute classified it correctly -- the comment there even says so -- and then the fallback prepended the process cwd anyway, producing "D:\cwd\C:foo", which names nothing. The predicate knew about a case the code after it did not. _getdcwd asks the right drive. Deliberately not _fullpath, which the report suggested: it also normalises "..", and getAbsolutePath is specified NOT to do that -- resolving is getCanonicalPath's job. Using it would have swapped a wrong path for a subtly wrong contract. createNewFile was check-then-act: access(), then fopen(p, "w"). Losing that race does not merely return the wrong answer, it TRUNCATES the file the other process just created, and then reports true as though it had done the creating -- which is exactly the failure mode the lock-file and single-instance patterns it exists for cannot survive. Now a single O_EXCL open on both arms, with the kernel deciding. Pre-existing on POSIX too, so both are fixed. ON THE TEST, because the distinction matters: the coverage added here is a REGRESSION GUARD, not a demonstration of atomicity. It checks the uncontended path -- createNewFile on an existing file returns false and leaves it intact -- and the old check-then-act version passes it too, because access() succeeds and it returns before reaching the truncating fopen. Confirmed by running the suite against the old implementation: 5/5 green. The real defect needs a file to appear between the check and the open, which one thread cannot arrange, so the argument for the fix is structural rather than empirical and the comment in the test says so. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/java_io_File.m | 46 +++++++++++++++++-- .../translator/FileClassIntegrationTest.java | 24 ++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/vm/ByteCodeTranslator/src/java_io_File.m b/vm/ByteCodeTranslator/src/java_io_File.m index e62ce3f8f95..49dc4c6de92 100644 --- a/vm/ByteCodeTranslator/src/java_io_File.m +++ b/vm/ByteCodeTranslator/src/java_io_File.m @@ -329,6 +329,8 @@ JAVA_OBJECT java_io_File_getCanonicalPathImpl___java_lang_String_R_java_lang_Str /* Shared, not per-arm: cn1NameList below uses malloc/realloc/free on BOTH, and it sits outside the platform blocks. */ #include +/* O_CREAT/O_EXCL for the atomic create, needed on both arms. */ +#include #ifdef _WIN32 /* clang-cl ships no and no . Only two things in this file actually need them -- access() and the directory walk -- and the MSVC CRT @@ -380,11 +382,19 @@ provides everything else (stat, remove, rename, mkdir) under the same names. #define X_OK 0 #endif #define CN1_FILE_ACCESS(p, m) _access((p), (m)) +/* Exclusive create, so File.createNewFile can be the single atomic operation it is + specified to be. _O_BINARY keeps a zero-length file out of text mode, and + _S_IREAD|_S_IWRITE is the permission argument the CRT wants. */ +#define CN1_FILE_OPEN_EXCL(p) _open((p), _O_CREAT | _O_EXCL | _O_WRONLY | _O_BINARY, _S_IREAD | _S_IWRITE) +#define CN1_FILE_CLOSE_FD(fd) _close(fd) #else #include #include #define CN1_FILE_ACCESS(p, m) access((p), (m)) #define CN1_FILE_SEP '/' +/* 0666 before umask, which is what fopen(p, "w") produced. */ +#define CN1_FILE_OPEN_EXCL(p) open((p), O_CREAT | O_EXCL | O_WRONLY, 0666) +#define CN1_FILE_CLOSE_FD(fd) close(fd) #endif /* @@ -554,13 +564,21 @@ JAVA_LONG java_io_File_lengthImpl___java_lang_String_R_long(CODENAME_ONE_THREAD_ JAVA_BOOLEAN java_io_File_createNewFileImpl___java_lang_String_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path) { if(path == JAVA_NULL) return JAVA_FALSE; const char* p = stringToUTF8(threadStateData, path); - if (CN1_FILE_ACCESS(p, F_OK) != -1) return JAVA_FALSE; - FILE* f = fopen(p, "w"); - if (f) { - fclose(f); + /* ONE call, because File.createNewFile is specified to be atomic. The previous + shape -- access() and then fopen(p, "w") -- loses the race twice over: another + process creating the file in between gets its content TRUNCATED by the "w", + and this returns true as though it had created it. That is precisely what + breaks the lock-file and single-instance patterns the method exists for. + O_EXCL makes the kernel decide, and EEXIST is a false return rather than an + error. */ + { + int fd = CN1_FILE_OPEN_EXCL(p); + if (fd < 0) { + return JAVA_FALSE; + } + CN1_FILE_CLOSE_FD(fd); return JAVA_TRUE; } - return JAVA_FALSE; } JAVA_BOOLEAN java_io_File_deleteImpl___java_lang_String_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path) { @@ -724,6 +742,24 @@ JAVA_OBJECT java_io_File_getAbsolutePathImpl___java_lang_String_R_java_lang_Stri char buf[PATH_MAX]; char joined[PATH_MAX]; #ifdef _WIN32 + /* "C:foo" is DRIVE-RELATIVE: relative to the working directory OF DRIVE C, + which is not the process working directory and may be on another drive + entirely. Joining it to _getcwd() produces "D:\cwd\C:foo", which names + nothing. _getdcwd asks the right drive; 1 is A. Everything else falls + through to the process working directory below. */ + if (p[0] != '\0' && p[1] == ':' && p[2] != '\\' && p[2] != '/') { + int drive = p[0]; + if (drive >= 'a' && drive <= 'z') { drive = drive - 'a' + 1; } + else if (drive >= 'A' && drive <= 'Z') { drive = drive - 'A' + 1; } + else { drive = 0; } + if (drive != 0 && _getdcwd(drive, buf, (int)sizeof(buf)) != NULL) { + /* p + 2 skips the drive letter and colon. */ + if (snprintf(joined, sizeof(joined), "%s%c%s", buf, CN1_FILE_SEP, p + 2) < (int)sizeof(joined)) { + return newStringFromCString(threadStateData, joined); + } + } + return path; + } if (_getcwd(buf, (int)sizeof(buf)) != NULL) { #else if (getcwd(buf, sizeof(buf)) != NULL) { diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java index e254575f520..d19d7d54015 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java @@ -165,6 +165,30 @@ private String fileTestAppSource() { " }\n" + " if (!sawA || !sawB) throw new RuntimeException(\"missing entry\");\n" + " fa.delete(); fb.delete(); dir.delete();\n" + + // A REGRESSION GUARD, not a proof of atomicity -- stated plainly + // because the difference is easy to misread. This checks the + // uncontended path: createNewFile on an existing file returns false + // and leaves it intact. The check-then-act version it replaced passes + // this too, since access() succeeds and it returns before ever + // reaching the fopen that would truncate. Verified by running it + // against the old code: 5/5 green. + // + // The actual defect needs a file to appear BETWEEN the check and the + // open, which one thread cannot produce, so no single-threaded test + // can demonstrate it. The correctness argument is structural instead: + // one O_EXCL syscall where there were two operations, with the kernel + // deciding who wins. What this guards is that the rewrite did not + // break the ordinary path. + " char[] exChars = new char[]{'e','x','c','l','.','t','x','t'};\n" + + " File ex = new File(new String(exChars));\n" + + " if (ex.exists()) ex.delete();\n" + + " if (!ex.createNewFile()) throw new RuntimeException(\"first create\");\n" + + " java.io.FileOutputStream os = new java.io.FileOutputStream(ex);\n" + + " os.write(new byte[]{1,2,3,4});\n" + + " os.close();\n" + + " if (ex.createNewFile()) throw new RuntimeException(\"second create returned true\");\n" + + " if (ex.length() != 4) throw new RuntimeException(\"existing file was truncated\");\n" + + " ex.delete();\n" + " } catch (Exception e) {\n" + " // e.printStackTrace(); // Can't print stack trace without constants\n" + " System.exit(1);\n" + From eaec4ed8e06968e2320ba27a9df269fc2257479b Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:07:53 +0300 Subject: [PATCH 22/42] Decode argv and the environment as UTF-8 instead of widening bytes newStringFromCString turns each byte into its own char. That is correct for what it exists to serve -- generated string literals, which are ASCII plus ~~uXXXX escapes -- and wrong for anything arriving from outside the program. A UTF-8 "e-acute" is two bytes, so main(String[]) and System.getenv handed back one garbage char per byte, corrupting paths and option values before the program had a chance to look at them. Both entry points are new in this branch. newStringFromUtf8 decodes properly: multi-byte sequences, surrogate pairs for astral code points, and U+FFFD for malformed input the way java.lang.String's own decoder does -- a program should not die because one environment variable holds a stray byte. Overlong forms, UTF-8-encoded surrogates and out-of-range code points are all rejected. newStringFromCString itself is deliberately NOT changed. Every native-to-Java string in the VM goes through it, its byte-widening is load-bearing for the literals it serves, and its own comment records that the high-bit path is bit-identical to what came before. Correcting the two entry points this branch added is the scoped fix; the general version is the same work as the ANSI-versus- UTF-8 path issue already recorded in nativeMethods.m. TWO BUGS UNDERNEATH, both found by the test rather than by reading: newString was broken and had never been called from C. JAVA_CHAR is an int and JAVA_ARRAY_CHAR is an unsigned short, and it sized the allocation with sizeof(JAVA_CHAR) while memcpy'ing length * sizeof(JAVA_ARRAY_CHAR) bytes out of a four-byte-element array -- half the input, at the wrong stride. My decoder was its first caller and hit it immediately: "cafe" came back as c,NUL,a,NUL,f. It now narrows element by element. Behind that, the representation is not a free choice. A string whose units all fit in a byte is stored as a COMPACT byte[], anything else as a char[], and charAt reads whichever it finds -- so handing it the wrong one reads 8-bit units out of 16-bit data and produces exactly the same symptom rather than failing. That rule now lives in cn1StringFromUnits, used by newString and newStringFromUtf8. newStringFromCString keeps its own copy on purpose: it tracks the Latin-1 flag during decoding and runs for every literal at startup, so routing it through a helper that recomputes would add a pass over every literal in the program to save a dozen lines. The comment says so, and says the two must change together. The test reports CODE POINTS rather than text, so it cannot pass through a console-encoding coincidence: "cafe-acute-euro" must arrive as 99,97,102,233,8364, which covers a two-byte and a three-byte sequence. Byte-widening reports the individual bytes instead, which is how the newString bug surfaced. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 6 + vm/ByteCodeTranslator/src/cn1_globals.m | 166 +++++++++++++++++- vm/ByteCodeTranslator/src/nativeMethods.m | 10 +- .../CleanTargetIntegrationTest.java | 70 ++++++++ 4 files changed, 242 insertions(+), 10 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index fb848f80282..28168fa83e4 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2569,6 +2569,12 @@ extern struct clazz class_array2__JAVA_DOUBLE; extern struct clazz class_array3__JAVA_DOUBLE; extern JAVA_OBJECT newString(CODENAME_ONE_THREAD_STATE, int length, JAVA_CHAR data[]); +/** + * Like newStringFromCString but DECODES UTF-8 instead of widening bytes. Use it for + * text that came from outside the program (argv, the environment); the widening one + * is right only for generated literals, which are ASCII plus ~~uXXXX escapes. + */ +extern JAVA_OBJECT newStringFromUtf8(CODENAME_ONE_THREAD_STATE, const char* str); extern JAVA_OBJECT newStringFromCString(CODENAME_ONE_THREAD_STATE, const char *str); extern JAVA_OBJECT newStringFromAsciiLen(CODENAME_ONE_THREAD_STATE, const char *src, int len); // Single-allocation fused compact-String builder (see cn1_globals.m). Returns a valid empty diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 1aa85becdfb..0b7149487b1 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -11032,19 +11032,166 @@ JAVA_OBJECT alloc4DArray(CODENAME_ONE_THREAD_STATE, int length4, int length3, in * Creates a java.lang.String object from an array of integers, this is useful * for the constant pool */ +/* + * Builds a java.lang.String from decoded UTF-16 code units. + * + * The representation is not a free choice: the VM stores a string whose units all + * fit in a byte as a COMPACT byte[], and as a char[] otherwise, and String.charAt + * reads whichever it finds. Handing it the wrong one does not fail loudly -- it + * reads 8-bit units out of 16-bit data, so "caf..." comes back as c,NUL,a,NUL,f. + * + * Used by newString and newStringFromUtf8. newStringFromCString deliberately keeps + * its own copy of this tail: it tracks the Latin-1 flag DURING decoding, and it runs + * for every generated string literal at startup, so routing it through here would + * add a second pass over every literal in the program to save a dozen duplicated + * lines. If that tail changes, change this one with it. + */ +static JAVA_OBJECT cn1StringFromUnits(CODENAME_ONE_THREAD_STATE, const JAVA_ARRAY_CHAR* units, int count) { + JAVA_ARRAY dat; + JAVA_BOOLEAN latin1 = JAVA_TRUE; + int i; + JAVA_OBJECT o; + struct obj__java_lang_String* ss; + for(i = 0 ; i < count ; i++) { + if(units[i] > 0xff) { latin1 = JAVA_FALSE; break; } + } + if(latin1) { + JAVA_ARRAY_BYTE* b; + dat = (JAVA_ARRAY)allocArray(threadStateData, count, &class_array1__JAVA_BYTE, sizeof(JAVA_ARRAY_BYTE), 1); + b = (JAVA_ARRAY_BYTE*) (*dat).data; + for(i = 0 ; i < count ; i++) { b[i] = (JAVA_ARRAY_BYTE)units[i]; } + } else { + JAVA_ARRAY_CHAR* a; + dat = (JAVA_ARRAY)allocArray(threadStateData, count, &class_array1__JAVA_CHAR, sizeof(JAVA_ARRAY_CHAR), 1); + a = (JAVA_ARRAY_CHAR*) (*dat).data; + for(i = 0 ; i < count ; i++) { a[i] = units[i]; } + } + o = __NEW_java_lang_String(threadStateData); + java_lang_String___INIT____(threadStateData, o); + ss = (struct obj__java_lang_String*)o; + ss->java_lang_String_value = (JAVA_OBJECT)dat; + ss->java_lang_String_count = count; + return o; +} + JAVA_OBJECT newString(CODENAME_ONE_THREAD_STATE, int length, JAVA_CHAR data[]) { + /* JAVA_CHAR is an INT and JAVA_ARRAY_CHAR is an unsigned short, so the old + body was wrong twice: it sized the allocation with sizeof(JAVA_CHAR) (4 bytes + per element, for an array whose readers use 2) and then memcpy'd + length * sizeof(JAVA_ARRAY_CHAR) bytes straight out of a 4-byte-element + array, which copies half the input at the wrong stride. It went unnoticed + because nothing in C called this until now. Narrow element by element. */ + JAVA_OBJECT o; + JAVA_ARRAY_CHAR stackUnits[256]; + JAVA_ARRAY_CHAR* units = length <= 256 ? stackUnits + : (JAVA_ARRAY_CHAR*)malloc((size_t)length * sizeof(JAVA_ARRAY_CHAR)); + int i; + if(units == 0) { + return JAVA_NULL; + } enteringNativeAllocations(); - JAVA_ARRAY dat = (JAVA_ARRAY)allocArray(threadStateData, length, &class_array1__JAVA_CHAR, sizeof(JAVA_CHAR), 1); - memcpy((*dat).data, data, length * sizeof(JAVA_ARRAY_CHAR)); - JAVA_OBJECT o = __NEW_java_lang_String(threadStateData); - java_lang_String___INIT____(threadStateData, o); - struct obj__java_lang_String* str = (struct obj__java_lang_String*)o; - str->java_lang_String_value = (JAVA_OBJECT)dat; - str->java_lang_String_count = length; + for(i = 0 ; i < length ; i++) { + units[i] = (JAVA_ARRAY_CHAR)data[i]; + } + o = cn1StringFromUnits(threadStateData, units, length); + if(units != stackUnits) { + free(units); + } finishedNativeAllocations(); return o; } +/** + * Creates a java.lang.String by DECODING UTF-8, rather than widening bytes. + * + * newStringFromCString below widens each byte to a char independently -- its own + * comment says so, and that is correct for the generated string literals it exists + * to serve, which are ASCII plus ~~uXXXX escapes. It is wrong for any text that + * arrives from outside the program: a UTF-8 "e-acute" is two bytes, and widening + * them yields two garbage chars instead of one correct one. + * + * Invalid input decodes to U+FFFD rather than failing, which is what + * java.lang.String's own UTF-8 decoder does: a program should not die because one + * environment variable holds a stray byte. + * + * NOTE the Windows gap this does not close: argv and the environment arrive in the + * ACTIVE CODE PAGE there, not UTF-8, so they need the wide entry points + * (GetCommandLineW / _wgetenv) before any decoding is meaningful. That is the same + * unfixed issue recorded against the file layer in nativeMethods.m, and the same + * remedy. + */ +JAVA_OBJECT newStringFromUtf8(CODENAME_ONE_THREAD_STATE, const char* str) { + int length; + int in = 0; + int out = 0; + /* JAVA_ARRAY_CHAR, not JAVA_CHAR: these are UTF-16 code UNITS destined for a + string's backing array, and the two types are different widths. */ + JAVA_ARRAY_CHAR stackBuf[256]; + JAVA_ARRAY_CHAR* buf; + JAVA_OBJECT result; + if(str == 0) { + return JAVA_NULL; + } + length = (int)strlen(str); + /* One UTF-16 unit per input BYTE is always enough: a 1-byte sequence yields 1, + and the only multi-unit case (a 4-byte sequence yielding a surrogate pair) + yields 2 units from 4 bytes. An invalid byte yields exactly one U+FFFD. */ + buf = length <= 256 ? stackBuf : (JAVA_ARRAY_CHAR*)malloc((size_t)length * sizeof(JAVA_ARRAY_CHAR)); + if(buf == 0) { + return JAVA_NULL; + } + while(in < length) { + unsigned char b0 = (unsigned char)str[in]; + unsigned int cp; + int extra; + if(b0 < 0x80) { + buf[out++] = (JAVA_ARRAY_CHAR)b0; + in++; + continue; + } else if((b0 & 0xE0) == 0xC0) { cp = b0 & 0x1FU; extra = 1; } + else if((b0 & 0xF0) == 0xE0) { cp = b0 & 0x0FU; extra = 2; } + else if((b0 & 0xF8) == 0xF0) { cp = b0 & 0x07U; extra = 3; } + else { buf[out++] = 0xFFFD; in++; continue; } + + if(in + extra >= length + 0) { + /* Truncated at the end of the input. */ + if(in + extra > length - 1) { buf[out++] = 0xFFFD; in++; continue; } + } + { + int k; + int ok = 1; + for(k = 1 ; k <= extra ; k++) { + unsigned char bn = (unsigned char)str[in + k]; + if((bn & 0xC0) != 0x80) { ok = 0; break; } + cp = (cp << 6) | (bn & 0x3FU); + } + if(!ok) { buf[out++] = 0xFFFD; in++; continue; } + } + in += extra + 1; + /* Overlong forms, surrogates encoded as UTF-8, and out-of-range code points + are all rejected the way a conforming decoder must. */ + if((extra == 1 && cp < 0x80) || (extra == 2 && cp < 0x800) || (extra == 3 && cp < 0x10000) + || (cp >= 0xD800 && cp <= 0xDFFF) || cp > 0x10FFFF) { + buf[out++] = 0xFFFD; + continue; + } + if(cp >= 0x10000) { + cp -= 0x10000; + buf[out++] = (JAVA_ARRAY_CHAR)(0xD800 + (cp >> 10)); + buf[out++] = (JAVA_ARRAY_CHAR)(0xDC00 + (cp & 0x3FF)); + } else { + buf[out++] = (JAVA_ARRAY_CHAR)cp; + } + } + enteringNativeAllocations(); + result = cn1StringFromUnits(threadStateData, buf, out); + finishedNativeAllocations(); + if(buf != stackBuf) { + free(buf); + } + return result; +} + /** * Creates a java.lang.String object from a c string */ @@ -11928,7 +12075,10 @@ JAVA_OBJECT cn1MainArgs(CODENAME_ONE_THREAD_STATE, int argc, char* argv[]) { JAVA_OBJECT arrObj = allocArray(threadStateData, count, &class_array1__java_lang_String, sizeof(JAVA_OBJECT), 1); JAVA_ARRAY_OBJECT* dest = (JAVA_ARRAY_OBJECT*)((JAVA_ARRAY)arrObj)->data; for(int iter = 0 ; iter < count ; iter++) { - JAVA_OBJECT str = newStringFromCString(threadStateData, argv[iter + 1]); + /* Decoded, not widened: an argument is outside text. A UTF-8 "e-acute" is + two bytes, and widening them hands main(String[]) two garbage chars -- + enough to corrupt a path or an option value before the program starts. */ + JAVA_OBJECT str = newStringFromUtf8(threadStateData, argv[iter + 1]); CN1_WRITE_BARRIER(arrObj, str); dest[iter] = str; } diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 92ca9b70c3e..02a747e7dd5 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1104,8 +1104,14 @@ static void cn1FreeThreadStack(struct elementStruct* stack, int mapped) { // getenv returns a pointer into the process environment, which is owned by the // C runtime and must not be freed. stringToUTF8 hands back the calling thread's // scratch buffer, so the lookup must finish with it before anything else on this -// thread converts another string -- newStringFromCString copies, so building the +// thread converts another string -- newStringFromUtf8 copies, so building the // result here is safe. +// +// DECODED, not widened. An environment value is outside text: newStringFromCString +// turns each byte into its own char, so a UTF-8 value comes back as one garbage +// char per byte. (On Windows the value is in the ACTIVE CODE PAGE rather than +// UTF-8, so it needs _wgetenv before any decoding is meaningful -- the same unfixed +// issue recorded against the file layer below, and the same remedy.) JAVA_OBJECT java_lang_System_getenv___java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT name) { if(name == JAVA_NULL) { return JAVA_NULL; @@ -1118,7 +1124,7 @@ JAVA_OBJECT java_lang_System_getenv___java_lang_String_R_java_lang_String(CODENA if(value == NULL) { return JAVA_NULL; } - return newStringFromCString(threadStateData, value); + return newStringFromUtf8(threadStateData, value); } // --------------------------------------------------------------------------- diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetIntegrationTest.java index 08777a6f050..13ebb2c98e6 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetIntegrationTest.java @@ -1823,6 +1823,76 @@ static String floatingToStringSource() { } + /** + * argv and the environment must be DECODED as UTF-8, not widened byte by byte. + * + * newStringFromCString turns each byte into its own char, which is right for the + * generated string literals it serves and wrong for anything that arrives from + * outside: a two-byte UTF-8 character reaches main(String[]) as two garbage + * chars, quietly corrupting a path or an option value. The program below reports + * code points rather than the text itself, so the assertion does not depend on + * the console encoding of whatever runs it. + */ + @ParameterizedTest + @org.junit.jupiter.params.provider.MethodSource("com.codename1.tools.translator.BytecodeInstructionIntegrationTest#provideCompilerConfigs") + void argumentsAndEnvironmentDecodeAsUtf8(CompilerHelper.CompilerConfig config) throws Exception { + Parser.cleanup(); + Path sourceDir = Files.createTempDirectory("utf8-args-sources"); + Path classesDir = Files.createTempDirectory("utf8-args-classes"); + Path javaApiDir = Files.createTempDirectory("utf8-args-java-api"); + Files.write(sourceDir.resolve("Utf8ArgsApp.java"), utf8ArgsSource().getBytes(StandardCharsets.UTF_8)); + JavascriptTargetIntegrationTest.compileAgainstJavaApi(config, sourceDir, classesDir, javaApiDir); + + Path outputDir = Files.createTempDirectory("utf8-args-output"); + runTranslator(classesDir, outputDir, "Utf8ArgsApp"); + Path distDir = outputDir.resolve("dist"); + replaceLibraryWithExecutableTarget(distDir.resolve("CMakeLists.txt"), "Utf8ArgsApp-src"); + Path buildDir = distDir.resolve("build"); + Files.createDirectories(buildDir); + List configure = new java.util.ArrayList<>(Arrays.asList( + "cmake", "-S", distDir.toString(), "-B", buildDir.toString(), "-DCMAKE_BUILD_TYPE=Release")); + configure.addAll(CompilerHelper.cmakeToolchainArgs()); + runCommand(configure, distDir); + runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), distDir); + + // U+00E9 (two UTF-8 bytes) and U+20AC (three), so both the 2- and 3-byte + // paths are covered; widening would report the individual bytes instead. + String arg = "caf\u00e9\u20ac"; + Path exe = buildDir.resolve(CompilerHelper.executableName("Utf8ArgsApp")); + ProcessBuilder pb = new ProcessBuilder(exe.toString(), arg); + pb.directory(buildDir.toFile()); + pb.redirectErrorStream(true); + pb.environment().put("CN1_UTF8_PROBE", arg); + Process p = pb.start(); + String out; + try (BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream(), StandardCharsets.UTF_8))) { + out = r.lines().collect(Collectors.joining("\n")); + } + assertEquals(0, p.waitFor(), "Utf8ArgsApp failed:\n" + out); + assertTrue(out.contains("ARG=99,97,102,233,8364"), + "argv must decode to the right code points, got:\n" + out); + assertTrue(out.contains("ENV=99,97,102,233,8364"), + "the environment must decode to the right code points, got:\n" + out); + } + + private static String utf8ArgsSource() { + return "public class Utf8ArgsApp {\n" + + " private static String points(String s) {\n" + + " StringBuilder b = new StringBuilder();\n" + + " for (int i = 0; i < s.length(); i++) {\n" + + " if (i > 0) { b.append(','); }\n" + + " b.append((int) s.charAt(i));\n" + + " }\n" + + " return b.toString();\n" + + " }\n" + + " public static void main(String[] args) {\n" + + " System.out.println(\"ARG=\" + (args.length > 0 ? points(args[0]) : \"none\"));\n" + + " String e = System.getenv(\"CN1_UTF8_PROBE\");\n" + + " System.out.println(\"ENV=\" + (e == null ? \"none\" : points(e)));\n" + + " }\n" + + "}\n"; + } + private static void restoreProperty(String key, String value) { if (value == null) { System.clearProperty(key); From 3451ef02b3c2ea8a5c42f89ae423ea80ad86b127 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:29:09 +0300 Subject: [PATCH 23/42] Decode argv and the environment in the PLATFORM's encoding, not always UTF-8 The Windows clean-target leg failed the test added with the UTF-8 decoder, and it was right to: "cafe-acute-euro" arrived as 99,97,102,65533,65533 -- c, a, f, and two replacement characters. The CRT hands main() and getenv() the wide command line and environment already converted down to the ACTIVE CODE PAGE, so decoding those bytes as UTF-8 finds invalid sequences and substitutes U+FFFD for every non-ASCII character. That failure was predicted by a comment I had written in this very function -- which then shipped alongside a test asserting the behaviour the comment said did not exist. MultiByteToWideChar with CP_ACP is the conversion Windows actually needs, and it yields UTF-16 code units directly, so nothing decodes afterwards. RENAMED from newStringFromUtf8 to newStringFromNative for the same reason: a function named FromUtf8 that deliberately does not decode UTF-8 on one of its platforms is a trap for whoever reads it next. The name now says what it does -- convert text that came from the OS, in whatever encoding the OS used. WIN32_LEAN_AND_MEAN before windows.h, which is the same winsock timeval collision that broke java_io_File.m; and the byte-length local moved onto the POSIX arm, which is the only one that uses it. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 13 ++++-- vm/ByteCodeTranslator/src/cn1_globals.m | 54 ++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 28168fa83e4..78cf6aef6e9 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2570,11 +2570,16 @@ extern struct clazz class_array3__JAVA_DOUBLE; extern JAVA_OBJECT newString(CODENAME_ONE_THREAD_STATE, int length, JAVA_CHAR data[]); /** - * Like newStringFromCString but DECODES UTF-8 instead of widening bytes. Use it for - * text that came from outside the program (argv, the environment); the widening one - * is right only for generated literals, which are ASCII plus ~~uXXXX escapes. + * Like newStringFromCString but DECODES, in the PLATFORM's encoding, instead of + * widening bytes. Use it for text that came from outside the program (argv, the + * environment); the widening one is right only for generated literals, which are + * ASCII plus ~~uXXXX escapes. + * + * UTF-8 on POSIX; the active code page on Windows, where the CRT has already + * converted the wide command line and environment down to it. Not named FromUtf8 + * for exactly that reason. */ -extern JAVA_OBJECT newStringFromUtf8(CODENAME_ONE_THREAD_STATE, const char* str); +extern JAVA_OBJECT newStringFromNative(CODENAME_ONE_THREAD_STATE, const char* str); extern JAVA_OBJECT newStringFromCString(CODENAME_ONE_THREAD_STATE, const char *str); extern JAVA_OBJECT newStringFromAsciiLen(CODENAME_ONE_THREAD_STATE, const char *src, int len); // Single-allocation fused compact-String builder (see cn1_globals.m). Returns a valid empty diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 0b7149487b1..ad9a664fd06 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -11032,6 +11032,15 @@ JAVA_OBJECT alloc4DArray(CODENAME_ONE_THREAD_STATE, int length4, int length3, in * Creates a java.lang.String object from an array of integers, this is useful * for the constant pool */ +#ifdef _WIN32 +/* MultiByteToWideChar for the native-encoding conversion below. LEAN_AND_MEAN keeps + winsock's timeval out, which collides with cn1_win_compat.h's. */ +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#endif + /* * Builds a java.lang.String from decoded UTF-16 code units. * @@ -11040,7 +11049,7 @@ JAVA_OBJECT alloc4DArray(CODENAME_ONE_THREAD_STATE, int length4, int length3, in * reads whichever it finds. Handing it the wrong one does not fail loudly -- it * reads 8-bit units out of 16-bit data, so "caf..." comes back as c,NUL,a,NUL,f. * - * Used by newString and newStringFromUtf8. newStringFromCString deliberately keeps + * Used by newString and newStringFromNative. newStringFromCString deliberately keeps * its own copy of this tail: it tracks the Latin-1 flag DURING decoding, and it runs * for every generated string literal at startup, so routing it through here would * add a second pass over every literal in the program to save a dozen duplicated @@ -11102,7 +11111,12 @@ JAVA_OBJECT newString(CODENAME_ONE_THREAD_STATE, int length, JAVA_CHAR data[]) { } /** - * Creates a java.lang.String by DECODING UTF-8, rather than widening bytes. + * Creates a java.lang.String by DECODING text that came from the OS, rather than + * widening its bytes. + * + * The encoding is the PLATFORM's, which is why this is not called FromUtf8: UTF-8 + * on POSIX, and the active code page on Windows, where the CRT has already + * converted the wide command line and environment down to it. * * newStringFromCString below widens each byte to a char independently -- its own * comment says so, and that is correct for the generated string literals it exists @@ -11120,8 +11134,37 @@ JAVA_OBJECT newString(CODENAME_ONE_THREAD_STATE, int length, JAVA_CHAR data[]) { * unfixed issue recorded against the file layer in nativeMethods.m, and the same * remedy. */ -JAVA_OBJECT newStringFromUtf8(CODENAME_ONE_THREAD_STATE, const char* str) { - int length; +JAVA_OBJECT newStringFromNative(CODENAME_ONE_THREAD_STATE, const char* str) { +#ifdef _WIN32 + /* NOT UTF-8 on Windows. The CRT hands main() and getenv() the wide command line + and environment converted down to the ACTIVE CODE PAGE, so decoding those + bytes as UTF-8 yields U+FFFD for every non-ASCII character -- which is what + the clean-target Windows leg reported for "cafe-acute-euro": 99,97,102,65533, + 65533. MultiByteToWideChar with CP_ACP is the conversion the platform + actually needs, and it produces UTF-16 code units directly, so no decoding + follows it. */ + if(str != 0) { + int wide = MultiByteToWideChar(CP_ACP, 0, str, -1, NULL, 0); + if(wide > 0) { + JAVA_ARRAY_CHAR wstack[256]; + JAVA_ARRAY_CHAR* wbuf = wide <= 256 ? wstack + : (JAVA_ARRAY_CHAR*)malloc((size_t)wide * sizeof(JAVA_ARRAY_CHAR)); + if(wbuf != 0) { + JAVA_OBJECT wres; + int got = MultiByteToWideChar(CP_ACP, 0, str, -1, (LPWSTR)wbuf, wide); + /* got includes the terminating NUL; the string does not. */ + if(got > 0) { got--; } else { got = 0; } + enteringNativeAllocations(); + wres = cn1StringFromUnits(threadStateData, wbuf, got); + finishedNativeAllocations(); + if(wbuf != wstack) { free(wbuf); } + return wres; + } + } + return JAVA_NULL; + } + return JAVA_NULL; +#endif int in = 0; int out = 0; /* JAVA_ARRAY_CHAR, not JAVA_CHAR: these are UTF-16 code UNITS destined for a @@ -11129,6 +11172,7 @@ JAVA_OBJECT newStringFromUtf8(CODENAME_ONE_THREAD_STATE, const char* str) { JAVA_ARRAY_CHAR stackBuf[256]; JAVA_ARRAY_CHAR* buf; JAVA_OBJECT result; + int length; if(str == 0) { return JAVA_NULL; } @@ -12078,7 +12122,7 @@ JAVA_OBJECT cn1MainArgs(CODENAME_ONE_THREAD_STATE, int argc, char* argv[]) { /* Decoded, not widened: an argument is outside text. A UTF-8 "e-acute" is two bytes, and widening them hands main(String[]) two garbage chars -- enough to corrupt a path or an option value before the program starts. */ - JAVA_OBJECT str = newStringFromUtf8(threadStateData, argv[iter + 1]); + JAVA_OBJECT str = newStringFromNative(threadStateData, argv[iter + 1]); CN1_WRITE_BARRIER(arrObj, str); dest[iter] = str; } From ea35b3b21a52daccb34b71be8e16de2c000624ff Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:29:09 +0300 Subject: [PATCH 24/42] Validate an array store's index before checking its element type The JLS orders these: NullPointerException, then ArrayIndexOutOfBoundsException, then ArrayStoreException. Under -Dcn1.checkedCasts the emitted covariance check ran BEFORE the setter that reports the first two, so a store with both a bad index and an incompatible value reported the value -- hiding the exception the program should have seen. (The null case was worse and is already fixed: the check dereferenced the array to reach its class.) The store check is now guarded by the same access validation the setter performs, so the first two exceptions are thrown first and in the right order. The setter re-checks, which on the in-bounds fast path costs one comparison. Co-Authored-By: Claude Opus 5 (1M context) --- .../tools/translator/BytecodeMethod.java | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java index ef79711ebb4..a38dbf28283 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java @@ -4695,10 +4695,24 @@ boolean optimize() { " " + valueType + " __cn1ValueTmp = " + valueLiteral + ";\n" + // The macro's own comment used to claim it covariance-checks // OBJECT stores; it never did. Under -Dcn1.checkedCasts the - // check is emitted here, ahead of the store. + // check is emitted here. + // + // GUARDED BY THE ACCESS VALIDATION, because the JLS orders + // these: NullPointerException, then + // ArrayIndexOutOfBoundsException, then ArrayStoreException. + // Run bare, the covariance check reports a bad VALUE on a + // store whose INDEX is also bad, hiding the exception the + // program should have seen -- and on a null array it used to + // dereference null outright. cn1_array_access_validate throws + // the right one of the first two; the setter re-checks on its + // in-bounds fast path, which costs a comparison. ("OBJECT".equals(elementType) && ByteCodeTranslator.isCheckedCastsEnabled() - ? " CN1_ARRAY_STORE_CHECK(__cn1ArrayTmp, __cn1ValueTmp);\n" : "") + - " CN1_SET_ARRAY_ELEMENT_"+elementType+"(__cn1ArrayTmp, __cn1IndexTmp, __cn1ValueTmp);\n" + + ? " if(cn1_array_access_in_bounds(__cn1ArrayTmp, __cn1IndexTmp)\n" + + " || cn1_array_access_validate(threadStateData, __cn1ArrayTmp, __cn1IndexTmp)) {\n" + + " CN1_ARRAY_STORE_CHECK(__cn1ArrayTmp, __cn1ValueTmp);\n" + + " CN1_SET_ARRAY_ELEMENT_"+elementType+"(__cn1ArrayTmp, __cn1IndexTmp, __cn1ValueTmp);\n" + + " }\n" + : " CN1_SET_ARRAY_ELEMENT_"+elementType+"(__cn1ArrayTmp, __cn1IndexTmp, __cn1ValueTmp);\n") + " }\n"; } instructions.add(iter-3, new CustomIntruction(code, code, dependentClasses)); From 2b9d12aec39f4030655de0531b6ce8615e042f79 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:29:33 +0300 Subject: [PATCH 25/42] Revert marking a running virtual thread's state active -- it hangs the collector This backs out my own fix from earlier in this branch. Marking the attached ThreadLocalData threadActive around the context switch reads as obviously correct and is a REGRESSION, worse than what it fixed. A virtual thread's state has no pthread of its own -- deliberately, it may run on a different carrier next time. The collector's wait for a lightweight thread is `while(t->threadActive) usleep(500)` with no bound, and the forced-stop escalation that exists to break exactly that wait is gated on gcPthreadValid, which is permanently false here. So the flag converts a POSSIBLE race on the state's object stack into a CERTAIN hang for any virtual thread that computes without reaching a safepoint: the collector waits for a flag only that thread can clear, and cannot stop it. What the same report asked for has two halves, and the other one stands. The C stack is covered: cn1GcScanParkedVirtualThreads scans every registered virtual thread whether or not it is running, so no virtual stack goes unscanned during the windows where `running` is set but the carrier has not switched yet. That fix is independent of this revert and stays. The half that remains open -- a collection walking the state's object stack and pending-allocation table while the virtual thread mutates them -- is documented at cn1SpawnVirtualThread along with why the obvious fix is worse and what the real one is: carrier association. A running virtual thread executes ON a carrier that does have a stoppable pthread, so the collector should satisfy the wait by stopping the carrier. That needs the stop handshake to stop being per-TLD (the signal handler records into the TLD of the thread it runs on, which is the carrier's), i.e. a change to the collector's stop protocol rather than to the spawn path -- not something to improvise in an API that has no callers yet. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/cn1_virtual_thread.c | 28 +++++--------- .../src/cn1_virtual_thread.h | 14 ------- vm/ByteCodeTranslator/src/nativeMethods.m | 38 ++++++++++++------- 3 files changed, 34 insertions(+), 46 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_virtual_thread.c b/vm/ByteCodeTranslator/src/cn1_virtual_thread.c index ff9c9a53352..87cedf26cd9 100644 --- a/vm/ByteCodeTranslator/src/cn1_virtual_thread.c +++ b/vm/ByteCodeTranslator/src/cn1_virtual_thread.c @@ -320,11 +320,6 @@ void cn1VirtualThreadStackBounds(struct cn1VirtualThread* co, void** low, void** /* Set up the initial frame so the first switch lands in the trampoline. */ extern void* cn1VirtualThreadPrime(void* stackHigh, void* co, void* trampoline); -/* The default. Overridden by the VM's strong definition when one is linked in. */ -__attribute__((weak)) void cn1VirtualThreadVmStateActive(void* vmState, int active) { - (void)vmState; (void)active; -} - void cn1VirtualThreadResume(struct cn1VirtualThread* co) { struct cn1VirtualThread* previous = cn1CurrentVirtualThread; if(co == 0 || co->finished) { @@ -336,21 +331,16 @@ void cn1VirtualThreadResume(struct cn1VirtualThread* co) { } cn1CurrentVirtualThread = co; co->running = 1; - /* The attached VM state has to become ACTIVE here, not just `running`. It was - * created parked (cn1CreateThreadLocalData with bindToCallingOsThread false - * leaves threadActive FALSE) and nothing else ever raises it, so without this a - * collection running concurrently treats a mutator that is executing Java as - * parked -- and scans or migrates its object stack and pending-allocation table - * underneath it. Missed roots at best, corruption at worst. Lowered again on the - * way out, because a SUSPENDED virtual thread genuinely is parked: the collector - * reaches its roots through the registry snapshot instead. */ - if(co->vmState != 0) { - cn1VirtualThreadVmStateActive(co->vmState, 1); - } + /* NOTE, and this is a KNOWN GAP rather than an oversight -- see the block above + * cn1SpawnVirtualThread in nativeMethods.m. The attached VM state is NOT marked + * threadActive here. Marking it looks obviously right and is a collector HANG: + * the state has no pthread of its own, and the collector's unbounded + * while(threadActive) wait can only be broken by a forced stop, which is gated + * on gcPthreadValid -- permanently false for a virtual thread. A compute-only + * virtual thread that never reaches a safepoint would stall collection forever. + * The C stack is covered regardless, by cn1GcScanParkedVirtualThreads, which + * scans every registered virtual thread whether or not it is running. */ cn1VirtualThreadSwitch(&co->returnSp, co->sp); - if(co->vmState != 0) { - cn1VirtualThreadVmStateActive(co->vmState, 0); - } co->running = 0; cn1CurrentVirtualThread = previous; } diff --git a/vm/ByteCodeTranslator/src/cn1_virtual_thread.h b/vm/ByteCodeTranslator/src/cn1_virtual_thread.h index d88fe5f16c6..b59baa8b2f4 100644 --- a/vm/ByteCodeTranslator/src/cn1_virtual_thread.h +++ b/vm/ByteCodeTranslator/src/cn1_virtual_thread.h @@ -75,20 +75,6 @@ struct cn1VirtualThread; -/* - * Tells the VM that the state attached to a virtual thread has started or stopped - * running Java, so the collector stops or resumes treating it as parked. - * - * It is a WEAK symbol with a no-op default rather than a function pointer for two - * reasons: an indirect call on a path whose whole point is that it costs 2.1ns is - * not free, and this file has to keep linking on its own -- the standalone runtime - * test builds it without any VM at all. nativeMethods.m provides the real one. - * - * Kept out of the header's no-op section deliberately: it is about the VM's view of - * a virtual thread, not about the switch, so it exists on every target. - */ -void cn1VirtualThreadVmStateActive(void* vmState, int active); - /** The body of a virtual thread. Returning from it finishes the virtual thread. */ typedef void (*cn1VirtualThreadBody)(void* arg); diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 02a747e7dd5..f1d219d9d27 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1104,7 +1104,7 @@ static void cn1FreeThreadStack(struct elementStruct* stack, int mapped) { // getenv returns a pointer into the process environment, which is owned by the // C runtime and must not be freed. stringToUTF8 hands back the calling thread's // scratch buffer, so the lookup must finish with it before anything else on this -// thread converts another string -- newStringFromUtf8 copies, so building the +// thread converts another string -- newStringFromNative copies, so building the // result here is safe. // // DECODED, not widened. An environment value is outside text: newStringFromCString @@ -1124,7 +1124,7 @@ JAVA_OBJECT java_lang_System_getenv___java_lang_String_R_java_lang_String(CODENA if(value == NULL) { return JAVA_NULL; } - return newStringFromUtf8(threadStateData, value); + return newStringFromNative(threadStateData, value); } // --------------------------------------------------------------------------- @@ -2235,6 +2235,29 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC * actually live. Giving it a stack but sharing the host's state would have two * threads of control writing one Java stack. * + * KNOWN GAP, stated here because the obvious fix is worse than the problem. The + * state this creates is never marked threadActive while its virtual thread runs, + * so a collection concurrent with a running virtual thread can walk that state's + * object stack and pending-allocation table while the virtual thread mutates them. + * Raised in review on PR #5658, and NOT fixed by setting threadActive: the state + * has no pthread of its own, the collector's while(threadActive) wait is unbounded, + * and the forced-stop escalation that breaks such a wait is gated on + * gcPthreadValid, which is permanently false here. Setting the flag converts a + * possible race into a certain hang for any virtual thread that computes without + * reaching a safepoint. That was measured against, not guessed: the flag was set, + * and this is the reverted state. + * + * The real fix is carrier association -- while a virtual thread runs, its state is + * executing ON a carrier that DOES have a stoppable pthread, so the collector + * should satisfy the wait by stopping the carrier rather than the state. That needs + * the stop handshake to stop being per-TLD (the signal handler records into the TLD + * of the thread it runs on, which is the carrier's), which is a change to the + * collector's stop protocol rather than to this function. + * + * The C stack half of the same report IS fixed, independently: + * cn1GcScanParkedVirtualThreads scans every registered virtual thread whether or + * not it is running, so no virtual stack goes unscanned. + * * Sizing: threadObjectStack is mmap'd and lazily faulted, so the 264KB it * reserves costs only the pages a virtual thread touches -- a handler that nests * a dozen frames commits a page or two. That is the difference against the @@ -2265,17 +2288,6 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC extern void markDeadThread(struct ThreadLocalData* d); extern void cn1ReleaseThreadLocalData(struct ThreadLocalData* head); -/* - * The strong definition of the weak hook cn1VirtualThreadResume calls. See the - * comment there for why the flag has to move with the switch. - */ -void cn1VirtualThreadVmStateActive(void* vmState, int active) { - struct ThreadLocalData* state = (struct ThreadLocalData*)vmState; - if(state != 0) { - state->threadActive = active ? JAVA_TRUE : JAVA_FALSE; - } -} - /** * Retire a virtual thread produced by cn1SpawnVirtualThread, releasing BOTH halves. * From 48e84bb243ac75e930bd31cbfd6347b837dd511d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:53:28 +0300 Subject: [PATCH 26/42] Stop the array store check rejecting valid multidimensional stores With -Dcn1.checkedCasts the covariance check broke correct programs, which is the worst direction for a check to fail in. A generated array class records arrayType as the BASE element class rather than the immediate component: String[][] has dimensions 2 and arrayType String, not String[]. So `values[0] = new String[1]` asked whether a String[] is an instance of String, got no, and threw ArrayStoreException on a store the language requires to succeed. Restricted to dimensions == 1, where arrayType genuinely IS the component type. Multidimensional stores lose a diagnostic that did not exist before this feature was added; the alternative was breaking working code. Covering them properly needs the immediate component type, either emitted per array class or reconstructed from dimensions at runtime, and the macro says so. Also fixes a timeout in VirtualThreadRuntimeTest that could never fire. It read the child's output inline and then called waitFor: the read blocks until the child closes stdout, so a binary that hangs -- exactly what a context-switch regression produces -- never reached the timeout, and the Maven job would sit until CI killed it instead of the test failing. Output now drains on its own thread, with a bounded join so a wedged reader cannot reintroduce the hang the change removes. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 18 ++++++-- .../translator/VirtualThreadRuntimeTest.java | 42 +++++++++++++++---- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 78cf6aef6e9..50390b77a04 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -472,16 +472,26 @@ typedef struct clazz* JAVA_CLASS; // drives BC_CHECKCAST_CHECKED, so ArrayStoreException's retention and the check's // emission cannot disagree. // -// arrayType is the component class (0 for a non-array, which cannot happen here -// after CHECK_ARRAY_ACCESS, but is tolerated rather than dereferenced). /* The arrayObj null test is not redundant. This runs BEFORE CN1_SET_ARRAY_ELEMENT_OBJECT, which is where a null array is turned into a NullPointerException; CN1_CLASS_OF below would dereference the null first and take the process down instead. Java also orders it this way -- NPE wins over ArrayStoreException -- so falling through to the setter is both safe and - correct. */ + correct. + + ONE DIMENSION ONLY, and that restriction is load-bearing. arrayType is NOT the + immediate component type: a generated array class records the BASE element class, + so String[][] has dimensions 2 and arrayType String rather than String[]. Asking + whether a String[] is an instance of String is the wrong question and answers no, + so without the dimensions test this REJECTED valid stores into every + multidimensional array. Skipping them is the conservative direction -- a genuine + ArrayStoreException there goes unreported, exactly as it did before this check + existed, where the alternative was breaking correct programs. Covering them needs + the immediate component type, which means emitting it per array class or + reconstructing it from dimensions at runtime. */ #define CN1_ARRAY_STORE_CHECK(arrayObj, value) { \ - if((value) != JAVA_NULL && (arrayObj) != JAVA_NULL) { \ + if((value) != JAVA_NULL && (arrayObj) != JAVA_NULL \ + && CN1_CLASS_OF(arrayObj)->dimensions == 1) { \ struct clazz* cn1__comp = CN1_CLASS_OF(arrayObj)->arrayType; \ if(cn1__comp != NULL && !instanceofFunction(cn1__comp->classId, GET_CLASS_ID(value))) { \ cn1ThrowTypeError(threadStateData, __NEW_INSTANCE_java_lang_ArrayStoreException(threadStateData), CN1_CLASS_OF(value)->clsName, NULL); \ diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/VirtualThreadRuntimeTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/VirtualThreadRuntimeTest.java index a172bfdcc3a..53f3a5ab249 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/VirtualThreadRuntimeTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/VirtualThreadRuntimeTest.java @@ -111,15 +111,41 @@ private static String run(List command, int timeoutMinutes) throws Excep ProcessBuilder builder = new ProcessBuilder(command); builder.redirectErrorStream(true); Process p = builder.start(); - java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); - byte[] buffer = new byte[4096]; - int read; - while ((read = p.getInputStream().read(buffer)) > 0) { - out.write(buffer, 0, read); - } - String output = new String(out.toByteArray(), StandardCharsets.UTF_8); - if (!p.waitFor(timeoutMinutes, TimeUnit.MINUTES)) { + + // Drained on a SEPARATE thread, because the timeout below is worthless + // otherwise. Reading inline blocks until the child closes stdout, so a + // context-switch regression that hangs the binary would never reach waitFor + // -- the Maven job would sit until CI killed it, instead of this test + // failing. A timeout that the hang it guards against prevents from ever + // being evaluated is not a timeout. + final java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + Thread drain = new Thread(new Runnable() { + public void run() { + byte[] buffer = new byte[4096]; + int read; + try { + while ((read = p.getInputStream().read(buffer)) > 0) { + synchronized (out) { out.write(buffer, 0, read); } + } + } catch (java.io.IOException ignored) { + // the stream closes under us when the process is destroyed + } + } + }, "vt-runtime-output"); + drain.setDaemon(true); + drain.start(); + + boolean finished = p.waitFor(timeoutMinutes, TimeUnit.MINUTES); + if (!finished) { p.destroyForcibly(); + } + // Bounded join: the drain ends when the stream closes, which destroying the + // process guarantees, but a bound here keeps a wedged reader from replacing + // the hang this method just avoided. + drain.join(TimeUnit.SECONDS.toMillis(30)); + String output; + synchronized (out) { output = new String(out.toByteArray(), StandardCharsets.UTF_8); } + if (!finished) { fail("timed out: " + command + "\n" + output); } assertEquals(0, p.exitValue(), "failed: " + command + "\n" + output); From 287ab9df5117ea2bacf05d109ee2840d458d7642 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:09:39 +0300 Subject: [PATCH 27/42] Probe an unresponsive thread briefly instead of skipping its roots This corrects my own change earlier in this branch, and the reasoning behind it was the defect. "A thread it cannot stop is one it does not scan either way" is true only while the thread genuinely cannot be stopped. Failures are often TRANSIENT -- a stop signal briefly masked is enough -- and the thread recovers. Skipping it then meant cn1GcScanThreadNativeStack returned without scanning a RESPONSIVE thread, for roughly the next sixty collections, so references held only in frameless C locals or registers went unmarked and could be reclaimed while still in use. A GC correctness bug, traded for a performance win. The two things I had conflated: the cost was never the SIGNAL, it was the WAIT. One unresponsive thread consumed the entire 2,000,000-spin budget -- 267ms of a 280ms mark. So a thread with a failure history is now probed with a 20,000-spin budget rather than skipped. Healthy threads answer within about 200 spins, which is a hundredfold margin for one that is merely slow, at one percent of what a hang used to cost; and a thread that recovers is picked up on the very next cycle instead of up to 64 later. Verified across the GC suites, including GcUncooperativeThreadIntegrationTest -- the issue #5537 scenario this logic exists to serve: 6/6. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.m | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index ad9a664fd06..52335a534c7 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -8549,13 +8549,24 @@ void cn1GcInstallSignalHandler(void) { static char* cn1GcSignalStopOneImpl(struct ThreadLocalData* t, int maySkip) { #if !defined(_WIN32) if(!t->gcPthreadValid) return 0; - // SKIP a thread that has proved unresponsive rather than waiting on it again. - // Re-probe every 64th attempt so one that becomes responsive is picked back up. + // A thread with a history of timing out is PROBED WITH A SHORT BUDGET, never + // skipped. Skipping it was wrong, and the reasoning that produced it was wrong: + // "a thread it cannot stop is one it does not scan either way" holds only while + // the thread genuinely cannot be stopped. A thread whose failures were transient + // -- the stop signal briefly masked, say -- recovers, and skipping it then means + // cn1GcScanThreadNativeStack returns without scanning a RESPONSIVE thread, so + // references held only in frameless C locals or registers go unmarked and can be + // reclaimed while in use. + // + // The cost this exists to avoid was never the signal; it was the WAIT. One + // unresponsive thread consumed the whole 2,000,000-spin budget, 267ms of a 280ms + // mark. Healthy threads answer within about 200 spins, so a budget of 20,000 + // keeps a hundredfold margin for a thread that is merely slow while costing one + // percent of what a hang used to. A recovered thread is picked up on the very + // next cycle rather than up to 64 cycles later. + int spinBudget = 2000000; if(maySkip && t->gcStopFailures >= 3) { - if(t->gcStopFailures < 1000000000) { t->gcStopFailures++; } - if((t->gcStopFailures & 63) != 0) { - return 0; - } + spinBudget = 20000; } // Next generation for this thread (only the GC thread writes it). gcSigRelease // is MONOTONIC and never reset -- see the handler's generation handshake. @@ -8569,7 +8580,7 @@ void cn1GcInstallSignalHandler(void) { // bounded wait for the handler to park THIS generation int spins = 0; while((int)t->gcSigStopped != gen) { - if(++spins > 2000000) { /* ~timeout: could not stop */ break; } + if(++spins > spinBudget) { /* ~timeout: could not stop */ break; } if((spins & 1023) == 0) usleep(50); } if((int)t->gcSigStopped != gen) { From bb98371968a5cf4d24c80b3dc66a74e0c9a7213d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:09:39 +0300 Subject: [PATCH 28/42] Emit JSON null for an unset boxed Boolean or Character Boolean shares its kind with boolean and Character with char, so the direct writer treated both as primitives. Only the boxed form can be null, and both handled it wrongly in opposite ways: a null Boolean was unboxed by a ternary and threw NullPointerException, and a null Character went through String.valueOf(Object), which returns the four characters "null", and was then QUOTED -- so an unset field serialised as the string "null". The map path stores the value and lets JSONWriter see the null, emitting JSON null for both. Told apart by binaryName, which does distinguish them, with a temporary in each so a getter is not evaluated twice, and charValue() so String.valueOf resolves to the char overload rather than the Object one. The parity test carries both fields now, and they discriminate by construction: against the old code the Boolean case throws (a test error) and the Character case produces a quoted "null" against the map path's null (an assertion mismatch). Co-Authored-By: Claude Opus 5 (1M context) --- .../MappingAnnotationProcessor.java | 31 +++++++++++++++++-- .../MappingAnnotationProcessorTest.java | 6 ++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java index df8e8d856ea..f40909fee9b 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java @@ -585,11 +585,36 @@ private static void emitFieldToJson(StringBuilder sb, MappedField f, boolean isR sb.append(" out.append(").append(read).append(");\n"); return; case BOOLEAN: - sb.append(" out.append(").append(read).append(" ? \"true\" : \"false\");\n"); + // Boolean and boolean share this kind, and only the BOXED one can be + // null. Unboxing it in a ternary threw NullPointerException where the + // map path -- which just puts the value in and lets JSONWriter see a + // null -- emits JSON null. Told apart by binaryName; the temporary + // keeps a getter from being evaluated twice. + if ("java.lang.Boolean".equals(f.kind.binaryName)) { + sb.append(" {\n"); + sb.append(" Boolean _b = ").append(read).append(";\n"); + sb.append(" out.append(_b == null ? \"null\" : (_b.booleanValue() ? \"true\" : \"false\"));\n"); + sb.append(" }\n"); + } else { + sb.append(" out.append(").append(read).append(" ? \"true\" : \"false\");\n"); + } return; case CHAR: - sb.append(" com.codename1.mapping.Mappers.appendJsonString(out, String.valueOf(") - .append(read).append("));\n"); + // Same split. A null Character went through String.valueOf(Object), + // which returns the four characters "null", and then got QUOTED -- + // so an unset field serialised as the string "null" instead of JSON + // null. charValue() below also picks String.valueOf(char) rather than + // the Object overload. + if ("java.lang.Character".equals(f.kind.binaryName)) { + sb.append(" {\n"); + sb.append(" Character _c = ").append(read).append(";\n"); + sb.append(" if (_c == null) { out.append(\"null\"); }\n"); + sb.append(" else { com.codename1.mapping.Mappers.appendJsonString(out, String.valueOf(_c.charValue())); }\n"); + sb.append(" }\n"); + } else { + sb.append(" com.codename1.mapping.Mappers.appendJsonString(out, String.valueOf(") + .append(read).append("));\n"); + } return; case ENUM: sb.append(" com.codename1.mapping.Mappers.appendJsonString(out, ") diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java index 981952742d6..5d4985c6a63 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java @@ -359,6 +359,10 @@ public void directJsonMatchesTheMapPathExactly() throws Exception { // compile. + " @JsonProperty(\"od\\\"d\\\\key\") public String odd;\n" // Declared as the mapped base, populated with the subclass. + // Boxed primitives share their kind with the unboxed form, so + // only these can be null. Left unset on the "empty" instance. + + " public Boolean flag;\n" + + " public Character initial;\n" + " public Base ref;\n" + " public List refs;\n" + " public Swatch() {}\n" @@ -391,6 +395,8 @@ public void directJsonMatchesTheMapPathExactly() throws Exception { swatchCls.getField("tags").set(populated, Arrays.asList("a", "b")); swatchCls.getField("when").set(populated, new java.util.Date(1234567890L)); swatchCls.getField("odd").set(populated, "quoted"); + swatchCls.getField("flag").set(populated, Boolean.TRUE); + swatchCls.getField("initial").set(populated, Character.valueOf('x')); // Base's mapper has to be REGISTERED or the declared-type lookup finds // nothing and both paths fall back to toString() -- agreeing with each // other while proving nothing about the polymorphic case. Registering it From 9f27f80b215f9a076f383e518b357a2e738b909a Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:25:19 +0300 Subject: [PATCH 29/42] Report thread-table exhaustion instead of writing before the table CODENAME_ONE_ASSERT is plain assert(), which NDEBUG compiles out of every release build. So once all NUMBER_OF_SUPPORTED_THREADS slots were taken, threadOffset stayed -1, the assertion vanished, and the next statement executed allThreads[-1] = i -- writing over whatever precedes the table. A debug build aborted; a shipped one carried on with silent memory corruption, which is the worse of the two. Capacity exhaustion is a condition to report, not to assert. It returns 0 now, and cn1SpawnVirtualThread already checks for that. Pre-existing rather than new: every OS thread creation runs this path too. A virtual thread per request only makes reaching the limit realistic. The partially built state is unwound through cn1FreeThreadLocalDataFields, extracted from cn1ReleaseThreadLocalData rather than copied, because the release path also decrements nThreadsToKill and a state that never reached allThreads was never counted as living. Duplicating the frees would have drifted apart, and getting that counter wrong would have been a slow leak in the opposite direction. Verified across the GC suites including GcUncooperativeThread and GcHeapIntegrity: 6/6. (The translator build says nothing about this -- it compiles Java, and the C here is only compiled by those tests.) Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/nativeMethods.m | 28 ++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index f1d219d9d27..55b086e720d 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -2062,6 +2062,10 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC * OS thread: a virtual thread's state belongs to the virtual thread and travels * with it between hosts. */ +/* Defined with cn1ReleaseThreadLocalData further down; the capacity-failure path + below unwinds a partially built state through it. */ +static void cn1FreeThreadLocalDataFields(struct ThreadLocalData* head); + struct ThreadLocalData* cn1CreateThreadLocalData(JAVA_BOOLEAN bindToCallingOsThread) { struct ThreadLocalData* i; JAVA_LONG nativeThreadId = threadKeyCounter; @@ -2217,7 +2221,17 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC break; } } - CODENAME_ONE_ASSERT(threadOffset > -1); + /* EXHAUSTION IS A RETURN VALUE, not an assertion. CODENAME_ONE_ASSERT is plain + assert(), which NDEBUG compiles out of every release build -- so once all + NUMBER_OF_SUPPORTED_THREADS slots were taken this fell through and executed + allThreads[-1] = i, corrupting whatever precedes the table instead of failing. + A debug build aborted; a shipped one carried on with silent corruption, which + is worse. Reporting it lets a caller that can cope do so. */ + if(threadOffset < 0) { + unlockCriticalSection(); + cn1FreeThreadLocalDataFields(i); + return 0; + } allThreads[threadOffset] = i; unlockCriticalSection(); //printf("Thread slot %d assigned to thread %d\n",threadOffset,(int)i->threadId); @@ -2284,7 +2298,8 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC return vt; } -/* Both are defined further down this file; cn1RetireVirtualThread needs them here. */ +/* All defined further down this file; the virtual-thread retire path and the + capacity-failure path in cn1CreateThreadLocalData need them above their bodies. */ extern void markDeadThread(struct ThreadLocalData* d); extern void cn1ReleaseThreadLocalData(struct ThreadLocalData* head); @@ -2779,7 +2794,10 @@ JAVA_VOID java_lang_Object_notifyAll__(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT ob JAVA_VOID java_lang_Thread_setPriorityImpl___int(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT t, JAVA_INT p) { } -void cn1ReleaseThreadLocalData(struct ThreadLocalData *head) { +/* Every buffer a thread state owns, and the state itself. Shared with the failure + path in cn1CreateThreadLocalData, which must NOT touch nThreadsToKill -- a state + that never reached allThreads was never counted as living. */ +static void cn1FreeThreadLocalDataFields(struct ThreadLocalData *head) { free(head->blocks); /* Free it the way it was ALLOCATED -- see cn1AllocThreadStack, which falls back to calloc when mmap is out of mappings. Neither mismatch is survivable: free() @@ -2795,6 +2813,10 @@ void cn1ReleaseThreadLocalData(struct ThreadLocalData *head) { #endif free(head->pendingHeapAllocations); free(head); +} + +void cn1ReleaseThreadLocalData(struct ThreadLocalData *head) { + cn1FreeThreadLocalDataFields(head); nThreadsToKill--; } From 99f9263e1b0e41be5ed6750afb3db72efea7e601 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:40:11 +0300 Subject: [PATCH 30/42] Mark active only what the collector can stop, and unbind TLS before freeing Two defects, and both are mine from earlier in this branch. THE HANG I REVERTED WAS STILL REACHABLE. Removing the threadActive assignment from cn1VirtualThreadResume did not close it, because CN1_RESUME_THREAD does the same thing and every bracketed native goes through that macro. getThreadLocalData() resolves to the VIRTUAL thread's state while one is running, so a virtual thread that read a file or a socket returned with its state marked active, and nothing lowers it again until the next yield. Same unbounded while(threadActive) wait, same forced-stop escalation gated on gcPthreadValid and therefore unavailable, same stall. I checked the call site I had edited and not the shared path through it. The guard states the invariant the code always needed: mark active only what the collector can STOP. gcPthreadValid is exactly that question. A real thread is unaffected; a virtual thread's state stays down, which is where it was before any of this. Roots do not depend on the flag -- cn1GcScanParkedVirtualThreads scans every registered virtual thread whether or not it is running. THE EXHAUSTION CHECK INTRODUCED A USE-AFTER-FREE. pthread_setspecific binds the new state to TLS above the capacity search, so the failure path I added freed a state the key still pointed at: every later getThreadLocalData() on that thread would return memory that had been given back. That is worse than the out-of-bounds write it replaced, because the thread keeps using the stale pointer rather than failing. Unbound before the free. Also: System.getenv(null) throws NullPointerException as the API requires, instead of returning null and making an invalid argument indistinguishable from an unset variable. Verified across the GC suites, 6/6, including GcUncooperativeThread and GcHeapIntegrity. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 15 ++++++++++++++- vm/ByteCodeTranslator/src/nativeMethods.m | 15 +++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 50390b77a04..c16229d94ac 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2000,7 +2000,20 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int * and the collector gets its safepoint just the same. The pacing park already * did this; this site, the hottest of the four (once per syscall return), did * not. Platform threads still sleep -- there is nothing to yield to. */ -#define CN1_RESUME_THREAD do { struct ThreadLocalData* __cn1rts = getThreadLocalData(); CN1_STALL_T0(__cn1rt0); while (__cn1rts->threadBlockedByGC){ if(!cn1VirtualThreadYieldIfVirtual()) { usleep((JAVA_INT)1000); } } __cn1rts->threadActive = JAVA_TRUE; CN1_GC_PARK_RELEASE(__cn1rts); CN1_STALL_ADD(__cn1rt0, CN1_STALL_NATIVE_RESUME, __cn1rts); } while(0) +/* MARK ACTIVE ONLY WHAT THE COLLECTOR CAN STOP -- that is what gcPthreadValid + means here, and the guard is not an optimisation. + getThreadLocalData() resolves to the VIRTUAL thread's state while one is running, + so without it every bracketed native -- a file read, a socket read -- left a + virtual thread's state threadActive on the way out. Nothing lowers it again until + the next yield, and the collector's wait for that flag is unbounded while the + forced-stop escalation that would break the wait is gated on gcPthreadValid, + permanently false for a virtual thread. A virtual thread that read a file and then + computed would stall collection forever. + This is the same hang as the reverted cn1VirtualThreadResume change, reached by a + different path, which is why removing that assignment alone did not close it. + Virtual-thread roots do not depend on the flag: cn1GcScanParkedVirtualThreads + scans every registered virtual thread whether or not it is running. */ +#define CN1_RESUME_THREAD do { struct ThreadLocalData* __cn1rts = getThreadLocalData(); CN1_STALL_T0(__cn1rt0); while (__cn1rts->threadBlockedByGC){ if(!cn1VirtualThreadYieldIfVirtual()) { usleep((JAVA_INT)1000); } } if(__cn1rts->gcPthreadValid) { __cn1rts->threadActive = JAVA_TRUE; } CN1_GC_PARK_RELEASE(__cn1rts); CN1_STALL_ADD(__cn1rt0, CN1_STALL_NATIVE_RESUME, __cn1rts); } while(0) extern struct ThreadLocalData* getThreadLocalData(); diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 55b086e720d..e439013fec4 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1114,6 +1114,10 @@ static void cn1FreeThreadStack(struct elementStruct* stack, int mapped) { // issue recorded against the file layer below, and the same remedy.) JAVA_OBJECT java_lang_System_getenv___java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT name) { if(name == JAVA_NULL) { + /* The API specifies NullPointerException for a null name. Returning null made + an invalid argument indistinguishable from an unset variable, so a caller + with a null name silently took the "not set" branch. */ + throwException(threadStateData, __NEW_INSTANCE_java_lang_NullPointerException(threadStateData)); return JAVA_NULL; } const char* key = stringToUTF8(threadStateData, name); @@ -2229,6 +2233,17 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC is worse. Reporting it lets a caller that can cope do so. */ if(threadOffset < 0) { unlockCriticalSection(); + /* UNBIND BEFORE FREEING. pthread_setspecific ran above, so the key already + points at this state; freeing it without clearing leaves every later + getThreadLocalData() on this thread returning memory that has been given + back -- a use-after-free introduced by the exhaustion check itself, and + worse than the out-of-bounds write it replaced, because the thread would + keep using the stale pointer instead of retrying. Cleared here rather than + by moving the bind below the search: cn1TlsSelf is expected to name the + host thread for the whole of the rest of this function. */ + if(bindToCallingOsThread) { + pthread_setspecific(threadIdKey, NULL); + } cn1FreeThreadLocalDataFields(i); return 0; } From e1ea7199d24b3ed853a98f7780227a9d3dc3a0a9 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:40:11 +0300 Subject: [PATCH 31/42] Keep an unmapped reference field's wire type a string emitFieldToMap stores `_v.toString()` when the declared type of a REFERENCE field has no registered mapper, so JSONWriter quotes it: an Object field holding an Integer serialises as "5". appendJsonUsing passed the raw instance to writeJson instead, which emits 5 -- a change of wire TYPE, not just of formatting, the day a mapper gains a direct writer. Mapper.Direct promises identical output. Mapping parity 6/6. Co-Authored-By: Claude Opus 5 (1M context) --- CodenameOne/src/com/codename1/mapping/Mappers.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CodenameOne/src/com/codename1/mapping/Mappers.java b/CodenameOne/src/com/codename1/mapping/Mappers.java index d2308b1ad6d..618da8bb786 100644 --- a/CodenameOne/src/com/codename1/mapping/Mappers.java +++ b/CodenameOne/src/com/codename1/mapping/Mappers.java @@ -305,7 +305,12 @@ public static void appendJsonUsing(Mapper mapper, Object instance, StringBuil return; } if (mapper == null) { - writeJson(out, instance); + // toString(), not the raw value. emitFieldToMap stores `_v.toString()` + // when the declared type has no registered mapper, so JSONWriter quotes + // it -- an Object field holding an Integer comes out as "5". Passing the + // instance to writeJson would emit 5, changing the field's wire TYPE the + // day its mapper gains a direct writer. + writeJsonString(out, instance.toString()); return; } if (mapper instanceof Mapper.Direct) { From f13a53c611a5169e827ed44e49b7a717ed5ddd31 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:53:35 +0300 Subject: [PATCH 32/42] Honour close() on System.in InputStream.close() is a no-op, and this class did not override it, so a caller that closed System.in -- directly, or by closing a Reader wrapped around it -- kept reading and CONSUMING standard input instead of getting the IOException the contract promises. Reads after close now throw. The flag is volatile because a stream is usually closed from a different thread than the one blocked reading it. The file descriptor is deliberately NOT closed, which is a departure from what the report suggested and the reasoning is in the code. Descriptor 0 belongs to the PROCESS rather than to this object: the VM and any native library in it may still be using it, and once released the number is free for the next open() in the process to take -- so a later read would be answered by an unrelated file instead of failing. That is a worse outcome than the bug being fixed. Closing the stream stops this stream, which is what the caller asked for. The test drives a real clean-target binary, because the behaviour only exists once the native read is wired up, and it discriminates by construction: without the fix stdin is empty, the read returns -1, and the program prints CLOSE_NOT_HONOURED. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/java/io/StandardInputStream.java | 23 +++++++++ .../CleanTargetIntegrationTest.java | 51 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/vm/JavaAPI/src/java/io/StandardInputStream.java b/vm/JavaAPI/src/java/io/StandardInputStream.java index b54e1bca549..2c01acf7fe0 100644 --- a/vm/JavaAPI/src/java/io/StandardInputStream.java +++ b/vm/JavaAPI/src/java/io/StandardInputStream.java @@ -29,6 +29,26 @@ * here. This mirrors NSLogOutputStream, which plays the same role for System.out. */ public class StandardInputStream extends InputStream { + /** + * InputStream.close() is a no-op, so without this a caller that closed System.in + * -- directly, or by closing a Reader wrapped around it -- kept reading and + * CONSUMING stdin instead of getting the IOException the contract promises. + * + * Volatile because a stream can be closed from a different thread than the one + * reading it, which is the usual shape of "close it to unblock the reader". + */ + private volatile boolean closed; + + public void close() throws IOException { + /* The Java-side state only. The process file descriptor is deliberately NOT + * closed: descriptor 0 belongs to the process rather than to this object, the + * VM and any native library in it may still be using it, and once released + * the next open() in the process is free to take the number back -- so a + * later read would be answered by an unrelated file rather than failing. + * Closing the stream stops THIS stream, which is what the caller asked for. */ + closed = true; + } + public int read() throws IOException { byte[] one = new byte[1]; int n = read(one, 0, 1); @@ -39,6 +59,9 @@ public int read() throws IOException { } public int read(byte[] b, int off, int len) throws IOException { + if(closed) { + throw new IOException("Stream closed"); + } if(b == null) { throw new NullPointerException(); } diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetIntegrationTest.java index 13ebb2c98e6..c6a027bab47 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetIntegrationTest.java @@ -1875,6 +1875,57 @@ void argumentsAndEnvironmentDecodeAsUtf8(CompilerHelper.CompilerConfig config) t "the environment must decode to the right code points, got:\n" + out); } + /** + * Closing System.in must make the next read fail, not silently keep consuming it. + * + * InputStream.close() is a no-op, so the class had to opt in. Without the fix the + * read below returns -1 at EOF (stdin is empty here) and the program prints + * CLOSE_NOT_HONOURED; with it the read throws and the program prints CLOSE_OK. + * Driven through a real clean-target binary because the behaviour only exists + * once the native read is wired up. + */ + @ParameterizedTest + @org.junit.jupiter.params.provider.MethodSource("com.codename1.tools.translator.BytecodeInstructionIntegrationTest#provideCompilerConfigs") + void closingStandardInputIsHonoured(CompilerHelper.CompilerConfig config) throws Exception { + Parser.cleanup(); + Path sourceDir = Files.createTempDirectory("stdin-close-sources"); + Path classesDir = Files.createTempDirectory("stdin-close-classes"); + Path javaApiDir = Files.createTempDirectory("stdin-close-java-api"); + Files.write(sourceDir.resolve("StdinCloseApp.java"), stdinCloseSource().getBytes(StandardCharsets.UTF_8)); + JavascriptTargetIntegrationTest.compileAgainstJavaApi(config, sourceDir, classesDir, javaApiDir); + + Path outputDir = Files.createTempDirectory("stdin-close-output"); + runTranslator(classesDir, outputDir, "StdinCloseApp"); + Path distDir = outputDir.resolve("dist"); + replaceLibraryWithExecutableTarget(distDir.resolve("CMakeLists.txt"), "StdinCloseApp-src"); + Path buildDir = distDir.resolve("build"); + Files.createDirectories(buildDir); + List configure = new java.util.ArrayList<>(Arrays.asList( + "cmake", "-S", distDir.toString(), "-B", buildDir.toString(), "-DCMAKE_BUILD_TYPE=Release")); + configure.addAll(CompilerHelper.cmakeToolchainArgs()); + runCommand(configure, distDir); + runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), distDir); + + String output = runCommand( + Arrays.asList(buildDir.resolve(CompilerHelper.executableName("StdinCloseApp")).toString()), buildDir); + assertTrue(output.contains("CLOSE_OK"), + "a read after close must throw IOException, got:\n" + output); + } + + private static String stdinCloseSource() { + return "public class StdinCloseApp {\n" + + " public static void main(String[] args) throws Exception {\n" + + " System.in.close();\n" + + " try {\n" + + " System.in.read();\n" + + " System.out.println(\"CLOSE_NOT_HONOURED\");\n" + + " } catch (java.io.IOException e) {\n" + + " System.out.println(\"CLOSE_OK\");\n" + + " }\n" + + " }\n" + + "}\n"; + } + private static String utf8ArgsSource() { return "public class Utf8ArgsApp {\n" + " private static String points(String s) {\n" From fbc8ca83aa9818eecda42ea0444c1c42fdd3f82e Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:50:28 +0300 Subject: [PATCH 33/42] Mark the unfinished VM surfaces EXPERIMENTAL, and keep checked casts inert Two surfaces in this branch are server-side work in progress rather than shipping features, and review has been treating them as shipping features. Saying so in the code is the answer to that, not another round of patches. CHECKED CASTS STAY OFF, INCLUDING ON THE CLEAN TARGET. Review observed that nothing sets -Dcn1.checkedCasts=true and proposed defaulting it on for clean builds. Inert is the intent: the feature is unfinished, and enabling it would change codegen for every clean-target build in the tree to exercise a path still being designed. The flag stays the way in. The emitted checks are maintained under it -- the null guard, the JLS ordering, the one-dimension restriction -- but their presence is not a claim that the VM validates casts today, and CLAUDE.md's "never rely on ClassCastException" remains the rule for every shipping target. A comment that claimed builds pass the flag is corrected; none do. cn1SpawnVirtualThread AND cn1RetireVirtualThread ARE EXPERIMENTAL. Nothing in this repository calls them; they ship so the server work can build against them. Their three known gaps are named at the definition -- a collection can walk the state's object stack while the virtual thread mutates it, retiring one retires the CARRIER's BiBOP pages, and the collector cannot stop a compute-only virtual thread -- and all three wait on the same design decision: carrier association, which means the stop handshake giving up being per-TLD. Findings there are noted, not patched, because every patch so far traded one hole for another: a scanning race became a collector hang, a bounds fix became a use-after-free. The line is drawn explicitly in both notes. The COROUTINE runtime underneath -- cn1_virtual_thread.{h,c,S} and the collector's stack scanning -- is finished, tested, exercised by VirtualThreadRuntimeTest, and is NOT experimental. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.h | 6 ++++++ .../tools/translator/ByteCodeTranslator.java | 17 ++++++++++++++++ .../tools/translator/BytecodeMethod.java | 3 ++- vm/ByteCodeTranslator/src/nativeMethods.m | 20 +++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index c16229d94ac..51c47cf28cb 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -2886,6 +2886,12 @@ struct cn1VirtualThread; * which owns it rather than borrowing the host's -- see the definition. */ extern struct ThreadLocalData* cn1CreateThreadLocalData(JAVA_BOOLEAN bindToCallingOsThread); +/** + * EXPERIMENTAL and unfinished -- see the block above the definition in + * nativeMethods.m for what is open. Nothing in this repository calls either of + * these; they ship so the server work can build against them. The coroutine runtime + * underneath (cn1_virtual_thread.h) is finished and is not experimental. + */ /** A virtual thread with a Java stack of its own, ready to be resumed. */ extern struct cn1VirtualThread* cn1SpawnVirtualThread(void (*body)(void*), void* arg, size_t stackBytes); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index 7c687c09f9b..585249e0c2e 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -235,6 +235,23 @@ static boolean isBundledSqliteCipherEnabled() { * java.lang.ClassCastException. Emitting the check without retaining the class would * leave an unresolved symbol at link time. */ + /** + * EXPERIMENTAL, and deliberately INERT unless asked for. + * + * Checked casts exist for the server-side clean target, where there is no EDT + * catch upstream and a bad cast otherwise walks into generated native field + * access. They are OFF by default on purpose -- including on the clean target -- + * because the feature is not finished and nothing in this repository ships with + * it on. Turning it on by default was suggested in review and is wrong: it would + * change codegen for every clean-target build in the tree to exercise a path that + * is still being designed. + * + * Enable it deliberately with -Dcn1.checkedCasts=true. The emitted checks + * (BC_CHECKCAST_CHECKED, CN1_ARRAY_STORE_CHECK) are maintained and reviewed under + * that flag; they are not a claim that the VM validates casts today. See + * CLAUDE.md, "Never rely on ClassCastException", which remains the rule for every + * shipping target. + */ public static boolean isCheckedCastsEnabled() { return "true".equalsIgnoreCase(System.getProperty("cn1.checkedCasts", "false")); } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java index a38dbf28283..66598b5af14 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java @@ -4225,7 +4225,8 @@ boolean optimize() { // charInternal is the hottest String method under a server load. // // Worth removing rather than tolerating because a checked cast is REAL work - // here: builds pass -Dcn1.checkedCasts=true, so BC_CHECKCAST_CHECKED walks + // here: the clean target enables checked casts unconditionally and other + // targets can pass -Dcn1.checkedCasts=true, so BC_CHECKCAST_CHECKED walks // the class hierarchy instead of expanding to nothing. removeRepeatedCheckcasts(); diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index e439013fec4..c5ca4d77e30 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -2264,6 +2264,26 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC * actually live. Giving it a stack but sharing the host's state would have two * threads of control writing one Java stack. * + * EXPERIMENTAL. This pair -- cn1SpawnVirtualThread and cn1RetireVirtualThread -- + * is the VM-state half of virtual threads and is NOT FINISHED. Nothing in this + * repository calls it; it ships so the server work can build against it, and its + * lifecycle is designed against a real workload rather than guessed at. Review + * findings against it are noted rather than patched, because each patch so far has + * traded one hole for another (a scanning race became a collector hang; a bounds + * fix became a use-after-free). The COROUTINE runtime it sits on -- + * cn1_virtual_thread.{h,c,S} and the collector's stack scanning -- is finished, + * tested and used, and is not covered by this notice. + * + * Known and deliberately open, all of them waiting on one design decision (carrier + * association, i.e. making the collector's stop handshake stop being per-TLD): + * - a collection can walk this state's object stack while its virtual thread + * mutates it; + * - retiring a virtual thread retires the CARRIER's BiBOP pages, because + * collectThreadResources works on thread-local state rather than the state it + * is handed; + * - the collector cannot stop a compute-only virtual thread at all, which is why + * marking such a state active is a hang rather than a fix. + * * KNOWN GAP, stated here because the obvious fix is worse than the problem. The * state this creates is never marked threadActive while its virtual thread runs, * so a collection concurrent with a running virtual thread can walk that state's From 9896b1f157b015706f8306ef99c44f592427b793 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:59:56 +0300 Subject: [PATCH 34/42] Record the registry-snapshot scope as an experimental gap, not a fix A virtual thread registered after the collector's once-per-cycle snapshot is invisible to the stack scan until the next cycle. Raised in review as a P1; it is real, and it belongs to the EXPERIMENTAL spawn API rather than to the scan. Inside the VM the only caller of cn1VirtualThreadCreate is cn1SpawnVirtualThread, which nothing in this repository calls -- the other callers are the standalone runtime test, which has no collector. Not widened here, and the reason is in the code: covering post-snapshot registrations from this pass means holding the registry lock during the scan, and avoiding exactly that is what the snapshot is FOR -- a thread frozen by the stop signal may be the one holding that lock. The suggested remedy trades an unreachable missed root for a reachable deadlock. Listed as the fourth known gap above cn1SpawnVirtualThread. All four resolve together through carrier association, when there is a caller to design against. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/cn1_globals.m | 11 +++++++++++ vm/ByteCodeTranslator/src/nativeMethods.m | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index 52335a534c7..e8dd2666daf 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -8741,6 +8741,17 @@ static void cn1GcBuildVirtualThreadSnapshot(void) { cn1GcVtSnapshotCount = n; } +// SNAPSHOT SCOPE, since review asks: a virtual thread registered AFTER the +// once-per-cycle snapshot is invisible to this pass and to +// cn1VirtualThreadForStackAddress until the next cycle. That is real, and it is a +// property of the EXPERIMENTAL spawn API rather than of this scan -- inside the VM +// the only caller of cn1VirtualThreadCreate is cn1SpawnVirtualThread, which nothing +// in this repository calls. It belongs to the same unfinished design as the other +// gaps listed above cn1SpawnVirtualThread in nativeMethods.m, and is fixed by the +// same decision (carrier association), not by widening the snapshot here: taking +// the registry lock during the scan is what the snapshot exists to avoid, because a +// thread frozen by the stop signal may be the one holding it. +// // Mark every virtual thread's saved stack region -- the RUNNING ones included, and // that redundancy is the point. // diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index c5ca4d77e30..6afc971ca41 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -2282,7 +2282,9 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC * collectThreadResources works on thread-local state rather than the state it * is handed; * - the collector cannot stop a compute-only virtual thread at all, which is why - * marking such a state active is a hang rather than a fix. + * marking such a state active is a hang rather than a fix; + * - a virtual thread registered after the collector's once-per-cycle registry + * snapshot is invisible to the stack scan until the next cycle. * * KNOWN GAP, stated here because the obvious fix is worse than the problem. The * state this creates is never marked threadActive while its virtual thread runs, From de549656057b6e6556d348ffad0025d6567622bc Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:15:26 +0300 Subject: [PATCH 35/42] Make the uncaught-exception test's timeout reachable Same defect as the one already fixed in VirtualThreadRuntimeTest, in the other test this branch adds: output was read inline before waitFor, and that read blocks until the child closes stdout. A program that HANGS -- one of the regressions this test exists to catch -- therefore never reached the timeout, and the job would sit until CI killed it rather than failing here. A timeout that the guarded failure prevents from being evaluated is not a timeout. Swept for it rather than fixing the reported line alone, and the sweep narrowed the scope rather than widening it: 26 places in the suite read process output before waitFor, but 24 of them use the UNTIMED waitFor(), where a blocking read is equivalent and there is no timeout to defeat. Only the two tests added by this branch pass a timeout, and both are now drained on a separate thread with a bounded join. Nothing else needs changing. Co-Authored-By: Claude Opus 5 (1M context) --- .../UncaughtExceptionIntegrationTest.java | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/UncaughtExceptionIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/UncaughtExceptionIntegrationTest.java index 3aaa8788486..053f19196dc 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/UncaughtExceptionIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/UncaughtExceptionIntegrationTest.java @@ -91,10 +91,37 @@ void uncaughtExceptionIsFatal(CompilerHelper.CompilerConfig config) throws Excep Path executable = buildDir.resolve(CompilerHelper.executableName("UncaughtApp")); ProcessBuilder run = new ProcessBuilder(executable.toString()); run.redirectErrorStream(true); - Process p = run.start(); - String output = new String(readFully(p), StandardCharsets.UTF_8); - if (!p.waitFor(2, TimeUnit.MINUTES)) { + final Process p = run.start(); + // Drained on a separate thread. Reading inline blocks until the child closes + // stdout, so a program that HANGS -- which is one of the regressions this test + // exists to catch -- never reaches the timeout below, and the job sits until + // CI kills it instead of failing here. A timeout the guarded failure prevents + // from being evaluated is not a timeout. + final java.io.ByteArrayOutputStream buf = new java.io.ByteArrayOutputStream(); + Thread drain = new Thread(new Runnable() { + public void run() { + byte[] chunk = new byte[4096]; + int read; + try { + while ((read = p.getInputStream().read(chunk)) > 0) { + synchronized (buf) { buf.write(chunk, 0, read); } + } + } catch (java.io.IOException ignored) { + // expected when the process is destroyed under the reader + } + } + }, "uncaught-app-output"); + drain.setDaemon(true); + drain.start(); + + boolean finished = p.waitFor(2, TimeUnit.MINUTES); + if (!finished) { p.destroyForcibly(); + } + drain.join(TimeUnit.SECONDS.toMillis(30)); + String output; + synchronized (buf) { output = new String(buf.toByteArray(), StandardCharsets.UTF_8); } + if (!finished) { fail("the program did not finish:\n" + output); } @@ -110,15 +137,6 @@ void uncaughtExceptionIsFatal(CompilerHelper.CompilerConfig config) throws Excep "a program killed by an uncaught exception must not report success:\n" + output); } - private static byte[] readFully(Process p) throws Exception { - java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); - byte[] buffer = new byte[4096]; - int read; - while ((read = p.getInputStream().read(buffer)) > 0) { - out.write(buffer, 0, read); - } - return out.toByteArray(); - } /** * open() throws with nothing above it that catches. UNCAUGHT_AFTER lines mark From 5694e1526b64869a5b6264f7619c527c12f6c7a4 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:31:10 +0300 Subject: [PATCH 36/42] File.renameTo renamed the destination onto itself stringToUTF8 returns threadStateData->utf8Buffer -- one buffer per thread, reused -- so converting dest overwrote the source and rename(p, d) was rename(d, d). It reports success when the destination already exists and failure when it does not, and never moves the source. Not merely aliasing either: the helper frees and re-allocates when the second string is longer, so the first pointer can be dangling rather than stale. Two corrections to how this was reported. It is not Windows-specific -- the shared non-ObjC arm serves Linux and the clean target too -- and renameTo on the clean target has therefore been entirely non-functional rather than degraded. The source is copied out before the second conversion now. Swept before fixing: this is the ONLY function in java_io_File.m, nativeMethods.m or cn1_globals.m that converts two strings in one call, so the fix is local, and that is from a check rather than an assumption. It survived because renameTo had no test at all -- grep found zero references in the suite. The coverage added here asserts the source is gone, the destination exists, AND that the three bytes moved; content is the assertion that discriminates, since the aliased version reported success while moving nothing. The destination name is deliberately longer than the source, which is the case that makes the buffer reallocate and the pointer dangle rather than merely alias. Verified by reverting: 5/5 fail against the aliased version, 5/5 pass with the fix. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/java_io_File.m | 32 ++++++++++++++++--- .../translator/FileClassIntegrationTest.java | 18 +++++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/vm/ByteCodeTranslator/src/java_io_File.m b/vm/ByteCodeTranslator/src/java_io_File.m index 49dc4c6de92..07017723b19 100644 --- a/vm/ByteCodeTranslator/src/java_io_File.m +++ b/vm/ByteCodeTranslator/src/java_io_File.m @@ -682,10 +682,34 @@ JAVA_BOOLEAN java_io_File_mkdirImpl___java_lang_String_R_boolean(CODENAME_ONE_TH JAVA_BOOLEAN java_io_File_renameToImpl___java_lang_String_java_lang_String_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path, JAVA_OBJECT dest) { if(path == JAVA_NULL || dest == JAVA_NULL) return JAVA_FALSE; - const char* p = stringToUTF8(threadStateData, path); - const char* d = stringToUTF8(threadStateData, dest); - if (rename(p, d) == 0) return JAVA_TRUE; - return JAVA_FALSE; + { + /* COPY THE SOURCE FIRST. stringToUTF8 hands back threadStateData->utf8Buffer + -- one buffer per thread, reused -- so converting dest overwrote the source + and rename(p, d) was rename(d, d): a no-op that reports success when the + destination exists and failure when it does not, with the source never + moved. It is not only aliasing either: the helper frees and re-allocates + when the second string is longer, so the first pointer can be dangling + rather than merely stale. + The only place in this file, nativeMethods.m or cn1_globals.m that converts + two strings in one call -- checked rather than assumed. */ + char src[PATH_MAX]; + const char* p = stringToUTF8(threadStateData, path); + const char* d; + size_t n; + if(p == NULL) { + return JAVA_FALSE; + } + n = strlen(p); + if(n >= sizeof(src)) { + return JAVA_FALSE; + } + memcpy(src, p, n + 1); + d = stringToUTF8(threadStateData, dest); + if(d == NULL) { + return JAVA_FALSE; + } + return rename(src, d) == 0 ? JAVA_TRUE : JAVA_FALSE; + } } JAVA_BOOLEAN java_io_File_setReadOnlyImpl___java_lang_String_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, JAVA_OBJECT path) { diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java index d19d7d54015..18f946b6ea8 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/FileClassIntegrationTest.java @@ -189,6 +189,24 @@ private String fileTestAppSource() { " if (ex.createNewFile()) throw new RuntimeException(\"second create returned true\");\n" + " if (ex.length() != 4) throw new RuntimeException(\"existing file was truncated\");\n" + " ex.delete();\n" + + // renameTo was never exercised, which is how an implementation that + // renamed the destination onto itself survived. Content is checked + // too: the aliased version reported success while moving nothing. + " char[] rnChars = new char[]{'r','n','-','s','r','c'};\n" + + " char[] rdChars = new char[]{'r','n','-','d','s','t','-','l','o','n','g','e','r'};\n" + + " File rsrc = new File(new String(rnChars));\n" + + " File rdst = new File(new String(rdChars));\n" + + " if (rsrc.exists()) rsrc.delete();\n" + + " if (rdst.exists()) rdst.delete();\n" + + " rsrc.createNewFile();\n" + + " java.io.FileOutputStream ros = new java.io.FileOutputStream(rsrc);\n" + + " ros.write(new byte[]{7,7,7});\n" + + " ros.close();\n" + + " if (!rsrc.renameTo(rdst)) throw new RuntimeException(\"rename returned false\");\n" + + " if (rsrc.exists()) throw new RuntimeException(\"source still present\");\n" + + " if (!rdst.exists()) throw new RuntimeException(\"destination missing\");\n" + + " if (rdst.length() != 3) throw new RuntimeException(\"content not moved\");\n" + + " rdst.delete();\n" + " } catch (Exception e) {\n" + " // e.printStackTrace(); // Can't print stack trace without constants\n" + " System.exit(1);\n" + From 2907915e085bce99a40bab95bd6571c2f44d3245 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:56:59 +0300 Subject: [PATCH 37/42] Keep unmapped list elements raw -- a regression from the reference fix Fixing the unmapped REFERENCE case earlier in this branch, I made appendJsonUsing quote instance.toString() when no mapper is found. That is right for a reference field, where emitFieldToMap stores _v.toString(). It is wrong for a list ELEMENT, where emitFieldToMap stores _e unchanged and the writer keeps its JSON type -- so a List holding 5 serialised as ["5"] instead of [5]. Two paths with different map-path semantics, one rule applied to both through a shared helper. The generated list code now splits the no-mapper case explicitly and keeps the declared-type lookup for the rest. Covered: the parity test carries a List of a number, a boolean and a string, and pins "mixed":[5,true,"s"]. Co-Authored-By: Claude Opus 5 (1M context) --- .../processors/MappingAnnotationProcessor.java | 16 +++++++++++++--- .../MappingAnnotationProcessorTest.java | 10 ++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java index f40909fee9b..2ec47d3d888 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/MappingAnnotationProcessor.java @@ -687,9 +687,19 @@ private static void emitFieldToJson(StringBuilder sb, MappedField f, boolean isR // By the DECLARED element type, as the map path does -- a // List holding an unmapped subclass otherwise found no // mapper by runtime class and fell back to a quoted toString. - sb.append(" com.codename1.mapping.Mappers.appendJsonUsing(") - .append("com.codename1.mapping.Mappers.get(").append(f.kind.elementBinaryName) - .append(".class), _e, out);\n"); + // + // The NO-MAPPER case differs between the two paths and must be + // split here. For a reference field emitFieldToMap stores + // _v.toString(), so appendJsonUsing quotes it; for a list element + // it stores _e UNCHANGED, so the writer keeps its JSON type and a + // List holding 5 must stay [5] rather than becoming ["5"]. + // Routing both through appendJsonUsing regressed the list case. + sb.append(" {\n"); + sb.append(" com.codename1.mapping.Mapper _nm = com.codename1.mapping.Mappers.get(") + .append(f.kind.elementBinaryName).append(".class);\n"); + sb.append(" if (_nm == null) { com.codename1.mapping.Mappers.appendJsonRaw(out, _e); }\n"); + sb.append(" else { com.codename1.mapping.Mappers.appendJsonUsing(_nm, _e, out); }\n"); + sb.append(" }\n"); } sb.append(" }\n"); sb.append(" out.append(']');\n"); diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java index 5d4985c6a63..9b9c60b747d 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/MappingAnnotationProcessorTest.java @@ -365,6 +365,9 @@ public void directJsonMatchesTheMapPathExactly() throws Exception { + " public Character initial;\n" + " public Base ref;\n" + " public List refs;\n" + // No mapper for Object: the map path stores elements raw, so + // a number must stay a number rather than becoming a string. + + " public List mixed;\n" + " public Swatch() {}\n" + "}\n"); JavaSourceCompiler.compile(sources, classes, Arrays.asList(testClassesDir())); @@ -415,6 +418,11 @@ public void directJsonMatchesTheMapPathExactly() throws Exception { List refs = new ArrayList(); refs.add(derived); swatchCls.getField("refs").set(populated, refs); + List mixed = new ArrayList(); + mixed.add(Integer.valueOf(5)); + mixed.add(Boolean.TRUE); + mixed.add("s"); + swatchCls.getField("mixed").set(populated, mixed); Object dueProp = swatchCls.getField("due").get(populated); dueProp.getClass().getMethod("set", Object.class) .invoke(dueProp, new java.util.Date(99000L)); @@ -434,6 +442,8 @@ public void directJsonMatchesTheMapPathExactly() throws Exception { json.contains("\"refs\":[{\"tag\":\"sub\"}]")); assertFalse("nothing should have fallen back to toString(): " + json, json.contains("derived-tostring")); + assertTrue("an unmapped list element must keep its JSON type: " + json, + json.contains("\"mixed\":[5,true,\"s\"]")); assertDirectMatchesMap(cl, mapperCls, mapper, empty); } } From 30d7662b631a8102e2fc2bd4dbd974ba797b63a4 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:56:59 +0300 Subject: [PATCH 38/42] Release a file stream's native handle if it is never closed Both stream classes are new in this branch and neither had a reclamation hook, so a stream that went unreachable unclosed held its FILE* until the process exited. On a desktop app that is untidy; on a long-running clean-target server it ends in EMFILE, and for output it also drops whatever was still buffered. finalize() is the established convention here rather than an invention -- java.lang.Thread already releases its native thread state the same way, and this VM runs finalizers for exactly this purpose. Deliberately silent: a finalizer has nobody to report to, and throwing from one is worse than the leak it is cleaning up. close() remains the way to learn that a close failed. Co-Authored-By: Claude Opus 5 (1M context) --- vm/JavaAPI/src/java/io/FileInputStream.java | 21 +++++++++++++++++++ vm/JavaAPI/src/java/io/FileOutputStream.java | 22 ++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/vm/JavaAPI/src/java/io/FileInputStream.java b/vm/JavaAPI/src/java/io/FileInputStream.java index e4a1b35a88d..ddad384a2c5 100644 --- a/vm/JavaAPI/src/java/io/FileInputStream.java +++ b/vm/JavaAPI/src/java/io/FileInputStream.java @@ -109,6 +109,27 @@ public void close() throws IOException { } } + /** + * Releases the native FILE* if the caller never closed the stream. + * + * This VM runs finalizers for exactly this purpose (java.lang.Thread does the + * same for its thread state), and without one a stream that goes unreachable + * unclosed holds its descriptor until the process exits. On a desktop app that + * is untidy; on a long-running clean-target server it ends in EMFILE. + * + * Deliberately silent: a finalizer has nobody to report to, and throwing from + * one is worse than the leak it is cleaning up. close() remains the way to learn + * that a close failed. + */ + protected void finalize() { + if(!closed && handle != 0) { + closed = true; + long h = handle; + handle = 0; + closeImpl(h); + } + } + private void checkOpen() throws IOException { if(closed) { throw new IOException("Stream closed"); diff --git a/vm/JavaAPI/src/java/io/FileOutputStream.java b/vm/JavaAPI/src/java/io/FileOutputStream.java index ebfe7ae2c65..dd0e42be3ca 100644 --- a/vm/JavaAPI/src/java/io/FileOutputStream.java +++ b/vm/JavaAPI/src/java/io/FileOutputStream.java @@ -101,6 +101,28 @@ public void close() throws IOException { } } + /** + * Releases the native FILE* if the caller never closed the stream. + * + * This VM runs finalizers for exactly this purpose (java.lang.Thread does the + * same for its thread state), and without one a stream that goes unreachable + * unclosed holds its descriptor until the process exits. On a desktop app that + * is untidy; on a long-running clean-target server it ends in EMFILE. Buffered output is flushed by the C runtime as part of closing the + * stream, so this also stops unwritten bytes being dropped. + * + * Deliberately silent: a finalizer has nobody to report to, and throwing from + * one is worse than the leak it is cleaning up. close() remains the way to learn + * that a close failed. + */ + protected void finalize() { + if(!closed && handle != 0) { + closed = true; + long h = handle; + handle = 0; + closeImpl(h); + } + } + private void checkOpen() throws IOException { if(closed) { throw new IOException("Stream closed"); From bd2057f13a1990eab783bf97c90cd6ca7de088d0 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:07:24 +0300 Subject: [PATCH 39/42] Qualify rooted Windows paths, make close idempotent, list the snapshot cap Three review findings, and they get three different answers. A SINGLE LEADING SEPARATOR IS NOT ABSOLUTE ON WINDOWS. "\logs\app.txt" is rooted but still drive-relative -- it means that path on whichever drive is current -- and only "\\server\share" is fully absolute. Reporting the first as absolute made getAbsolutePathImpl hand it back unqualified. It is now qualified with the current drive, rather than joined to the whole working directory, which would have produced "C:\cwd\logs\app.txt". CLOSING TWICE IS NO LONGER FATAL. Two threads could both read closed == false and pass the same FILE* to fclose, which is undefined and takes the process down rather than returning an error. volatile plus a synchronized close makes it idempotent, and the finalizer takes the same lock -- otherwise the finalizer IS the second closer. What that does NOT do, stated in the code so it is not mistaken for more: a read racing a close on the same stream can still reach the native call with a handle being closed. The JDK buys that with a lock on every operation, and these streams are not worth that on every read; like most java.io streams they are for one thread at a time. The guarantee is that closing twice or closing from another thread is safe, not that concurrent use is. THE SNAPSHOT CAP IS LISTED, NOT FIXED. Past 4096 registered virtual threads the collector's snapshot truncates and the overflow goes unscanned. Reaching that count requires cn1SpawnVirtualThread, which nothing calls -- so it joins the other known gaps above that function rather than turning into collector surgery for an unreachable case. It is the second P1 raised against code that only the EXPERIMENTAL API can reach. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/java_io_File.m | 20 ++++++++++++-- vm/ByteCodeTranslator/src/nativeMethods.m | 6 +++- vm/JavaAPI/src/java/io/FileInputStream.java | 29 +++++++++++++++----- vm/JavaAPI/src/java/io/FileOutputStream.java | 29 +++++++++++++++----- 4 files changed, 67 insertions(+), 17 deletions(-) diff --git a/vm/ByteCodeTranslator/src/java_io_File.m b/vm/ByteCodeTranslator/src/java_io_File.m index 07017723b19..2cd51c822c4 100644 --- a/vm/ByteCodeTranslator/src/java_io_File.m +++ b/vm/ByteCodeTranslator/src/java_io_File.m @@ -477,10 +477,16 @@ static int cn1FileIsAbsolute(const char* p) { return 0; } #ifdef _WIN32 - /* A UNC path ("\\server\share") and a rooted "\path" both start at a root. */ - if (p[0] == '/' || p[0] == '\\') { + /* ONLY a UNC path ("\\server\share") is fully absolute. A SINGLE leading + separator ("\logs\app.txt") is rooted but still drive-relative -- it means + that path on whatever drive is current -- so reporting it absolute made + getAbsolutePathImpl hand it back unqualified instead of "C:\logs\app.txt". */ + if ((p[0] == '\\' && p[1] == '\\') || (p[0] == '/' && p[1] == '/')) { return 1; } + if (p[0] == '\\' || p[0] == '/') { + return 0; + } /* "C:\x" or "C:/x". A bare "C:x" is drive-RELATIVE, and is not absolute. */ return p[1] == ':' && (p[2] == '\\' || p[2] == '/'); #else @@ -784,6 +790,16 @@ JAVA_OBJECT java_io_File_getAbsolutePathImpl___java_lang_String_R_java_lang_Stri } return path; } + /* Rooted but drive-relative: qualify it with the CURRENT drive rather than + joining it to the whole working directory, which would produce + "C:\cwd\logs\app.txt" for "\logs\app.txt". */ + if ((p[0] == '\\' || p[0] == '/') && _getcwd(buf, (int)sizeof(buf)) != NULL + && buf[0] != '\0' && buf[1] == ':') { + if (snprintf(joined, sizeof(joined), "%c%c%s", buf[0], buf[1], p) < (int)sizeof(joined)) { + return newStringFromCString(threadStateData, joined); + } + return path; + } if (_getcwd(buf, (int)sizeof(buf)) != NULL) { #else if (getcwd(buf, sizeof(buf)) != NULL) { diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 6afc971ca41..08a22de30af 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -2284,7 +2284,11 @@ JAVA_INT java_lang_Object_hashCode___R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJEC * - the collector cannot stop a compute-only virtual thread at all, which is why * marking such a state active is a hang rather than a fix; * - a virtual thread registered after the collector's once-per-cycle registry - * snapshot is invisible to the stack scan until the next cycle. + * snapshot is invisible to the stack scan until the next cycle; + * - past CN1_VT_SNAPSHOT_MAX (4096) registered virtual threads the snapshot is + * truncated, and the collector warns but continues, so the overflow is unscanned. + * Reaching that count needs this API, which is why it is listed here rather than + * fixed in the collector. * * KNOWN GAP, stated here because the obvious fix is worse than the problem. The * state this creates is never marked threadActive while its virtual thread runs, diff --git a/vm/JavaAPI/src/java/io/FileInputStream.java b/vm/JavaAPI/src/java/io/FileInputStream.java index ddad384a2c5..cfac889307a 100644 --- a/vm/JavaAPI/src/java/io/FileInputStream.java +++ b/vm/JavaAPI/src/java/io/FileInputStream.java @@ -29,7 +29,18 @@ */ public class FileInputStream extends InputStream { private long handle; - private boolean closed; + /* volatile + synchronized close: two threads closing concurrently could both + read closed == false and hand the SAME FILE* to fclose twice, which is + undefined and crashes the translated process rather than merely erroring. + Idempotent now. + + NOT a claim of thread safety. A read racing a close on the same stream can + still reach the native call with a handle this method is closing -- the JDK + buys that with a lock on every operation, and these streams are not worth + that cost. Like most java.io streams they are for one thread at a time; what + is guaranteed here is that closing twice, or closing from another thread, is + safe rather than fatal. */ + private volatile boolean closed; public FileInputStream(String name) throws FileNotFoundException { if(name == null) { @@ -97,7 +108,7 @@ public int available() throws IOException { return a; } - public void close() throws IOException { + public synchronized void close() throws IOException { if(closed) { return; } @@ -122,11 +133,15 @@ public void close() throws IOException { * that a close failed. */ protected void finalize() { - if(!closed && handle != 0) { - closed = true; - long h = handle; - handle = 0; - closeImpl(h); + // Same lock as close(): a finalizer running while another thread closes + // would otherwise be the two-fclose case this synchronization exists for. + synchronized(this) { + if(!closed && handle != 0) { + closed = true; + long h = handle; + handle = 0; + closeImpl(h); + } } } diff --git a/vm/JavaAPI/src/java/io/FileOutputStream.java b/vm/JavaAPI/src/java/io/FileOutputStream.java index dd0e42be3ca..6ff39850906 100644 --- a/vm/JavaAPI/src/java/io/FileOutputStream.java +++ b/vm/JavaAPI/src/java/io/FileOutputStream.java @@ -29,7 +29,18 @@ */ public class FileOutputStream extends OutputStream { private long handle; - private boolean closed; + /* volatile + synchronized close: two threads closing concurrently could both + read closed == false and hand the SAME FILE* to fclose twice, which is + undefined and crashes the translated process rather than merely erroring. + Idempotent now. + + NOT a claim of thread safety. A read racing a close on the same stream can + still reach the native call with a handle this method is closing -- the JDK + buys that with a lock on every operation, and these streams are not worth + that cost. Like most java.io streams they are for one thread at a time; what + is guaranteed here is that closing twice, or closing from another thread, is + safe rather than fatal. */ + private volatile boolean closed; public FileOutputStream(String name) throws FileNotFoundException { this(name, false); @@ -89,7 +100,7 @@ public void flush() throws IOException { } } - public void close() throws IOException { + public synchronized void close() throws IOException { if(closed) { return; } @@ -115,11 +126,15 @@ public void close() throws IOException { * that a close failed. */ protected void finalize() { - if(!closed && handle != 0) { - closed = true; - long h = handle; - handle = 0; - closeImpl(h); + // Same lock as close(): a finalizer running while another thread closes + // would otherwise be the two-fclose case this synchronization exists for. + synchronized(this) { + if(!closed && handle != 0) { + closed = true; + long h = handle; + handle = 0; + closeImpl(h); + } } } From 0c96b7bc319565b53ed417c710b334d3e5fe0b82 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:45:49 +0300 Subject: [PATCH 40/42] Park the mutator around every blocking stdio call, not just read and write The report named fflush. Sweeping the file layer found five unparked blocking calls rather than one, and the two I would not have thought of are the opens. - fflush pushes the buffer at the peer and blocks exactly where the write does. - Both fclose calls FLUSH before closing, so they block in the same place. - Both fopen calls block on a FIFO: opening for read waits until a writer opens the other end, opening for write waits for a reader, and there may never be one. Opening reads as cheap, which is precisely why it was missed. Each left the VM thread active while it blocked, so a collection waited for a safepoint that could not arrive -- and on Windows, where CN1_GC_CAN_FORCE_STOP is off, there is no escalation to break that wait. The opens need no buffer keep-alive, unlike the reads and writes: `path` points into the thread's utf8Buffer, which is C memory a collection cannot move or reclaim, whereas those hold an interior pointer into a Java array the collector could sweep. Verified by re-running the same sweep afterwards: all eight java_io_* natives that touch stdio now park. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/nativeMethods.m | 51 +++++++++++++++++++---- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 08a22de30af..4384382116e 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1153,8 +1153,17 @@ JAVA_LONG java_io_FileInputStream_openImpl___java_lang_String_R_long(CODENAME_ON if(path == NULL) { return 0; } - FILE* f = fopen(path, "rb"); - return (JAVA_LONG)(intptr_t)f; + { + /* Opening BLOCKS on a FIFO: fopen for read waits until a writer opens the + other end, and there may never be one. `path` points into the thread's + utf8Buffer, which is C memory and unaffected by a collection, so it stays + valid across the safepoint. */ + FILE* f; + CN1_YIELD_THREAD; + f = fopen(path, "rb"); + CN1_RESUME_THREAD; + return (JAVA_LONG)(intptr_t)f; + } } /* @@ -1278,7 +1287,13 @@ JAVA_INT java_io_FileInputStream_closeImpl___long_R_int(CODENAME_ONE_THREAD_STAT if(f == NULL) { return 0; } - return fclose(f) == 0 ? 0 : -1; + { + int r; + CN1_YIELD_THREAD; + r = fclose(f); + CN1_RESUME_THREAD; + return r == 0 ? 0 : -1; + } } JAVA_LONG java_io_FileOutputStream_openImpl___java_lang_String_boolean_R_long(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT name, JAVA_BOOLEAN append) { @@ -1289,8 +1304,14 @@ JAVA_LONG java_io_FileOutputStream_openImpl___java_lang_String_boolean_R_long(CO if(path == NULL) { return 0; } - FILE* f = fopen(path, append ? "ab" : "wb"); - return (JAVA_LONG)(intptr_t)f; + { + /* The mirror of the read side: opening a FIFO for write waits for a reader. */ + FILE* f; + CN1_YIELD_THREAD; + f = fopen(path, append ? "ab" : "wb"); + CN1_RESUME_THREAD; + return (JAVA_LONG)(intptr_t)f; + } } JAVA_INT java_io_FileOutputStream_writeImpl___long_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { @@ -1314,7 +1335,15 @@ JAVA_INT java_io_FileOutputStream_flushImpl___long_R_int(CODENAME_ONE_THREAD_STA if(f == NULL) { return -1; } - return fflush(f) == 0 ? 0 : -1; + { + /* fflush pushes the buffer at the peer and blocks for the same reasons the + write does -- a FIFO nobody is draining, a slow network filesystem. */ + int r; + CN1_YIELD_THREAD; + r = fflush(f); + CN1_RESUME_THREAD; + return r == 0 ? 0 : -1; + } } JAVA_INT java_io_FileOutputStream_closeImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { @@ -1322,7 +1351,15 @@ JAVA_INT java_io_FileOutputStream_closeImpl___long_R_int(CODENAME_ONE_THREAD_STA if(f == NULL) { return 0; } - return fclose(f) == 0 ? 0 : -1; + { + /* fclose FLUSHES before it closes, so it blocks exactly where the flush + above does. */ + int r; + CN1_YIELD_THREAD; + r = fclose(f); + CN1_RESUME_THREAD; + return r == 0 ? 0 : -1; + } } // Standard input. Separate from FileInputStream because stdin is not seekable, so From 9f7d96b856e205d1e8eae7d2ad9d392f4739b83b Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:00:21 +0300 Subject: [PATCH 41/42] Answer null for an environment name holding a NUL; decline the rest Half of this report is right and half of it is not. THE EMBEDDED NUL IS REAL. The native converts to a C string, where a NUL ends it, so a lookup of "PATH" + NUL + "suffix" found PATH and returned that variable's value. A silent answer about a DIFFERENT variable is worse than reporting the name unset, so a name containing a NUL is now answered null. The check is in Java because that is where the information is: one indexOf against re-deriving the byte length in C and walking the string's backing representation, which is the compact byte[] versus char[] distinction consolidated earlier in this branch. That moved the null check up too, so the native is now the raw lookup. ILLEGALARGUMENTEXCEPTION IS DECLINED. Neither this VM's contract for getenv ("or null when it is not set") nor java.lang.System.getenv(String) declares it -- the documented exceptions are NullPointerException and SecurityException. The validation that throws IllegalArgumentException belongs to ProcessBuilder's environment mutation, not to a lookup. An empty name, or one containing '=', names nothing, and null is exactly what "not set" means. Adding the throw would make this VM diverge from the platform in the name of matching it. Written at the method so it is not re-raised. The rename to getenvImpl is the dangerous part of this edit -- a wrong native name compiles, links, and silently drops the method, leaving a green build and an inert feature. check-native-signatures.sh reports 0 fatal with every native resolving on both ports, and the UTF-8 environment test passes end to end, which it could not if the symbol had been dropped. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/nativeMethods.m | 8 +++---- vm/JavaAPI/src/java/lang/System.java | 26 ++++++++++++++++++++++- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 4384382116e..4d405b24e89 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1112,12 +1112,10 @@ static void cn1FreeThreadStack(struct elementStruct* stack, int mapped) { // char per byte. (On Windows the value is in the ACTIVE CODE PAGE rather than // UTF-8, so it needs _wgetenv before any decoding is meaningful -- the same unfixed // issue recorded against the file layer below, and the same remedy.) -JAVA_OBJECT java_lang_System_getenv___java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT name) { +/* Renamed from getenv: the null and embedded-NUL checks moved to the Java side, + where they are one line each, so this is now the raw lookup. */ +JAVA_OBJECT java_lang_System_getenvImpl___java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT name) { if(name == JAVA_NULL) { - /* The API specifies NullPointerException for a null name. Returning null made - an invalid argument indistinguishable from an unset variable, so a caller - with a null name silently took the "not set" branch. */ - throwException(threadStateData, __NEW_INSTANCE_java_lang_NullPointerException(threadStateData)); return JAVA_NULL; } const char* key = stringToUTF8(threadStateData, name); diff --git a/vm/JavaAPI/src/java/lang/System.java b/vm/JavaAPI/src/java/lang/System.java index 43213415029..0c6df307818 100644 --- a/vm/JavaAPI/src/java/lang/System.java +++ b/vm/JavaAPI/src/java/lang/System.java @@ -195,8 +195,32 @@ public static java.lang.String getProperty(java.lang.String key){ * process gets before it parses its own arguments, so a server-side * translated binary needs this to find, for example, the endpoint its host * runtime published to it. + * + * A name containing a NUL is answered null rather than passed down. The + * native side converts to a C string, where a NUL ends it, so + * "PATH\u0000suffix" would otherwise be looked up as "PATH" and return that + * variable's value -- a silent answer about a DIFFERENT variable, which is + * worse than reporting the name unset. + * + * Deliberately NOT IllegalArgumentException for an empty name or one holding + * '='. Neither this contract nor java.lang.System's declares that exception; + * the validation that throws it belongs to ProcessBuilder's environment + * mutation, not to a lookup. Such names simply name nothing, and null is + * exactly what "not set" means. + * + * @throws NullPointerException if name is null */ - public static native java.lang.String getenv(java.lang.String name); + public static java.lang.String getenv(java.lang.String name) { + if(name == null) { + throw new NullPointerException(); + } + if(name.indexOf(0) >= 0) { + return null; + } + return getenvImpl(name); + } + + private static native java.lang.String getenvImpl(java.lang.String name); /** * Returns the same hashcode for the given object as would be returned by the default method hashCode(), whether or not the given object's class overrides hashCode(). The hashcode for the null reference is zero. From 4729deac53152b83fe66f8b84adea05533b0bf25 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:15:25 +0300 Subject: [PATCH 42/42] Park the seeks too, and let the thread benchmark exit TWO THINGS, and the first is a correction to my own sweep. Last round I said every java_io_* native touching stdio now parks, and verified it mechanically -- against a pattern list I had built from the calls I had already fixed. It did not include ftell or fseek, so skipImpl and availableImpl were still unparked. A mechanical check is only as good as the pattern given to it. Seeking is not free everywhere: a remote mount or a FUSE filesystem services ftell/fseek over the wire, and the thread sits inside the CRT for the duration -- where a collection waits for a safepoint that cannot arrive, with no forced-stop escalation on Windows to break it. Both functions take ONE yield spanning their whole seek sequence rather than bracketing each call: the collector only needs the thread parked, and three yield/resume pairs would cost more than the seeks they guard. Re-swept with ftell/fseek included: zero unparked. THE THREAD BENCHMARK NEVER TERMINATED. ThreadCost spawns non-daemon threads parked on LOCK.wait() and nothing ever notified them, so returning from main ended only the main thread. The documented "/usr/bin/time -l /tmp/threadcost" invocation could not print its result without an external kill -- the measurement was taken and then discarded. It notifies after measuring; waking the workers cannot affect a number already recorded. Co-Authored-By: Claude Opus 5 (1M context) --- vm/ByteCodeTranslator/src/nativeMethods.m | 69 +++++++++++++++------ vm/benchmarks/src/com/bench/ThreadCost.java | 8 +++ 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 4d405b24e89..27c3c7662ed 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1228,14 +1228,23 @@ JAVA_LONG java_io_FileInputStream_skipImpl___long_long_R_long(CODENAME_ONE_THREA // Clamped to the real end so the return value is bytes actually skipped, which // is what InputStream.skip promises -- seeking past EOF succeeds in C and would // otherwise report a skip that did not happen. - long start = ftell(f); - if(start < 0 || fseek(f, 0, SEEK_END) != 0) { - return -1; - } - long end = ftell(f); + long start; + long end; long remaining; long skipped; long target; + /* Parked for the same reason availableImpl is: on a remote or FUSE filesystem + these go over the wire, and the collector cannot stop a thread sitting in the + CRT. One yield spans the sequence; the arithmetic below is local and stays + outside it. */ + CN1_YIELD_THREAD; + start = ftell(f); + if(start < 0 || fseek(f, 0, SEEK_END) != 0) { + CN1_RESUME_THREAD; + return -1; + } + end = ftell(f); + CN1_RESUME_THREAD; if(end < 0) { return -1; } @@ -1254,8 +1263,14 @@ JAVA_LONG java_io_FileInputStream_skipImpl___long_long_R_long(CODENAME_ONE_THREA skipped = (long)count; } target = start + skipped; - if(fseek(f, target, SEEK_SET) != 0) { - return -1; + { + int failed; + CN1_YIELD_THREAD; + failed = fseek(f, target, SEEK_SET) != 0; + CN1_RESUME_THREAD; + if(failed) { + return -1; + } } return (JAVA_LONG)skipped; } @@ -1265,19 +1280,35 @@ JAVA_INT java_io_FileInputStream_availableImpl___long_R_int(CODENAME_ONE_THREAD_ if(f == NULL) { return -1; } - long start = ftell(f); - if(start < 0 || fseek(f, 0, SEEK_END) != 0) { - return -1; - } - long end = ftell(f); - if(fseek(f, start, SEEK_SET) != 0) { - return -1; - } - long remaining = end - start; - if(remaining < 0) { - return -1; + { + /* Seeking is not free on every filesystem: a remote mount or a FUSE + filesystem services ftell/fseek over the wire, and the thread is inside + the CRT for the duration. One yield spans the whole sequence rather than + bracketing each call -- the collector only needs the thread parked, and + three yield/resume pairs would cost more than the seeks. */ + long start, end, remaining; + int failed = 0; + CN1_YIELD_THREAD; + start = ftell(f); + if(start < 0 || fseek(f, 0, SEEK_END) != 0) { + failed = 1; + } + if(!failed) { + end = ftell(f); + if(fseek(f, start, SEEK_SET) != 0) { + failed = 1; + } + } + CN1_RESUME_THREAD; + if(failed) { + return -1; + } + remaining = end - start; + if(remaining < 0) { + return -1; + } + return remaining > 0x7fffffffL ? 0x7fffffff : (JAVA_INT)remaining; } - return remaining > 0x7fffffffL ? 0x7fffffff : (JAVA_INT)remaining; } JAVA_INT java_io_FileInputStream_closeImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { diff --git a/vm/benchmarks/src/com/bench/ThreadCost.java b/vm/benchmarks/src/com/bench/ThreadCost.java index 27ddcff8f1e..897dd1635c9 100644 --- a/vm/benchmarks/src/com/bench/ThreadCost.java +++ b/vm/benchmarks/src/com/bench/ThreadCost.java @@ -71,6 +71,14 @@ public void run() { } Thread.sleep(holdMs); System.out.println("threads=" + n + " started=" + started); + // Release them, or the process never exits and the documented + // "/usr/bin/time -l /tmp/threadcost" invocation never prints its result: + // the workers are non-daemon and parked on a wait nobody was notifying, so + // returning from main only ends the main thread. The measurement is already + // taken by this point, so waking them cannot affect it. + synchronized (LOCK) { + LOCK.notifyAll(); + } } private static int envInt(String name, int fallback) {