From 9bbc4f2650ca1a4cf82bd251403f49924576753b Mon Sep 17 00:00:00 2001 From: Frotty Date: Sun, 16 Aug 2026 16:19:17 +0200 Subject: [PATCH 1/4] Run a standard library program on Lua Nothing did. Every test putting the library on that target compiled only, because three packages could not initialise: Colors, Vectors and GameTimer each call a native the runtime shim did not define. StringCase made a fourth once the program itself ran. That is how the empty allocation in #1239 survived - the paths which would have caught it were compiled and never executed. The four natives are byte-accurate rather than approximations, since a test comparing targets is worth nothing if they disagree by construction: StringHash is the same lookup2 as Wc3StringHash and pinned against it for a fixed corpus, StringCase folds only ascii. A timer keeps what it was started with and never fires: a test drives its own program rather than waiting on game time, and a package which starts one at init needs the call to succeed. Success is read off stdout, and a program with the library gets the library's own testSuccess, which is empty - so such a test could only ever be reported as not having succeeded, whatever it did. The harness now installs one which prints, as the generated fallback does without the library. FastHashMap against the standard library runs on Lua as a result, rather than being compiled and assumed. --- .../wurstscript/tests/FastHashMapTests.java | 2 +- .../wurstscript/tests/Wc3StringHashTest.java | 15 +++ .../wurstscript/tests/WurstScriptTest.java | 5 + .../src/test/resources/luaruntime/wc3shim.lua | 102 ++++++++++++++++++ 4 files changed, 123 insertions(+), 1 deletion(-) diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/FastHashMapTests.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/FastHashMapTests.java index f98c1a09b..446610d5b 100644 --- a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/FastHashMapTests.java +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/FastHashMapTests.java @@ -413,7 +413,7 @@ public void fastHashMapAgainstTheStandardLibrary() { */ @Test public void fastHashMapAgainstTheStandardLibraryLua() throws IOException { - test().withStdLib().testLua(true) + test().withStdLib().testLua(true).executeProg() .lines(withStandardLibrary(program(fastHashMap(), INT_INSTANCE, USE_WITH_COLLISION))); assertSpecialisedClassesAllocateTheirFields( Files.toString(new File("test-output/lua/FastHashMapTests_fastHashMapAgainstTheStandardLibraryLua.lua"), diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java index a466a8cf5..164260af2 100644 --- a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java @@ -87,4 +87,19 @@ public void everyContinuationByteHashesApart() { } assertEquals(hashes.size(), 64, "all 64 continuation bytes should hash apart"); } + + /** + * The Lua test runtime carries a second implementation of this hash, because a program running + * there computes it too. Two transcriptions of one algorithm drift; these are the values that + * one produces, so a change to either side which parts them fails here. + */ + @Test + public void agreesWithTheLuaRuntimeImplementation() { + assertEquals(Wc3StringHash.hash(ILconstString.fromText("abc").getVal()), 1043745117); + assertEquals(Wc3StringHash.hash(ILconstString.fromText("Hello World").getVal()), -1563733934); + assertEquals(Wc3StringHash.hash(ILconstString.fromText("Units\\Human\\Footman.mdx").getVal()), 166547459); + assertEquals(Wc3StringHash.hash(ILconstString.fromText("ä").getVal()), 1899444195); + assertEquals(Wc3StringHash.hash(ILconstString.fromText("abcdefghijklmnop").getVal()), 190888648); + assertEquals(Wc3StringHash.hash(ILconstString.fromText("").getVal()), 0); + } } diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/WurstScriptTest.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/WurstScriptTest.java index a81e34ac4..ea88aad23 100644 --- a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/WurstScriptTest.java +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/WurstScriptTest.java @@ -556,6 +556,11 @@ private void translateAndTestLua(String name, boolean executeProg, WurstGui gui, } } chunk.append("dofile('").append(luaFile.getPath().replace('\\', '/')).append("');"); + // Success is read off stdout, so testSuccess has to print. A program with no + // standard library gets a generated fallback which does; one with the library gets + // the library's own, which is empty - so without this, a test on that target can + // only ever be reported as not having succeeded, whatever it did. + chunk.append("testSuccess = function() print('testSuccess') end;"); chunk.append("main()"); String[] args = { luaExecutable, diff --git a/de.peeeq.wurstscript/src/test/resources/luaruntime/wc3shim.lua b/de.peeeq.wurstscript/src/test/resources/luaruntime/wc3shim.lua index bf7199f5c..2c66343d9 100644 --- a/de.peeeq.wurstscript/src/test/resources/luaruntime/wc3shim.lua +++ b/de.peeeq.wurstscript/src/test/resources/luaruntime/wc3shim.lua @@ -60,6 +60,108 @@ end function I2S(i) return tostring(math.floor(i)) end function S2I(s) return math.floor(tonumber(s) or 0) end +-- StringHash, over bytes, as the game and the interpreter both compute it. Bob Jenkins' +-- lookup2, with the same normalisation: ascii letters upper-cased and a forward slash read as +-- a backslash. Kept in step with Wc3StringHash on the Java side. +local function mix(a, b, c) + local M = 0xFFFFFFFF + a = (a - b - c) & M; a = a ~ (c >> 13) + b = (b - c - a) & M; b = b ~ ((a << 8) & M) + c = (c - a - b) & M; c = c ~ (b >> 13) + a = (a - b - c) & M; a = a ~ (c >> 12) + b = (b - c - a) & M; b = b ~ ((a << 16) & M) + c = (c - a - b) & M; c = c ~ (b >> 5) + a = (a - b - c) & M; a = a ~ (c >> 3) + b = (b - c - a) & M; b = b ~ ((a << 10) & M) + c = (c - a - b) & M; c = c ~ (b >> 15) + return a, b, c +end + +function StringHash(s) + if s == nil or #s == 0 then + return 0 + end + local bytes = {} + for i = 1, #s do + local v = string.byte(s, i) + if v >= 97 and v <= 122 then + v = v - 32 + elseif v == 47 then + v = 92 + end + bytes[i] = v + end + local a, b, c = 0x9e3779b9, 0x9e3779b9, 0 + local len = #bytes + local i = 1 + local M = 0xFFFFFFFF + while len >= 12 do + a = (a + bytes[i] + (bytes[i+1] << 8) + (bytes[i+2] << 16) + (bytes[i+3] << 24)) & M + b = (b + bytes[i+4] + (bytes[i+5] << 8) + (bytes[i+6] << 16) + (bytes[i+7] << 24)) & M + c = (c + bytes[i+8] + (bytes[i+9] << 8) + (bytes[i+10] << 16) + (bytes[i+11] << 24)) & M + a, b, c = mix(a, b, c) + i = i + 12 + len = len - 12 + end + c = (c + #bytes) & M + -- The low byte of c holds the length, so the tail starts at the second. + if len >= 11 then c = (c + (bytes[i+10] << 24)) & M end + if len >= 10 then c = (c + (bytes[i+9] << 16)) & M end + if len >= 9 then c = (c + (bytes[i+8] << 8)) & M end + if len >= 8 then b = (b + (bytes[i+7] << 24)) & M end + if len >= 7 then b = (b + (bytes[i+6] << 16)) & M end + if len >= 6 then b = (b + (bytes[i+5] << 8)) & M end + if len >= 5 then b = (b + bytes[i+4]) & M end + if len >= 4 then a = (a + (bytes[i+3] << 24)) & M end + if len >= 3 then a = (a + (bytes[i+2] << 16)) & M end + if len >= 2 then a = (a + (bytes[i+1] << 8)) & M end + if len >= 1 then a = (a + bytes[i]) & M end + a, b, c = mix(a, b, c) + -- The game returns a signed 32 bit integer. + if c >= 0x80000000 then + c = c - 0x100000000 + end + return c +end + +-- Only ascii letters change case: the bytes of a multibyte character are not letters, and +-- folding one rewrites the character. Same rule as the interpreter's StringCase. +function StringCase(s, upperCase) + local out = {} + for i = 1, #s do + local v = string.byte(s, i) + if upperCase and v >= 97 and v <= 122 then + v = v - 32 + elseif not upperCase and v >= 65 and v <= 90 then + v = v + 32 + end + out[i] = string.char(v) + end + return table.concat(out) +end + +-- Locations are a plain pair; nothing in a test reads terrain from one. +function Location(x, y) return { x = x, y = y } end +function GetLocationX(loc) return loc.x end +function GetLocationY(loc) return loc.y end +function MoveLocation(loc, x, y) loc.x = x loc.y = y end +function RemoveLocation(loc) end + +-- Timers hold what they were started with and never fire: a test drives its own program rather +-- than waiting on game time, and a package which starts a timer at init only needs the call to +-- succeed. +function TimerStart(t, timeout, periodic, handler) + t.timeout = timeout + t.periodic = periodic + t.handler = handler +end +function TimerGetElapsed(t) return 0.0 end +function TimerGetRemaining(t) return t.timeout or 0.0 end +function TimerGetTimeout(t) return t.timeout or 0.0 end +function PauseTimer(t) end +function ResumeTimer(t) end +function DestroyTimer(t) end + -- Reforged player layout: 24 playable slots, neutrals at 24..27, 28 total. function GetBJMaxPlayers() return 24 end function GetBJMaxPlayerSlots() return 28 end From db333697af417b36f623c3888864bcde22c267aa Mon Sep 17 00:00:00 2001 From: Frotty Date: Sun, 16 Aug 2026 16:35:16 +0200 Subject: [PATCH 2/4] Terminate on success, and run both hashes in the parity test The injected testSuccess dropped the os.exit() the generated native does, so a program calling it before something which fails or loops would newly fail or time out after having already succeeded - a difference between the targets rather than a fix to one. The parity test compared the Java hash against numbers written down from a Lua run, which is not a parity test: a change breaking only the Lua side left it green. It runs the inputs through the shim now and compares the two, with the input escaped byte by byte so what Lua hashes is what Java was handed. --- .../wurstscript/tests/Wc3StringHashTest.java | 48 +++++++++++++++---- .../wurstscript/tests/WurstScriptTest.java | 4 +- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java index 164260af2..81385283b 100644 --- a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java @@ -15,7 +15,7 @@ * are only comparable where the library is defined - a string of whole characters - so that is what * is compared, and it is enough: the arithmetic is the same for every input, only the bytes differ. */ -public class Wc3StringHashTest { +public class Wc3StringHashTest extends WurstScriptTest { private static void agrees(String text) throws UnsupportedEncodingException { assertEquals(Wc3StringHash.hash(ILconstString.fromText(text).getVal()), @@ -90,16 +90,44 @@ public void everyContinuationByteHashesApart() { /** * The Lua test runtime carries a second implementation of this hash, because a program running - * there computes it too. Two transcriptions of one algorithm drift; these are the values that - * one produces, so a change to either side which parts them fails here. + * there computes it too. Two transcriptions of one algorithm drift, so this runs the inputs + * through both and compares — asserting the Java side against written-down numbers would stay + * green if only the Lua side changed, which is the case worth catching. */ @Test - public void agreesWithTheLuaRuntimeImplementation() { - assertEquals(Wc3StringHash.hash(ILconstString.fromText("abc").getVal()), 1043745117); - assertEquals(Wc3StringHash.hash(ILconstString.fromText("Hello World").getVal()), -1563733934); - assertEquals(Wc3StringHash.hash(ILconstString.fromText("Units\\Human\\Footman.mdx").getVal()), 166547459); - assertEquals(Wc3StringHash.hash(ILconstString.fromText("ä").getVal()), 1899444195); - assertEquals(Wc3StringHash.hash(ILconstString.fromText("abcdefghijklmnop").getVal()), 190888648); - assertEquals(Wc3StringHash.hash(ILconstString.fromText("").getVal()), 0); + public void agreesWithTheLuaRuntimeImplementation() throws Exception { + String[] inputs = {"abc", "Hello World", "Units\\Human\\Footman.mdx", "ä", + "abcdefghijklmnop", "", "MIXED/Case\\Path", "日本語"}; + + for (String input : inputs) { + assertEquals(luaHashOf(input), Wc3StringHash.hash(ILconstString.fromText(input).getVal()), + "hash of " + input); + } + } + + /** Runs the shim's StringHash on one input, as a program on that target would. */ + private int luaHashOf(String text) throws Exception { + // Escaped byte by byte, so what the shim hashes is what the Java side was handed rather + // than whatever the command line did to it. + StringBuilder literal = new StringBuilder(); + for (byte b : text.getBytes(java.nio.charset.StandardCharsets.UTF_8)) { + literal.append("\\").append(b & 0xFF); + } + String script = "dofile('src/test/resources/luaruntime/wc3shim.lua') " + + "print(StringHash('" + literal + "'))"; + Process p = new ProcessBuilder(getLuaExecutable(), "-e", script) + .redirectErrorStream(true) + .start(); + String out; + try (java.io.BufferedReader r = new java.io.BufferedReader( + new java.io.InputStreamReader(p.getInputStream(), java.nio.charset.StandardCharsets.UTF_8))) { + out = r.lines().collect(java.util.stream.Collectors.joining("\n")).trim(); + } + p.waitFor(30, java.util.concurrent.TimeUnit.SECONDS); + try { + return Integer.parseInt(out); + } catch (NumberFormatException e) { + throw new AssertionError("lua did not return a hash for \"" + text + "\": " + out); + } } } diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/WurstScriptTest.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/WurstScriptTest.java index ea88aad23..8ed7d2e71 100644 --- a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/WurstScriptTest.java +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/WurstScriptTest.java @@ -560,7 +560,7 @@ private void translateAndTestLua(String name, boolean executeProg, WurstGui gui, // standard library gets a generated fallback which does; one with the library gets // the library's own, which is empty - so without this, a test on that target can // only ever be reported as not having succeeded, whatever it did. - chunk.append("testSuccess = function() print('testSuccess') end;"); + chunk.append("testSuccess = function() print('testSuccess') os.exit() end;"); chunk.append("main()"); String[] args = { luaExecutable, @@ -706,7 +706,7 @@ private Thread collectStreamAsync(InputStream stream, StringBuilder out, return t; } - private String getLuaExecutable() { + protected String getLuaExecutable() { if (resolvedLuaExecutable != null) { return resolvedLuaExecutable; } From f78ca816710f0f2f303cfaf285f7bb06e6bbd38d Mon Sep 17 00:00:00 2001 From: Frotty Date: Sun, 16 Aug 2026 16:41:09 +0200 Subject: [PATCH 3/4] Name the reset which gives the second compile a clean cache The item said the caches being cleared around every test method is what gives the two compilations the same starting state. That is only half of it: the before-method reset covers the first compile, and the explicit clearAll between them covers the second. Naming the wrong one matters when diagnosing, because removing the in-test reset would invalidate the conclusion rather than explain the failure. --- .../tests/wurstscript/tests/Wc3StringHashTest.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java index 81385283b..dd3e30456 100644 --- a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java @@ -115,7 +115,15 @@ private int luaHashOf(String text) throws Exception { } String script = "dofile('src/test/resources/luaruntime/wc3shim.lua') " + "print(StringHash('" + literal + "'))"; - Process p = new ProcessBuilder(getLuaExecutable(), "-e", script) + String lua; + try { + lua = getLuaExecutable(); + } catch (IllegalStateException e) { + // Same handling as the normal execution path: a host without a working interpreter + // skips visibly rather than failing the class, since nothing here is being tested. + throw new org.testng.SkipException("Skipped the Lua half of the hash parity check: " + e.getMessage()); + } + Process p = new ProcessBuilder(lua, "-e", script) .redirectErrorStream(true) .start(); String out; From 246e4d6141de14d985069cab5c299559e3af2959 Mon Sep 17 00:00:00 2001 From: Frotty Date: Sun, 16 Aug 2026 17:15:45 +0200 Subject: [PATCH 4/4] Bound the parity test's Lua call with a deadline that can fire It read the process output to the end before waiting with a timeout, so a shim or a hash which hung never reached the wait: the read blocks until the stream closes, and a hung process does not close it. The timeout could only have fired after the hang had already stopped the suite. Drained on a thread of its own now, waited for with the deadline, and killed when it expires - the same shape as the main Lua harness, which had this same bug and this same fix. --- .../wurstscript/tests/Wc3StringHashTest.java | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java index dd3e30456..b4ba2fa09 100644 --- a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java @@ -126,12 +126,32 @@ private int luaHashOf(String text) throws Exception { Process p = new ProcessBuilder(lua, "-e", script) .redirectErrorStream(true) .start(); - String out; - try (java.io.BufferedReader r = new java.io.BufferedReader( - new java.io.InputStreamReader(p.getInputStream(), java.nio.charset.StandardCharsets.UTF_8))) { - out = r.lines().collect(java.util.stream.Collectors.joining("\n")).trim(); + // Drained on a thread of its own and waited for with a deadline. Reading to the end first + // waits for the process to close the stream, which a hung one never does - the timeout + // below would then be reached only after the hang had already stopped the suite. + StringBuilder output = new StringBuilder(); + Thread drain = new Thread(() -> { + try (java.io.BufferedReader r = new java.io.BufferedReader( + new java.io.InputStreamReader(p.getInputStream(), java.nio.charset.StandardCharsets.UTF_8))) { + String line; + while ((line = r.readLine()) != null) { + output.append(line).append('\n'); + } + } catch (java.io.IOException e) { + // The process was killed underneath the read; the timeout below reports it. + } + }); + drain.setDaemon(true); + drain.start(); + + if (!p.waitFor(30, java.util.concurrent.TimeUnit.SECONDS)) { + p.destroyForcibly(); + drain.join(5_000); + throw new AssertionError("lua did not finish hashing \"" + text + "\" within 30s"); } - p.waitFor(30, java.util.concurrent.TimeUnit.SECONDS); + drain.join(5_000); + + String out = output.toString().trim(); try { return Integer.parseInt(out); } catch (NumberFormatException e) {