From caebb0ecd78b280f602d0ca1c076a72c69379f4a Mon Sep 17 00:00:00 2001 From: Frotty Date: Sun, 16 Aug 2026 13:09:52 +0200 Subject: [PATCH 1/2] Hold an interpreted string as bytes, the way the game does Warcraft III counts and indexes a string in bytes, so a slice may stop between the bytes of one character, and Lua agrees because its strings are byte arrays. The interpreter held a Java string and counted UTF-16 code units, which is the same answer only for ascii. The standard library depends on the difference rather than avoiding it: it cuts a character in half on purpose to find out how the engine represents a partial slice, and slices a 64 character literal byte by byte to enumerate every continuation byte. Under UTF-16 that detection concludes the engine has no multibyte characters, so anything computed at compiletime is built from lengths the game will not agree with, and nothing reports it. The value is now held one char per byte, so Java's own length and substring give the game's answers. Text is encoded coming in and decoded going back out to a file or a screen; a half character has no text to decode to, which is the point. Two things follow from the representation. StringCase folds only ascii letters, because the bytes of a multibyte character are not letters and folding them rewrites the character. StringHash is computed over the bytes: the library's takes text and encodes it as UTF-8 itself, so it cannot hash half a character, and decoding first would collapse every partial slice onto one value and take the continuation bytes with it. The Lua test runtime never implemented StringLength or SubString, so the generated fallback raised an error and the Lua half of such a test could not have been passing for the reason it appeared to. Both are in the shim now. --- .../wurstio/CompiletimeFunctionRunner.java | 4 +- .../interpreter/CompiletimeNatives.java | 16 +-- .../jassinterpreter/JassInterpreter.java | 2 +- .../providers/AbilityProvider.java | 4 +- .../providers/GamecacheProvider.java | 2 +- .../providers/HashtableProvider.java | 2 +- .../providers/LuaEnsureTypeProvider.java | 2 +- .../providers/OutputProvider.java | 16 +-- .../providers/StringProvider.java | 40 +++--- .../providers/UnitProvider.java | 4 +- .../providers/WurstflectionProvider.java | 2 +- .../intermediatelang/ILconstString.java | 45 +++++- .../intermediatelang/Wc3StringHash.java | 96 +++++++++++++ .../interpreter/EvaluateExpr.java | 4 +- .../interpreter/ILInterpreter.java | 2 +- .../tests/CompiletimeNativesTest.java | 134 +++++++++--------- .../tests/StringByteSemanticsTests.java | 91 ++++++++++++ .../wurstscript/tests/Wc3StringHashTest.java | 90 ++++++++++++ .../src/test/resources/luaruntime/wc3shim.lua | 14 ++ 19 files changed, 452 insertions(+), 118 deletions(-) create mode 100644 de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/Wc3StringHash.java create mode 100644 de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/StringByteSemanticsTests.java create mode 100644 de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompiletimeFunctionRunner.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompiletimeFunctionRunner.java index e2b7e9243..f35396f22 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompiletimeFunctionRunner.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompiletimeFunctionRunner.java @@ -429,7 +429,7 @@ private ImExpr constantToExpr(Element trace, ILconst value, @Nullable ImType exp } else if (value instanceof ILconstReal) { return JassIm.ImRealVal("" + ((ILconstReal) value).getVal()); } else if (value instanceof ILconstString) { - return JassIm.ImStringVal(((ILconstString) value).getVal()); + return JassIm.ImStringVal(((ILconstString) value).text()); } else if (value instanceof ILconstNull) { return expectedType == null ? ImHelper.nullExpr() : JassIm.ImNull(expectedType.copy()); } else if (value instanceof ILconstTuple) { @@ -1043,7 +1043,7 @@ private ImExpr constantToExprHashtable(Element trace, ImVar htVar, IlConstHandle JassIm.ImVarAccess(htVar), JassIm.ImIntVal(key.getParentkey()), JassIm.ImIntVal(key.getChildkey()), - JassIm.ImStringVal(iv.getVal()) + JassIm.ImStringVal(iv.text()) ), false, CallType.NORMAL)); } else if (v instanceof ILconstBool) { ILconstBool iv = (ILconstBool) v; diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/CompiletimeNatives.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/CompiletimeNatives.java index 479a8228e..33f487300 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/CompiletimeNatives.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/CompiletimeNatives.java @@ -47,7 +47,7 @@ public CompiletimeNatives(ProgramStateIO globalState, WurstProjectConfigData pro private ILconstTuple makeKey(String key) { - return new ILconstTuple(new ILconstString(key)); + return new ILconstTuple(ILconstString.fromText(key)); } public ILconstTuple createObjectDefinition(ILconstString fileType, ILconstInt newUnitId, ILconstInt deriveFrom) { @@ -93,7 +93,7 @@ public void ObjectDefinition_setInt(ILconstTuple unitType, ILconstString modific public void ObjectDefinition_setString(ILconstTuple unitType, ILconstString modification, ILconstString value) { ObjMod.Obj od = globalState.getObjectDefinition(getKey(unitType)); - modifyObject(od, modification, ObjMod.ValType.STRING, War3String.valueOf(value.getVal())); + modifyObject(od, modification, ObjMod.ValType.STRING, War3String.valueOf(value.text())); } public void ObjectDefinition_setReal(ILconstTuple unitType, ILconstString modification, ILconstReal value) { @@ -114,7 +114,7 @@ public void ObjectDefinition_setLvlInt(ILconstTuple unitType, ILconstString modi public void ObjectDefinition_setLvlString(ILconstTuple unitType, ILconstString modification, ILconstInt level, ILconstString value) { ObjMod.Obj od = globalState.getObjectDefinition(getKey(unitType)); - modifyObject(od, modification, ObjMod.ValType.STRING, level.getVal(), War3String.valueOf(value.getVal())); + modifyObject(od, modification, ObjMod.ValType.STRING, level.getVal(), War3String.valueOf(value.text())); } public void ObjectDefinition_setLvlReal(ILconstTuple unitType, ILconstString modification, ILconstInt level, ILconstReal value) { @@ -135,7 +135,7 @@ public void ObjectDefinition_setLvlDataInt(ILconstTuple unitType, ILconstString public void ObjectDefinition_setLvlDataString(ILconstTuple unitType, ILconstString modification, ILconstInt level, ILconstInt dataPointer, ILconstString value) { ObjMod.Obj od = globalState.getObjectDefinition(getKey(unitType)); - modifyObject(od, modification, ObjMod.ValType.STRING, level.getVal(), dataPointer.getVal(), War3String.valueOf(value.getVal())); + modifyObject(od, modification, ObjMod.ValType.STRING, level.getVal(), dataPointer.getVal(), War3String.valueOf(value.text())); } public void ObjectDefinition_setLvlDataReal(ILconstTuple unitType, ILconstString modification, ILconstInt level, ILconstInt dataPointer, ILconstReal value) { @@ -185,15 +185,15 @@ private String getKey(ILconstTuple unitType) { } public void compileError(ILconstString msg) { - throw new InterpreterException(msg.getVal()); + throw new InterpreterException(msg.text()); } public ILconstString getMapName() { - return new ILconstString(projectConfigData.buildMapData().name()); + return ILconstString.fromText(projectConfigData.buildMapData().name()); } public ILconstString getBuildDate() { - return new ILconstString(LocalDateTime.now().truncatedTo(ChronoUnit.MINUTES).toString()); + return ILconstString.fromText(LocalDateTime.now().truncatedTo(ChronoUnit.MINUTES).toString()); } public ILconstBool isProductionBuild() { @@ -379,7 +379,7 @@ public ILconstString sqlite_column_string(ILconstInt statement, ILconstInt index // A SQL NULL maps to "" here; use sqlite_column_is_null to distinguish NULL // from an empty string / zero value. String val = rs.getString(index.getVal() + 1); - return new ILconstString(val == null ? "" : val); + return ILconstString.fromText(val == null ? "" : val); } catch (SQLException e) { throw new InterpreterException("Failed to get column string: " + e.getMessage()); } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/JassInterpreter.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/JassInterpreter.java index 6e3c574a0..f61a000f4 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/JassInterpreter.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/JassInterpreter.java @@ -386,7 +386,7 @@ public ILconst case_JassOpEquals(JassOpEquals jassOpEquals) { @Override public ILconst case_JassExprStringVal(JassExprStringVal e) { - return new ILconstString(e.getValS()); + return ILconstString.fromText(e.getValS()); } @Override diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/AbilityProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/AbilityProvider.java index d74c00313..9d82492c2 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/AbilityProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/AbilityProvider.java @@ -10,10 +10,10 @@ public AbilityProvider(AbstractInterpreter interpreter) { } public ILconstString BlzGetAbilityIcon(ILconstInt abilCode) { - return new ILconstString(""); + return ILconstString.fromText(""); } public ILconstString BlzGetAbilityExtendedTooltip(ILconstInt abilCode, ILconstInt level) { - return new ILconstString(""); + return ILconstString.fromText(""); } } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/GamecacheProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/GamecacheProvider.java index d7bd46ce7..6801dd33d 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/GamecacheProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/GamecacheProvider.java @@ -63,7 +63,7 @@ public ILconstReal GetStoredReal(IlConstHandle ht, ILconstString key1, ILconstSt } public ILconstString GetStoredString(IlConstHandle ht, ILconstString key1, ILconstString key2) { - return haveSaved(ht, key1, key2, ILconstString.class) ? load(ht, key1, key2, ILconstString.class) : new ILconstString(""); + return haveSaved(ht, key1, key2, ILconstString.class) ? load(ht, key1, key2, ILconstString.class) : ILconstString.fromText(""); } public ILconstBool GetStoredBoolean(IlConstHandle ht, ILconstString key1, ILconstString key2) { diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/HashtableProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/HashtableProvider.java index 1a901e1a8..e4ef842d9 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/HashtableProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/HashtableProvider.java @@ -74,7 +74,7 @@ public ILconstReal LoadReal(IlConstHandle ht, ILconstInt key1, ILconstInt key2) } public ILconstString LoadStr(IlConstHandle ht, ILconstInt key1, ILconstInt key2) { - return haveSaved(ht, key1, key2, ILconstString.class) ? load(ht, key1, key2, ILconstString.class) : new ILconstString(""); + return haveSaved(ht, key1, key2, ILconstString.class) ? load(ht, key1, key2, ILconstString.class) : ILconstString.fromText(""); } public ILconstBool LoadBoolean(IlConstHandle ht, ILconstInt key1, ILconstInt key2) { diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/LuaEnsureTypeProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/LuaEnsureTypeProvider.java index c4bab136b..c42e27f37 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/LuaEnsureTypeProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/LuaEnsureTypeProvider.java @@ -43,7 +43,7 @@ public ILconstString __wurst_rawToString(ILconstString x) { } public ILconstString __wurst_rawConcat(ILconstString x, ILconstString y) { - return new ILconstString(x.getVal() + y.getVal()); + return ILconstString.ofBytes(x.getVal() + y.getVal()); } public ILconstInt __wurst_rawFloorDivInt(ILconstInt a, ILconstInt b) { diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java index 083953019..c9cae968c 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java @@ -23,29 +23,29 @@ public void setOutStream(PrintStream outStream) { } public void DisplayTextToForce(IlConstHandle force, ILconstString msg) { - outStream.println(msg.getVal()); + outStream.println(msg.text()); } public void DisplayTimedTextToForce(IlConstHandle force, ILconstReal duration, ILconstString msg) { - outStream.println(msg.getVal()); + outStream.println(msg.text()); } public void DisplayTextToPlayer(IlConstHandle player, ILconstReal x, ILconstReal y, ILconstString msg) { - outStream.println(msg.getVal()); + outStream.println(msg.text()); } public void DisplayTimedTextToPlayer(IlConstHandle player, ILconstReal x, ILconstReal y, ILconstReal duration, ILconstString msg) { - outStream.println(msg.getVal()); + outStream.println(msg.text()); } @Implements(funcNames = {"BJDebugMsg", "println"}) public void println(ILconstString msg) { - outStream.println(msg.getVal()); + outStream.println(msg.text()); } public void $debugPrint(ILconstString msg) { - outStream.println(msg.getVal()); - throw new DebugPrintError(msg.getVal()); + outStream.println(msg.text()); + throw new DebugPrintError(msg.text()); } public void testSuccess() { @@ -53,6 +53,6 @@ public void testSuccess() { } public void testFail(ILconstString msg) { - throw new TestFailException(msg.getVal()); + throw new TestFailException(msg.text()); } } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/StringProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/StringProvider.java index 73e10def9..a8cea2ce7 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/StringProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/StringProvider.java @@ -1,16 +1,14 @@ package de.peeeq.wurstio.jassinterpreter.providers; import de.peeeq.wurstio.jassinterpreter.InterpreterException; -import de.peeeq.wurstscript.WLogger; import de.peeeq.wurstscript.intermediatelang.ILconstBool; import de.peeeq.wurstscript.intermediatelang.ILconstInt; import de.peeeq.wurstscript.intermediatelang.ILconstReal; import de.peeeq.wurstscript.intermediatelang.ILconstString; +import de.peeeq.wurstscript.intermediatelang.Wc3StringHash; import de.peeeq.wurstscript.intermediatelang.interpreter.AbstractInterpreter; -import net.moonlightflower.wc3libs.misc.StringHash; import org.apache.commons.lang.StringUtils; -import java.io.UnsupportedEncodingException; import java.math.RoundingMode; import java.text.NumberFormat; import java.util.Locale; @@ -24,7 +22,7 @@ public StringProvider(AbstractInterpreter interpreter) { } public ILconstString I2S(ILconstInt i) { - return new ILconstString("" + i.getVal()); + return ILconstString.fromText("" + i.getVal()); } private static final Pattern s2ipattern = Pattern.compile("([+\\-]?[0-9]+).*"); @@ -53,7 +51,7 @@ public ILconstReal S2R(ILconstString s) { } public ILconstString R2S(ILconstReal r) { - return new ILconstString("" + r.getVal()); + return ILconstString.fromText("" + r.getVal()); } public ILconstString R2SW(ILconstReal r, ILconstInt width, ILconstInt precision) { @@ -65,7 +63,7 @@ public ILconstString R2SW(ILconstReal r, ILconstInt width, ILconstInt precision) String s = formatter.format(r.getVal()); // pad to desired width s = StringUtils.rightPad(s, width.getVal()); - return new ILconstString(s); + return ILconstString.fromText(s); } public ILconstInt R2I(ILconstReal i) { @@ -81,12 +79,7 @@ public ILconstInt StringHash(ILconstString s) { if (s == null) { return new ILconstInt(0); } - try { - return new ILconstInt(StringHash.hash(s.getVal())); - } catch (UnsupportedEncodingException e) { - WLogger.severe(e); - } - return new ILconstInt(0); + return new ILconstInt(Wc3StringHash.hash(s.getVal())); } public ILconstInt StringLength(ILconstString string) { @@ -110,13 +103,26 @@ public ILconstString SubString(ILconstString istr, ILconstInt start, ILconstInt // since this is most likely a bug in your code, the interpreter will throw an exception instead: throw new InterpreterException("SubString called with start index " + start + " greater than string length " + str.length()); } - return new ILconstString(str.substring(s, e)); + return ILconstString.ofBytes(str.substring(s, e)); } + /** + * Only ascii letters change case. A string is a sequence of bytes, and the bytes of a multibyte + * character are not letters to case at all - folding them the way a latin-1 char would fold + * rewrites the character into a different one. + */ public ILconstString StringCase(ILconstString string, ILconstBool upperCase) { - return new ILconstString( - upperCase.getVal() ? - string.getVal().toUpperCase() - : string.getVal().toLowerCase()); + String bytes = string.getVal(); + StringBuilder result = new StringBuilder(bytes.length()); + for (int i = 0; i < bytes.length(); i++) { + char c = bytes.charAt(i); + if (upperCase.getVal() && c >= 'a' && c <= 'z') { + c -= 32; + } else if (!upperCase.getVal() && c >= 'A' && c <= 'Z') { + c += 32; + } + result.append(c); + } + return ILconstString.ofBytes(result.toString()); } } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/UnitProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/UnitProvider.java index 28eca76c7..a4a72a9f7 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/UnitProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/UnitProvider.java @@ -66,10 +66,10 @@ public ILconstReal GetUnitFacing(IlConstHandle unit) { public ILconstString GetUnitName(IlConstHandle unit) { if (unit == null) { - return new ILconstString(""); + return ILconstString.fromText(""); } UnitMock unitMock = (UnitMock) unit.getObj(); - return new ILconstString(ObjectHelper.objectIdIntToString(unitMock.unitid.getVal())); + return ILconstString.fromText(ObjectHelper.objectIdIntToString(unitMock.unitid.getVal())); } public ILconstInt GetUnitGoldCost(ILconstInt unitid) { diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/WurstflectionProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/WurstflectionProvider.java index 51830b7d9..4f86d0cd0 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/WurstflectionProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/WurstflectionProvider.java @@ -25,7 +25,7 @@ public ILconstString typeIdToTypeName(ILconstInt typeId) { int typeIdInt = typeId.getVal(); for (Map.Entry e : prog.attrTypeId().entrySet()) { if (e.getValue() == typeIdInt) { - ILconstString iLconstString = new ILconstString(calculateClassName(e.getKey())); + ILconstString iLconstString = ILconstString.fromText(calculateClassName(e.getKey())); return Optional.of(iLconstString) .orElseGet(() -> { throw new InterpreterException("Could not determine type name for id " + typeId); diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/ILconstString.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/ILconstString.java index fa63e6d86..c7bbf4acb 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/ILconstString.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/ILconstString.java @@ -3,23 +3,60 @@ import de.peeeq.wurstscript.types.WurstType; import de.peeeq.wurstscript.types.WurstTypeString; +import java.nio.charset.StandardCharsets; import java.util.Objects; +/** + * A string as Warcraft III has it: a sequence of bytes. {@code StringLength} counts them and + * {@code SubString} indexes them, so a slice may stop between the bytes of one character. Lua agrees, + * its strings being byte arrays too. + *

+ * The value is held one char per byte, so every character in it is below 256 and Java's own length + * and substring already give the game's answers. Text arriving from source or from the host has to be + * encoded on the way in ({@link #fromText}), and anything leaving the interpreter for a file or a + * screen decoded on the way out ({@link #text()}). A half character has no text to decode to, which is + * the point: it keeps its byte until the other half is added back. + */ public class ILconstString extends ILconstAbstract implements ILconstAddable { - private final String val; // including the quotes + /** One char per byte, so all chars are < 256. */ + private final String val; - public ILconstString(String strVal) { - this.val = strVal; + private ILconstString(String byteView) { + this.val = byteView; } + /** Encodes text, which Wurst source and the host both give as UTF-8. */ + public static ILconstString fromText(String text) { + return new ILconstString(encode(text)); + } + + /** Wraps bytes which are already one per char, as produced by slicing or joining. */ + public static ILconstString ofBytes(String byteView) { + return new ILconstString(byteView); + } + + static String encode(String text) { + return new String(text.getBytes(StandardCharsets.UTF_8), StandardCharsets.ISO_8859_1); + } + + static String decode(String byteView) { + return new String(byteView.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8); + } + + /** The bytes, one per char. This is what the string natives count and index. */ public String getVal() { return val; } + /** The text those bytes spell, for anything which leaves the interpreter. */ + public String text() { + return decode(val); + } + @Override public String print() { - return "\"" + val + "\""; + return "\"" + text() + "\""; } public WurstType getType() { diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/Wc3StringHash.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/Wc3StringHash.java new file mode 100644 index 000000000..c5630e991 --- /dev/null +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/Wc3StringHash.java @@ -0,0 +1,96 @@ +package de.peeeq.wurstscript.intermediatelang; + +/** + * {@code StringHash} over bytes. + *

+ * The library's version takes text and encodes it as UTF-8 itself, which cannot express a string + * holding half of a character - and half characters are exactly what the standard library hashes to + * find out how the engine represents them. Decoding first would collapse every partial slice onto one + * value, so the continuation bytes the standard library tells apart by hash would stop being + * distinguishable. + *

+ * The function is Bob Jenkins' lookup2, the same one the library implements; only the input differs. + * {@code Wc3StringHashTest} checks the two agree on text, where both are defined. + */ +public final class Wc3StringHash { + + private Wc3StringHash() { + } + + /** + * @param bytes one byte per char, all below 256 + */ + public static int hash(String bytes) { + if (bytes.isEmpty()) { + return 0; + } + byte[] normalized = new byte[bytes.length()]; + for (int i = 0; i < bytes.length(); i++) { + byte b = (byte) bytes.charAt(i); + // Case insensitive, and either slash names the same file. Both comparisons are on signed + // bytes, so anything above 127 - every byte of a multibyte character - is left alone. + if (b >= 'a' && b <= 'z') { + b -= 32; + } else if (b == '/') { + b = '\\'; + } + normalized[i] = b; + } + return lookup2(normalized); + } + + private static int lookup2(byte[] k) { + int a = 0x9e3779b9; + int b = 0x9e3779b9; + int c = 0; + int len = k.length; + int i = 0; + while (len >= 12) { + a += u(k[i]) + (u(k[i + 1]) << 8) + (u(k[i + 2]) << 16) + (u(k[i + 3]) << 24); + b += u(k[i + 4]) + (u(k[i + 5]) << 8) + (u(k[i + 6]) << 16) + (u(k[i + 7]) << 24); + c += u(k[i + 8]) + (u(k[i + 9]) << 8) + (u(k[i + 10]) << 16) + (u(k[i + 11]) << 24); + // mix + a -= b; a -= c; a ^= (c >>> 13); + b -= c; b -= a; b ^= (a << 8); + c -= a; c -= b; c ^= (b >>> 13); + a -= b; a -= c; a ^= (c >>> 12); + b -= c; b -= a; b ^= (a << 16); + c -= a; c -= b; c ^= (b >>> 5); + a -= b; a -= c; a ^= (c >>> 3); + b -= c; b -= a; b ^= (a << 10); + c -= a; c -= b; c ^= (b >>> 15); + i += 12; + len -= 12; + } + c += k.length; + // The low byte of c holds the length, so the tail starts at the second. + switch (len) { + case 11: c += u(k[i + 10]) << 24; + case 10: c += u(k[i + 9]) << 16; + case 9: c += u(k[i + 8]) << 8; + case 8: b += u(k[i + 7]) << 24; + case 7: b += u(k[i + 6]) << 16; + case 6: b += u(k[i + 5]) << 8; + case 5: b += u(k[i + 4]); + case 4: a += u(k[i + 3]) << 24; + case 3: a += u(k[i + 2]) << 16; + case 2: a += u(k[i + 1]) << 8; + case 1: a += u(k[i]); + default: + } + a -= b; a -= c; a ^= (c >>> 13); + b -= c; b -= a; b ^= (a << 8); + c -= a; c -= b; c ^= (b >>> 13); + a -= b; a -= c; a ^= (c >>> 12); + b -= c; b -= a; b ^= (a << 16); + c -= a; c -= b; c ^= (b >>> 5); + a -= b; a -= c; a ^= (c >>> 3); + b -= c; b -= a; b ^= (a << 10); + c -= a; c -= b; c ^= (b >>> 15); + return c; + } + + private static int u(byte b) { + return b & 0xFF; + } +} diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/EvaluateExpr.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/EvaluateExpr.java index df15e6d64..25a35c851 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/EvaluateExpr.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/EvaluateExpr.java @@ -142,7 +142,7 @@ public static ILaddress evaluateLvalue(ImStatementExpr e, ProgramState globalSta } public static ILconst eval(ImStringVal e, ProgramState globalState, LocalState localState) { - return new ILconstString(e.getValS()); + return ILconstString.fromText(e.getValS()); } public static ILconst eval(ImTupleExpr e, ProgramState globalState, LocalState localState) { @@ -334,7 +334,7 @@ public static ILconst eval(ImGetStackTrace e, ProgramState globalState, LocalState localState) { StringBuilder sb = new StringBuilder(); globalState.getStackFrames().appendTo(sb); - return new ILconstString(sb.toString()); + return ILconstString.fromText(sb.toString()); } public static ILconst eval(ImCompiletimeExpr expr, ProgramState globalState, LocalState localState) { diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/ILInterpreter.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/ILInterpreter.java index 467af1c5c..56d0b5f06 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/ILInterpreter.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/ILInterpreter.java @@ -623,7 +623,7 @@ public void runVoidFunc(ImFunction f, @Nullable Element trace) { ILconst[] args = {}; if (!f.getParameters().isEmpty()) { // this should only happen because of added stacktrace parameter - args = new ILconstString[]{new ILconstString("initial call")}; + args = new ILconstString[]{ILconstString.fromText("initial call")}; } runFunc(globalState, f, trace, args); } diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/CompiletimeNativesTest.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/CompiletimeNativesTest.java index 9f984c0bf..cb18bb1d5 100644 --- a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/CompiletimeNativesTest.java +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/CompiletimeNativesTest.java @@ -51,8 +51,8 @@ public void modifyObjectKeepsDifferentDataPointers() throws Exception { modifyObject.setAccessible(true); MetaFieldId unrealId = MetaFieldId.valueOf("unat"); - modifyObject.invoke(natives, obj, new ILconstString(unrealId.getVal()), ObjMod.ValType.UNREAL, 1, 0, War3Real.valueOf(1.0)); - modifyObject.invoke(natives, obj, new ILconstString(unrealId.getVal()), ObjMod.ValType.UNREAL, 1, 1, War3Real.valueOf(2.0)); + modifyObject.invoke(natives, obj, ILconstString.fromText(unrealId.getVal()), ObjMod.ValType.UNREAL, 1, 0, War3Real.valueOf(1.0)); + modifyObject.invoke(natives, obj, ILconstString.fromText(unrealId.getVal()), ObjMod.ValType.UNREAL, 1, 1, War3Real.valueOf(2.0)); List unrealMods = obj.getMods().stream() .filter(m -> m instanceof ObjMod.Obj.ExtendedMod) @@ -151,8 +151,8 @@ public void duplicateCodeObjectDefinitionsReportError() { int hfoo = ObjectHelper.objectIdStringToInt("hfoo"); int hf01 = ObjectHelper.objectIdStringToInt("hf01"); - natives.createObjectDefinition(new ILconstString("w3u"), new ILconstInt(hf01), new ILconstInt(hfoo)); - natives.createObjectDefinition(new ILconstString("w3u"), new ILconstInt(hf01), new ILconstInt(hfoo)); + natives.createObjectDefinition(ILconstString.fromText("w3u"), new ILconstInt(hf01), new ILconstInt(hfoo)); + natives.createObjectDefinition(ILconstString.fromText("w3u"), new ILconstInt(hf01), new ILconstInt(hfoo)); assertEquals(gui.getErrorCount(), 1); assertTrue(gui.getErrors().contains("Object definition with id hf01 is defined more than once.")); @@ -172,7 +172,7 @@ public void createObjectDefinitionDoesNotReportExistingMapObjectAsError() throws W3U.Obj existing = w3u.addObj(ObjId.valueOf("hf01"), ObjId.valueOf("hpea")); existing.addMod(new ObjMod.Obj.Mod(MetaFieldId.valueOf("unam"), ObjMod.ValType.STRING, War3String.valueOf("old map object"))); - natives.createObjectDefinition(new ILconstString("w3u"), new ILconstInt(hf01), new ILconstInt(hfoo)); + natives.createObjectDefinition(ILconstString.fromText("w3u"), new ILconstInt(hf01), new ILconstInt(hfoo)); assertEquals(gui.getErrorCount(), 0); assertEquals(w3u.getCustomObjs().size(), 1); @@ -189,10 +189,10 @@ public void sameIdObjectDefinitionsMergeModsWithoutDuplicateError() throws Excep CompiletimeNatives natives = new CompiletimeNatives(state, null, false); int hfoo = ObjectHelper.objectIdStringToInt("hfoo"); - var first = natives.createObjectDefinition(new ILconstString("w3u"), new ILconstInt(hfoo), new ILconstInt(hfoo)); - natives.ObjectDefinition_setString(first, new ILconstString("unam"), new ILconstString("first")); - var second = natives.createObjectDefinition(new ILconstString("w3u"), new ILconstInt(hfoo), new ILconstInt(hfoo)); - natives.ObjectDefinition_setString(second, new ILconstString("utip"), new ILconstString("second")); + var first = natives.createObjectDefinition(ILconstString.fromText("w3u"), new ILconstInt(hfoo), new ILconstInt(hfoo)); + natives.ObjectDefinition_setString(first, ILconstString.fromText("unam"), ILconstString.fromText("first")); + var second = natives.createObjectDefinition(ILconstString.fromText("w3u"), new ILconstInt(hfoo), new ILconstInt(hfoo)); + natives.ObjectDefinition_setString(second, ILconstString.fromText("utip"), ILconstString.fromText("second")); Method getDataStore = ProgramStateIO.class.getDeclaredMethod("getDataStore", String.class); getDataStore.setAccessible(true); @@ -208,15 +208,15 @@ public void sameIdObjectDefinitionsMergeModsWithoutDuplicateError() throws Excep @Test public void testCloseSqliteResourcesOnProviderClose() { CompiletimeNatives natives = new CompiletimeNatives(null, null, false); - ILconstInt connHandle = natives.sqlite_open(new ILconstString(":memory:")); - natives.sqlite_exec(connHandle, new ILconstString("CREATE TABLE Test (id INT);")); - ILconstInt stmtHandle = natives.sqlite_prepare(connHandle, new ILconstString("INSERT INTO Test VALUES (1);")); + ILconstInt connHandle = natives.sqlite_open(ILconstString.fromText(":memory:")); + natives.sqlite_exec(connHandle, ILconstString.fromText("CREATE TABLE Test (id INT);")); + ILconstInt stmtHandle = natives.sqlite_prepare(connHandle, ILconstString.fromText("INSERT INTO Test VALUES (1);")); natives.sqlite_step(stmtHandle); natives.close(); try { - natives.sqlite_prepare(connHandle, new ILconstString("SELECT * FROM Test;")); + natives.sqlite_prepare(connHandle, ILconstString.fromText("SELECT * FROM Test;")); org.testng.Assert.fail("Expected InterpreterException for invalid connection handle after close"); } catch (de.peeeq.wurstio.jassinterpreter.InterpreterException e) { assertTrue(e.getMessage().contains("Invalid SQLite connection handle")); @@ -230,13 +230,13 @@ public void testCloseSqliteResourcesFromProgramStateClose() throws Exception { CompiletimeNatives natives = new CompiletimeNatives(state, null, false); state.addNativeProvider(natives); - ILconstInt connHandle = natives.sqlite_open(new ILconstString(":memory:")); - natives.sqlite_exec(connHandle, new ILconstString("CREATE TABLE Test (id INT);")); + ILconstInt connHandle = natives.sqlite_open(ILconstString.fromText(":memory:")); + natives.sqlite_exec(connHandle, ILconstString.fromText("CREATE TABLE Test (id INT);")); state.close(); try { - natives.sqlite_prepare(connHandle, new ILconstString("SELECT * FROM Test;")); + natives.sqlite_prepare(connHandle, ILconstString.fromText("SELECT * FROM Test;")); org.testng.Assert.fail("Expected InterpreterException for invalid connection handle after ProgramState close"); } catch (de.peeeq.wurstio.jassinterpreter.InterpreterException e) { assertTrue(e.getMessage().contains("Invalid SQLite connection handle")); @@ -250,18 +250,18 @@ public void sqliteExecRunsMultiStatementScriptWithTrigger() { CompiletimeNatives natives = new CompiletimeNatives(state, null, false); state.addNativeProvider(natives); - ILconstInt db = natives.sqlite_open(new ILconstString(":memory:")); + ILconstInt db = natives.sqlite_open(ILconstString.fromText(":memory:")); // Multi-statement script including a trigger BEGIN...END body (whose inner ';' // terminators would break a naive splitter) and a bracket-quoted identifier // containing ';'. sqlite3_exec delegates to SQLite's own parser, so both work. - natives.sqlite_exec(db, new ILconstString( + natives.sqlite_exec(db, ILconstString.fromText( "CREATE TABLE src (id INTEGER);" + "CREATE TABLE dst (id INTEGER);" + "CREATE TRIGGER mirror AFTER INSERT ON src BEGIN INSERT INTO dst VALUES (NEW.id); END;" + "CREATE TABLE [we;ird] (x INTEGER);" + "INSERT INTO src VALUES (1); INSERT INTO src VALUES (2);")); - ILconstInt q = natives.sqlite_prepare(db, new ILconstString( + ILconstInt q = natives.sqlite_prepare(db, ILconstString.fromText( "SELECT (SELECT COUNT(*) FROM dst), " + "(SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='we;ird')")); assertTrue(natives.sqlite_step(q).getVal()); @@ -296,11 +296,11 @@ private static void assertInterpreterError(String expectedSubstring, Runnable ac @Test public void sqliteReadsBackEveryColumnTypeIncludingNull() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString( + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText( "CREATE TABLE T (i INTEGER, r REAL, s TEXT, n TEXT);" + "INSERT INTO T VALUES (42, 2.5, 'hello', NULL);")); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT i, r, s, n FROM T")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT i, r, s, n FROM T")); // column_count works before any step (metadata branch) ... assertEquals(n.sqlite_column_count(q).getVal(), 4); assertTrue(n.sqlite_step(q).getVal()); @@ -322,10 +322,10 @@ public void sqliteReadsBackEveryColumnTypeIncludingNull() { @Test public void sqliteResetRewindsSelectResultSet() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString( + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText( "CREATE TABLE T (id INTEGER); INSERT INTO T VALUES (10); INSERT INTO T VALUES (20);")); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT id FROM T ORDER BY id")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT id FROM T ORDER BY id")); assertTrue(n.sqlite_step(q).getVal()); assertEquals(n.sqlite_column_int(q, i(0)).getVal(), 10); assertTrue(n.sqlite_step(q).getVal()); @@ -341,9 +341,9 @@ public void sqliteResetRewindsSelectResultSet() { @Test public void sqliteColumnIntTruncates64BitValueToInt() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (v INTEGER); INSERT INTO T VALUES (5000000000);")); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT v FROM T")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (v INTEGER); INSERT INTO T VALUES (5000000000);")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT v FROM T")); assertTrue(n.sqlite_step(q).getVal()); // WurstScript int is 32-bit: a 64-bit INTEGER wraps to its low 32 bits (documented). assertEquals(n.sqlite_column_int(q, i(0)).getVal(), (int) 5000000000L); @@ -354,20 +354,20 @@ public void sqliteColumnIntTruncates64BitValueToInt() { @Test public void sqliteBindRoundTripAndRebindAfterStepReexecutes() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (i INTEGER, r REAL, s TEXT)")); - ILconstInt ins = n.sqlite_prepare(db, new ILconstString("INSERT INTO T VALUES (?, ?, ?)")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (i INTEGER, r REAL, s TEXT)")); + ILconstInt ins = n.sqlite_prepare(db, ILconstString.fromText("INSERT INTO T VALUES (?, ?, ?)")); n.sqlite_bind_int(ins, i(1), i(1)); n.sqlite_bind_real(ins, i(2), new ILconstReal(1.5f)); - n.sqlite_bind_string(ins, i(3), new ILconstString("a")); + n.sqlite_bind_string(ins, i(3), ILconstString.fromText("a")); assertFalse(n.sqlite_step(ins).getVal()); // rebind i and s WITHOUT a reset: must re-execute; r keeps its previous binding n.sqlite_bind_int(ins, i(1), i(2)); - n.sqlite_bind_string(ins, i(3), new ILconstString("b")); + n.sqlite_bind_string(ins, i(3), ILconstString.fromText("b")); assertFalse(n.sqlite_step(ins).getVal()); n.sqlite_finalize(ins); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT i, r, s FROM T ORDER BY i")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT i, r, s FROM T ORDER BY i")); assertTrue(n.sqlite_step(q).getVal()); assertEquals(n.sqlite_column_int(q, i(0)).getVal(), 1); assertEquals((double) n.sqlite_column_real(q, i(1)).getVal(), 1.5, 0.0); @@ -384,16 +384,16 @@ public void sqliteBindRoundTripAndRebindAfterStepReexecutes() { @Test public void sqliteClearBindingsResetsParametersToNull() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (a INTEGER, b TEXT)")); - ILconstInt ins = n.sqlite_prepare(db, new ILconstString("INSERT INTO T VALUES (?, ?)")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (a INTEGER, b TEXT)")); + ILconstInt ins = n.sqlite_prepare(db, ILconstString.fromText("INSERT INTO T VALUES (?, ?)")); n.sqlite_bind_int(ins, i(1), i(7)); - n.sqlite_bind_string(ins, i(2), new ILconstString("x")); + n.sqlite_bind_string(ins, i(2), ILconstString.fromText("x")); n.sqlite_clear_bindings(ins); assertFalse(n.sqlite_step(ins).getVal()); n.sqlite_finalize(ins); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT a, b FROM T")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT a, b FROM T")); assertTrue(n.sqlite_step(q).getVal()); assertTrue(n.sqlite_column_is_null(q, i(0)).getVal()); assertTrue(n.sqlite_column_is_null(q, i(1)).getVal()); @@ -407,16 +407,16 @@ public void sqliteClearBindingsAfterStepReexecutesWithoutReset() { // sqlite_bind_*, so a step after clearing re-runs the statement with NULL params // WITHOUT an explicit sqlite_reset. Otherwise the second step silently no-ops. CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (v INTEGER)")); - ILconstInt ins = n.sqlite_prepare(db, new ILconstString("INSERT INTO T VALUES (?)")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (v INTEGER)")); + ILconstInt ins = n.sqlite_prepare(db, ILconstString.fromText("INSERT INTO T VALUES (?)")); n.sqlite_bind_int(ins, i(1), i(100)); assertFalse(n.sqlite_step(ins).getVal()); // inserts 100 n.sqlite_clear_bindings(ins); // no explicit reset assertFalse(n.sqlite_step(ins).getVal()); // must re-execute → inserts NULL n.sqlite_finalize(ins); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT count(*), count(v) FROM T")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT count(*), count(v) FROM T")); assertTrue(n.sqlite_step(q).getVal()); assertEquals(n.sqlite_column_int(q, i(0)).getVal(), 2); // two rows total assertEquals(n.sqlite_column_int(q, i(1)).getVal(), 1); // one non-NULL (the 100) @@ -430,10 +430,10 @@ public void sqliteClearBindingsAfterStepDiscardsOldResultSet() { // clear_bindings must discard it so the next step RE-RUNS the query with the now // NULL parameter, rather than continuing to walk the stale result set. CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (v INTEGER)")); - n.sqlite_exec(db, new ILconstString("INSERT INTO T VALUES (1), (2), (3)")); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT v FROM T WHERE v <> ? ORDER BY v")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (v INTEGER)")); + n.sqlite_exec(db, ILconstString.fromText("INSERT INTO T VALUES (1), (2), (3)")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT v FROM T WHERE v <> ? ORDER BY v")); n.sqlite_bind_int(q, i(1), i(2)); // excludes 2 → rows {1, 3} assertTrue(n.sqlite_step(q).getVal()); assertEquals(n.sqlite_column_int(q, i(0)).getVal(), 1); @@ -448,9 +448,9 @@ public void sqliteClearBindingsAfterStepDiscardsOldResultSet() { @Test public void sqliteFinalizeInvalidatesStatementHandle() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (id INTEGER)")); - ILconstInt stmt = n.sqlite_prepare(db, new ILconstString("INSERT INTO T VALUES (1)")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (id INTEGER)")); + ILconstInt stmt = n.sqlite_prepare(db, ILconstString.fromText("INSERT INTO T VALUES (1)")); n.sqlite_finalize(stmt); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_step(stmt)); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_finalize(stmt)); @@ -460,12 +460,12 @@ public void sqliteFinalizeInvalidatesStatementHandle() { @Test public void sqliteCloseInvalidatesConnectionAndItsStatements() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (id INTEGER)")); - ILconstInt stmt = n.sqlite_prepare(db, new ILconstString("SELECT id FROM T")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (id INTEGER)")); + ILconstInt stmt = n.sqlite_prepare(db, ILconstString.fromText("SELECT id FROM T")); n.sqlite_close(db); - assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_prepare(db, new ILconstString("SELECT 1"))); - assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_exec(db, new ILconstString("SELECT 1"))); + assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_prepare(db, ILconstString.fromText("SELECT 1"))); + assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_exec(db, ILconstString.fromText("SELECT 1"))); assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_close(db)); // statements belonging to the closed connection were finalized/invalidated too assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_step(stmt)); @@ -474,30 +474,30 @@ public void sqliteCloseInvalidatesConnectionAndItsStatements() { @Test public void sqliteInvalidHandlesAndBadSqlThrow() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_bind_int(i(999), i(1), i(1))); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_bind_real(i(999), i(1), new ILconstReal(1f))); - assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_bind_string(i(999), i(1), new ILconstString("x"))); + assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_bind_string(i(999), i(1), ILconstString.fromText("x"))); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_step(i(999))); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_reset(i(999))); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_clear_bindings(i(999))); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_finalize(i(999))); assertInterpreterError("No result set", () -> n.sqlite_column_int(i(999), i(0))); assertInterpreterError("No result set", () -> n.sqlite_column_is_null(i(999), i(0))); - assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_prepare(i(999), new ILconstString("SELECT 1"))); - assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_exec(i(999), new ILconstString("SELECT 1"))); + assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_prepare(i(999), ILconstString.fromText("SELECT 1"))); + assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_exec(i(999), ILconstString.fromText("SELECT 1"))); assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_close(i(999))); - assertInterpreterError("Failed to prepare SQLite statement", () -> n.sqlite_prepare(db, new ILconstString("NOT VALID SQL"))); - assertInterpreterError("Failed to exec SQLite query", () -> n.sqlite_exec(db, new ILconstString("NOT VALID SQL"))); + assertInterpreterError("Failed to prepare SQLite statement", () -> n.sqlite_prepare(db, ILconstString.fromText("NOT VALID SQL"))); + assertInterpreterError("Failed to exec SQLite query", () -> n.sqlite_exec(db, ILconstString.fromText("NOT VALID SQL"))); n.sqlite_close(db); } @Test public void sqliteColumnAccessWithoutResultSetThrows() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (id INTEGER); INSERT INTO T VALUES (1)")); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT id FROM T")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (id INTEGER); INSERT INTO T VALUES (1)")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT id FROM T")); // reading a column before any step -> no result set yet assertInterpreterError("No result set", () -> n.sqlite_column_int(q, i(0))); assertInterpreterError("No result set", () -> n.sqlite_column_is_null(q, i(0))); @@ -509,17 +509,17 @@ public void sqliteColumnAccessWithoutResultSetThrows() { public void sqliteOpenRejectsFileUriWithQueryParameters() { CompiletimeNatives n = newSqliteNatives(); assertInterpreterError("query parameters are not allowed", - () -> n.sqlite_open(new ILconstString("file::memory:?enable_load_extension=true"))); + () -> n.sqlite_open(ILconstString.fromText("file::memory:?enable_load_extension=true"))); } @Test public void sqliteLoadExtensionIsBlocked() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); // extension loading is disabled on the connection, so load_extension is not authorized // (this is what closes the dlopen-arbitrary-native-code vector at compiletime). assertInterpreterError("not authorized", - () -> n.sqlite_exec(db, new ILconstString("SELECT load_extension('nonexistent_extension')"))); + () -> n.sqlite_exec(db, ILconstString.fromText("SELECT load_extension('nonexistent_extension')"))); n.sqlite_close(db); } diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/StringByteSemanticsTests.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/StringByteSemanticsTests.java new file mode 100644 index 000000000..8b1962068 --- /dev/null +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/StringByteSemanticsTests.java @@ -0,0 +1,91 @@ +package tests.wurstscript.tests; + +import org.testng.annotations.Test; + +/** + * Warcraft III treats a string as a sequence of bytes: {@code StringLength} counts bytes and + * {@code SubString} takes byte offsets, so a slice can cut a multibyte character in half. Lua agrees, + * its strings being byte arrays. The interpreter holds a Java string and counts UTF-16 code units, + * which is the same answer only for ascii. + *

+ * The standard library depends on the difference rather than avoiding it: {@code String.wurst} cuts a + * character in half on purpose to find out how the engine represents a partial slice, and slices a + * 64 character literal byte by byte to enumerate every continuation byte. Under UTF-16 that detection + * quietly concludes the engine has no multibyte characters, so anything computed at compiletime - + * object editor text, chunked data - is built from lengths the game will not agree with. + */ +public class StringByteSemanticsTests extends WurstScriptTest { + + private static String[] program(String... body) { + String[] head = { + "package test", + "native testSuccess()", + "@extern native StringLength(string s) returns int", + "@extern native SubString(string s, int start, int stop) returns string", + "@extern native StringHash(string s) returns int", + }; + String[] all = new String[head.length + body.length]; + System.arraycopy(head, 0, all, 0, head.length); + System.arraycopy(body, 0, all, head.length, body.length); + return all; + } + + /** + * Written as an escape rather than as itself, so what reaches the compiler does not depend on + * the encoding javac happens to read this file with. + */ + private static final String A_UMLAUT = "ä"; + + /** Two bytes in UTF-8, and the game counts bytes. */ + private static final String[] LENGTH_OF_A_TWO_BYTE_CHARACTER = program( + "init", + " if StringLength(\"" + A_UMLAUT + "\") == 2", + " testSuccess()" + ); + + @Test + public void lengthCountsBytes() { + testAssertOkLines(true, LENGTH_OF_A_TWO_BYTE_CHARACTER); + } + + @Test + public void lengthCountsBytesLua() { + test().testLua(true).executeProg().lines(LENGTH_OF_A_TWO_BYTE_CHARACTER); + } + + /** A slice may stop between the bytes of one character, which is how the stdlib probes. */ + private static final String[] SLICING_A_CHARACTER_IN_HALF = program( + "init", + " let half = SubString(\"ä\", 0, 1)", + " if StringLength(half) == 1 and half != \"ä\"", + " testSuccess()" + ); + + @Test + public void aSliceCanCutACharacterInHalf() { + testAssertOkLines(true, SLICING_A_CHARACTER_IN_HALF); + } + + @Test + public void aSliceCanCutACharacterInHalfLua() { + test().testLua(true).executeProg().lines(SLICING_A_CHARACTER_IN_HALF); + } + + /** The halves are the bytes of the whole, so putting them back gives it back. */ + private static final String[] HALVES_REJOIN = program( + "init", + " let s = \"äö\"", + " if SubString(s, 0, 2) + SubString(s, 2, 4) == s and StringLength(s) == 4", + " testSuccess()" + ); + + @Test + public void slicesRejoinIntoTheOriginal() { + testAssertOkLines(true, HALVES_REJOIN); + } + + @Test + public void slicesRejoinIntoTheOriginalLua() { + test().testLua(true).executeProg().lines(HALVES_REJOIN); + } +} diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java new file mode 100644 index 000000000..a466a8cf5 --- /dev/null +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java @@ -0,0 +1,90 @@ +package tests.wurstscript.tests; + +import de.peeeq.wurstscript.intermediatelang.ILconstString; +import de.peeeq.wurstscript.intermediatelang.Wc3StringHash; +import org.testng.annotations.Test; + +import java.io.UnsupportedEncodingException; +import java.util.Random; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; + +/** + * Checks the byte hash against the library's, which is the same function reached through text. They + * are only comparable where the library is defined - a string of whole characters - so that is what + * is compared, and it is enough: the arithmetic is the same for every input, only the bytes differ. + */ +public class Wc3StringHashTest { + + private static void agrees(String text) throws UnsupportedEncodingException { + assertEquals(Wc3StringHash.hash(ILconstString.fromText(text).getVal()), + net.moonlightflower.wc3libs.misc.StringHash.hash(text), + "hash of " + text); + } + + @Test + public void agreesOnShortStrings() throws UnsupportedEncodingException { + for (String s : new String[]{"", "a", "ab", "abc", "abcd", "abcde", "abcdef", "abcdefg", + "abcdefgh", "abcdefghi", "abcdefghij", "abcdefghijk", "abcdefghijkl", "abcdefghijklm"}) { + agrees(s); + } + } + + /** Case folding and the slash rule are part of the hash, not of the caller. */ + @Test + public void agreesOnStringsNeedingNormalisation() throws UnsupportedEncodingException { + for (String s : new String[]{"ABC", "AbC", "path/to/file", "path\\to\\file", + "Units\\Human\\Footman.mdx", "MIXED/Case\\Path"}) { + agrees(s); + } + } + + /** Multibyte text still agrees, because whole characters decode back to themselves. */ + @Test + public void agreesOnMultibyteText() throws UnsupportedEncodingException { + for (String s : new String[]{"ä", "äöü", "ЀЁЂЃ", "日本語", "😀", "aäböcü1234567890"}) { + agrees(s); + } + } + + @Test + public void agreesOnRandomAsciiOfEveryLength() throws UnsupportedEncodingException { + Random random = new Random(20260816); + for (int length = 0; length < 40; length++) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + sb.append((char) (32 + random.nextInt(95))); + } + agrees(sb.toString()); + } + } + + /** + * The reason for hashing bytes rather than text. Each half of a two byte character has to keep a + * hash of its own; decoding first would turn both into the same replacement character. + */ + @Test + public void halvesOfACharacterHashApart() { + String bytes = ILconstString.fromText("ä").getVal(); + String lead = bytes.substring(0, 1); + String continuation = bytes.substring(1, 2); + assertNotEquals(Wc3StringHash.hash(lead), Wc3StringHash.hash(continuation), + "the two bytes of a character must not hash alike"); + assertNotEquals(Wc3StringHash.hash(lead), Wc3StringHash.hash(bytes), + "half a character must not hash like the whole"); + } + + /** + * The standard library enumerates every continuation byte by slicing one literal and keeps them + * apart by hash, so all 64 have to be distinct. + */ + @Test + public void everyContinuationByteHashesApart() { + java.util.Set hashes = new java.util.HashSet<>(); + for (int b = 0x80; b <= 0xBF; b++) { + hashes.add(Wc3StringHash.hash(String.valueOf((char) b))); + } + assertEquals(hashes.size(), 64, "all 64 continuation bytes should hash apart"); + } +} diff --git a/de.peeeq.wurstscript/src/test/resources/luaruntime/wc3shim.lua b/de.peeeq.wurstscript/src/test/resources/luaruntime/wc3shim.lua index 8a124b211..bf7199f5c 100644 --- a/de.peeeq.wurstscript/src/test/resources/luaruntime/wc3shim.lua +++ b/de.peeeq.wurstscript/src/test/resources/luaruntime/wc3shim.lua @@ -46,6 +46,20 @@ function FourCC(s) + string.byte(s, 3)) * 256 + string.byte(s, 4) end +-- String natives. The game counts and indexes bytes, and a Lua string is a +-- byte array, so these are the plain Lua operations. SubString takes a +-- 0-based start and an exclusive end; string.sub is 1-based and inclusive. +function StringLength(s) + return #s +end + +function SubString(s, start, stop) + return string.sub(s, start + 1, stop) +end + +function I2S(i) return tostring(math.floor(i)) end +function S2I(s) return math.floor(tonumber(s) or 0) end + -- Reforged player layout: 24 playable slots, neutrals at 24..27, 28 total. function GetBJMaxPlayers() return 24 end function GetBJMaxPlayerSlots() return 28 end From 8fa85ac3dd08d2bee9b84747fe553e42050f8a4b Mon Sep 17 00:00:00 2001 From: Frotty Date: Sun, 16 Aug 2026 14:01:50 +0200 Subject: [PATCH 2/2] Decode strings crossing into SQLite, and refuse half a character in a literal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every value handed to the driver is text, so the four outbound calls decode the way the inbound read already did. Binding "ä" without it stored the bytes as though each were a character, and it came back a different string than the one that went in. A compiletime expression's result becomes a literal in the generated script, which is written as UTF-8, and neither Jass nor the escaping here can write a byte down numerically. Half a character therefore went in as the replacement character and came back three bytes long where the interpreter counted one. Refused with a message pointing at the fix instead of carried across at a different length. Whole characters cross unchanged, which is what the standard library does at compiletime. --- .../wurstio/CompiletimeFunctionRunner.java | 23 ++++++++++-- .../interpreter/CompiletimeNatives.java | 8 ++--- .../intermediatelang/ILconstString.java | 12 +++++++ .../tests/CompiletimeNativesTest.java | 35 ++++++++++++++++++ .../tests/StringByteSemanticsTests.java | 36 +++++++++++++++++++ 5 files changed, 108 insertions(+), 6 deletions(-) diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompiletimeFunctionRunner.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompiletimeFunctionRunner.java index f35396f22..6ef67034b 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompiletimeFunctionRunner.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompiletimeFunctionRunner.java @@ -421,6 +421,25 @@ private ImExpr constantToExpr(Element trace, ILconst value) { return constantToExpr(trace, value, null); } + /** + * The text for a string a compiletime expression produced, which becomes a literal in the + * generated script. + *

+ * A string held by the interpreter is a sequence of bytes and may hold half of a character - + * slicing one in half is a thing the standard library does deliberately. A literal cannot: the + * script is written as UTF-8 and neither Jass nor the escaping here can write a byte down + * numerically, so half a character would go in as the replacement character and come back out + * three bytes long. Refused rather than carried across at a different length. + */ + private String literalText(ILconstString value, Element trace) { + if (!value.isText()) { + throw new CompileError(trace, "A compiletime expression returned a string holding part of a" + + " multibyte character, which cannot be written into the generated script. Slice it" + + " where the program runs rather than at compiletime, or keep whole characters."); + } + return value.text(); + } + private ImExpr constantToExpr(Element trace, ILconst value, @Nullable ImType expectedType) { if (value instanceof ILconstBool) { return JassIm.ImBoolVal(((ILconstBool) value).getVal()); @@ -429,7 +448,7 @@ private ImExpr constantToExpr(Element trace, ILconst value, @Nullable ImType exp } else if (value instanceof ILconstReal) { return JassIm.ImRealVal("" + ((ILconstReal) value).getVal()); } else if (value instanceof ILconstString) { - return JassIm.ImStringVal(((ILconstString) value).text()); + return JassIm.ImStringVal(literalText((ILconstString) value, trace)); } else if (value instanceof ILconstNull) { return expectedType == null ? ImHelper.nullExpr() : JassIm.ImNull(expectedType.copy()); } else if (value instanceof ILconstTuple) { @@ -1043,7 +1062,7 @@ private ImExpr constantToExprHashtable(Element trace, ImVar htVar, IlConstHandle JassIm.ImVarAccess(htVar), JassIm.ImIntVal(key.getParentkey()), JassIm.ImIntVal(key.getChildkey()), - JassIm.ImStringVal(iv.text()) + JassIm.ImStringVal(literalText(iv, trace)) ), false, CallType.NORMAL)); } else if (v instanceof ILconstBool) { ILconstBool iv = (ILconstBool) v; diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/CompiletimeNatives.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/CompiletimeNatives.java index 33f487300..bae31c95c 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/CompiletimeNatives.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/CompiletimeNatives.java @@ -224,7 +224,7 @@ private PreparedStatement sqliteStatement(int handle) { } public ILconstInt sqlite_open(ILconstString path) { - String dbPath = path.getVal(); + String dbPath = path.text(); // SQLite "file:" URI paths can carry query parameters such as // "?enable_load_extension=true" that would turn on extension loading and let // load_extension() dlopen arbitrary native code on the build machine at compiletime. @@ -250,7 +250,7 @@ public ILconstInt sqlite_open(ILconstString path) { public ILconstInt sqlite_prepare(ILconstInt connection, ILconstString query) { Connection conn = sqliteConnection(connection.getVal()); try { - PreparedStatement stmt = conn.prepareStatement(query.getVal()); + PreparedStatement stmt = conn.prepareStatement(query.text()); int handle = ++sqliteHandleCounter; sqliteStatements.put(handle, stmt); sqliteStatementConnections.put(handle, connection.getVal()); @@ -300,7 +300,7 @@ public void sqlite_bind_real(ILconstInt statement, ILconstInt index, ILconstReal public void sqlite_bind_string(ILconstInt statement, ILconstInt index, ILconstString value) { PreparedStatement stmt = sqliteStatement(statement.getVal()); try { - stmt.setString(index.getVal(), value.getVal()); + stmt.setString(index.getVal(), value.text()); markStatementForReexecution(statement.getVal()); } catch (SQLException e) { throw new InterpreterException("Failed to bind string: " + e.getMessage()); @@ -476,7 +476,7 @@ public void sqlite_exec(ILconstInt connection, ILconstString query) { // and a hand-rolled splitter cannot correctly handle trigger BEGIN...END // bodies, CASE...END, or every identifier-quoting form ([id], `id`, "id"). SQLiteConnection sqliteConn = conn.unwrap(SQLiteConnection.class); - sqliteConn.getDatabase()._exec(query.getVal()); + sqliteConn.getDatabase()._exec(query.text()); } catch (SQLException e) { throw new InterpreterException("Failed to exec SQLite query: " + e.getMessage()); } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/ILconstString.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/ILconstString.java index c7bbf4acb..5bee28e36 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/ILconstString.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/ILconstString.java @@ -54,6 +54,18 @@ public String text() { return decode(val); } + /** + * Whether the bytes spell text at all. + *

+ * Half of a character does not, and it cannot leave the interpreter: it has to become a literal + * in the generated script, which is written as UTF-8 and escapes nothing numerically, so the byte + * has no way to be written down. Decoding it anyway turns it into a replacement character, which + * is three bytes where the interpreter counted one. + */ + public boolean isText() { + return encode(decode(val)).equals(val); + } + @Override public String print() { return "\"" + text() + "\""; diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/CompiletimeNativesTest.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/CompiletimeNativesTest.java index cb18bb1d5..c19764dc7 100644 --- a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/CompiletimeNativesTest.java +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/CompiletimeNativesTest.java @@ -319,6 +319,41 @@ public void sqliteReadsBackEveryColumnTypeIncludingNull() { n.sqlite_close(db); } + /** + * A string is held as bytes inside the interpreter and as text by the driver, so every value + * crossing into SQLite has to be decoded and everything read back encoded again. Binding text + * without decoding stores the bytes as though each were a character, and it comes back a + * different string than the one that went in. + */ + @Test + public void sqliteRoundTripsNonAsciiText() { + CompiletimeNatives n = newSqliteNatives(); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (s TEXT)")); + + ILconstInt insert = n.sqlite_prepare(db, ILconstString.fromText("INSERT INTO T VALUES (?)")); + ILconstString written = ILconstString.fromText("Grüße 日本"); + n.sqlite_bind_string(insert, i(1), written); + n.sqlite_step(insert); + n.sqlite_finalize(insert); + + ILconstInt read = n.sqlite_prepare(db, ILconstString.fromText("SELECT s FROM T")); + assertTrue(n.sqlite_step(read).getVal()); + ILconstString readBack = n.sqlite_column_string(read, i(0)); + assertEquals(readBack.text(), "Grüße 日本"); + // and the same bytes, so a length taken either side of the round trip agrees + assertEquals(readBack.getVal(), written.getVal()); + n.sqlite_finalize(read); + + // a non-ascii literal in the SQL itself takes the same path + ILconstInt matched = n.sqlite_prepare(db, + ILconstString.fromText("SELECT count(*) FROM T WHERE s = 'Grüße 日本'")); + assertTrue(n.sqlite_step(matched).getVal()); + assertEquals(n.sqlite_column_int(matched, i(0)).getVal(), 1); + n.sqlite_finalize(matched); + n.sqlite_close(db); + } + @Test public void sqliteResetRewindsSelectResultSet() { CompiletimeNatives n = newSqliteNatives(); diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/StringByteSemanticsTests.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/StringByteSemanticsTests.java index 8b1962068..585968090 100644 --- a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/StringByteSemanticsTests.java +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/StringByteSemanticsTests.java @@ -88,4 +88,40 @@ public void slicesRejoinIntoTheOriginal() { public void slicesRejoinIntoTheOriginalLua() { test().testLua(true).executeProg().lines(HALVES_REJOIN); } + + /** + * Whole characters cross from compiletime into the generated script and keep their length, so the + * value the interpreter computed is the value the program runs with. + */ + @Test + public void aCompiletimeStringKeepsItsLengthAfterTransforms() { + test().executeProg().runCompiletimeFunctions(true).lines(program( + "function compiletime(string s) returns string", + " return s", + "constant string GREETING = compiletime(\"h" + A_UMLAUT + "llo\")", + "init", + " if StringLength(GREETING) == 6 and GREETING == \"h" + A_UMLAUT + "llo\"", + " testSuccess()" + )); + } + + /** + * Half a character cannot. It has to become a literal in a script written as UTF-8, and neither + * Jass nor the escaping has a way to write a byte down numerically, so it would go in as the + * replacement character and come back out three bytes long rather than one. Refused instead, and + * this pins that it is refused rather than silently carried across at a different length. + */ + @Test + public void aCompiletimeStringHoldingHalfACharacterIsRefused() { + test().executeProg().runCompiletimeFunctions(true) + .expectError("part of a multibyte character") + .lines(program( + "function compiletime(string s) returns string", + " return s", + "constant string HALF = compiletime(SubString(\"" + A_UMLAUT + "\", 0, 1))", + "init", + " if StringLength(HALF) == 1", + " testSuccess()" + )); + } }