diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompiletimeFunctionRunner.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompiletimeFunctionRunner.java index e2b7e9243..6ef67034b 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompiletimeFunctionRunner.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompiletimeFunctionRunner.java @@ -421,6 +421,25 @@ private ImExpr constantToExpr(Element trace, ILconst value) { return constantToExpr(trace, value, null); } + /** + * The text for a string a compiletime expression produced, which becomes a literal in the + * generated script. + *

+ * A string held by the interpreter is a sequence of bytes and may hold half of a character - + * slicing one in half is a thing the standard library does deliberately. A literal cannot: the + * script is written as UTF-8 and neither Jass nor the escaping here can write a byte down + * numerically, so half a character would go in as the replacement character and come back out + * three bytes long. Refused rather than carried across at a different length. + */ + private String literalText(ILconstString value, Element trace) { + if (!value.isText()) { + throw new CompileError(trace, "A compiletime expression returned a string holding part of a" + + " multibyte character, which cannot be written into the generated script. Slice it" + + " where the program runs rather than at compiletime, or keep whole characters."); + } + return value.text(); + } + private ImExpr constantToExpr(Element trace, ILconst value, @Nullable ImType expectedType) { if (value instanceof ILconstBool) { return JassIm.ImBoolVal(((ILconstBool) value).getVal()); @@ -429,7 +448,7 @@ private ImExpr constantToExpr(Element trace, ILconst value, @Nullable ImType exp } else if (value instanceof ILconstReal) { return JassIm.ImRealVal("" + ((ILconstReal) value).getVal()); } else if (value instanceof ILconstString) { - return JassIm.ImStringVal(((ILconstString) value).getVal()); + return JassIm.ImStringVal(literalText((ILconstString) value, trace)); } else if (value instanceof ILconstNull) { return expectedType == null ? ImHelper.nullExpr() : JassIm.ImNull(expectedType.copy()); } else if (value instanceof ILconstTuple) { @@ -1043,7 +1062,7 @@ private ImExpr constantToExprHashtable(Element trace, ImVar htVar, IlConstHandle JassIm.ImVarAccess(htVar), JassIm.ImIntVal(key.getParentkey()), JassIm.ImIntVal(key.getChildkey()), - JassIm.ImStringVal(iv.getVal()) + JassIm.ImStringVal(literalText(iv, trace)) ), false, CallType.NORMAL)); } else if (v instanceof ILconstBool) { ILconstBool iv = (ILconstBool) v; diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/CompiletimeNatives.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/CompiletimeNatives.java index 479a8228e..bae31c95c 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/CompiletimeNatives.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/CompiletimeNatives.java @@ -47,7 +47,7 @@ public CompiletimeNatives(ProgramStateIO globalState, WurstProjectConfigData pro private ILconstTuple makeKey(String key) { - return new ILconstTuple(new ILconstString(key)); + return new ILconstTuple(ILconstString.fromText(key)); } public ILconstTuple createObjectDefinition(ILconstString fileType, ILconstInt newUnitId, ILconstInt deriveFrom) { @@ -93,7 +93,7 @@ public void ObjectDefinition_setInt(ILconstTuple unitType, ILconstString modific public void ObjectDefinition_setString(ILconstTuple unitType, ILconstString modification, ILconstString value) { ObjMod.Obj od = globalState.getObjectDefinition(getKey(unitType)); - modifyObject(od, modification, ObjMod.ValType.STRING, War3String.valueOf(value.getVal())); + modifyObject(od, modification, ObjMod.ValType.STRING, War3String.valueOf(value.text())); } public void ObjectDefinition_setReal(ILconstTuple unitType, ILconstString modification, ILconstReal value) { @@ -114,7 +114,7 @@ public void ObjectDefinition_setLvlInt(ILconstTuple unitType, ILconstString modi public void ObjectDefinition_setLvlString(ILconstTuple unitType, ILconstString modification, ILconstInt level, ILconstString value) { ObjMod.Obj od = globalState.getObjectDefinition(getKey(unitType)); - modifyObject(od, modification, ObjMod.ValType.STRING, level.getVal(), War3String.valueOf(value.getVal())); + modifyObject(od, modification, ObjMod.ValType.STRING, level.getVal(), War3String.valueOf(value.text())); } public void ObjectDefinition_setLvlReal(ILconstTuple unitType, ILconstString modification, ILconstInt level, ILconstReal value) { @@ -135,7 +135,7 @@ public void ObjectDefinition_setLvlDataInt(ILconstTuple unitType, ILconstString public void ObjectDefinition_setLvlDataString(ILconstTuple unitType, ILconstString modification, ILconstInt level, ILconstInt dataPointer, ILconstString value) { ObjMod.Obj od = globalState.getObjectDefinition(getKey(unitType)); - modifyObject(od, modification, ObjMod.ValType.STRING, level.getVal(), dataPointer.getVal(), War3String.valueOf(value.getVal())); + modifyObject(od, modification, ObjMod.ValType.STRING, level.getVal(), dataPointer.getVal(), War3String.valueOf(value.text())); } public void ObjectDefinition_setLvlDataReal(ILconstTuple unitType, ILconstString modification, ILconstInt level, ILconstInt dataPointer, ILconstReal value) { @@ -185,15 +185,15 @@ private String getKey(ILconstTuple unitType) { } public void compileError(ILconstString msg) { - throw new InterpreterException(msg.getVal()); + throw new InterpreterException(msg.text()); } public ILconstString getMapName() { - return new ILconstString(projectConfigData.buildMapData().name()); + return ILconstString.fromText(projectConfigData.buildMapData().name()); } public ILconstString getBuildDate() { - return new ILconstString(LocalDateTime.now().truncatedTo(ChronoUnit.MINUTES).toString()); + return ILconstString.fromText(LocalDateTime.now().truncatedTo(ChronoUnit.MINUTES).toString()); } public ILconstBool isProductionBuild() { @@ -224,7 +224,7 @@ private PreparedStatement sqliteStatement(int handle) { } public ILconstInt sqlite_open(ILconstString path) { - String dbPath = path.getVal(); + String dbPath = path.text(); // SQLite "file:" URI paths can carry query parameters such as // "?enable_load_extension=true" that would turn on extension loading and let // load_extension() dlopen arbitrary native code on the build machine at compiletime. @@ -250,7 +250,7 @@ public ILconstInt sqlite_open(ILconstString path) { public ILconstInt sqlite_prepare(ILconstInt connection, ILconstString query) { Connection conn = sqliteConnection(connection.getVal()); try { - PreparedStatement stmt = conn.prepareStatement(query.getVal()); + PreparedStatement stmt = conn.prepareStatement(query.text()); int handle = ++sqliteHandleCounter; sqliteStatements.put(handle, stmt); sqliteStatementConnections.put(handle, connection.getVal()); @@ -300,7 +300,7 @@ public void sqlite_bind_real(ILconstInt statement, ILconstInt index, ILconstReal public void sqlite_bind_string(ILconstInt statement, ILconstInt index, ILconstString value) { PreparedStatement stmt = sqliteStatement(statement.getVal()); try { - stmt.setString(index.getVal(), value.getVal()); + stmt.setString(index.getVal(), value.text()); markStatementForReexecution(statement.getVal()); } catch (SQLException e) { throw new InterpreterException("Failed to bind string: " + e.getMessage()); @@ -379,7 +379,7 @@ public ILconstString sqlite_column_string(ILconstInt statement, ILconstInt index // A SQL NULL maps to "" here; use sqlite_column_is_null to distinguish NULL // from an empty string / zero value. String val = rs.getString(index.getVal() + 1); - return new ILconstString(val == null ? "" : val); + return ILconstString.fromText(val == null ? "" : val); } catch (SQLException e) { throw new InterpreterException("Failed to get column string: " + e.getMessage()); } @@ -476,7 +476,7 @@ public void sqlite_exec(ILconstInt connection, ILconstString query) { // and a hand-rolled splitter cannot correctly handle trigger BEGIN...END // bodies, CASE...END, or every identifier-quoting form ([id], `id`, "id"). SQLiteConnection sqliteConn = conn.unwrap(SQLiteConnection.class); - sqliteConn.getDatabase()._exec(query.getVal()); + sqliteConn.getDatabase()._exec(query.text()); } catch (SQLException e) { throw new InterpreterException("Failed to exec SQLite query: " + e.getMessage()); } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/JassInterpreter.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/JassInterpreter.java index 6e3c574a0..f61a000f4 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/JassInterpreter.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/JassInterpreter.java @@ -386,7 +386,7 @@ public ILconst case_JassOpEquals(JassOpEquals jassOpEquals) { @Override public ILconst case_JassExprStringVal(JassExprStringVal e) { - return new ILconstString(e.getValS()); + return ILconstString.fromText(e.getValS()); } @Override diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/AbilityProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/AbilityProvider.java index d74c00313..9d82492c2 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/AbilityProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/AbilityProvider.java @@ -10,10 +10,10 @@ public AbilityProvider(AbstractInterpreter interpreter) { } public ILconstString BlzGetAbilityIcon(ILconstInt abilCode) { - return new ILconstString(""); + return ILconstString.fromText(""); } public ILconstString BlzGetAbilityExtendedTooltip(ILconstInt abilCode, ILconstInt level) { - return new ILconstString(""); + return ILconstString.fromText(""); } } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/GamecacheProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/GamecacheProvider.java index d7bd46ce7..6801dd33d 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/GamecacheProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/GamecacheProvider.java @@ -63,7 +63,7 @@ public ILconstReal GetStoredReal(IlConstHandle ht, ILconstString key1, ILconstSt } public ILconstString GetStoredString(IlConstHandle ht, ILconstString key1, ILconstString key2) { - return haveSaved(ht, key1, key2, ILconstString.class) ? load(ht, key1, key2, ILconstString.class) : new ILconstString(""); + return haveSaved(ht, key1, key2, ILconstString.class) ? load(ht, key1, key2, ILconstString.class) : ILconstString.fromText(""); } public ILconstBool GetStoredBoolean(IlConstHandle ht, ILconstString key1, ILconstString key2) { diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/HashtableProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/HashtableProvider.java index 1a901e1a8..e4ef842d9 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/HashtableProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/HashtableProvider.java @@ -74,7 +74,7 @@ public ILconstReal LoadReal(IlConstHandle ht, ILconstInt key1, ILconstInt key2) } public ILconstString LoadStr(IlConstHandle ht, ILconstInt key1, ILconstInt key2) { - return haveSaved(ht, key1, key2, ILconstString.class) ? load(ht, key1, key2, ILconstString.class) : new ILconstString(""); + return haveSaved(ht, key1, key2, ILconstString.class) ? load(ht, key1, key2, ILconstString.class) : ILconstString.fromText(""); } public ILconstBool LoadBoolean(IlConstHandle ht, ILconstInt key1, ILconstInt key2) { diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/LuaEnsureTypeProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/LuaEnsureTypeProvider.java index c4bab136b..c42e27f37 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/LuaEnsureTypeProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/LuaEnsureTypeProvider.java @@ -43,7 +43,7 @@ public ILconstString __wurst_rawToString(ILconstString x) { } public ILconstString __wurst_rawConcat(ILconstString x, ILconstString y) { - return new ILconstString(x.getVal() + y.getVal()); + return ILconstString.ofBytes(x.getVal() + y.getVal()); } public ILconstInt __wurst_rawFloorDivInt(ILconstInt a, ILconstInt b) { diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java index 083953019..c9cae968c 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java @@ -23,29 +23,29 @@ public void setOutStream(PrintStream outStream) { } public void DisplayTextToForce(IlConstHandle force, ILconstString msg) { - outStream.println(msg.getVal()); + outStream.println(msg.text()); } public void DisplayTimedTextToForce(IlConstHandle force, ILconstReal duration, ILconstString msg) { - outStream.println(msg.getVal()); + outStream.println(msg.text()); } public void DisplayTextToPlayer(IlConstHandle player, ILconstReal x, ILconstReal y, ILconstString msg) { - outStream.println(msg.getVal()); + outStream.println(msg.text()); } public void DisplayTimedTextToPlayer(IlConstHandle player, ILconstReal x, ILconstReal y, ILconstReal duration, ILconstString msg) { - outStream.println(msg.getVal()); + outStream.println(msg.text()); } @Implements(funcNames = {"BJDebugMsg", "println"}) public void println(ILconstString msg) { - outStream.println(msg.getVal()); + outStream.println(msg.text()); } public void $debugPrint(ILconstString msg) { - outStream.println(msg.getVal()); - throw new DebugPrintError(msg.getVal()); + outStream.println(msg.text()); + throw new DebugPrintError(msg.text()); } public void testSuccess() { @@ -53,6 +53,6 @@ public void testSuccess() { } public void testFail(ILconstString msg) { - throw new TestFailException(msg.getVal()); + throw new TestFailException(msg.text()); } } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/StringProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/StringProvider.java index 73e10def9..a8cea2ce7 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/StringProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/StringProvider.java @@ -1,16 +1,14 @@ package de.peeeq.wurstio.jassinterpreter.providers; import de.peeeq.wurstio.jassinterpreter.InterpreterException; -import de.peeeq.wurstscript.WLogger; import de.peeeq.wurstscript.intermediatelang.ILconstBool; import de.peeeq.wurstscript.intermediatelang.ILconstInt; import de.peeeq.wurstscript.intermediatelang.ILconstReal; import de.peeeq.wurstscript.intermediatelang.ILconstString; +import de.peeeq.wurstscript.intermediatelang.Wc3StringHash; import de.peeeq.wurstscript.intermediatelang.interpreter.AbstractInterpreter; -import net.moonlightflower.wc3libs.misc.StringHash; import org.apache.commons.lang.StringUtils; -import java.io.UnsupportedEncodingException; import java.math.RoundingMode; import java.text.NumberFormat; import java.util.Locale; @@ -24,7 +22,7 @@ public StringProvider(AbstractInterpreter interpreter) { } public ILconstString I2S(ILconstInt i) { - return new ILconstString("" + i.getVal()); + return ILconstString.fromText("" + i.getVal()); } private static final Pattern s2ipattern = Pattern.compile("([+\\-]?[0-9]+).*"); @@ -53,7 +51,7 @@ public ILconstReal S2R(ILconstString s) { } public ILconstString R2S(ILconstReal r) { - return new ILconstString("" + r.getVal()); + return ILconstString.fromText("" + r.getVal()); } public ILconstString R2SW(ILconstReal r, ILconstInt width, ILconstInt precision) { @@ -65,7 +63,7 @@ public ILconstString R2SW(ILconstReal r, ILconstInt width, ILconstInt precision) String s = formatter.format(r.getVal()); // pad to desired width s = StringUtils.rightPad(s, width.getVal()); - return new ILconstString(s); + return ILconstString.fromText(s); } public ILconstInt R2I(ILconstReal i) { @@ -81,12 +79,7 @@ public ILconstInt StringHash(ILconstString s) { if (s == null) { return new ILconstInt(0); } - try { - return new ILconstInt(StringHash.hash(s.getVal())); - } catch (UnsupportedEncodingException e) { - WLogger.severe(e); - } - return new ILconstInt(0); + return new ILconstInt(Wc3StringHash.hash(s.getVal())); } public ILconstInt StringLength(ILconstString string) { @@ -110,13 +103,26 @@ public ILconstString SubString(ILconstString istr, ILconstInt start, ILconstInt // since this is most likely a bug in your code, the interpreter will throw an exception instead: throw new InterpreterException("SubString called with start index " + start + " greater than string length " + str.length()); } - return new ILconstString(str.substring(s, e)); + return ILconstString.ofBytes(str.substring(s, e)); } + /** + * Only ascii letters change case. A string is a sequence of bytes, and the bytes of a multibyte + * character are not letters to case at all - folding them the way a latin-1 char would fold + * rewrites the character into a different one. + */ public ILconstString StringCase(ILconstString string, ILconstBool upperCase) { - return new ILconstString( - upperCase.getVal() ? - string.getVal().toUpperCase() - : string.getVal().toLowerCase()); + String bytes = string.getVal(); + StringBuilder result = new StringBuilder(bytes.length()); + for (int i = 0; i < bytes.length(); i++) { + char c = bytes.charAt(i); + if (upperCase.getVal() && c >= 'a' && c <= 'z') { + c -= 32; + } else if (!upperCase.getVal() && c >= 'A' && c <= 'Z') { + c += 32; + } + result.append(c); + } + return ILconstString.ofBytes(result.toString()); } } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/UnitProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/UnitProvider.java index 28eca76c7..a4a72a9f7 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/UnitProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/UnitProvider.java @@ -66,10 +66,10 @@ public ILconstReal GetUnitFacing(IlConstHandle unit) { public ILconstString GetUnitName(IlConstHandle unit) { if (unit == null) { - return new ILconstString(""); + return ILconstString.fromText(""); } UnitMock unitMock = (UnitMock) unit.getObj(); - return new ILconstString(ObjectHelper.objectIdIntToString(unitMock.unitid.getVal())); + return ILconstString.fromText(ObjectHelper.objectIdIntToString(unitMock.unitid.getVal())); } public ILconstInt GetUnitGoldCost(ILconstInt unitid) { diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/WurstflectionProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/WurstflectionProvider.java index 51830b7d9..4f86d0cd0 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/WurstflectionProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/WurstflectionProvider.java @@ -25,7 +25,7 @@ public ILconstString typeIdToTypeName(ILconstInt typeId) { int typeIdInt = typeId.getVal(); for (Map.Entry e : prog.attrTypeId().entrySet()) { if (e.getValue() == typeIdInt) { - ILconstString iLconstString = new ILconstString(calculateClassName(e.getKey())); + ILconstString iLconstString = ILconstString.fromText(calculateClassName(e.getKey())); return Optional.of(iLconstString) .orElseGet(() -> { throw new InterpreterException("Could not determine type name for id " + typeId); diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/ILconstString.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/ILconstString.java index fa63e6d86..5bee28e36 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/ILconstString.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/ILconstString.java @@ -3,23 +3,72 @@ import de.peeeq.wurstscript.types.WurstType; import de.peeeq.wurstscript.types.WurstTypeString; +import java.nio.charset.StandardCharsets; import java.util.Objects; +/** + * A string as Warcraft III has it: a sequence of bytes. {@code StringLength} counts them and + * {@code SubString} indexes them, so a slice may stop between the bytes of one character. Lua agrees, + * its strings being byte arrays too. + *

+ * The value is held one char per byte, so every character in it is below 256 and Java's own length + * and substring already give the game's answers. Text arriving from source or from the host has to be + * encoded on the way in ({@link #fromText}), and anything leaving the interpreter for a file or a + * screen decoded on the way out ({@link #text()}). A half character has no text to decode to, which is + * the point: it keeps its byte until the other half is added back. + */ public class ILconstString extends ILconstAbstract implements ILconstAddable { - private final String val; // including the quotes + /** One char per byte, so all chars are < 256. */ + private final String val; - public ILconstString(String strVal) { - this.val = strVal; + private ILconstString(String byteView) { + this.val = byteView; } + /** Encodes text, which Wurst source and the host both give as UTF-8. */ + public static ILconstString fromText(String text) { + return new ILconstString(encode(text)); + } + + /** Wraps bytes which are already one per char, as produced by slicing or joining. */ + public static ILconstString ofBytes(String byteView) { + return new ILconstString(byteView); + } + + static String encode(String text) { + return new String(text.getBytes(StandardCharsets.UTF_8), StandardCharsets.ISO_8859_1); + } + + static String decode(String byteView) { + return new String(byteView.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8); + } + + /** The bytes, one per char. This is what the string natives count and index. */ public String getVal() { return val; } + /** The text those bytes spell, for anything which leaves the interpreter. */ + public String text() { + return decode(val); + } + + /** + * Whether the bytes spell text at all. + *

+ * Half of a character does not, and it cannot leave the interpreter: it has to become a literal + * in the generated script, which is written as UTF-8 and escapes nothing numerically, so the byte + * has no way to be written down. Decoding it anyway turns it into a replacement character, which + * is three bytes where the interpreter counted one. + */ + public boolean isText() { + return encode(decode(val)).equals(val); + } + @Override public String print() { - return "\"" + val + "\""; + return "\"" + text() + "\""; } public WurstType getType() { diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/Wc3StringHash.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/Wc3StringHash.java new file mode 100644 index 000000000..c5630e991 --- /dev/null +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/Wc3StringHash.java @@ -0,0 +1,96 @@ +package de.peeeq.wurstscript.intermediatelang; + +/** + * {@code StringHash} over bytes. + *

+ * The library's version takes text and encodes it as UTF-8 itself, which cannot express a string + * holding half of a character - and half characters are exactly what the standard library hashes to + * find out how the engine represents them. Decoding first would collapse every partial slice onto one + * value, so the continuation bytes the standard library tells apart by hash would stop being + * distinguishable. + *

+ * The function is Bob Jenkins' lookup2, the same one the library implements; only the input differs. + * {@code Wc3StringHashTest} checks the two agree on text, where both are defined. + */ +public final class Wc3StringHash { + + private Wc3StringHash() { + } + + /** + * @param bytes one byte per char, all below 256 + */ + public static int hash(String bytes) { + if (bytes.isEmpty()) { + return 0; + } + byte[] normalized = new byte[bytes.length()]; + for (int i = 0; i < bytes.length(); i++) { + byte b = (byte) bytes.charAt(i); + // Case insensitive, and either slash names the same file. Both comparisons are on signed + // bytes, so anything above 127 - every byte of a multibyte character - is left alone. + if (b >= 'a' && b <= 'z') { + b -= 32; + } else if (b == '/') { + b = '\\'; + } + normalized[i] = b; + } + return lookup2(normalized); + } + + private static int lookup2(byte[] k) { + int a = 0x9e3779b9; + int b = 0x9e3779b9; + int c = 0; + int len = k.length; + int i = 0; + while (len >= 12) { + a += u(k[i]) + (u(k[i + 1]) << 8) + (u(k[i + 2]) << 16) + (u(k[i + 3]) << 24); + b += u(k[i + 4]) + (u(k[i + 5]) << 8) + (u(k[i + 6]) << 16) + (u(k[i + 7]) << 24); + c += u(k[i + 8]) + (u(k[i + 9]) << 8) + (u(k[i + 10]) << 16) + (u(k[i + 11]) << 24); + // mix + a -= b; a -= c; a ^= (c >>> 13); + b -= c; b -= a; b ^= (a << 8); + c -= a; c -= b; c ^= (b >>> 13); + a -= b; a -= c; a ^= (c >>> 12); + b -= c; b -= a; b ^= (a << 16); + c -= a; c -= b; c ^= (b >>> 5); + a -= b; a -= c; a ^= (c >>> 3); + b -= c; b -= a; b ^= (a << 10); + c -= a; c -= b; c ^= (b >>> 15); + i += 12; + len -= 12; + } + c += k.length; + // The low byte of c holds the length, so the tail starts at the second. + switch (len) { + case 11: c += u(k[i + 10]) << 24; + case 10: c += u(k[i + 9]) << 16; + case 9: c += u(k[i + 8]) << 8; + case 8: b += u(k[i + 7]) << 24; + case 7: b += u(k[i + 6]) << 16; + case 6: b += u(k[i + 5]) << 8; + case 5: b += u(k[i + 4]); + case 4: a += u(k[i + 3]) << 24; + case 3: a += u(k[i + 2]) << 16; + case 2: a += u(k[i + 1]) << 8; + case 1: a += u(k[i]); + default: + } + a -= b; a -= c; a ^= (c >>> 13); + b -= c; b -= a; b ^= (a << 8); + c -= a; c -= b; c ^= (b >>> 13); + a -= b; a -= c; a ^= (c >>> 12); + b -= c; b -= a; b ^= (a << 16); + c -= a; c -= b; c ^= (b >>> 5); + a -= b; a -= c; a ^= (c >>> 3); + b -= c; b -= a; b ^= (a << 10); + c -= a; c -= b; c ^= (b >>> 15); + return c; + } + + private static int u(byte b) { + return b & 0xFF; + } +} diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/EvaluateExpr.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/EvaluateExpr.java index df15e6d64..25a35c851 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/EvaluateExpr.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/EvaluateExpr.java @@ -142,7 +142,7 @@ public static ILaddress evaluateLvalue(ImStatementExpr e, ProgramState globalSta } public static ILconst eval(ImStringVal e, ProgramState globalState, LocalState localState) { - return new ILconstString(e.getValS()); + return ILconstString.fromText(e.getValS()); } public static ILconst eval(ImTupleExpr e, ProgramState globalState, LocalState localState) { @@ -334,7 +334,7 @@ public static ILconst eval(ImGetStackTrace e, ProgramState globalState, LocalState localState) { StringBuilder sb = new StringBuilder(); globalState.getStackFrames().appendTo(sb); - return new ILconstString(sb.toString()); + return ILconstString.fromText(sb.toString()); } public static ILconst eval(ImCompiletimeExpr expr, ProgramState globalState, LocalState localState) { diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/ILInterpreter.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/ILInterpreter.java index 467af1c5c..56d0b5f06 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/ILInterpreter.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/ILInterpreter.java @@ -623,7 +623,7 @@ public void runVoidFunc(ImFunction f, @Nullable Element trace) { ILconst[] args = {}; if (!f.getParameters().isEmpty()) { // this should only happen because of added stacktrace parameter - args = new ILconstString[]{new ILconstString("initial call")}; + args = new ILconstString[]{ILconstString.fromText("initial call")}; } runFunc(globalState, f, trace, args); } diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/CompiletimeNativesTest.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/CompiletimeNativesTest.java index 9f984c0bf..c19764dc7 100644 --- a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/CompiletimeNativesTest.java +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/CompiletimeNativesTest.java @@ -51,8 +51,8 @@ public void modifyObjectKeepsDifferentDataPointers() throws Exception { modifyObject.setAccessible(true); MetaFieldId unrealId = MetaFieldId.valueOf("unat"); - modifyObject.invoke(natives, obj, new ILconstString(unrealId.getVal()), ObjMod.ValType.UNREAL, 1, 0, War3Real.valueOf(1.0)); - modifyObject.invoke(natives, obj, new ILconstString(unrealId.getVal()), ObjMod.ValType.UNREAL, 1, 1, War3Real.valueOf(2.0)); + modifyObject.invoke(natives, obj, ILconstString.fromText(unrealId.getVal()), ObjMod.ValType.UNREAL, 1, 0, War3Real.valueOf(1.0)); + modifyObject.invoke(natives, obj, ILconstString.fromText(unrealId.getVal()), ObjMod.ValType.UNREAL, 1, 1, War3Real.valueOf(2.0)); List unrealMods = obj.getMods().stream() .filter(m -> m instanceof ObjMod.Obj.ExtendedMod) @@ -151,8 +151,8 @@ public void duplicateCodeObjectDefinitionsReportError() { int hfoo = ObjectHelper.objectIdStringToInt("hfoo"); int hf01 = ObjectHelper.objectIdStringToInt("hf01"); - natives.createObjectDefinition(new ILconstString("w3u"), new ILconstInt(hf01), new ILconstInt(hfoo)); - natives.createObjectDefinition(new ILconstString("w3u"), new ILconstInt(hf01), new ILconstInt(hfoo)); + natives.createObjectDefinition(ILconstString.fromText("w3u"), new ILconstInt(hf01), new ILconstInt(hfoo)); + natives.createObjectDefinition(ILconstString.fromText("w3u"), new ILconstInt(hf01), new ILconstInt(hfoo)); assertEquals(gui.getErrorCount(), 1); assertTrue(gui.getErrors().contains("Object definition with id hf01 is defined more than once.")); @@ -172,7 +172,7 @@ public void createObjectDefinitionDoesNotReportExistingMapObjectAsError() throws W3U.Obj existing = w3u.addObj(ObjId.valueOf("hf01"), ObjId.valueOf("hpea")); existing.addMod(new ObjMod.Obj.Mod(MetaFieldId.valueOf("unam"), ObjMod.ValType.STRING, War3String.valueOf("old map object"))); - natives.createObjectDefinition(new ILconstString("w3u"), new ILconstInt(hf01), new ILconstInt(hfoo)); + natives.createObjectDefinition(ILconstString.fromText("w3u"), new ILconstInt(hf01), new ILconstInt(hfoo)); assertEquals(gui.getErrorCount(), 0); assertEquals(w3u.getCustomObjs().size(), 1); @@ -189,10 +189,10 @@ public void sameIdObjectDefinitionsMergeModsWithoutDuplicateError() throws Excep CompiletimeNatives natives = new CompiletimeNatives(state, null, false); int hfoo = ObjectHelper.objectIdStringToInt("hfoo"); - var first = natives.createObjectDefinition(new ILconstString("w3u"), new ILconstInt(hfoo), new ILconstInt(hfoo)); - natives.ObjectDefinition_setString(first, new ILconstString("unam"), new ILconstString("first")); - var second = natives.createObjectDefinition(new ILconstString("w3u"), new ILconstInt(hfoo), new ILconstInt(hfoo)); - natives.ObjectDefinition_setString(second, new ILconstString("utip"), new ILconstString("second")); + var first = natives.createObjectDefinition(ILconstString.fromText("w3u"), new ILconstInt(hfoo), new ILconstInt(hfoo)); + natives.ObjectDefinition_setString(first, ILconstString.fromText("unam"), ILconstString.fromText("first")); + var second = natives.createObjectDefinition(ILconstString.fromText("w3u"), new ILconstInt(hfoo), new ILconstInt(hfoo)); + natives.ObjectDefinition_setString(second, ILconstString.fromText("utip"), ILconstString.fromText("second")); Method getDataStore = ProgramStateIO.class.getDeclaredMethod("getDataStore", String.class); getDataStore.setAccessible(true); @@ -208,15 +208,15 @@ public void sameIdObjectDefinitionsMergeModsWithoutDuplicateError() throws Excep @Test public void testCloseSqliteResourcesOnProviderClose() { CompiletimeNatives natives = new CompiletimeNatives(null, null, false); - ILconstInt connHandle = natives.sqlite_open(new ILconstString(":memory:")); - natives.sqlite_exec(connHandle, new ILconstString("CREATE TABLE Test (id INT);")); - ILconstInt stmtHandle = natives.sqlite_prepare(connHandle, new ILconstString("INSERT INTO Test VALUES (1);")); + ILconstInt connHandle = natives.sqlite_open(ILconstString.fromText(":memory:")); + natives.sqlite_exec(connHandle, ILconstString.fromText("CREATE TABLE Test (id INT);")); + ILconstInt stmtHandle = natives.sqlite_prepare(connHandle, ILconstString.fromText("INSERT INTO Test VALUES (1);")); natives.sqlite_step(stmtHandle); natives.close(); try { - natives.sqlite_prepare(connHandle, new ILconstString("SELECT * FROM Test;")); + natives.sqlite_prepare(connHandle, ILconstString.fromText("SELECT * FROM Test;")); org.testng.Assert.fail("Expected InterpreterException for invalid connection handle after close"); } catch (de.peeeq.wurstio.jassinterpreter.InterpreterException e) { assertTrue(e.getMessage().contains("Invalid SQLite connection handle")); @@ -230,13 +230,13 @@ public void testCloseSqliteResourcesFromProgramStateClose() throws Exception { CompiletimeNatives natives = new CompiletimeNatives(state, null, false); state.addNativeProvider(natives); - ILconstInt connHandle = natives.sqlite_open(new ILconstString(":memory:")); - natives.sqlite_exec(connHandle, new ILconstString("CREATE TABLE Test (id INT);")); + ILconstInt connHandle = natives.sqlite_open(ILconstString.fromText(":memory:")); + natives.sqlite_exec(connHandle, ILconstString.fromText("CREATE TABLE Test (id INT);")); state.close(); try { - natives.sqlite_prepare(connHandle, new ILconstString("SELECT * FROM Test;")); + natives.sqlite_prepare(connHandle, ILconstString.fromText("SELECT * FROM Test;")); org.testng.Assert.fail("Expected InterpreterException for invalid connection handle after ProgramState close"); } catch (de.peeeq.wurstio.jassinterpreter.InterpreterException e) { assertTrue(e.getMessage().contains("Invalid SQLite connection handle")); @@ -250,18 +250,18 @@ public void sqliteExecRunsMultiStatementScriptWithTrigger() { CompiletimeNatives natives = new CompiletimeNatives(state, null, false); state.addNativeProvider(natives); - ILconstInt db = natives.sqlite_open(new ILconstString(":memory:")); + ILconstInt db = natives.sqlite_open(ILconstString.fromText(":memory:")); // Multi-statement script including a trigger BEGIN...END body (whose inner ';' // terminators would break a naive splitter) and a bracket-quoted identifier // containing ';'. sqlite3_exec delegates to SQLite's own parser, so both work. - natives.sqlite_exec(db, new ILconstString( + natives.sqlite_exec(db, ILconstString.fromText( "CREATE TABLE src (id INTEGER);" + "CREATE TABLE dst (id INTEGER);" + "CREATE TRIGGER mirror AFTER INSERT ON src BEGIN INSERT INTO dst VALUES (NEW.id); END;" + "CREATE TABLE [we;ird] (x INTEGER);" + "INSERT INTO src VALUES (1); INSERT INTO src VALUES (2);")); - ILconstInt q = natives.sqlite_prepare(db, new ILconstString( + ILconstInt q = natives.sqlite_prepare(db, ILconstString.fromText( "SELECT (SELECT COUNT(*) FROM dst), " + "(SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='we;ird')")); assertTrue(natives.sqlite_step(q).getVal()); @@ -296,11 +296,11 @@ private static void assertInterpreterError(String expectedSubstring, Runnable ac @Test public void sqliteReadsBackEveryColumnTypeIncludingNull() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString( + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText( "CREATE TABLE T (i INTEGER, r REAL, s TEXT, n TEXT);" + "INSERT INTO T VALUES (42, 2.5, 'hello', NULL);")); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT i, r, s, n FROM T")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT i, r, s, n FROM T")); // column_count works before any step (metadata branch) ... assertEquals(n.sqlite_column_count(q).getVal(), 4); assertTrue(n.sqlite_step(q).getVal()); @@ -319,13 +319,48 @@ public void sqliteReadsBackEveryColumnTypeIncludingNull() { n.sqlite_close(db); } + /** + * A string is held as bytes inside the interpreter and as text by the driver, so every value + * crossing into SQLite has to be decoded and everything read back encoded again. Binding text + * without decoding stores the bytes as though each were a character, and it comes back a + * different string than the one that went in. + */ + @Test + public void sqliteRoundTripsNonAsciiText() { + CompiletimeNatives n = newSqliteNatives(); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (s TEXT)")); + + ILconstInt insert = n.sqlite_prepare(db, ILconstString.fromText("INSERT INTO T VALUES (?)")); + ILconstString written = ILconstString.fromText("Grüße 日本"); + n.sqlite_bind_string(insert, i(1), written); + n.sqlite_step(insert); + n.sqlite_finalize(insert); + + ILconstInt read = n.sqlite_prepare(db, ILconstString.fromText("SELECT s FROM T")); + assertTrue(n.sqlite_step(read).getVal()); + ILconstString readBack = n.sqlite_column_string(read, i(0)); + assertEquals(readBack.text(), "Grüße 日本"); + // and the same bytes, so a length taken either side of the round trip agrees + assertEquals(readBack.getVal(), written.getVal()); + n.sqlite_finalize(read); + + // a non-ascii literal in the SQL itself takes the same path + ILconstInt matched = n.sqlite_prepare(db, + ILconstString.fromText("SELECT count(*) FROM T WHERE s = 'Grüße 日本'")); + assertTrue(n.sqlite_step(matched).getVal()); + assertEquals(n.sqlite_column_int(matched, i(0)).getVal(), 1); + n.sqlite_finalize(matched); + n.sqlite_close(db); + } + @Test public void sqliteResetRewindsSelectResultSet() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString( + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText( "CREATE TABLE T (id INTEGER); INSERT INTO T VALUES (10); INSERT INTO T VALUES (20);")); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT id FROM T ORDER BY id")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT id FROM T ORDER BY id")); assertTrue(n.sqlite_step(q).getVal()); assertEquals(n.sqlite_column_int(q, i(0)).getVal(), 10); assertTrue(n.sqlite_step(q).getVal()); @@ -341,9 +376,9 @@ public void sqliteResetRewindsSelectResultSet() { @Test public void sqliteColumnIntTruncates64BitValueToInt() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (v INTEGER); INSERT INTO T VALUES (5000000000);")); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT v FROM T")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (v INTEGER); INSERT INTO T VALUES (5000000000);")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT v FROM T")); assertTrue(n.sqlite_step(q).getVal()); // WurstScript int is 32-bit: a 64-bit INTEGER wraps to its low 32 bits (documented). assertEquals(n.sqlite_column_int(q, i(0)).getVal(), (int) 5000000000L); @@ -354,20 +389,20 @@ public void sqliteColumnIntTruncates64BitValueToInt() { @Test public void sqliteBindRoundTripAndRebindAfterStepReexecutes() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (i INTEGER, r REAL, s TEXT)")); - ILconstInt ins = n.sqlite_prepare(db, new ILconstString("INSERT INTO T VALUES (?, ?, ?)")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (i INTEGER, r REAL, s TEXT)")); + ILconstInt ins = n.sqlite_prepare(db, ILconstString.fromText("INSERT INTO T VALUES (?, ?, ?)")); n.sqlite_bind_int(ins, i(1), i(1)); n.sqlite_bind_real(ins, i(2), new ILconstReal(1.5f)); - n.sqlite_bind_string(ins, i(3), new ILconstString("a")); + n.sqlite_bind_string(ins, i(3), ILconstString.fromText("a")); assertFalse(n.sqlite_step(ins).getVal()); // rebind i and s WITHOUT a reset: must re-execute; r keeps its previous binding n.sqlite_bind_int(ins, i(1), i(2)); - n.sqlite_bind_string(ins, i(3), new ILconstString("b")); + n.sqlite_bind_string(ins, i(3), ILconstString.fromText("b")); assertFalse(n.sqlite_step(ins).getVal()); n.sqlite_finalize(ins); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT i, r, s FROM T ORDER BY i")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT i, r, s FROM T ORDER BY i")); assertTrue(n.sqlite_step(q).getVal()); assertEquals(n.sqlite_column_int(q, i(0)).getVal(), 1); assertEquals((double) n.sqlite_column_real(q, i(1)).getVal(), 1.5, 0.0); @@ -384,16 +419,16 @@ public void sqliteBindRoundTripAndRebindAfterStepReexecutes() { @Test public void sqliteClearBindingsResetsParametersToNull() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (a INTEGER, b TEXT)")); - ILconstInt ins = n.sqlite_prepare(db, new ILconstString("INSERT INTO T VALUES (?, ?)")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (a INTEGER, b TEXT)")); + ILconstInt ins = n.sqlite_prepare(db, ILconstString.fromText("INSERT INTO T VALUES (?, ?)")); n.sqlite_bind_int(ins, i(1), i(7)); - n.sqlite_bind_string(ins, i(2), new ILconstString("x")); + n.sqlite_bind_string(ins, i(2), ILconstString.fromText("x")); n.sqlite_clear_bindings(ins); assertFalse(n.sqlite_step(ins).getVal()); n.sqlite_finalize(ins); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT a, b FROM T")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT a, b FROM T")); assertTrue(n.sqlite_step(q).getVal()); assertTrue(n.sqlite_column_is_null(q, i(0)).getVal()); assertTrue(n.sqlite_column_is_null(q, i(1)).getVal()); @@ -407,16 +442,16 @@ public void sqliteClearBindingsAfterStepReexecutesWithoutReset() { // sqlite_bind_*, so a step after clearing re-runs the statement with NULL params // WITHOUT an explicit sqlite_reset. Otherwise the second step silently no-ops. CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (v INTEGER)")); - ILconstInt ins = n.sqlite_prepare(db, new ILconstString("INSERT INTO T VALUES (?)")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (v INTEGER)")); + ILconstInt ins = n.sqlite_prepare(db, ILconstString.fromText("INSERT INTO T VALUES (?)")); n.sqlite_bind_int(ins, i(1), i(100)); assertFalse(n.sqlite_step(ins).getVal()); // inserts 100 n.sqlite_clear_bindings(ins); // no explicit reset assertFalse(n.sqlite_step(ins).getVal()); // must re-execute → inserts NULL n.sqlite_finalize(ins); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT count(*), count(v) FROM T")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT count(*), count(v) FROM T")); assertTrue(n.sqlite_step(q).getVal()); assertEquals(n.sqlite_column_int(q, i(0)).getVal(), 2); // two rows total assertEquals(n.sqlite_column_int(q, i(1)).getVal(), 1); // one non-NULL (the 100) @@ -430,10 +465,10 @@ public void sqliteClearBindingsAfterStepDiscardsOldResultSet() { // clear_bindings must discard it so the next step RE-RUNS the query with the now // NULL parameter, rather than continuing to walk the stale result set. CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (v INTEGER)")); - n.sqlite_exec(db, new ILconstString("INSERT INTO T VALUES (1), (2), (3)")); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT v FROM T WHERE v <> ? ORDER BY v")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (v INTEGER)")); + n.sqlite_exec(db, ILconstString.fromText("INSERT INTO T VALUES (1), (2), (3)")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT v FROM T WHERE v <> ? ORDER BY v")); n.sqlite_bind_int(q, i(1), i(2)); // excludes 2 → rows {1, 3} assertTrue(n.sqlite_step(q).getVal()); assertEquals(n.sqlite_column_int(q, i(0)).getVal(), 1); @@ -448,9 +483,9 @@ public void sqliteClearBindingsAfterStepDiscardsOldResultSet() { @Test public void sqliteFinalizeInvalidatesStatementHandle() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (id INTEGER)")); - ILconstInt stmt = n.sqlite_prepare(db, new ILconstString("INSERT INTO T VALUES (1)")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (id INTEGER)")); + ILconstInt stmt = n.sqlite_prepare(db, ILconstString.fromText("INSERT INTO T VALUES (1)")); n.sqlite_finalize(stmt); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_step(stmt)); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_finalize(stmt)); @@ -460,12 +495,12 @@ public void sqliteFinalizeInvalidatesStatementHandle() { @Test public void sqliteCloseInvalidatesConnectionAndItsStatements() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (id INTEGER)")); - ILconstInt stmt = n.sqlite_prepare(db, new ILconstString("SELECT id FROM T")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (id INTEGER)")); + ILconstInt stmt = n.sqlite_prepare(db, ILconstString.fromText("SELECT id FROM T")); n.sqlite_close(db); - assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_prepare(db, new ILconstString("SELECT 1"))); - assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_exec(db, new ILconstString("SELECT 1"))); + assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_prepare(db, ILconstString.fromText("SELECT 1"))); + assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_exec(db, ILconstString.fromText("SELECT 1"))); assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_close(db)); // statements belonging to the closed connection were finalized/invalidated too assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_step(stmt)); @@ -474,30 +509,30 @@ public void sqliteCloseInvalidatesConnectionAndItsStatements() { @Test public void sqliteInvalidHandlesAndBadSqlThrow() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_bind_int(i(999), i(1), i(1))); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_bind_real(i(999), i(1), new ILconstReal(1f))); - assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_bind_string(i(999), i(1), new ILconstString("x"))); + assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_bind_string(i(999), i(1), ILconstString.fromText("x"))); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_step(i(999))); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_reset(i(999))); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_clear_bindings(i(999))); assertInterpreterError("Invalid SQLite statement handle", () -> n.sqlite_finalize(i(999))); assertInterpreterError("No result set", () -> n.sqlite_column_int(i(999), i(0))); assertInterpreterError("No result set", () -> n.sqlite_column_is_null(i(999), i(0))); - assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_prepare(i(999), new ILconstString("SELECT 1"))); - assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_exec(i(999), new ILconstString("SELECT 1"))); + assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_prepare(i(999), ILconstString.fromText("SELECT 1"))); + assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_exec(i(999), ILconstString.fromText("SELECT 1"))); assertInterpreterError("Invalid SQLite connection handle", () -> n.sqlite_close(i(999))); - assertInterpreterError("Failed to prepare SQLite statement", () -> n.sqlite_prepare(db, new ILconstString("NOT VALID SQL"))); - assertInterpreterError("Failed to exec SQLite query", () -> n.sqlite_exec(db, new ILconstString("NOT VALID SQL"))); + assertInterpreterError("Failed to prepare SQLite statement", () -> n.sqlite_prepare(db, ILconstString.fromText("NOT VALID SQL"))); + assertInterpreterError("Failed to exec SQLite query", () -> n.sqlite_exec(db, ILconstString.fromText("NOT VALID SQL"))); n.sqlite_close(db); } @Test public void sqliteColumnAccessWithoutResultSetThrows() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); - n.sqlite_exec(db, new ILconstString("CREATE TABLE T (id INTEGER); INSERT INTO T VALUES (1)")); - ILconstInt q = n.sqlite_prepare(db, new ILconstString("SELECT id FROM T")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); + n.sqlite_exec(db, ILconstString.fromText("CREATE TABLE T (id INTEGER); INSERT INTO T VALUES (1)")); + ILconstInt q = n.sqlite_prepare(db, ILconstString.fromText("SELECT id FROM T")); // reading a column before any step -> no result set yet assertInterpreterError("No result set", () -> n.sqlite_column_int(q, i(0))); assertInterpreterError("No result set", () -> n.sqlite_column_is_null(q, i(0))); @@ -509,17 +544,17 @@ public void sqliteColumnAccessWithoutResultSetThrows() { public void sqliteOpenRejectsFileUriWithQueryParameters() { CompiletimeNatives n = newSqliteNatives(); assertInterpreterError("query parameters are not allowed", - () -> n.sqlite_open(new ILconstString("file::memory:?enable_load_extension=true"))); + () -> n.sqlite_open(ILconstString.fromText("file::memory:?enable_load_extension=true"))); } @Test public void sqliteLoadExtensionIsBlocked() { CompiletimeNatives n = newSqliteNatives(); - ILconstInt db = n.sqlite_open(new ILconstString(":memory:")); + ILconstInt db = n.sqlite_open(ILconstString.fromText(":memory:")); // extension loading is disabled on the connection, so load_extension is not authorized // (this is what closes the dlopen-arbitrary-native-code vector at compiletime). assertInterpreterError("not authorized", - () -> n.sqlite_exec(db, new ILconstString("SELECT load_extension('nonexistent_extension')"))); + () -> n.sqlite_exec(db, ILconstString.fromText("SELECT load_extension('nonexistent_extension')"))); n.sqlite_close(db); } diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/StringByteSemanticsTests.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/StringByteSemanticsTests.java new file mode 100644 index 000000000..585968090 --- /dev/null +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/StringByteSemanticsTests.java @@ -0,0 +1,127 @@ +package tests.wurstscript.tests; + +import org.testng.annotations.Test; + +/** + * Warcraft III treats a string as a sequence of bytes: {@code StringLength} counts bytes and + * {@code SubString} takes byte offsets, so a slice can cut a multibyte character in half. Lua agrees, + * its strings being byte arrays. The interpreter holds a Java string and counts UTF-16 code units, + * which is the same answer only for ascii. + *

+ * The standard library depends on the difference rather than avoiding it: {@code String.wurst} cuts a + * character in half on purpose to find out how the engine represents a partial slice, and slices a + * 64 character literal byte by byte to enumerate every continuation byte. Under UTF-16 that detection + * quietly concludes the engine has no multibyte characters, so anything computed at compiletime - + * object editor text, chunked data - is built from lengths the game will not agree with. + */ +public class StringByteSemanticsTests extends WurstScriptTest { + + private static String[] program(String... body) { + String[] head = { + "package test", + "native testSuccess()", + "@extern native StringLength(string s) returns int", + "@extern native SubString(string s, int start, int stop) returns string", + "@extern native StringHash(string s) returns int", + }; + String[] all = new String[head.length + body.length]; + System.arraycopy(head, 0, all, 0, head.length); + System.arraycopy(body, 0, all, head.length, body.length); + return all; + } + + /** + * Written as an escape rather than as itself, so what reaches the compiler does not depend on + * the encoding javac happens to read this file with. + */ + private static final String A_UMLAUT = "ä"; + + /** Two bytes in UTF-8, and the game counts bytes. */ + private static final String[] LENGTH_OF_A_TWO_BYTE_CHARACTER = program( + "init", + " if StringLength(\"" + A_UMLAUT + "\") == 2", + " testSuccess()" + ); + + @Test + public void lengthCountsBytes() { + testAssertOkLines(true, LENGTH_OF_A_TWO_BYTE_CHARACTER); + } + + @Test + public void lengthCountsBytesLua() { + test().testLua(true).executeProg().lines(LENGTH_OF_A_TWO_BYTE_CHARACTER); + } + + /** A slice may stop between the bytes of one character, which is how the stdlib probes. */ + private static final String[] SLICING_A_CHARACTER_IN_HALF = program( + "init", + " let half = SubString(\"ä\", 0, 1)", + " if StringLength(half) == 1 and half != \"ä\"", + " testSuccess()" + ); + + @Test + public void aSliceCanCutACharacterInHalf() { + testAssertOkLines(true, SLICING_A_CHARACTER_IN_HALF); + } + + @Test + public void aSliceCanCutACharacterInHalfLua() { + test().testLua(true).executeProg().lines(SLICING_A_CHARACTER_IN_HALF); + } + + /** The halves are the bytes of the whole, so putting them back gives it back. */ + private static final String[] HALVES_REJOIN = program( + "init", + " let s = \"äö\"", + " if SubString(s, 0, 2) + SubString(s, 2, 4) == s and StringLength(s) == 4", + " testSuccess()" + ); + + @Test + public void slicesRejoinIntoTheOriginal() { + testAssertOkLines(true, HALVES_REJOIN); + } + + @Test + public void slicesRejoinIntoTheOriginalLua() { + test().testLua(true).executeProg().lines(HALVES_REJOIN); + } + + /** + * Whole characters cross from compiletime into the generated script and keep their length, so the + * value the interpreter computed is the value the program runs with. + */ + @Test + public void aCompiletimeStringKeepsItsLengthAfterTransforms() { + test().executeProg().runCompiletimeFunctions(true).lines(program( + "function compiletime(string s) returns string", + " return s", + "constant string GREETING = compiletime(\"h" + A_UMLAUT + "llo\")", + "init", + " if StringLength(GREETING) == 6 and GREETING == \"h" + A_UMLAUT + "llo\"", + " testSuccess()" + )); + } + + /** + * Half a character cannot. It has to become a literal in a script written as UTF-8, and neither + * Jass nor the escaping has a way to write a byte down numerically, so it would go in as the + * replacement character and come back out three bytes long rather than one. Refused instead, and + * this pins that it is refused rather than silently carried across at a different length. + */ + @Test + public void aCompiletimeStringHoldingHalfACharacterIsRefused() { + test().executeProg().runCompiletimeFunctions(true) + .expectError("part of a multibyte character") + .lines(program( + "function compiletime(string s) returns string", + " return s", + "constant string HALF = compiletime(SubString(\"" + A_UMLAUT + "\", 0, 1))", + "init", + " if StringLength(HALF) == 1", + " testSuccess()" + )); + } +} diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java new file mode 100644 index 000000000..a466a8cf5 --- /dev/null +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/Wc3StringHashTest.java @@ -0,0 +1,90 @@ +package tests.wurstscript.tests; + +import de.peeeq.wurstscript.intermediatelang.ILconstString; +import de.peeeq.wurstscript.intermediatelang.Wc3StringHash; +import org.testng.annotations.Test; + +import java.io.UnsupportedEncodingException; +import java.util.Random; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; + +/** + * Checks the byte hash against the library's, which is the same function reached through text. They + * are only comparable where the library is defined - a string of whole characters - so that is what + * is compared, and it is enough: the arithmetic is the same for every input, only the bytes differ. + */ +public class Wc3StringHashTest { + + private static void agrees(String text) throws UnsupportedEncodingException { + assertEquals(Wc3StringHash.hash(ILconstString.fromText(text).getVal()), + net.moonlightflower.wc3libs.misc.StringHash.hash(text), + "hash of " + text); + } + + @Test + public void agreesOnShortStrings() throws UnsupportedEncodingException { + for (String s : new String[]{"", "a", "ab", "abc", "abcd", "abcde", "abcdef", "abcdefg", + "abcdefgh", "abcdefghi", "abcdefghij", "abcdefghijk", "abcdefghijkl", "abcdefghijklm"}) { + agrees(s); + } + } + + /** Case folding and the slash rule are part of the hash, not of the caller. */ + @Test + public void agreesOnStringsNeedingNormalisation() throws UnsupportedEncodingException { + for (String s : new String[]{"ABC", "AbC", "path/to/file", "path\\to\\file", + "Units\\Human\\Footman.mdx", "MIXED/Case\\Path"}) { + agrees(s); + } + } + + /** Multibyte text still agrees, because whole characters decode back to themselves. */ + @Test + public void agreesOnMultibyteText() throws UnsupportedEncodingException { + for (String s : new String[]{"ä", "äöü", "ЀЁЂЃ", "日本語", "😀", "aäböcü1234567890"}) { + agrees(s); + } + } + + @Test + public void agreesOnRandomAsciiOfEveryLength() throws UnsupportedEncodingException { + Random random = new Random(20260816); + for (int length = 0; length < 40; length++) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + sb.append((char) (32 + random.nextInt(95))); + } + agrees(sb.toString()); + } + } + + /** + * The reason for hashing bytes rather than text. Each half of a two byte character has to keep a + * hash of its own; decoding first would turn both into the same replacement character. + */ + @Test + public void halvesOfACharacterHashApart() { + String bytes = ILconstString.fromText("ä").getVal(); + String lead = bytes.substring(0, 1); + String continuation = bytes.substring(1, 2); + assertNotEquals(Wc3StringHash.hash(lead), Wc3StringHash.hash(continuation), + "the two bytes of a character must not hash alike"); + assertNotEquals(Wc3StringHash.hash(lead), Wc3StringHash.hash(bytes), + "half a character must not hash like the whole"); + } + + /** + * The standard library enumerates every continuation byte by slicing one literal and keeps them + * apart by hash, so all 64 have to be distinct. + */ + @Test + public void everyContinuationByteHashesApart() { + java.util.Set hashes = new java.util.HashSet<>(); + for (int b = 0x80; b <= 0xBF; b++) { + hashes.add(Wc3StringHash.hash(String.valueOf((char) b))); + } + assertEquals(hashes.size(), 64, "all 64 continuation bytes should hash apart"); + } +} diff --git a/de.peeeq.wurstscript/src/test/resources/luaruntime/wc3shim.lua b/de.peeeq.wurstscript/src/test/resources/luaruntime/wc3shim.lua index 8a124b211..bf7199f5c 100644 --- a/de.peeeq.wurstscript/src/test/resources/luaruntime/wc3shim.lua +++ b/de.peeeq.wurstscript/src/test/resources/luaruntime/wc3shim.lua @@ -46,6 +46,20 @@ function FourCC(s) + string.byte(s, 3)) * 256 + string.byte(s, 4) end +-- String natives. The game counts and indexes bytes, and a Lua string is a +-- byte array, so these are the plain Lua operations. SubString takes a +-- 0-based start and an exclusive end; string.sub is 1-based and inclusive. +function StringLength(s) + return #s +end + +function SubString(s, start, stop) + return string.sub(s, start + 1, stop) +end + +function I2S(i) return tostring(math.floor(i)) end +function S2I(s) return math.floor(tonumber(s) or 0) end + -- Reforged player layout: 24 playable slots, neutrals at 24..27, 28 total. function GetBJMaxPlayers() return 24 end function GetBJMaxPlayerSlots() return 28 end