Hold an interpreted string as bytes, the way the game does

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompiletimeFunctionRunner.java

@@ -421,6 +421,25 @@ private ImExpr constantToExpr(Element trace, ILconst value) {
  return constantToExpr(trace, value, null);
  }
 
+ /**
+ * The text for a string a compiletime expression produced, which becomes a literal in the
+ * generated script.
+ * <p>
+ * A string held by the interpreter is a sequence of bytes and may hold half of a character -
+ * slicing one in half is a thing the standard library does deliberately. A literal cannot: the
+ * script is written as UTF-8 and neither Jass nor the escaping here can write a byte down
+ * numerically, so half a character would go in as the replacement character and come back out
+ * three bytes long. Refused rather than carried across at a different length.
+ */
+ private String literalText(ILconstString value, Element trace) {
+ if (!value.isText()) {
+ throw new CompileError(trace, "A compiletime expression returned a string holding part of a"
+ + " multibyte character, which cannot be written into the generated script. Slice it"
+ + " where the program runs rather than at compiletime, or keep whole characters.");
+ }
+ return value.text();
+ }
+
  private ImExpr constantToExpr(Element trace, ILconst value, @Nullable ImType expectedType) {
  if (value instanceof ILconstBool) {
  return JassIm.ImBoolVal(((ILconstBool) value).getVal());
@@ -429,7 +448,7 @@ private ImExpr constantToExpr(Element trace, ILconst value, @Nullable ImType exp
  } else if (value instanceof ILconstReal) {
  return JassIm.ImRealVal("" + ((ILconstReal) value).getVal());
  } else if (value instanceof ILconstString) {
- return JassIm.ImStringVal(((ILconstString) value).getVal());
+ return JassIm.ImStringVal(literalText((ILconstString) value, trace));
  } else if (value instanceof ILconstNull) {
  return expectedType == null ? ImHelper.nullExpr() : JassIm.ImNull(expectedType.copy());
  } else if (value instanceof ILconstTuple) {
@@ -1043,7 +1062,7 @@ private ImExpr constantToExprHashtable(Element trace, ImVar htVar, IlConstHandle
  JassIm.ImVarAccess(htVar),
  JassIm.ImIntVal(key.getParentkey()),
  JassIm.ImIntVal(key.getChildkey()),
- JassIm.ImStringVal(iv.getVal())
+ JassIm.ImStringVal(literalText(iv, trace))
  ), false, CallType.NORMAL));
  } else if (v instanceof ILconstBool) {
  ILconstBool iv = (ILconstBool) v;

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/CompiletimeNatives.java

@@ -47,7 +47,7 @@ public CompiletimeNatives(ProgramStateIO globalState, WurstProjectConfigData pro
 
 
  private ILconstTuple makeKey(String key) {
- return new ILconstTuple(new ILconstString(key));
+ return new ILconstTuple(ILconstString.fromText(key));
  }
 
  public ILconstTuple createObjectDefinition(ILconstString fileType, ILconstInt newUnitId, ILconstInt deriveFrom) {
@@ -93,7 +93,7 @@ public void ObjectDefinition_setInt(ILconstTuple unitType, ILconstString modific
 
  public void ObjectDefinition_setString(ILconstTuple unitType, ILconstString modification, ILconstString value) {
  ObjMod.Obj od = globalState.getObjectDefinition(getKey(unitType));
- modifyObject(od, modification, ObjMod.ValType.STRING, War3String.valueOf(value.getVal()));
+ modifyObject(od, modification, ObjMod.ValType.STRING, War3String.valueOf(value.text()));
  }
 
  public void ObjectDefinition_setReal(ILconstTuple unitType, ILconstString modification, ILconstReal value) {
@@ -114,7 +114,7 @@ public void ObjectDefinition_setLvlInt(ILconstTuple unitType, ILconstString modi
 
  public void ObjectDefinition_setLvlString(ILconstTuple unitType, ILconstString modification, ILconstInt level, ILconstString value) {
  ObjMod.Obj od = globalState.getObjectDefinition(getKey(unitType));
- modifyObject(od, modification, ObjMod.ValType.STRING, level.getVal(), War3String.valueOf(value.getVal()));
+ modifyObject(od, modification, ObjMod.ValType.STRING, level.getVal(), War3String.valueOf(value.text()));
  }
 
  public void ObjectDefinition_setLvlReal(ILconstTuple unitType, ILconstString modification, ILconstInt level, ILconstReal value) {
@@ -135,7 +135,7 @@ public void ObjectDefinition_setLvlDataInt(ILconstTuple unitType, ILconstString
 
  public void ObjectDefinition_setLvlDataString(ILconstTuple unitType, ILconstString modification, ILconstInt level, ILconstInt dataPointer, ILconstString value) {
  ObjMod.Obj od = globalState.getObjectDefinition(getKey(unitType));
- modifyObject(od, modification, ObjMod.ValType.STRING, level.getVal(), dataPointer.getVal(), War3String.valueOf(value.getVal()));
+ modifyObject(od, modification, ObjMod.ValType.STRING, level.getVal(), dataPointer.getVal(), War3String.valueOf(value.text()));
  }
 
  public void ObjectDefinition_setLvlDataReal(ILconstTuple unitType, ILconstString modification, ILconstInt level, ILconstInt dataPointer, ILconstReal value) {
@@ -185,15 +185,15 @@ private String getKey(ILconstTuple unitType) {
  }
 
  public void compileError(ILconstString msg) {
- throw new InterpreterException(msg.getVal());
+ throw new InterpreterException(msg.text());
  }
 
  public ILconstString getMapName() {
- return new ILconstString(projectConfigData.buildMapData().name());
+ return ILconstString.fromText(projectConfigData.buildMapData().name());
  }
 
  public ILconstString getBuildDate() {
- return new ILconstString(LocalDateTime.now().truncatedTo(ChronoUnit.MINUTES).toString());
+ return ILconstString.fromText(LocalDateTime.now().truncatedTo(ChronoUnit.MINUTES).toString());
  }
 
  public ILconstBool isProductionBuild() {
@@ -224,7 +224,7 @@ private PreparedStatement sqliteStatement(int handle) {
  }
 
  public ILconstInt sqlite_open(ILconstString path) {
- String dbPath = path.getVal();
+ String dbPath = path.text();
  // SQLite "file:" URI paths can carry query parameters such as
  // "?enable_load_extension=true" that would turn on extension loading and let
  // load_extension() dlopen arbitrary native code on the build machine at compiletime.
@@ -250,7 +250,7 @@ public ILconstInt sqlite_open(ILconstString path) {
  public ILconstInt sqlite_prepare(ILconstInt connection, ILconstString query) {
  Connection conn = sqliteConnection(connection.getVal());
  try {
- PreparedStatement stmt = conn.prepareStatement(query.getVal());
+ PreparedStatement stmt = conn.prepareStatement(query.text());
  int handle = ++sqliteHandleCounter;
  sqliteStatements.put(handle, stmt);
  sqliteStatementConnections.put(handle, connection.getVal());
@@ -300,7 +300,7 @@ public void sqlite_bind_real(ILconstInt statement, ILconstInt index, ILconstReal
  public void sqlite_bind_string(ILconstInt statement, ILconstInt index, ILconstString value) {
  PreparedStatement stmt = sqliteStatement(statement.getVal());
  try {
- stmt.setString(index.getVal(), value.getVal());
+ stmt.setString(index.getVal(), value.text());
  markStatementForReexecution(statement.getVal());
  } catch (SQLException e) {
  throw new InterpreterException("Failed to bind string: " + e.getMessage());
@@ -379,7 +379,7 @@ public ILconstString sqlite_column_string(ILconstInt statement, ILconstInt index
  // A SQL NULL maps to "" here; use sqlite_column_is_null to distinguish NULL
  // from an empty string / zero value.
  String val = rs.getString(index.getVal() + 1);
- return new ILconstString(val == null ? "" : val);
+ return ILconstString.fromText(val == null ? "" : val);
  } catch (SQLException e) {
  throw new InterpreterException("Failed to get column string: " + e.getMessage());
  }
@@ -476,7 +476,7 @@ public void sqlite_exec(ILconstInt connection, ILconstString query) {
  // and a hand-rolled splitter cannot correctly handle trigger BEGIN...END
  // bodies, CASE...END, or every identifier-quoting form ([id], `id`, "id").
  SQLiteConnection sqliteConn = conn.unwrap(SQLiteConnection.class);
- sqliteConn.getDatabase()._exec(query.getVal());
+ sqliteConn.getDatabase()._exec(query.text());
  } catch (SQLException e) {
  throw new InterpreterException("Failed to exec SQLite query: " + e.getMessage());
  }

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/JassInterpreter.java

@@ -386,7 +386,7 @@ public ILconst case_JassOpEquals(JassOpEquals jassOpEquals) {
 
  @Override
  public ILconst case_JassExprStringVal(JassExprStringVal e) {
- return new ILconstString(e.getValS());
+ return ILconstString.fromText(e.getValS());
  }
 
  @Override

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/AbilityProvider.java

@@ -10,10 +10,10 @@ public AbilityProvider(AbstractInterpreter interpreter) {
  }
 
  public ILconstString BlzGetAbilityIcon(ILconstInt abilCode) {
- return new ILconstString("");
+ return ILconstString.fromText("");
  }
 
  public ILconstString BlzGetAbilityExtendedTooltip(ILconstInt abilCode, ILconstInt level) {
- return new ILconstString("");
+ return ILconstString.fromText("");
  }
 }

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/GamecacheProvider.java

@@ -63,7 +63,7 @@ public ILconstReal GetStoredReal(IlConstHandle ht, ILconstString key1, ILconstSt
  }
 
  public ILconstString GetStoredString(IlConstHandle ht, ILconstString key1, ILconstString key2) {
- return haveSaved(ht, key1, key2, ILconstString.class) ? load(ht, key1, key2, ILconstString.class) : new ILconstString("");
+ return haveSaved(ht, key1, key2, ILconstString.class) ? load(ht, key1, key2, ILconstString.class) : ILconstString.fromText("");
  }
 
  public ILconstBool GetStoredBoolean(IlConstHandle ht, ILconstString key1, ILconstString key2) {

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/HashtableProvider.java

@@ -74,7 +74,7 @@ public ILconstReal LoadReal(IlConstHandle ht, ILconstInt key1, ILconstInt key2)
  }
 
  public ILconstString LoadStr(IlConstHandle ht, ILconstInt key1, ILconstInt key2) {
- return haveSaved(ht, key1, key2, ILconstString.class) ? load(ht, key1, key2, ILconstString.class) : new ILconstString("");
+ return haveSaved(ht, key1, key2, ILconstString.class) ? load(ht, key1, key2, ILconstString.class) : ILconstString.fromText("");
  }
 
  public ILconstBool LoadBoolean(IlConstHandle ht, ILconstInt key1, ILconstInt key2) {

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/LuaEnsureTypeProvider.java

@@ -43,7 +43,7 @@ public ILconstString __wurst_rawToString(ILconstString x) {
  }
 
  public ILconstString __wurst_rawConcat(ILconstString x, ILconstString y) {
- return new ILconstString(x.getVal() + y.getVal());
+ return ILconstString.ofBytes(x.getVal() + y.getVal());
  }
 
  public ILconstInt __wurst_rawFloorDivInt(ILconstInt a, ILconstInt b) {

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java

@@ -23,36 +23,36 @@ public void setOutStream(PrintStream outStream) {
  }
 
  public void DisplayTextToForce(IlConstHandle force, ILconstString msg) {
- outStream.println(msg.getVal());
+ outStream.println(msg.text());
  }
 
  public void DisplayTimedTextToForce(IlConstHandle force, ILconstReal duration, ILconstString msg) {
- outStream.println(msg.getVal());
+ outStream.println(msg.text());
  }
 
  public void DisplayTextToPlayer(IlConstHandle player, ILconstReal x, ILconstReal y, ILconstString msg) {
- outStream.println(msg.getVal());
+ outStream.println(msg.text());
  }
 
  public void DisplayTimedTextToPlayer(IlConstHandle player, ILconstReal x, ILconstReal y, ILconstReal duration, ILconstString msg) {
- outStream.println(msg.getVal());
+ outStream.println(msg.text());
  }
 
  @Implements(funcNames = {"BJDebugMsg", "println"})
  public void println(ILconstString msg) {
- outStream.println(msg.getVal());
+ outStream.println(msg.text());
  }
 
  public void $debugPrint(ILconstString msg) {
- outStream.println(msg.getVal());
- throw new DebugPrintError(msg.getVal());
+ outStream.println(msg.text());
+ throw new DebugPrintError(msg.text());
  }
 
  public void testSuccess() {
  throw TestSuccessException.instance;
  }
 
  public void testFail(ILconstString msg) {
- throw new TestFailException(msg.getVal());
+ throw new TestFailException(msg.text());
  }
 }

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/StringProvider.java

@@ -1,16 +1,14 @@
 package de.peeeq.wurstio.jassinterpreter.providers;
 
 import de.peeeq.wurstio.jassinterpreter.InterpreterException;
-import de.peeeq.wurstscript.WLogger;
 import de.peeeq.wurstscript.intermediatelang.ILconstBool;
 import de.peeeq.wurstscript.intermediatelang.ILconstInt;
 import de.peeeq.wurstscript.intermediatelang.ILconstReal;
 import de.peeeq.wurstscript.intermediatelang.ILconstString;
+import de.peeeq.wurstscript.intermediatelang.Wc3StringHash;
 import de.peeeq.wurstscript.intermediatelang.interpreter.AbstractInterpreter;
-import net.moonlightflower.wc3libs.misc.StringHash;
 import org.apache.commons.lang.StringUtils;
 
-import java.io.UnsupportedEncodingException;
 import java.math.RoundingMode;
 import java.text.NumberFormat;
 import java.util.Locale;
@@ -24,7 +22,7 @@ public StringProvider(AbstractInterpreter interpreter) {
  }
 
  public ILconstString I2S(ILconstInt i) {
- return new ILconstString("" + i.getVal());
+ return ILconstString.fromText("" + i.getVal());
  }
 
  private static final Pattern s2ipattern = Pattern.compile("([+\\-]?[0-9]+).*");
@@ -53,7 +51,7 @@ public ILconstReal S2R(ILconstString s) {
  }
 
  public ILconstString R2S(ILconstReal r) {
- return new ILconstString("" + r.getVal());
+ return ILconstString.fromText("" + r.getVal());
  }
 
  public ILconstString R2SW(ILconstReal r, ILconstInt width, ILconstInt precision) {
@@ -65,7 +63,7 @@ public ILconstString R2SW(ILconstReal r, ILconstInt width, ILconstInt precision)
  String s = formatter.format(r.getVal());
  // pad to desired width
  s = StringUtils.rightPad(s, width.getVal());
- return new ILconstString(s);
+ return ILconstString.fromText(s);
  }
 
  public ILconstInt R2I(ILconstReal i) {
@@ -81,12 +79,7 @@ public ILconstInt StringHash(ILconstString s) {
  if (s == null) {
  return new ILconstInt(0);
  }
- try {
- return new ILconstInt(StringHash.hash(s.getVal()));
- } catch (UnsupportedEncodingException e) {
- WLogger.severe(e);
- }
- return new ILconstInt(0);
+ return new ILconstInt(Wc3StringHash.hash(s.getVal()));
  }
 
  public ILconstInt StringLength(ILconstString string) {
@@ -110,13 +103,26 @@ public ILconstString SubString(ILconstString istr, ILconstInt start, ILconstInt
  // since this is most likely a bug in your code, the interpreter will throw an exception instead:
  throw new InterpreterException("SubString called with start index " + start + " greater than string length " + str.length());
  }
- return new ILconstString(str.substring(s, e));
+ return ILconstString.ofBytes(str.substring(s, e));
  }
 
+ /**
+ * Only ascii letters change case. A string is a sequence of bytes, and the bytes of a multibyte
+ * character are not letters to case at all - folding them the way a latin-1 char would fold
+ * rewrites the character into a different one.
+ */
  public ILconstString StringCase(ILconstString string, ILconstBool upperCase) {
- return new ILconstString(
- upperCase.getVal() ?
- string.getVal().toUpperCase()
- : string.getVal().toLowerCase());
+ String bytes = string.getVal();
+ StringBuilder result = new StringBuilder(bytes.length());
+ for (int i = 0; i < bytes.length(); i++) {
+ char c = bytes.charAt(i);
+ if (upperCase.getVal() && c >= 'a' && c <= 'z') {
+ c -= 32;
+ } else if (!upperCase.getVal() && c >= 'A' && c <= 'Z') {
+ c += 32;
+ }
+ result.append(c);
+ }
+ return ILconstString.ofBytes(result.toString());
  }
 }

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/UnitProvider.java

@@ -66,10 +66,10 @@ public ILconstReal GetUnitFacing(IlConstHandle unit) {
 
  public ILconstString GetUnitName(IlConstHandle unit) {
  if (unit == null) {
- return new ILconstString("");
+ return ILconstString.fromText("");
  }
  UnitMock unitMock = (UnitMock) unit.getObj();
- return new ILconstString(ObjectHelper.objectIdIntToString(unitMock.unitid.getVal()));
+ return ILconstString.fromText(ObjectHelper.objectIdIntToString(unitMock.unitid.getVal()));
  }
 
  public ILconstInt GetUnitGoldCost(ILconstInt unitid) {

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/WurstflectionProvider.java

@@ -25,7 +25,7 @@ public ILconstString typeIdToTypeName(ILconstInt typeId) {
  int typeIdInt = typeId.getVal();
  for (Map.Entry<ImClass, Integer> e : prog.attrTypeId().entrySet()) {
  if (e.getValue() == typeIdInt) {
- ILconstString iLconstString = new ILconstString(calculateClassName(e.getKey()));
+ ILconstString iLconstString = ILconstString.fromText(calculateClassName(e.getKey()));
  return Optional.of(iLconstString)
  .orElseGet(() -> {
  throw new InterpreterException("Could not determine type name for id " + typeId);