From 01acf199c0b15b5da07e90aaba7bfd90849761eb Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 21 Aug 2026 17:46:08 -0700 Subject: [PATCH 1/2] GitHub Issue 1432: Cannot delete a multi-choice option on a source type --- api/src/org/labkey/api/data/SQLFragment.java | 2999 +++++++++--------- 1 file changed, 1500 insertions(+), 1499 deletions(-) diff --git a/api/src/org/labkey/api/data/SQLFragment.java b/api/src/org/labkey/api/data/SQLFragment.java index 52f0d894a08..1f89bf965ad 100644 --- a/api/src/org/labkey/api/data/SQLFragment.java +++ b/api/src/org/labkey/api/data/SQLFragment.java @@ -1,1499 +1,1500 @@ -/* - * Copyright (c) 2008-2026 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.data; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.query.AliasManager; -import org.labkey.api.query.FieldKey; -import org.labkey.api.settings.AppProps; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JdbcUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.logging.LogHelper; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import static org.labkey.api.query.ExprColumn.STR_TABLE_ALIAS; - -/// A composable SQL builder that pairs SQL text with its JDBC parameter values, ensuring -/// they travel together through query construction. Implements [Appendable] and -/// [CharSequence] for fluent assembly of SQL statements. -/// -/// Provides type-safe `appendValue()` methods for inlining literals of common -/// types (integers, strings, dates, GUIDs, etc.) and `add()` methods for binding -/// JDBC `?` parameters. Fragments can be composed via `append(SQLFragment)` to -/// merge both SQL text and parameter lists. -/// -/// Supports Common Table Expressions (CTEs) through -/// [#addCommonTableExpression(SqlDialect, Object, String, SQLFragment)], which -/// manages deduplication, token substitution, and correct ordering of WITH clauses -/// across nested and combined fragments. -/// -/// Enforces basic SQL injection safeguards by rejecting unmatched quotes and -/// semicolons in appended text. -public class SQLFragment implements Appendable, CharSequence -{ - private static final Logger LOG = LogHelper.getLogger(SQLFragment.class, "SQL injection safety net diagnostics"); - - public static final String FEATUREFLAG_DISABLE_STRICT_CHECKS = "SQLFragmentDisableStrictChecks"; - - private String sql; - private StringBuilder sb = null; - private List params; // TODO: Should be List - - private final List tempTokens = new ArrayList<>(); // Hold refs to ensure they're not GC'd - - // use ordered map to make sql generation more deterministic (see collectCommonTableExpressions()) - private LinkedHashMap commonTableExpressionsMap = null; - - private static class CTE - { - CTE(@NotNull SqlDialect dialect, @NotNull String name) - { - this.dialect = dialect; - this.preferredName = name; - tokens.add("/*$*/" + GUID.makeGUID() + ":" + name + "/*$*/"); - } - - CTE(@NotNull SqlDialect dialect, @NotNull String name, SQLFragment sqlf, boolean recursive) - { - this(dialect, name); - this.sqlf = sqlf; - this.recursive = recursive; - } - - CTE(CTE from) - { - this.dialect = from.dialect; - this.preferredName = from.preferredName; - this.tokens.addAll(from.tokens); - this.sqlf = from.sqlf; - this.recursive = from.recursive; - } - - public CTE copy(boolean deep) - { - CTE copy = new CTE(this); - if (deep) - copy.sqlf = new SQLFragment().append(copy.sqlf); - return copy; - } - - private String token() - { - return tokens.iterator().next(); - } - - private final @NotNull SqlDialect dialect; - final String preferredName; - boolean recursive = false; // NOTE this is dialect dependant (getSql() does not take a dialect) - final Set tokens = new TreeSet<>(); - SQLFragment sqlf = null; - } - - public SQLFragment() - { - sql = ""; - } - - public SQLFragment(CharSequence charseq, @Nullable List params) - { - if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || - (StringUtils.countMatches(charseq, '\"') % 2) != 0 || - StringUtils.contains(charseq, ';')) - { - throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); - } - - this.sql = charseq.toString(); - if (null != params) - this.params = new ArrayList<>(params); - } - - - public SQLFragment(CharSequence sql, Object... params) - { - this(sql, Arrays.asList(params)); - } - - - public SQLFragment(SQLFragment other) - { - this(other,false); - } - - - public SQLFragment(SQLFragment other, boolean deep) - { - sql = other.getSqlCharSequence().toString(); - if (null != other.params) - addAll(other.params); - if (null != other.commonTableExpressionsMap && !other.commonTableExpressionsMap.isEmpty()) - { - if (null == this.commonTableExpressionsMap) - this.commonTableExpressionsMap = new LinkedHashMap<>(); - for (Map.Entry e : other.commonTableExpressionsMap.entrySet()) - { - CTE cte = e.getValue().copy(deep); - this.commonTableExpressionsMap.put(e.getKey(),cte); - } - } - this.tempTokens.addAll(other.tempTokens); - } - - - @Override - public boolean isEmpty() - { - return (null == sb || sb.isEmpty()) && (sql == null || sql.isEmpty()); - } - - - /* same as getSQL() but without CTE handling */ - public String getRawSQL() - { - return null != sb ? sb.toString() : null != sql ? sql : ""; - } - - /* - * Directly set the current SQL. - * - * This is useful for wrapping existing SQL, for instance adding a cast - * Obviously parameter number and order must remain unchanged - * - * This can also be used for processing sql scripts (e.g. module .sql update scripts) - */ - public SQLFragment setSqlUnsafe(String unsafe) - { - this.sql = unsafe; - this.sb = null; - return this; - } - - public static SQLFragment unsafe(String unsafe) - { - return new SQLFragment().setSqlUnsafe(unsafe); - } - - - private String replaceCteTokens(String self, String select, List> ctes) - { - for (Pair pair : ctes) - { - String alias = pair.first; - CTE cte = pair.second; - for (String token : cte.tokens) - { - select = Strings.CS.replace(select, token, alias); - } - } - if (null != self) - select = Strings.CS.replace(select, "$SELF$", self); - return select; - } - - - private List collectCommonTableExpressions() - { - List list = new ArrayList<>(); - _collectCommonTableExpressions(list); - return list; - } - - private void _collectCommonTableExpressions(List list) - { - if (null != commonTableExpressionsMap) - { - commonTableExpressionsMap.values().forEach(cte -> cte.sqlf._collectCommonTableExpressions(list)); - list.addAll(commonTableExpressionsMap.values()); - } - } - - - public String getSQL() - { - if (null == commonTableExpressionsMap || commonTableExpressionsMap.isEmpty()) - return null != sb ? sb.toString() : null != sql ? sql : ""; - - List commonTableExpressions = collectCommonTableExpressions(); - assert !commonTableExpressions.isEmpty(); - - boolean recursive = commonTableExpressions.stream() - .anyMatch(cte -> cte.recursive); - StringBuilder ret = new StringBuilder("WITH" + (recursive ? " RECURSIVE" : "")); - - // generate final aliases for each CTE */ - SqlDialect dialect = Objects.requireNonNull(commonTableExpressions.getFirst().dialect); - AliasManager am = new AliasManager(dialect); - List> ctes = commonTableExpressions.stream() - .map(cte -> new Pair<>(am.decideAlias(cte.preferredName),cte)) - .collect(Collectors.toList()); - - String comma = "\n/*CTE*/\n\t"; - for (Pair p : ctes) - { - String alias = p.first; - CTE cte = p.second; - SQLFragment expr = cte.sqlf; - String sql = expr._getOwnSql(alias, ctes); - ret.append(comma).append(alias).append(" AS (").append(sql).append(")"); - comma = "\n,/*CTE*/\n\t"; - } - ret.append("\n"); - - String select = _getOwnSql( null, ctes ); - ret.append(replaceCteTokens(null, select, ctes)); - return ret.toString(); - } - - - private String _getOwnSql(String alias, List> ctes) - { - String ownSql = null != sb ? sb.toString() : null != this.sql ? this.sql : ""; - return replaceCteTokens(alias, ownSql, ctes); - } - - - static Pattern markerPattern = Pattern.compile("/\\*\\$\\*/.*/\\*\\$\\*/"); - - /* This is not an exhaustive .equals() test, but it give pretty good confidence that these statements are the same */ - static boolean debugCompareSQL(SQLFragment sql1, SQLFragment sql2) - { - String select1 = sql1.getRawSQL(); - String select2 = sql2.getRawSQL(); - - if ((null == sql1.commonTableExpressionsMap || sql1.commonTableExpressionsMap.isEmpty()) && - (null == sql2.commonTableExpressionsMap || sql2.commonTableExpressionsMap.isEmpty())) - return select1.equals(select2); - - select1 = markerPattern.matcher(select1).replaceAll("CTE"); - select2 = markerPattern.matcher(select2).replaceAll("CTE"); - if (!select1.equals(select2)) - return false; - - Set ctes1 = sql1.commonTableExpressionsMap.values().stream() - .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) - .collect(Collectors.toSet()); - Set ctes2 = sql2.commonTableExpressionsMap.values().stream() - .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) - .collect(Collectors.toSet()); - return ctes1.equals(ctes2); - } - - - // It is a little confusing that getString() does not return the same charsequence that this object purports to - // represent. However, this is a good "display value" for this object. - // see getSqlCharSequence() - @NotNull - public String toString() - { - return "SQLFragment@" + System.identityHashCode(this) + "\n" + toDebugString(); - } - - // Not recommended -- this uses the LabKey scope to dictate parsing of identifiers and string literals... which - // might not be correct for the incoming SQL. - @Deprecated // Use the variant below that takes a SqlDialect - public String toDebugString() - { - return toDebugString(DbScope.getLabKeyScope().getSqlDialect()); - } - - public String toDebugString(SqlDialect dialect) - { - return JdbcUtil.format(this, dialect); - } - - public List getParams() - { - var ctes = collectCommonTableExpressions(); - List ret = new ArrayList<>(); - - for (var cte : ctes) - ret.addAll(cte.sqlf.getParamsNoCTEs()); - ret.addAll(getParamsNoCTEs()); - return Collections.unmodifiableList(ret); - } - - - public List> getParamsWithFragments() - { - var ctes = collectCommonTableExpressions(); - List> ret = new ArrayList<>(); - - for (CTE cte : ctes) - { - if (null != cte.sqlf && null != cte.sqlf.params) - { - for (int i = 0; i < cte.sqlf.params.size(); i++) - { - ret.add(new Pair<>(cte.sqlf, i)); - } - } - } - - if (null != params) - { - for (int i = 0; i < params.size(); i++) - { - ret.add(new Pair<>(this, i)); - } - } - return ret; - } - - private final static Object[] EMPTY_ARRAY = new Object[0]; - - public Object[] getParamsArray() - { - return null == params ? EMPTY_ARRAY : params.toArray(); - } - - public List getParamsNoCTEs() - { - return params == null ? Collections.emptyList() : Collections.unmodifiableList(params); - } - - private List getMutableParams() - { - if (!(params instanceof ArrayList)) - { - List t = new ArrayList<>(); - if (params != null) - t.addAll(params); - params = t; - } - return params; - } - - - private StringBuilder getStringBuilder() - { - if (null == sb) - sb = new StringBuilder(null==sql?"":sql); - return sb; - } - - - @Override - public SQLFragment append(CharSequence charseq) - { - if (null == charseq) - return this; - - if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || - (StringUtils.countMatches(charseq, '\"') % 2) != 0 || - StringUtils.contains(charseq, ';')) - { - throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); - } - - getStringBuilder().append(charseq); - return this; - } - - public SQLFragment appendIdentifier(DatabaseIdentifier id) - { - return append(id.getSql()); - } - - /** Functionally the same as append(CharSequence). This method just has different asserts */ - public SQLFragment appendIdentifier(CharSequence charseq) - { - if (null == charseq) - return this; - if (charseq instanceof SQLFragment sqlf) - { - if (0 != sqlf.getParamsArray().length) - throw new IllegalStateException("Unexpected SQL in appendIdentifier()"); - charseq = sqlf.getRawSQL(); - } - - String identifier = charseq.toString().strip(); - - if (STR_TABLE_ALIAS.equals(identifier)) - { - getStringBuilder().append(identifier); - return this; - } - - boolean malformed; - boolean quoteWrapped = identifier.length() >= 2 && identifier.startsWith("\"") && identifier.endsWith("\""); - if (quoteWrapped) - malformed = (StringUtils.countMatches(identifier, '\"') % 2) != 0; - else if (identifier.length() >= 2 && identifier.startsWith("`") && identifier.endsWith("`")) - malformed = (StringUtils.countMatches(identifier, '`') % 2) != 0; - else - malformed = StringUtils.containsAny(identifier, "*/\\'\"`?;- \t\n"); - if (malformed) - throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) value appears to be incorrectly formatted: " + identifier); - - // A quote-wrapped value must be a well-formed quoted identifier, or dotted sequence of them. - if (quoteWrapped && !isQuotedIdentifierSequence(identifier)) - { - if (AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - LOG.warn("appendIdentifier strict pre-quoted check would have rejected (flag-on, allowed): {}", identifier); - else - { - LOG.warn("appendIdentifier strict pre-quoted check rejected (flag-off): {}", identifier); - throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) pre-quoted value is not a well-formed quoted identifier (or dotted sequence of them): " + identifier); - } - } - - getStringBuilder().append(charseq); - return this; - } - - // True iff the value is one or more double-quote-delimited identifiers joined by single dots, - // e.g. "a", "a"."b", "schema"."table". A literal quote within a segment must be escaped as "". - // A '.' inside quotes is part of the name; a '.' between segments is a separator. Anything else - // outside the quotes -- stray text, a lone (breakout) quote, an unterminated segment -- is rejected. - private static boolean isQuotedIdentifierSequence(String s) - { - int i = 0; - int n = s.length(); - while (i < n) - { - if (s.charAt(i) != '"') // each segment must open with a quote - return false; - i++; - boolean closed = false; - while (i < n) - { - if (s.charAt(i) == '"') - { - if (i + 1 < n && s.charAt(i + 1) == '"') // escaped "" -- part of the name - { - i += 2; - continue; - } - i++; // closing quote - closed = true; - break; - } - i++; // any other char is legal inside quotes - } - if (!closed) // ran off the end without a closing quote - return false; - if (i == n) // end of a valid final segment - return true; - if (s.charAt(i) != '.') // segments must be separated by a single dot - return false; - i++; // consume the separator and require another segment - } - return false; // trailing dot with no following segment - } - - // just to save some typing - public SQLFragment appendDottedIdentifiers(CharSequence table, DatabaseIdentifier col) - { - return appendIdentifier(table).append(".").appendIdentifier(col); - } - - // just to save some typing - public SQLFragment appendDottedIdentifiers(CharSequence... ids) - { - var dot = ""; - for (var id : ids) - { - append(dot).appendIdentifier(id); - dot = "."; - } - return this; - } - - /** append End Of Statement */ - public SQLFragment appendEOS() - { - getStringBuilder().append(";\n"); - return this; - } - - - @Override - public SQLFragment append(CharSequence csq, int start, int end) - { - append(csq.subSequence(start, end)); - return this; - } - - /** Adds the container's ID as an in-line string constant to the SQL */ - public SQLFragment appendValue(Container c) - { - if (null == c) - return appendNull(); - return appendValue(c, null); - } - - public SQLFragment appendValue(@NotNull Container c, SqlDialect dialect) - { - appendValue(c.getEntityId(), dialect); - String name = c.getName(); - if (!StringUtils.containsAny(name,"*/\\'\"?")) - append("/* ").append(name).append(" */"); - return this; - } - - public SQLFragment appendNull() - { - getStringBuilder().append("NULL"); - return this; - } - - public SQLFragment appendValue(Boolean B, @NotNull SqlDialect dialect) - { - if (null == B) - return append("CAST(NULL AS ").append(dialect.getBooleanDataType()).append(")"); - getStringBuilder().append(B ? dialect.getBooleanTRUE() : dialect.getBooleanFALSE()); - return this; - } - - public SQLFragment appendValue(Integer I) - { - if (null == I) - return appendNull(); - getStringBuilder().append(I.intValue()); - return this; - } - - public SQLFragment appendValue(int i) - { - getStringBuilder().append(i); - return this; - } - - - public SQLFragment appendValue(Long L) - { - if (null == L) - return appendNull(); - getStringBuilder().append((long)L); - return this; - } - - public SQLFragment appendValue(long l) - { - getStringBuilder().append(l); - return this; - } - - public SQLFragment appendValue(Float F) - { - if (null == F) - return appendNull(); - return appendValue(F.floatValue()); - } - - public SQLFragment appendValue(float f) - { - if (Float.isFinite(f)) - { - getStringBuilder().append(f); - } - else - { - getStringBuilder().append("?"); - add(f); - } - return this; - } - - public SQLFragment appendValue(Double D) - { - if (null == D) - return appendNull(); - else - return appendValue(D.doubleValue()); - } - - public SQLFragment appendValue(double d) - { - if (Double.isFinite(d)) - { - getStringBuilder().append(d); - } - else - { - getStringBuilder().append("?"); - add(d); - } - return this; - } - - public SQLFragment appendValue(Number N) - { - if (null == N) - return appendNull(); - - if (N instanceof Quantity q) - N = q.value(); - - if (N instanceof BigDecimal || N instanceof BigInteger || N instanceof Long) - { - getStringBuilder().append(N); - } - else if (Double.isFinite(N.doubleValue())) - { - getStringBuilder().append(N); - } - else - { - getStringBuilder().append(" ? "); - add(N); - } - return this; - } - - public final SQLFragment appendNowTimestamp() - { - return appendValue(new NowTimestamp()); - } - - // Issue 27534: Stop using {fn now()} in function declarations - // Issue 48864: Query Table's use of web server time can cause discrepancies in created/modified timestamps - public final SQLFragment appendValue(NowTimestamp now) - { - if (null == now) - return appendNull(); - getStringBuilder().append("CURRENT_TIMESTAMP"); - return this; - } - - public final SQLFragment appendValue(java.util.Date d) - { - if (null == d) - return appendNull(); - if (d.getClass() == java.util.Date.class) - getStringBuilder().append("{ts '").append(new Timestamp(d.getTime())).append("'}"); - else if (d.getClass() == java.sql.Timestamp.class) - getStringBuilder().append("{ts '").append(d).append("'}"); - else if (d.getClass() == java.sql.Date.class) - getStringBuilder().append("{d '").append(d).append("'}"); - else - throw new IllegalStateException("Unexpected date type: " + d.getClass().getName()); - return this; - } - - public SQLFragment appendValue(GUID g) - { - return appendValue(g, null); - } - - public SQLFragment appendValue(GUID g, SqlDialect d) - { - if (null == g) - return appendNull(); - // doesn't need StringHandler, just hex and hyphen - String sqlGUID = "'" + g + "'"; - // I'm testing dialect type, because some dialects do not support getGuidType(), and postgers uses VARCHAR anyway - if (null != d && d.isSqlServer()) - getStringBuilder().append("CAST(").append(sqlGUID).append(" AS UNIQUEIDENTIFIER)"); - else - getStringBuilder().append(sqlGUID); - return this; - } - - public SQLFragment appendValue(Enum e) - { - if (null == e) - return appendNull(); - String name = e.name(); - // Enum.name() returns a legal Java identifier per JLS, so none of these characters can appear - // in practice. Defense in depth: reject anything SQL-active rather than only the apostrophe. - if (StringUtils.containsAny(name, "'\"\\;\r\n")) - throw new IllegalStateException("Unexpected character in Enum.name(): " + name); - getStringBuilder().append("'").append(name).append("'"); - return this; - } - - public SQLFragment append(FieldKey fk) - { - if (null == fk) - return appendNull(); - append(String.valueOf(fk)); - return this; - } - - - /** Adds the object as a JDBC parameter value */ - public SQLFragment add(Object p) - { - getMutableParams().add(p); - return this; - } - - public SQLFragment add(Object p, JdbcType type) - { - getMutableParams().add(new Parameter.TypedValue(p, type)); - return this; - } - - /** Adds the objects as JDBC parameter values */ - public SQLFragment addAll(Collection l) - { - getMutableParams().addAll(l); - return this; - } - - - /** Adds the objects as JDBC parameter values */ - public SQLFragment addAll(Object... values) - { - if (values == null) - return this; - addAll(Arrays.asList(values)); - return this; - } - - - /** Sets the parameter at the index to the object's value */ - public void set(int i, Object p) - { - getMutableParams().set(i,p); - } - - /** Append both the SQL and the parameters from the other SQLFragment to this SQLFragment */ - public SQLFragment append(SQLFragment f) - { - if (null != f.sb) - getStringBuilder().append(f.sb); - else - getStringBuilder().append(f.sql); - if (null != f.params) - addAll(f.params); - mergeCommonTableExpressions(f); - tempTokens.addAll(f.tempTokens); - return this; - } - - public SQLFragment append(@NotNull Iterable fragments, @NotNull String separator) - { - String s = ""; - for (SQLFragment fragment : fragments) - { - append(s); - s = separator; - append(fragment); - } - return this; - } - - // return boolean so this can be used in an assert. passing in a dialect is not ideal, but parsing comments out - // before submitting the fragment is not reliable and holding statements & comments separately (to eliminate the - // need to parse them) isn't particularly easy... so punt for now. - public boolean appendComment(String comment, SqlDialect dialect) - { - if (dialect.supportsComments()) - { - StringBuilder sb = getStringBuilder(); - int len = sb.length(); - if (len > 0 && sb.charAt(len-1) != '\n') - sb.append('\n'); - sb.append("\n-- "); - boolean truncated = comment.length() > 1000; - if (truncated) - comment = StringUtilsLabKey.leftSurrogatePairFriendly(comment, 1000); - // Strip CR/LF so an embedded newline can't terminate the `--` comment and turn the rest - // of the payload into live SQL. - comment = comment.replace('\r', ' ').replace('\n', ' '); - sb.append(comment); - if (StringUtils.countMatches(comment, "'")%2==1) - sb.append("'"); - if (truncated) - sb.append("..."); - sb.append('\n'); - } - return true; - } - - - /** see also append(TableInfo, String alias) */ - public SQLFragment append(TableInfo table) - { - SQLFragment s = table.getSQLName(); - if (s != null) - return append(s); - - String alias = table.getSqlDialect().makeLegalIdentifier(table.getName()); - return append(table.getFromSQL(alias)); - } - - /** Add a table/query to the SQL with an alias, as used in a FROM clause */ - public SQLFragment append(TableInfo table, String alias) - { - return append(table.getFromSQL(alias)); - } - - /** Add to the SQL */ - @Override - public SQLFragment append(char ch) - { - getStringBuilder().append(ch); - return this; - } - - /** This is like appendValue(CharSequence s), but force use of literal syntax - * CAUTIONARY NOTE: String literals in PostgresSQL are tricky because of overloaded functions - * array_agg('string') fails array_agg('string'::VARCHAR) works - * json_object('{}) works json_object('string'::VARCHAR) fails - * In the case of json_object() it expects TEXT. Postgres will promote 'json' to TEXT, but not 'json'::VARCHAR - */ - public SQLFragment appendStringLiteral(CharSequence s, @NotNull SqlDialect d) - { - if (null==s) - return appendNull(); - getStringBuilder().append(d.getStringHandler().quoteStringLiteral(s.toString())); - return this; - } - - /** Add to the SQL as either an in-line string literal or as a JDBC parameter depending on whether it would need escaping */ - public SQLFragment appendValue(CharSequence s) - { - return appendValue(s, null); - } - - public SQLFragment appendValue(CharSequence s, SqlDialect d) - { - if (null==s) - return appendNull(); - if (null==d || s.length() > 200) - return append("?").add(s.toString()); - appendStringLiteral(s, d); - return this; - } - - public SQLFragment appendInClause(@NotNull Collection params, SqlDialect dialect) - { - dialect.appendInClauseSql(this, params); - return this; - } - - public CharSequence getSqlCharSequence() - { - if (null != sb) - { - return sb; - } - return sql; - } - - public void insert(int index, SQLFragment sql) - { - if (!sql.getParams().isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with parameters - they must be inserted/merged separately"); - } - if (sql.commonTableExpressionsMap != null && !sql.commonTableExpressionsMap.isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with CTEs - they must be inserted/merged separately"); - } - if (!tempTokens.isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with temp tokens - they must be inserted/merged separately"); - } - getStringBuilder().insert(index, sql.getRawSQL()); - } - - /** Insert into the SQL */ - public void insert(int index, String str) - { - if ((StringUtils.countMatches(str, '\'') % 2) != 0 || - (StringUtils.countMatches(str, '\"') % 2) != 0 || - StringUtils.contains(str, ';')) - { - throw new IllegalArgumentException("SQLFragment.insert(int,String) does not allow semicolons or unmatched quotes"); - } - - getStringBuilder().insert(index, str); - } - - /** Insert this SQLFragment's SQL and parameters at the start of the existing SQL and parameters */ - public void prepend(SQLFragment sql) - { - getStringBuilder().insert(0, sql.getSqlCharSequence().toString()); - if (null != sql.params) - getMutableParams().addAll(0, sql.params); - mergeCommonTableExpressions(sql); - } - - - public int indexOf(String str) - { - return getStringBuilder().indexOf(str); - } - - - // Display query in "English" (display SQL with params substituted) - // with a little more work could probably be made to be SQL legal - public String getFilterText() - { - String sql = getSQL().replaceFirst("WHERE ", ""); - List params = getParams(); - for (Object param1 : params) - { - String param = param1.toString(); - param = param.replaceAll("\\\\", "\\\\\\\\"); - param = param.replaceAll("\\$", "\\\\\\$"); - sql = sql.replaceFirst("\\?", param); - } - return sql.replace("\"", ""); - } - - - @Override - public char charAt(int index) - { - return getSqlCharSequence().charAt(index); - } - - @Override - public int length() - { - return getSqlCharSequence().length(); - } - - @Override - public @NotNull CharSequence subSequence(int start, int end) - { - return getSqlCharSequence().subSequence(start, end); - } - - /** - * KEY is used as a faster way to look for equivalent CTE expressions. - * returning a name here allows us to potentially merge CTE at add time - * - * if you don't have a key you can just use sqlf.toString() - */ - public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf) - { - return addCommonTableExpression(dialect, key, proposedName, sqlf, false); - } - - public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf, boolean recursive) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - CTE prev = commonTableExpressionsMap.get(key); - if (null != prev) - return prev.token(); - CTE cte = new CTE(dialect, proposedName, sqlf, recursive); - commonTableExpressionsMap.put(key, cte); - return cte.token(); - } - - public String createCommonTableExpressionToken(SqlDialect dialect, Object key, String proposedName) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - CTE prev = commonTableExpressionsMap.get(key); - if (null != prev) - throw new IllegalStateException("Cannot create CTE token from already used key."); - CTE cte = new CTE(dialect ,proposedName); - commonTableExpressionsMap.put(key, cte); - return cte.token(); - } - - public void setCommonTableExpressionSql(Object key, SQLFragment sqlf, boolean recursive) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - - if (null != sqlf.commonTableExpressionsMap && !sqlf.commonTableExpressionsMap.isEmpty()) - { - // Need to merge CTEs up; this.cte depends on newSql.ctes, so they need to come first - SQLFragment newSql = new SQLFragment(sqlf); - LinkedHashMap toMap = new LinkedHashMap<>(newSql.commonTableExpressionsMap); - for (Map.Entry e : commonTableExpressionsMap.entrySet()) - { - CTE from = e.getValue(); - CTE to = toMap.get(e.getKey()); - if (null != to) - to.tokens.addAll(from.tokens); - else - toMap.put(e.getKey(), from.copy(false)); - } - - commonTableExpressionsMap = toMap; - newSql.commonTableExpressionsMap = null; - sqlf = newSql; - } - - CTE cte = commonTableExpressionsMap == null ? null : commonTableExpressionsMap.get(key); - if (null == cte) - throw new IllegalStateException("CTE not found."); - cte.sqlf = sqlf; - cte.recursive = recursive; - } - - - private void mergeCommonTableExpressions(SQLFragment sqlFrom) - { - if (null == sqlFrom.commonTableExpressionsMap || sqlFrom.commonTableExpressionsMap.isEmpty()) - return; - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - for (Map.Entry e : sqlFrom.commonTableExpressionsMap.entrySet()) - { - CTE from = e.getValue(); - CTE to = commonTableExpressionsMap.get(e.getKey()); - if (null != to) - to.tokens.addAll(from.tokens); - else - commonTableExpressionsMap.put(e.getKey(), from.copy(false)); - } - } - - - public void addTempToken(Object tempToken) - { - tempTokens.add(tempToken); - } - - public void addTempTokens(SQLFragment other) - { - tempTokens.add(other.tempTokens); - } - - public static SQLFragment prettyPrint(SQLFragment from) - { - SQLFragment sqlf = new SQLFragment(from); - - String s = from.getSqlCharSequence().toString(); - StringBuilder sb = new StringBuilder(s.length() + 200); - String[] lines = StringUtils.split(s, '\n'); - int indent = 0; - - for (String line : lines) - { - String t = line.trim(); - - if (t.isEmpty()) - continue; - - if (t.startsWith("-- params = b.getParams(); - assertEquals(2,params.size()); - assertEquals(5, params.get(0)); - assertEquals("xxyzzy", params.get(1)); - - - SQLFragment c = new SQLFragment(b); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT a FROM b WHERE x=?) - SELECT * FROM CTE WHERE y=?""", - c.getSQL()); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT a FROM b WHERE x=5) - SELECT * FROM CTE WHERE y='xxyzzy'""", - filterDebugString(c.toDebugString())); - params = c.getParams(); - assertEquals(2,params.size()); - assertEquals(5, params.get(0)); - assertEquals("xxyzzy", params.get(1)); - - - // combining - - SQLFragment sqlf = new SQLFragment(); - String token = sqlf.addCommonTableExpression(dialect, "KEY_A", "cte1", new SQLFragment("SELECT * FROM a")); - sqlf.append("SELECT * FROM ").append(token).append(" _1"); - - assertEquals(""" - WITH - /*CTE*/ - \tcte1 AS (SELECT * FROM a) - SELECT * FROM cte1 _1""", - sqlf.getSQL()); - - SQLFragment sqlf2 = new SQLFragment(); - String token2 = sqlf2.addCommonTableExpression(dialect, "KEY_A", "cte2", new SQLFragment("SELECT * FROM a")); - sqlf2.append("SELECT * FROM ").append(token2).append(" _2"); - assertEquals(""" - WITH - /*CTE*/ - \tcte2 AS (SELECT * FROM a) - SELECT * FROM cte2 _2""", - sqlf2.getSQL()); - - SQLFragment sqlf3 = new SQLFragment(); - String token3 = sqlf3.addCommonTableExpression(dialect, "KEY_B", "cte3", new SQLFragment("SELECT * FROM b")); - sqlf3.append("SELECT * FROM ").append(token3).append(" _3"); - assertEquals(""" - WITH - /*CTE*/ - \tcte3 AS (SELECT * FROM b) - SELECT * FROM cte3 _3""", - sqlf3.getSQL()); - - SQLFragment union = new SQLFragment(); - union.append(sqlf); - union.append("\nUNION\n"); - union.append(sqlf2); - union.append("\nUNION\n"); - union.append(sqlf3); - assertEquals(""" - WITH - /*CTE*/ - \tcte1 AS (SELECT * FROM a) - ,/*CTE*/ - \tcte3 AS (SELECT * FROM b) - SELECT * FROM cte1 _1 - UNION - SELECT * FROM cte1 _2 - UNION - SELECT * FROM cte3 _3""", - union.getSQL()); - } - - @Test - public void nested_cte() - { - // one-level cte using cteToken (CTE fragment 'a' does not contain a CTE) - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); - assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); - SQLFragment b = new SQLFragment(); - String cteToken = b.addCommonTableExpression(dialect, new Object(), "CTE", a); - b.append("SELECT * FROM ").append(cteToken).append(" WHERE p=?").add("parameterTWO"); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) - SELECT * FROM CTE WHERE p='parameterTWO'""", - filterDebugString(b.toDebugString())); - assertEquals("parameterONE", b.getParams().getFirst()); - } - - // two-level cte using cteTokens (CTE fragment 'b' contains a CTE of fragment a) - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); - assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterTWO"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - c.append("SELECT * FROM ").append(cteTokenB).append(" WHERE i=?").add(3); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterTWO') - SELECT * FROM B_ WHERE i=3""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(3, params.size()); - assertEquals("parameterONE", params.get(0)); - assertEquals("parameterTWO", params.get(1)); - assertEquals(3, params.get(2)); - } - - // Same as previous but top-level query has both a nested and non-nested CTE - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); - SQLFragment a2 = new SQLFragment("SELECT 2 as i, 'Atwo' as s, CAST(? AS VARCHAR) as p", "parameterAtwo"); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - String cteTokenA2 = c.addCommonTableExpression(dialect, new Object(), "A2_", a2); - c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') - ,/*CTE*/ - \tA2_ AS (SELECT 2 as i, 'Atwo' as s, CAST('parameterAtwo' AS VARCHAR) as p) - SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(4, params.size()); - assertEquals("parameterAone", params.get(0)); - assertEquals("parameterB", params.get(1)); - assertEquals("parameterAtwo", params.get(2)); - assertEquals(4, params.get(3)); - } - - // Same as previous but two of the CTEs are the same and should be collapsed (e.g. imagine a container filter implemented with a CTE) - // TODO, we only collapse CTEs that are siblings - { - SQLFragment cf = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, "CTE_KEY_CF", "A_", cf); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - String cteTokenA2 = c.addCommonTableExpression(dialect, "CTE_KEY_CF", "A2_", cf); - c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') - ,/*CTE*/ - \tA2_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(4, params.size()); - assertEquals("parameterAone", params.get(0)); - assertEquals("parameterB", params.get(1)); - assertEquals("parameterAone", params.get(2)); - assertEquals(4, params.get(3)); - } - } - - - private void shouldFail(Runnable r) - { - try - { - r.run(); - fail("Expected IllegalArgumentException"); - } - catch (IllegalArgumentException e) - { - // expected - } - } - - - @Test - public void testIllegalArgument() - { - shouldFail(() -> new SQLFragment(";")); - shouldFail(() -> new SQLFragment().append(";")); - shouldFail(() -> new SQLFragment("AND name='")); - shouldFail(() -> new SQLFragment().append("AND name = '")); - shouldFail(() -> new SQLFragment().append("AND name = 'Robert'); DROP TABLE Students; --")); - - shouldFail(() -> new SQLFragment().appendIdentifier("column name")); - shouldFail(() -> new SQLFragment().appendIdentifier("?")); - shouldFail(() -> new SQLFragment().appendIdentifier(";")); - shouldFail(() -> new SQLFragment().appendIdentifier("\"column\"name\"")); - } - - - String mysqlQuoteIdentifier(String id) - { - return "`" + id.replace("`", "``") + "`"; - } - - @Test - public void testMysql() - { - // OK - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("mysql")); - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my`sql")); - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my\"sql")); - - // not OK - shouldFail(() -> new SQLFragment().appendIdentifier("`")); - shouldFail(() -> new SQLFragment().appendIdentifier("`a`a`")); - } - - @Test - public void testAppendCommentStripsNewlines() - { - // PR1-P3: an embedded newline in a comment payload must not terminate the `-- ` comment - // line and expose the trailing text as live SQL. - if (!dialect.supportsComments()) - return; - - SQLFragment sqlf = new SQLFragment(); - sqlf.appendComment("hello\nDROP TABLE x;--", dialect); - String sql = sqlf.getSQL(); - - assertFalse("appendComment leaked an embedded newline into emitted SQL: " + sql, sql.contains("hello\nDROP")); - assertTrue("appendComment should keep the payload on one comment line: " + sql, sql.contains("hello DROP TABLE x;--")); - - // CR is stripped too - SQLFragment sqlf2 = new SQLFragment(); - sqlf2.appendComment("hi\rDROP TABLE x;--", dialect); - assertFalse("appendComment leaked an embedded CR: " + sqlf2.getSQL(), sqlf2.getSQL().contains("hi\rDROP")); - } - - @Test - public void testAppendIdentifierPreQuotedDottedAccepted() - { - // PR2-G1: a dotted sequence of individually well-formed quoted identifiers is legitimate -- - // e.g. a fully-qualified "schema"."table" returned by TableInfo.getSelectName(). The strict - // check validates each quoted segment rather than blanket-rejecting any interior dot. - new SQLFragment().appendIdentifier("\"schema\".\"table\""); - new SQLFragment().appendIdentifier("\"a\".\"b\".\"c\""); - } - - @Test - public void testAppendIdentifierPreQuotedDotInsideQuotesAccepted() - { - // PR2-G1: a `.` *inside* the quotes is part of a single identifier's name, not a separator. - // This is the metadata-name case (e.g. a calculated column's generated class name) that the - // earlier strict check wrongly rejected. - new SQLFragment().appendIdentifier("\"name.with.dots\""); - new SQLFragment().appendIdentifier("\"org.labkey.query.sql.CalculatedExpressionColumn6b8f\""); - } - - @Test - public void testAppendIdentifierPreQuotedInteriorQuoteRejected() - { - // PR2-G1: an interior `"` that isn't part of the standard `""` doubling is a breakout. - // Even quote count alone is insufficient. - shouldFail(() -> new SQLFragment().appendIdentifier("\"foo\"bar\"baz\"")); - // A classic breakout: close the identifier early, then inject. The lone interior quote - // (after `foo`) is followed by non-separator text, so it must be rejected. - shouldFail(() -> new SQLFragment().appendIdentifier("\"foo\"; DROP TABLE x; --\"")); - // Stray text after a valid segment (not a `.` separator). - shouldFail(() -> new SQLFragment().appendIdentifier("\"a\" \"b\"")); - // Trailing separator with no following segment. - shouldFail(() -> new SQLFragment().appendIdentifier("\"a\".\"")); - // Empty segment between separators. - shouldFail(() -> new SQLFragment().appendIdentifier("\"a\"..\"b\"")); - } - - @Test - public void testAppendIdentifierPreQuotedValidCases() - { - // PR2-G1: legitimate pre-quoted identifiers still pass. - new SQLFragment().appendIdentifier("\"my_table\""); // simple quoted - new SQLFragment().appendIdentifier("\"with a space\""); // whitespace inside quotes is fine - new SQLFragment().appendIdentifier("\"foo\"\"bar\""); // embedded literal " via "" doubling - new SQLFragment().appendIdentifier("\"contains\"\"more\"\"doubles\""); - // Doubled interior quotes around a dot decode to a single identifier named weird"."name -- - // distinct from the two-segment "weird"."name" -- and cannot break out, so it's accepted. - new SQLFragment().appendIdentifier("\"weird\"\".\"\"name\""); - } - } - - @Override - public boolean equals(Object obj) - { - if (!(obj instanceof SQLFragment other)) - { - return false; - } - return getSQL().equals(other.getSQL()) && getParams().equals(other.getParams()); - } - - @Override - public int hashCode() - { - return Objects.hash(getSQL(), getParams()); - } - - /** - * Joins the SQLFragments in the provided {@code Iterable} into a single SQLFragment. The SQL is joined by string - * concatenation using the provided separator. The parameters are combined to form the new parameter list. - * - * @param fragments SQLFragments to join together - * @param separator Separator to use - * @return A new SQLFragment that joins all the SQLFragments - */ - public static SQLFragment join(Iterable fragments, SQLFragment separator) - { - SQLFragment join = new SQLFragment(); - boolean first = true; - - for (SQLFragment fragment : fragments) - { - if (first) - first = false; - else - join.append(separator); - join.append(fragment); - } - - return join; - } -} +/* + * Copyright (c) 2008-2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.data; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.query.AliasManager; +import org.labkey.api.query.FieldKey; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JdbcUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.labkey.api.query.ExprColumn.STR_TABLE_ALIAS; + +/// A composable SQL builder that pairs SQL text with its JDBC parameter values, ensuring +/// they travel together through query construction. Implements [Appendable] and +/// [CharSequence] for fluent assembly of SQL statements. +/// +/// Provides type-safe `appendValue()` methods for inlining literals of common +/// types (integers, strings, dates, GUIDs, etc.) and `add()` methods for binding +/// JDBC `?` parameters. Fragments can be composed via `append(SQLFragment)` to +/// merge both SQL text and parameter lists. +/// +/// Supports Common Table Expressions (CTEs) through +/// [#addCommonTableExpression(SqlDialect, Object, String, SQLFragment)], which +/// manages deduplication, token substitution, and correct ordering of WITH clauses +/// across nested and combined fragments. +/// +/// Enforces basic SQL injection safeguards by rejecting unmatched quotes and +/// semicolons in appended text. +public class SQLFragment implements Appendable, CharSequence +{ + private static final Logger LOG = LogHelper.getLogger(SQLFragment.class, "SQL injection safety net diagnostics"); + + public static final String FEATUREFLAG_DISABLE_STRICT_CHECKS = "SQLFragmentDisableStrictChecks"; + + private String sql; + private StringBuilder sb = null; + private List params; // TODO: Should be List + + private final List tempTokens = new ArrayList<>(); // Hold refs to ensure they're not GC'd + + // use ordered map to make sql generation more deterministic (see collectCommonTableExpressions()) + private LinkedHashMap commonTableExpressionsMap = null; + + private static class CTE + { + CTE(@NotNull SqlDialect dialect, @NotNull String name) + { + this.dialect = dialect; + this.preferredName = name; + tokens.add("/*$*/" + GUID.makeGUID() + ":" + name + "/*$*/"); + } + + CTE(@NotNull SqlDialect dialect, @NotNull String name, SQLFragment sqlf, boolean recursive) + { + this(dialect, name); + this.sqlf = sqlf; + this.recursive = recursive; + } + + CTE(CTE from) + { + this.dialect = from.dialect; + this.preferredName = from.preferredName; + this.tokens.addAll(from.tokens); + this.sqlf = from.sqlf; + this.recursive = from.recursive; + } + + public CTE copy(boolean deep) + { + CTE copy = new CTE(this); + if (deep) + copy.sqlf = new SQLFragment().append(copy.sqlf); + return copy; + } + + private String token() + { + return tokens.iterator().next(); + } + + private final @NotNull SqlDialect dialect; + final String preferredName; + boolean recursive = false; // NOTE this is dialect dependant (getSql() does not take a dialect) + final Set tokens = new TreeSet<>(); + SQLFragment sqlf = null; + } + + public SQLFragment() + { + sql = ""; + } + + public SQLFragment(CharSequence charseq, @Nullable List params) + { + if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || + (StringUtils.countMatches(charseq, '\"') % 2) != 0 || + StringUtils.contains(charseq, ';')) + { + throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); + } + + this.sql = charseq.toString(); + if (null != params) + this.params = new ArrayList<>(params); + } + + + public SQLFragment(CharSequence sql, Object... params) + { + this(sql, Arrays.asList(params)); + } + + + public SQLFragment(SQLFragment other) + { + this(other,false); + } + + + public SQLFragment(SQLFragment other, boolean deep) + { + sql = other.getSqlCharSequence().toString(); + if (null != other.params) + addAll(other.params); + if (null != other.commonTableExpressionsMap && !other.commonTableExpressionsMap.isEmpty()) + { + if (null == this.commonTableExpressionsMap) + this.commonTableExpressionsMap = new LinkedHashMap<>(); + for (Map.Entry e : other.commonTableExpressionsMap.entrySet()) + { + CTE cte = e.getValue().copy(deep); + this.commonTableExpressionsMap.put(e.getKey(),cte); + } + } + this.tempTokens.addAll(other.tempTokens); + } + + + @Override + public boolean isEmpty() + { + return (null == sb || sb.isEmpty()) && (sql == null || sql.isEmpty()); + } + + + /* same as getSQL() but without CTE handling */ + public String getRawSQL() + { + return null != sb ? sb.toString() : null != sql ? sql : ""; + } + + /* + * Directly set the current SQL. + * + * This is useful for wrapping existing SQL, for instance adding a cast + * Obviously parameter number and order must remain unchanged + * + * This can also be used for processing sql scripts (e.g. module .sql update scripts) + */ + public SQLFragment setSqlUnsafe(String unsafe) + { + this.sql = unsafe; + this.sb = null; + return this; + } + + public static SQLFragment unsafe(String unsafe) + { + return new SQLFragment().setSqlUnsafe(unsafe); + } + + + private String replaceCteTokens(String self, String select, List> ctes) + { + for (Pair pair : ctes) + { + String alias = pair.first; + CTE cte = pair.second; + for (String token : cte.tokens) + { + select = Strings.CS.replace(select, token, alias); + } + } + if (null != self) + select = Strings.CS.replace(select, "$SELF$", self); + return select; + } + + + private List collectCommonTableExpressions() + { + List list = new ArrayList<>(); + _collectCommonTableExpressions(list); + return list; + } + + private void _collectCommonTableExpressions(List list) + { + if (null != commonTableExpressionsMap) + { + commonTableExpressionsMap.values().forEach(cte -> cte.sqlf._collectCommonTableExpressions(list)); + list.addAll(commonTableExpressionsMap.values()); + } + } + + + public String getSQL() + { + if (null == commonTableExpressionsMap || commonTableExpressionsMap.isEmpty()) + return null != sb ? sb.toString() : null != sql ? sql : ""; + + List commonTableExpressions = collectCommonTableExpressions(); + assert !commonTableExpressions.isEmpty(); + + boolean recursive = commonTableExpressions.stream() + .anyMatch(cte -> cte.recursive); + StringBuilder ret = new StringBuilder("WITH" + (recursive ? " RECURSIVE" : "")); + + // generate final aliases for each CTE */ + SqlDialect dialect = Objects.requireNonNull(commonTableExpressions.getFirst().dialect); + AliasManager am = new AliasManager(dialect); + List> ctes = commonTableExpressions.stream() + .map(cte -> new Pair<>(am.decideAlias(cte.preferredName),cte)) + .collect(Collectors.toList()); + + String comma = "\n/*CTE*/\n\t"; + for (Pair p : ctes) + { + String alias = p.first; + CTE cte = p.second; + SQLFragment expr = cte.sqlf; + String sql = expr._getOwnSql(alias, ctes); + ret.append(comma).append(alias).append(" AS (").append(sql).append(")"); + comma = "\n,/*CTE*/\n\t"; + } + ret.append("\n"); + + String select = _getOwnSql( null, ctes ); + ret.append(replaceCteTokens(null, select, ctes)); + return ret.toString(); + } + + + private String _getOwnSql(String alias, List> ctes) + { + String ownSql = null != sb ? sb.toString() : null != this.sql ? this.sql : ""; + return replaceCteTokens(alias, ownSql, ctes); + } + + + static Pattern markerPattern = Pattern.compile("/\\*\\$\\*/.*/\\*\\$\\*/"); + + /* This is not an exhaustive .equals() test, but it give pretty good confidence that these statements are the same */ + static boolean debugCompareSQL(SQLFragment sql1, SQLFragment sql2) + { + String select1 = sql1.getRawSQL(); + String select2 = sql2.getRawSQL(); + + if ((null == sql1.commonTableExpressionsMap || sql1.commonTableExpressionsMap.isEmpty()) && + (null == sql2.commonTableExpressionsMap || sql2.commonTableExpressionsMap.isEmpty())) + return select1.equals(select2); + + select1 = markerPattern.matcher(select1).replaceAll("CTE"); + select2 = markerPattern.matcher(select2).replaceAll("CTE"); + if (!select1.equals(select2)) + return false; + + Set ctes1 = sql1.commonTableExpressionsMap.values().stream() + .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) + .collect(Collectors.toSet()); + Set ctes2 = sql2.commonTableExpressionsMap.values().stream() + .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) + .collect(Collectors.toSet()); + return ctes1.equals(ctes2); + } + + + // It is a little confusing that getString() does not return the same charsequence that this object purports to + // represent. However, this is a good "display value" for this object. + // see getSqlCharSequence() + @NotNull + public String toString() + { + return "SQLFragment@" + System.identityHashCode(this) + "\n" + toDebugString(); + } + + // Not recommended -- this uses the LabKey scope to dictate parsing of identifiers and string literals... which + // might not be correct for the incoming SQL. + @Deprecated // Use the variant below that takes a SqlDialect + public String toDebugString() + { + return toDebugString(DbScope.getLabKeyScope().getSqlDialect()); + } + + public String toDebugString(SqlDialect dialect) + { + return JdbcUtil.format(this, dialect); + } + + public List getParams() + { + var ctes = collectCommonTableExpressions(); + List ret = new ArrayList<>(); + + for (var cte : ctes) + ret.addAll(cte.sqlf.getParamsNoCTEs()); + ret.addAll(getParamsNoCTEs()); + return Collections.unmodifiableList(ret); + } + + + public List> getParamsWithFragments() + { + var ctes = collectCommonTableExpressions(); + List> ret = new ArrayList<>(); + + for (CTE cte : ctes) + { + if (null != cte.sqlf && null != cte.sqlf.params) + { + for (int i = 0; i < cte.sqlf.params.size(); i++) + { + ret.add(new Pair<>(cte.sqlf, i)); + } + } + } + + if (null != params) + { + for (int i = 0; i < params.size(); i++) + { + ret.add(new Pair<>(this, i)); + } + } + return ret; + } + + private final static Object[] EMPTY_ARRAY = new Object[0]; + + public Object[] getParamsArray() + { + return null == params ? EMPTY_ARRAY : params.toArray(); + } + + public List getParamsNoCTEs() + { + return params == null ? Collections.emptyList() : Collections.unmodifiableList(params); + } + + private List getMutableParams() + { + if (!(params instanceof ArrayList)) + { + List t = new ArrayList<>(); + if (params != null) + t.addAll(params); + params = t; + } + return params; + } + + + private StringBuilder getStringBuilder() + { + if (null == sb) + sb = new StringBuilder(null==sql?"":sql); + return sb; + } + + + @Override + public SQLFragment append(CharSequence charseq) + { + if (null == charseq) + return this; + + if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || + (StringUtils.countMatches(charseq, '\"') % 2) != 0 || + StringUtils.contains(charseq, ';')) + { + throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); + } + + getStringBuilder().append(charseq); + return this; + } + + public SQLFragment appendIdentifier(DatabaseIdentifier id) + { + return append(id.getSql()); + } + + /** Functionally the same as append(CharSequence). This method just has different asserts */ + public SQLFragment appendIdentifier(CharSequence charseq) + { + if (null == charseq) + return this; + if (charseq instanceof SQLFragment sqlf) + { + if (0 != sqlf.getParamsArray().length) + throw new IllegalStateException("Unexpected SQL in appendIdentifier()"); + charseq = sqlf.getRawSQL(); + } + + String identifier = charseq.toString().strip(); + + if (STR_TABLE_ALIAS.equals(identifier)) + { + getStringBuilder().append(identifier); + return this; + } + + boolean malformed; + boolean quoteWrapped = identifier.length() >= 2 && identifier.startsWith("\"") && identifier.endsWith("\""); + if (quoteWrapped) + malformed = (StringUtils.countMatches(identifier, '\"') % 2) != 0; + else if (identifier.length() >= 2 && identifier.startsWith("`") && identifier.endsWith("`")) + malformed = (StringUtils.countMatches(identifier, '`') % 2) != 0; + else + malformed = StringUtils.containsAny(identifier, "*/\\'\"`?;- \t\n"); + if (malformed) + throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) value appears to be incorrectly formatted: " + identifier); + + // A quote-wrapped value must be a well-formed quoted identifier, or dotted sequence of them. + if (quoteWrapped && !isQuotedIdentifierSequence(identifier)) + { + if (AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + LOG.warn("appendIdentifier strict pre-quoted check would have rejected (flag-on, allowed): {}", identifier); + else + { + LOG.warn("appendIdentifier strict pre-quoted check rejected (flag-off): {}", identifier); + throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) pre-quoted value is not a well-formed quoted identifier (or dotted sequence of them): " + identifier); + } + } + + getStringBuilder().append(charseq); + return this; + } + + // True iff the value is one or more double-quote-delimited identifiers joined by single dots, + // e.g. "a", "a"."b", "schema"."table". A literal quote within a segment must be escaped as "". + // A '.' inside quotes is part of the name; a '.' between segments is a separator. Anything else + // outside the quotes -- stray text, a lone (breakout) quote, an unterminated segment -- is rejected. + private static boolean isQuotedIdentifierSequence(String s) + { + int i = 0; + int n = s.length(); + while (i < n) + { + if (s.charAt(i) != '"') // each segment must open with a quote + return false; + i++; + boolean closed = false; + while (i < n) + { + if (s.charAt(i) == '"') + { + if (i + 1 < n && s.charAt(i + 1) == '"') // escaped "" -- part of the name + { + i += 2; + continue; + } + i++; // closing quote + closed = true; + break; + } + i++; // any other char is legal inside quotes + } + if (!closed) // ran off the end without a closing quote + return false; + if (i == n) // end of a valid final segment + return true; + if (s.charAt(i) != '.') // segments must be separated by a single dot + return false; + i++; // consume the separator and require another segment + } + return false; // trailing dot with no following segment + } + + // just to save some typing + public SQLFragment appendDottedIdentifiers(CharSequence table, DatabaseIdentifier col) + { + return appendIdentifier(table).append(".").appendIdentifier(col); + } + + // just to save some typing + public SQLFragment appendDottedIdentifiers(CharSequence... ids) + { + var dot = ""; + for (var id : ids) + { + append(dot).appendIdentifier(id); + dot = "."; + } + return this; + } + + /** append End Of Statement */ + public SQLFragment appendEOS() + { + getStringBuilder().append(";\n"); + return this; + } + + + @Override + public SQLFragment append(CharSequence csq, int start, int end) + { + append(csq.subSequence(start, end)); + return this; + } + + /** Adds the container's ID as an in-line string constant to the SQL */ + public SQLFragment appendValue(Container c) + { + if (null == c) + return appendNull(); + return appendValue(c, null); + } + + public SQLFragment appendValue(@NotNull Container c, SqlDialect dialect) + { + appendValue(c.getEntityId(), dialect); + String name = c.getName(); + if (!StringUtils.containsAny(name,"*/\\'\"?")) + append("/* ").append(name).append(" */"); + return this; + } + + public SQLFragment appendNull() + { + getStringBuilder().append("NULL"); + return this; + } + + public SQLFragment appendValue(Boolean B, @NotNull SqlDialect dialect) + { + if (null == B) + return append("CAST(NULL AS ").append(dialect.getBooleanDataType()).append(")"); + getStringBuilder().append(B ? dialect.getBooleanTRUE() : dialect.getBooleanFALSE()); + return this; + } + + public SQLFragment appendValue(Integer I) + { + if (null == I) + return appendNull(); + getStringBuilder().append(I.intValue()); + return this; + } + + public SQLFragment appendValue(int i) + { + getStringBuilder().append(i); + return this; + } + + + public SQLFragment appendValue(Long L) + { + if (null == L) + return appendNull(); + getStringBuilder().append((long)L); + return this; + } + + public SQLFragment appendValue(long l) + { + getStringBuilder().append(l); + return this; + } + + public SQLFragment appendValue(Float F) + { + if (null == F) + return appendNull(); + return appendValue(F.floatValue()); + } + + public SQLFragment appendValue(float f) + { + if (Float.isFinite(f)) + { + getStringBuilder().append(f); + } + else + { + getStringBuilder().append("?"); + add(f); + } + return this; + } + + public SQLFragment appendValue(Double D) + { + if (null == D) + return appendNull(); + else + return appendValue(D.doubleValue()); + } + + public SQLFragment appendValue(double d) + { + if (Double.isFinite(d)) + { + getStringBuilder().append(d); + } + else + { + getStringBuilder().append("?"); + add(d); + } + return this; + } + + public SQLFragment appendValue(Number N) + { + if (null == N) + return appendNull(); + + if (N instanceof Quantity q) + N = q.value(); + + if (N instanceof BigDecimal || N instanceof BigInteger || N instanceof Long) + { + getStringBuilder().append(N); + } + else if (Double.isFinite(N.doubleValue())) + { + getStringBuilder().append(N); + } + else + { + getStringBuilder().append(" ? "); + add(N); + } + return this; + } + + public final SQLFragment appendNowTimestamp() + { + return appendValue(new NowTimestamp()); + } + + // Issue 27534: Stop using {fn now()} in function declarations + // Issue 48864: Query Table's use of web server time can cause discrepancies in created/modified timestamps + public final SQLFragment appendValue(NowTimestamp now) + { + if (null == now) + return appendNull(); + getStringBuilder().append("CURRENT_TIMESTAMP"); + return this; + } + + public final SQLFragment appendValue(java.util.Date d) + { + if (null == d) + return appendNull(); + if (d.getClass() == java.util.Date.class) + getStringBuilder().append("{ts '").append(new Timestamp(d.getTime())).append("'}"); + else if (d.getClass() == java.sql.Timestamp.class) + getStringBuilder().append("{ts '").append(d).append("'}"); + else if (d.getClass() == java.sql.Date.class) + getStringBuilder().append("{d '").append(d).append("'}"); + else + throw new IllegalStateException("Unexpected date type: " + d.getClass().getName()); + return this; + } + + public SQLFragment appendValue(GUID g) + { + return appendValue(g, null); + } + + public SQLFragment appendValue(GUID g, SqlDialect d) + { + if (null == g) + return appendNull(); + // doesn't need StringHandler, just hex and hyphen + String sqlGUID = "'" + g + "'"; + // I'm testing dialect type, because some dialects do not support getGuidType(), and postgers uses VARCHAR anyway + if (null != d && d.isSqlServer()) + getStringBuilder().append("CAST(").append(sqlGUID).append(" AS UNIQUEIDENTIFIER)"); + else + getStringBuilder().append(sqlGUID); + return this; + } + + public SQLFragment appendValue(Enum e) + { + if (null == e) + return appendNull(); + String name = e.name(); + // Enum.name() returns a legal Java identifier per JLS, so none of these characters can appear + // in practice. Defense in depth: reject anything SQL-active rather than only the apostrophe. + if (StringUtils.containsAny(name, "'\"\\;\r\n")) + throw new IllegalStateException("Unexpected character in Enum.name(): " + name); + getStringBuilder().append("'").append(name).append("'"); + return this; + } + + public SQLFragment append(FieldKey fk) + { + if (null == fk) + return appendNull(); + append(String.valueOf(fk)); + return this; + } + + + /** Adds the object as a JDBC parameter value */ + public SQLFragment add(Object p) + { + getMutableParams().add(p); + return this; + } + + public SQLFragment add(Object p, JdbcType type) + { + getMutableParams().add(new Parameter.TypedValue(p, type)); + return this; + } + + /** Adds the objects as JDBC parameter values */ + public SQLFragment addAll(Collection l) + { + getMutableParams().addAll(l); + return this; + } + + + /** Adds the objects as JDBC parameter values */ + public SQLFragment addAll(Object... values) + { + if (values == null) + return this; + addAll(Arrays.asList(values)); + return this; + } + + + /** Sets the parameter at the index to the object's value */ + public void set(int i, Object p) + { + getMutableParams().set(i,p); + } + + /** Append both the SQL and the parameters from the other SQLFragment to this SQLFragment */ + public SQLFragment append(SQLFragment f) + { + if (null != f.sb) + getStringBuilder().append(f.sb); + else + getStringBuilder().append(f.sql); + if (null != f.params) + addAll(f.params); + mergeCommonTableExpressions(f); + tempTokens.addAll(f.tempTokens); + return this; + } + + public SQLFragment append(@NotNull Iterable fragments, @NotNull String separator) + { + String s = ""; + for (SQLFragment fragment : fragments) + { + append(s); + s = separator; + append(fragment); + } + return this; + } + + // return boolean so this can be used in an assert. passing in a dialect is not ideal, but parsing comments out + // before submitting the fragment is not reliable and holding statements & comments separately (to eliminate the + // need to parse them) isn't particularly easy... so punt for now. + public boolean appendComment(String comment, SqlDialect dialect) + { + if (dialect.supportsComments()) + { + StringBuilder sb = getStringBuilder(); + int len = sb.length(); + if (len > 0 && sb.charAt(len-1) != '\n') + sb.append('\n'); + sb.append("\n-- "); + boolean truncated = comment.length() > 1000; + if (truncated) + comment = StringUtilsLabKey.leftSurrogatePairFriendly(comment, 1000); + // Strip CR/LF so an embedded newline can't terminate the `--` comment and turn the rest + // of the payload into live SQL. + comment = comment.replace('\r', ' ').replace('\n', ' '); + sb.append(comment); + if (StringUtils.countMatches(comment, "'")%2==1) + sb.append("'"); + if (truncated) + sb.append("..."); + sb.append('\n'); + } + return true; + } + + + /** see also append(TableInfo, String alias) */ + public SQLFragment append(TableInfo table) + { + SQLFragment s = table.getSQLName(); + if (s != null) + return append(s); + + // GitHub 1432: Cannot delete a multi-choice option on a source type + String alias = AliasManager.makeLegalName(table.getName(), table.getSchema().getSqlDialect()); + return append(table.getFromSQL(alias)); + } + + /** Add a table/query to the SQL with an alias, as used in a FROM clause */ + public SQLFragment append(TableInfo table, String alias) + { + return append(table.getFromSQL(alias)); + } + + /** Add to the SQL */ + @Override + public SQLFragment append(char ch) + { + getStringBuilder().append(ch); + return this; + } + + /** This is like appendValue(CharSequence s), but force use of literal syntax + * CAUTIONARY NOTE: String literals in PostgresSQL are tricky because of overloaded functions + * array_agg('string') fails array_agg('string'::VARCHAR) works + * json_object('{}) works json_object('string'::VARCHAR) fails + * In the case of json_object() it expects TEXT. Postgres will promote 'json' to TEXT, but not 'json'::VARCHAR + */ + public SQLFragment appendStringLiteral(CharSequence s, @NotNull SqlDialect d) + { + if (null==s) + return appendNull(); + getStringBuilder().append(d.getStringHandler().quoteStringLiteral(s.toString())); + return this; + } + + /** Add to the SQL as either an in-line string literal or as a JDBC parameter depending on whether it would need escaping */ + public SQLFragment appendValue(CharSequence s) + { + return appendValue(s, null); + } + + public SQLFragment appendValue(CharSequence s, SqlDialect d) + { + if (null==s) + return appendNull(); + if (null==d || s.length() > 200) + return append("?").add(s.toString()); + appendStringLiteral(s, d); + return this; + } + + public SQLFragment appendInClause(@NotNull Collection params, SqlDialect dialect) + { + dialect.appendInClauseSql(this, params); + return this; + } + + public CharSequence getSqlCharSequence() + { + if (null != sb) + { + return sb; + } + return sql; + } + + public void insert(int index, SQLFragment sql) + { + if (!sql.getParams().isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with parameters - they must be inserted/merged separately"); + } + if (sql.commonTableExpressionsMap != null && !sql.commonTableExpressionsMap.isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with CTEs - they must be inserted/merged separately"); + } + if (!tempTokens.isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with temp tokens - they must be inserted/merged separately"); + } + getStringBuilder().insert(index, sql.getRawSQL()); + } + + /** Insert into the SQL */ + public void insert(int index, String str) + { + if ((StringUtils.countMatches(str, '\'') % 2) != 0 || + (StringUtils.countMatches(str, '\"') % 2) != 0 || + StringUtils.contains(str, ';')) + { + throw new IllegalArgumentException("SQLFragment.insert(int,String) does not allow semicolons or unmatched quotes"); + } + + getStringBuilder().insert(index, str); + } + + /** Insert this SQLFragment's SQL and parameters at the start of the existing SQL and parameters */ + public void prepend(SQLFragment sql) + { + getStringBuilder().insert(0, sql.getSqlCharSequence().toString()); + if (null != sql.params) + getMutableParams().addAll(0, sql.params); + mergeCommonTableExpressions(sql); + } + + + public int indexOf(String str) + { + return getStringBuilder().indexOf(str); + } + + + // Display query in "English" (display SQL with params substituted) + // with a little more work could probably be made to be SQL legal + public String getFilterText() + { + String sql = getSQL().replaceFirst("WHERE ", ""); + List params = getParams(); + for (Object param1 : params) + { + String param = param1.toString(); + param = param.replaceAll("\\\\", "\\\\\\\\"); + param = param.replaceAll("\\$", "\\\\\\$"); + sql = sql.replaceFirst("\\?", param); + } + return sql.replace("\"", ""); + } + + + @Override + public char charAt(int index) + { + return getSqlCharSequence().charAt(index); + } + + @Override + public int length() + { + return getSqlCharSequence().length(); + } + + @Override + public @NotNull CharSequence subSequence(int start, int end) + { + return getSqlCharSequence().subSequence(start, end); + } + + /** + * KEY is used as a faster way to look for equivalent CTE expressions. + * returning a name here allows us to potentially merge CTE at add time + * + * if you don't have a key you can just use sqlf.toString() + */ + public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf) + { + return addCommonTableExpression(dialect, key, proposedName, sqlf, false); + } + + public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf, boolean recursive) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + CTE prev = commonTableExpressionsMap.get(key); + if (null != prev) + return prev.token(); + CTE cte = new CTE(dialect, proposedName, sqlf, recursive); + commonTableExpressionsMap.put(key, cte); + return cte.token(); + } + + public String createCommonTableExpressionToken(SqlDialect dialect, Object key, String proposedName) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + CTE prev = commonTableExpressionsMap.get(key); + if (null != prev) + throw new IllegalStateException("Cannot create CTE token from already used key."); + CTE cte = new CTE(dialect ,proposedName); + commonTableExpressionsMap.put(key, cte); + return cte.token(); + } + + public void setCommonTableExpressionSql(Object key, SQLFragment sqlf, boolean recursive) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + + if (null != sqlf.commonTableExpressionsMap && !sqlf.commonTableExpressionsMap.isEmpty()) + { + // Need to merge CTEs up; this.cte depends on newSql.ctes, so they need to come first + SQLFragment newSql = new SQLFragment(sqlf); + LinkedHashMap toMap = new LinkedHashMap<>(newSql.commonTableExpressionsMap); + for (Map.Entry e : commonTableExpressionsMap.entrySet()) + { + CTE from = e.getValue(); + CTE to = toMap.get(e.getKey()); + if (null != to) + to.tokens.addAll(from.tokens); + else + toMap.put(e.getKey(), from.copy(false)); + } + + commonTableExpressionsMap = toMap; + newSql.commonTableExpressionsMap = null; + sqlf = newSql; + } + + CTE cte = commonTableExpressionsMap == null ? null : commonTableExpressionsMap.get(key); + if (null == cte) + throw new IllegalStateException("CTE not found."); + cte.sqlf = sqlf; + cte.recursive = recursive; + } + + + private void mergeCommonTableExpressions(SQLFragment sqlFrom) + { + if (null == sqlFrom.commonTableExpressionsMap || sqlFrom.commonTableExpressionsMap.isEmpty()) + return; + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + for (Map.Entry e : sqlFrom.commonTableExpressionsMap.entrySet()) + { + CTE from = e.getValue(); + CTE to = commonTableExpressionsMap.get(e.getKey()); + if (null != to) + to.tokens.addAll(from.tokens); + else + commonTableExpressionsMap.put(e.getKey(), from.copy(false)); + } + } + + + public void addTempToken(Object tempToken) + { + tempTokens.add(tempToken); + } + + public void addTempTokens(SQLFragment other) + { + tempTokens.add(other.tempTokens); + } + + public static SQLFragment prettyPrint(SQLFragment from) + { + SQLFragment sqlf = new SQLFragment(from); + + String s = from.getSqlCharSequence().toString(); + StringBuilder sb = new StringBuilder(s.length() + 200); + String[] lines = StringUtils.split(s, '\n'); + int indent = 0; + + for (String line : lines) + { + String t = line.trim(); + + if (t.isEmpty()) + continue; + + if (t.startsWith("-- params = b.getParams(); + assertEquals(2,params.size()); + assertEquals(5, params.get(0)); + assertEquals("xxyzzy", params.get(1)); + + + SQLFragment c = new SQLFragment(b); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT a FROM b WHERE x=?) + SELECT * FROM CTE WHERE y=?""", + c.getSQL()); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT a FROM b WHERE x=5) + SELECT * FROM CTE WHERE y='xxyzzy'""", + filterDebugString(c.toDebugString())); + params = c.getParams(); + assertEquals(2,params.size()); + assertEquals(5, params.get(0)); + assertEquals("xxyzzy", params.get(1)); + + + // combining + + SQLFragment sqlf = new SQLFragment(); + String token = sqlf.addCommonTableExpression(dialect, "KEY_A", "cte1", new SQLFragment("SELECT * FROM a")); + sqlf.append("SELECT * FROM ").append(token).append(" _1"); + + assertEquals(""" + WITH + /*CTE*/ + \tcte1 AS (SELECT * FROM a) + SELECT * FROM cte1 _1""", + sqlf.getSQL()); + + SQLFragment sqlf2 = new SQLFragment(); + String token2 = sqlf2.addCommonTableExpression(dialect, "KEY_A", "cte2", new SQLFragment("SELECT * FROM a")); + sqlf2.append("SELECT * FROM ").append(token2).append(" _2"); + assertEquals(""" + WITH + /*CTE*/ + \tcte2 AS (SELECT * FROM a) + SELECT * FROM cte2 _2""", + sqlf2.getSQL()); + + SQLFragment sqlf3 = new SQLFragment(); + String token3 = sqlf3.addCommonTableExpression(dialect, "KEY_B", "cte3", new SQLFragment("SELECT * FROM b")); + sqlf3.append("SELECT * FROM ").append(token3).append(" _3"); + assertEquals(""" + WITH + /*CTE*/ + \tcte3 AS (SELECT * FROM b) + SELECT * FROM cte3 _3""", + sqlf3.getSQL()); + + SQLFragment union = new SQLFragment(); + union.append(sqlf); + union.append("\nUNION\n"); + union.append(sqlf2); + union.append("\nUNION\n"); + union.append(sqlf3); + assertEquals(""" + WITH + /*CTE*/ + \tcte1 AS (SELECT * FROM a) + ,/*CTE*/ + \tcte3 AS (SELECT * FROM b) + SELECT * FROM cte1 _1 + UNION + SELECT * FROM cte1 _2 + UNION + SELECT * FROM cte3 _3""", + union.getSQL()); + } + + @Test + public void nested_cte() + { + // one-level cte using cteToken (CTE fragment 'a' does not contain a CTE) + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); + assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); + SQLFragment b = new SQLFragment(); + String cteToken = b.addCommonTableExpression(dialect, new Object(), "CTE", a); + b.append("SELECT * FROM ").append(cteToken).append(" WHERE p=?").add("parameterTWO"); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) + SELECT * FROM CTE WHERE p='parameterTWO'""", + filterDebugString(b.toDebugString())); + assertEquals("parameterONE", b.getParams().getFirst()); + } + + // two-level cte using cteTokens (CTE fragment 'b' contains a CTE of fragment a) + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); + assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterTWO"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + c.append("SELECT * FROM ").append(cteTokenB).append(" WHERE i=?").add(3); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterTWO') + SELECT * FROM B_ WHERE i=3""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(3, params.size()); + assertEquals("parameterONE", params.get(0)); + assertEquals("parameterTWO", params.get(1)); + assertEquals(3, params.get(2)); + } + + // Same as previous but top-level query has both a nested and non-nested CTE + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); + SQLFragment a2 = new SQLFragment("SELECT 2 as i, 'Atwo' as s, CAST(? AS VARCHAR) as p", "parameterAtwo"); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + String cteTokenA2 = c.addCommonTableExpression(dialect, new Object(), "A2_", a2); + c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') + ,/*CTE*/ + \tA2_ AS (SELECT 2 as i, 'Atwo' as s, CAST('parameterAtwo' AS VARCHAR) as p) + SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(4, params.size()); + assertEquals("parameterAone", params.get(0)); + assertEquals("parameterB", params.get(1)); + assertEquals("parameterAtwo", params.get(2)); + assertEquals(4, params.get(3)); + } + + // Same as previous but two of the CTEs are the same and should be collapsed (e.g. imagine a container filter implemented with a CTE) + // TODO, we only collapse CTEs that are siblings + { + SQLFragment cf = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, "CTE_KEY_CF", "A_", cf); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + String cteTokenA2 = c.addCommonTableExpression(dialect, "CTE_KEY_CF", "A2_", cf); + c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') + ,/*CTE*/ + \tA2_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(4, params.size()); + assertEquals("parameterAone", params.get(0)); + assertEquals("parameterB", params.get(1)); + assertEquals("parameterAone", params.get(2)); + assertEquals(4, params.get(3)); + } + } + + + private void shouldFail(Runnable r) + { + try + { + r.run(); + fail("Expected IllegalArgumentException"); + } + catch (IllegalArgumentException e) + { + // expected + } + } + + + @Test + public void testIllegalArgument() + { + shouldFail(() -> new SQLFragment(";")); + shouldFail(() -> new SQLFragment().append(";")); + shouldFail(() -> new SQLFragment("AND name='")); + shouldFail(() -> new SQLFragment().append("AND name = '")); + shouldFail(() -> new SQLFragment().append("AND name = 'Robert'); DROP TABLE Students; --")); + + shouldFail(() -> new SQLFragment().appendIdentifier("column name")); + shouldFail(() -> new SQLFragment().appendIdentifier("?")); + shouldFail(() -> new SQLFragment().appendIdentifier(";")); + shouldFail(() -> new SQLFragment().appendIdentifier("\"column\"name\"")); + } + + + String mysqlQuoteIdentifier(String id) + { + return "`" + id.replace("`", "``") + "`"; + } + + @Test + public void testMysql() + { + // OK + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("mysql")); + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my`sql")); + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my\"sql")); + + // not OK + shouldFail(() -> new SQLFragment().appendIdentifier("`")); + shouldFail(() -> new SQLFragment().appendIdentifier("`a`a`")); + } + + @Test + public void testAppendCommentStripsNewlines() + { + // PR1-P3: an embedded newline in a comment payload must not terminate the `-- ` comment + // line and expose the trailing text as live SQL. + if (!dialect.supportsComments()) + return; + + SQLFragment sqlf = new SQLFragment(); + sqlf.appendComment("hello\nDROP TABLE x;--", dialect); + String sql = sqlf.getSQL(); + + assertFalse("appendComment leaked an embedded newline into emitted SQL: " + sql, sql.contains("hello\nDROP")); + assertTrue("appendComment should keep the payload on one comment line: " + sql, sql.contains("hello DROP TABLE x;--")); + + // CR is stripped too + SQLFragment sqlf2 = new SQLFragment(); + sqlf2.appendComment("hi\rDROP TABLE x;--", dialect); + assertFalse("appendComment leaked an embedded CR: " + sqlf2.getSQL(), sqlf2.getSQL().contains("hi\rDROP")); + } + + @Test + public void testAppendIdentifierPreQuotedDottedAccepted() + { + // PR2-G1: a dotted sequence of individually well-formed quoted identifiers is legitimate -- + // e.g. a fully-qualified "schema"."table" returned by TableInfo.getSelectName(). The strict + // check validates each quoted segment rather than blanket-rejecting any interior dot. + new SQLFragment().appendIdentifier("\"schema\".\"table\""); + new SQLFragment().appendIdentifier("\"a\".\"b\".\"c\""); + } + + @Test + public void testAppendIdentifierPreQuotedDotInsideQuotesAccepted() + { + // PR2-G1: a `.` *inside* the quotes is part of a single identifier's name, not a separator. + // This is the metadata-name case (e.g. a calculated column's generated class name) that the + // earlier strict check wrongly rejected. + new SQLFragment().appendIdentifier("\"name.with.dots\""); + new SQLFragment().appendIdentifier("\"org.labkey.query.sql.CalculatedExpressionColumn6b8f\""); + } + + @Test + public void testAppendIdentifierPreQuotedInteriorQuoteRejected() + { + // PR2-G1: an interior `"` that isn't part of the standard `""` doubling is a breakout. + // Even quote count alone is insufficient. + shouldFail(() -> new SQLFragment().appendIdentifier("\"foo\"bar\"baz\"")); + // A classic breakout: close the identifier early, then inject. The lone interior quote + // (after `foo`) is followed by non-separator text, so it must be rejected. + shouldFail(() -> new SQLFragment().appendIdentifier("\"foo\"; DROP TABLE x; --\"")); + // Stray text after a valid segment (not a `.` separator). + shouldFail(() -> new SQLFragment().appendIdentifier("\"a\" \"b\"")); + // Trailing separator with no following segment. + shouldFail(() -> new SQLFragment().appendIdentifier("\"a\".\"")); + // Empty segment between separators. + shouldFail(() -> new SQLFragment().appendIdentifier("\"a\"..\"b\"")); + } + + @Test + public void testAppendIdentifierPreQuotedValidCases() + { + // PR2-G1: legitimate pre-quoted identifiers still pass. + new SQLFragment().appendIdentifier("\"my_table\""); // simple quoted + new SQLFragment().appendIdentifier("\"with a space\""); // whitespace inside quotes is fine + new SQLFragment().appendIdentifier("\"foo\"\"bar\""); // embedded literal " via "" doubling + new SQLFragment().appendIdentifier("\"contains\"\"more\"\"doubles\""); + // Doubled interior quotes around a dot decode to a single identifier named weird"."name -- + // distinct from the two-segment "weird"."name" -- and cannot break out, so it's accepted. + new SQLFragment().appendIdentifier("\"weird\"\".\"\"name\""); + } + } + + @Override + public boolean equals(Object obj) + { + if (!(obj instanceof SQLFragment other)) + { + return false; + } + return getSQL().equals(other.getSQL()) && getParams().equals(other.getParams()); + } + + @Override + public int hashCode() + { + return Objects.hash(getSQL(), getParams()); + } + + /** + * Joins the SQLFragments in the provided {@code Iterable} into a single SQLFragment. The SQL is joined by string + * concatenation using the provided separator. The parameters are combined to form the new parameter list. + * + * @param fragments SQLFragments to join together + * @param separator Separator to use + * @return A new SQLFragment that joins all the SQLFragments + */ + public static SQLFragment join(Iterable fragments, SQLFragment separator) + { + SQLFragment join = new SQLFragment(); + boolean first = true; + + for (SQLFragment fragment : fragments) + { + if (first) + first = false; + else + join.append(separator); + join.append(fragment); + } + + return join; + } +} From c8c552d751376f8e9507ec0bd77ca51e69930325 Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 21 Aug 2026 18:56:07 -0700 Subject: [PATCH 2/2] selenium test --- api/src/org/labkey/api/data/SQLFragment.java | 3000 +++++++++--------- 1 file changed, 1500 insertions(+), 1500 deletions(-) diff --git a/api/src/org/labkey/api/data/SQLFragment.java b/api/src/org/labkey/api/data/SQLFragment.java index 1f89bf965ad..b088e446351 100644 --- a/api/src/org/labkey/api/data/SQLFragment.java +++ b/api/src/org/labkey/api/data/SQLFragment.java @@ -1,1500 +1,1500 @@ -/* - * Copyright (c) 2008-2026 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.data; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.query.AliasManager; -import org.labkey.api.query.FieldKey; -import org.labkey.api.settings.AppProps; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JdbcUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.logging.LogHelper; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import static org.labkey.api.query.ExprColumn.STR_TABLE_ALIAS; - -/// A composable SQL builder that pairs SQL text with its JDBC parameter values, ensuring -/// they travel together through query construction. Implements [Appendable] and -/// [CharSequence] for fluent assembly of SQL statements. -/// -/// Provides type-safe `appendValue()` methods for inlining literals of common -/// types (integers, strings, dates, GUIDs, etc.) and `add()` methods for binding -/// JDBC `?` parameters. Fragments can be composed via `append(SQLFragment)` to -/// merge both SQL text and parameter lists. -/// -/// Supports Common Table Expressions (CTEs) through -/// [#addCommonTableExpression(SqlDialect, Object, String, SQLFragment)], which -/// manages deduplication, token substitution, and correct ordering of WITH clauses -/// across nested and combined fragments. -/// -/// Enforces basic SQL injection safeguards by rejecting unmatched quotes and -/// semicolons in appended text. -public class SQLFragment implements Appendable, CharSequence -{ - private static final Logger LOG = LogHelper.getLogger(SQLFragment.class, "SQL injection safety net diagnostics"); - - public static final String FEATUREFLAG_DISABLE_STRICT_CHECKS = "SQLFragmentDisableStrictChecks"; - - private String sql; - private StringBuilder sb = null; - private List params; // TODO: Should be List - - private final List tempTokens = new ArrayList<>(); // Hold refs to ensure they're not GC'd - - // use ordered map to make sql generation more deterministic (see collectCommonTableExpressions()) - private LinkedHashMap commonTableExpressionsMap = null; - - private static class CTE - { - CTE(@NotNull SqlDialect dialect, @NotNull String name) - { - this.dialect = dialect; - this.preferredName = name; - tokens.add("/*$*/" + GUID.makeGUID() + ":" + name + "/*$*/"); - } - - CTE(@NotNull SqlDialect dialect, @NotNull String name, SQLFragment sqlf, boolean recursive) - { - this(dialect, name); - this.sqlf = sqlf; - this.recursive = recursive; - } - - CTE(CTE from) - { - this.dialect = from.dialect; - this.preferredName = from.preferredName; - this.tokens.addAll(from.tokens); - this.sqlf = from.sqlf; - this.recursive = from.recursive; - } - - public CTE copy(boolean deep) - { - CTE copy = new CTE(this); - if (deep) - copy.sqlf = new SQLFragment().append(copy.sqlf); - return copy; - } - - private String token() - { - return tokens.iterator().next(); - } - - private final @NotNull SqlDialect dialect; - final String preferredName; - boolean recursive = false; // NOTE this is dialect dependant (getSql() does not take a dialect) - final Set tokens = new TreeSet<>(); - SQLFragment sqlf = null; - } - - public SQLFragment() - { - sql = ""; - } - - public SQLFragment(CharSequence charseq, @Nullable List params) - { - if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || - (StringUtils.countMatches(charseq, '\"') % 2) != 0 || - StringUtils.contains(charseq, ';')) - { - throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); - } - - this.sql = charseq.toString(); - if (null != params) - this.params = new ArrayList<>(params); - } - - - public SQLFragment(CharSequence sql, Object... params) - { - this(sql, Arrays.asList(params)); - } - - - public SQLFragment(SQLFragment other) - { - this(other,false); - } - - - public SQLFragment(SQLFragment other, boolean deep) - { - sql = other.getSqlCharSequence().toString(); - if (null != other.params) - addAll(other.params); - if (null != other.commonTableExpressionsMap && !other.commonTableExpressionsMap.isEmpty()) - { - if (null == this.commonTableExpressionsMap) - this.commonTableExpressionsMap = new LinkedHashMap<>(); - for (Map.Entry e : other.commonTableExpressionsMap.entrySet()) - { - CTE cte = e.getValue().copy(deep); - this.commonTableExpressionsMap.put(e.getKey(),cte); - } - } - this.tempTokens.addAll(other.tempTokens); - } - - - @Override - public boolean isEmpty() - { - return (null == sb || sb.isEmpty()) && (sql == null || sql.isEmpty()); - } - - - /* same as getSQL() but without CTE handling */ - public String getRawSQL() - { - return null != sb ? sb.toString() : null != sql ? sql : ""; - } - - /* - * Directly set the current SQL. - * - * This is useful for wrapping existing SQL, for instance adding a cast - * Obviously parameter number and order must remain unchanged - * - * This can also be used for processing sql scripts (e.g. module .sql update scripts) - */ - public SQLFragment setSqlUnsafe(String unsafe) - { - this.sql = unsafe; - this.sb = null; - return this; - } - - public static SQLFragment unsafe(String unsafe) - { - return new SQLFragment().setSqlUnsafe(unsafe); - } - - - private String replaceCteTokens(String self, String select, List> ctes) - { - for (Pair pair : ctes) - { - String alias = pair.first; - CTE cte = pair.second; - for (String token : cte.tokens) - { - select = Strings.CS.replace(select, token, alias); - } - } - if (null != self) - select = Strings.CS.replace(select, "$SELF$", self); - return select; - } - - - private List collectCommonTableExpressions() - { - List list = new ArrayList<>(); - _collectCommonTableExpressions(list); - return list; - } - - private void _collectCommonTableExpressions(List list) - { - if (null != commonTableExpressionsMap) - { - commonTableExpressionsMap.values().forEach(cte -> cte.sqlf._collectCommonTableExpressions(list)); - list.addAll(commonTableExpressionsMap.values()); - } - } - - - public String getSQL() - { - if (null == commonTableExpressionsMap || commonTableExpressionsMap.isEmpty()) - return null != sb ? sb.toString() : null != sql ? sql : ""; - - List commonTableExpressions = collectCommonTableExpressions(); - assert !commonTableExpressions.isEmpty(); - - boolean recursive = commonTableExpressions.stream() - .anyMatch(cte -> cte.recursive); - StringBuilder ret = new StringBuilder("WITH" + (recursive ? " RECURSIVE" : "")); - - // generate final aliases for each CTE */ - SqlDialect dialect = Objects.requireNonNull(commonTableExpressions.getFirst().dialect); - AliasManager am = new AliasManager(dialect); - List> ctes = commonTableExpressions.stream() - .map(cte -> new Pair<>(am.decideAlias(cte.preferredName),cte)) - .collect(Collectors.toList()); - - String comma = "\n/*CTE*/\n\t"; - for (Pair p : ctes) - { - String alias = p.first; - CTE cte = p.second; - SQLFragment expr = cte.sqlf; - String sql = expr._getOwnSql(alias, ctes); - ret.append(comma).append(alias).append(" AS (").append(sql).append(")"); - comma = "\n,/*CTE*/\n\t"; - } - ret.append("\n"); - - String select = _getOwnSql( null, ctes ); - ret.append(replaceCteTokens(null, select, ctes)); - return ret.toString(); - } - - - private String _getOwnSql(String alias, List> ctes) - { - String ownSql = null != sb ? sb.toString() : null != this.sql ? this.sql : ""; - return replaceCteTokens(alias, ownSql, ctes); - } - - - static Pattern markerPattern = Pattern.compile("/\\*\\$\\*/.*/\\*\\$\\*/"); - - /* This is not an exhaustive .equals() test, but it give pretty good confidence that these statements are the same */ - static boolean debugCompareSQL(SQLFragment sql1, SQLFragment sql2) - { - String select1 = sql1.getRawSQL(); - String select2 = sql2.getRawSQL(); - - if ((null == sql1.commonTableExpressionsMap || sql1.commonTableExpressionsMap.isEmpty()) && - (null == sql2.commonTableExpressionsMap || sql2.commonTableExpressionsMap.isEmpty())) - return select1.equals(select2); - - select1 = markerPattern.matcher(select1).replaceAll("CTE"); - select2 = markerPattern.matcher(select2).replaceAll("CTE"); - if (!select1.equals(select2)) - return false; - - Set ctes1 = sql1.commonTableExpressionsMap.values().stream() - .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) - .collect(Collectors.toSet()); - Set ctes2 = sql2.commonTableExpressionsMap.values().stream() - .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) - .collect(Collectors.toSet()); - return ctes1.equals(ctes2); - } - - - // It is a little confusing that getString() does not return the same charsequence that this object purports to - // represent. However, this is a good "display value" for this object. - // see getSqlCharSequence() - @NotNull - public String toString() - { - return "SQLFragment@" + System.identityHashCode(this) + "\n" + toDebugString(); - } - - // Not recommended -- this uses the LabKey scope to dictate parsing of identifiers and string literals... which - // might not be correct for the incoming SQL. - @Deprecated // Use the variant below that takes a SqlDialect - public String toDebugString() - { - return toDebugString(DbScope.getLabKeyScope().getSqlDialect()); - } - - public String toDebugString(SqlDialect dialect) - { - return JdbcUtil.format(this, dialect); - } - - public List getParams() - { - var ctes = collectCommonTableExpressions(); - List ret = new ArrayList<>(); - - for (var cte : ctes) - ret.addAll(cte.sqlf.getParamsNoCTEs()); - ret.addAll(getParamsNoCTEs()); - return Collections.unmodifiableList(ret); - } - - - public List> getParamsWithFragments() - { - var ctes = collectCommonTableExpressions(); - List> ret = new ArrayList<>(); - - for (CTE cte : ctes) - { - if (null != cte.sqlf && null != cte.sqlf.params) - { - for (int i = 0; i < cte.sqlf.params.size(); i++) - { - ret.add(new Pair<>(cte.sqlf, i)); - } - } - } - - if (null != params) - { - for (int i = 0; i < params.size(); i++) - { - ret.add(new Pair<>(this, i)); - } - } - return ret; - } - - private final static Object[] EMPTY_ARRAY = new Object[0]; - - public Object[] getParamsArray() - { - return null == params ? EMPTY_ARRAY : params.toArray(); - } - - public List getParamsNoCTEs() - { - return params == null ? Collections.emptyList() : Collections.unmodifiableList(params); - } - - private List getMutableParams() - { - if (!(params instanceof ArrayList)) - { - List t = new ArrayList<>(); - if (params != null) - t.addAll(params); - params = t; - } - return params; - } - - - private StringBuilder getStringBuilder() - { - if (null == sb) - sb = new StringBuilder(null==sql?"":sql); - return sb; - } - - - @Override - public SQLFragment append(CharSequence charseq) - { - if (null == charseq) - return this; - - if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || - (StringUtils.countMatches(charseq, '\"') % 2) != 0 || - StringUtils.contains(charseq, ';')) - { - throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); - } - - getStringBuilder().append(charseq); - return this; - } - - public SQLFragment appendIdentifier(DatabaseIdentifier id) - { - return append(id.getSql()); - } - - /** Functionally the same as append(CharSequence). This method just has different asserts */ - public SQLFragment appendIdentifier(CharSequence charseq) - { - if (null == charseq) - return this; - if (charseq instanceof SQLFragment sqlf) - { - if (0 != sqlf.getParamsArray().length) - throw new IllegalStateException("Unexpected SQL in appendIdentifier()"); - charseq = sqlf.getRawSQL(); - } - - String identifier = charseq.toString().strip(); - - if (STR_TABLE_ALIAS.equals(identifier)) - { - getStringBuilder().append(identifier); - return this; - } - - boolean malformed; - boolean quoteWrapped = identifier.length() >= 2 && identifier.startsWith("\"") && identifier.endsWith("\""); - if (quoteWrapped) - malformed = (StringUtils.countMatches(identifier, '\"') % 2) != 0; - else if (identifier.length() >= 2 && identifier.startsWith("`") && identifier.endsWith("`")) - malformed = (StringUtils.countMatches(identifier, '`') % 2) != 0; - else - malformed = StringUtils.containsAny(identifier, "*/\\'\"`?;- \t\n"); - if (malformed) - throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) value appears to be incorrectly formatted: " + identifier); - - // A quote-wrapped value must be a well-formed quoted identifier, or dotted sequence of them. - if (quoteWrapped && !isQuotedIdentifierSequence(identifier)) - { - if (AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - LOG.warn("appendIdentifier strict pre-quoted check would have rejected (flag-on, allowed): {}", identifier); - else - { - LOG.warn("appendIdentifier strict pre-quoted check rejected (flag-off): {}", identifier); - throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) pre-quoted value is not a well-formed quoted identifier (or dotted sequence of them): " + identifier); - } - } - - getStringBuilder().append(charseq); - return this; - } - - // True iff the value is one or more double-quote-delimited identifiers joined by single dots, - // e.g. "a", "a"."b", "schema"."table". A literal quote within a segment must be escaped as "". - // A '.' inside quotes is part of the name; a '.' between segments is a separator. Anything else - // outside the quotes -- stray text, a lone (breakout) quote, an unterminated segment -- is rejected. - private static boolean isQuotedIdentifierSequence(String s) - { - int i = 0; - int n = s.length(); - while (i < n) - { - if (s.charAt(i) != '"') // each segment must open with a quote - return false; - i++; - boolean closed = false; - while (i < n) - { - if (s.charAt(i) == '"') - { - if (i + 1 < n && s.charAt(i + 1) == '"') // escaped "" -- part of the name - { - i += 2; - continue; - } - i++; // closing quote - closed = true; - break; - } - i++; // any other char is legal inside quotes - } - if (!closed) // ran off the end without a closing quote - return false; - if (i == n) // end of a valid final segment - return true; - if (s.charAt(i) != '.') // segments must be separated by a single dot - return false; - i++; // consume the separator and require another segment - } - return false; // trailing dot with no following segment - } - - // just to save some typing - public SQLFragment appendDottedIdentifiers(CharSequence table, DatabaseIdentifier col) - { - return appendIdentifier(table).append(".").appendIdentifier(col); - } - - // just to save some typing - public SQLFragment appendDottedIdentifiers(CharSequence... ids) - { - var dot = ""; - for (var id : ids) - { - append(dot).appendIdentifier(id); - dot = "."; - } - return this; - } - - /** append End Of Statement */ - public SQLFragment appendEOS() - { - getStringBuilder().append(";\n"); - return this; - } - - - @Override - public SQLFragment append(CharSequence csq, int start, int end) - { - append(csq.subSequence(start, end)); - return this; - } - - /** Adds the container's ID as an in-line string constant to the SQL */ - public SQLFragment appendValue(Container c) - { - if (null == c) - return appendNull(); - return appendValue(c, null); - } - - public SQLFragment appendValue(@NotNull Container c, SqlDialect dialect) - { - appendValue(c.getEntityId(), dialect); - String name = c.getName(); - if (!StringUtils.containsAny(name,"*/\\'\"?")) - append("/* ").append(name).append(" */"); - return this; - } - - public SQLFragment appendNull() - { - getStringBuilder().append("NULL"); - return this; - } - - public SQLFragment appendValue(Boolean B, @NotNull SqlDialect dialect) - { - if (null == B) - return append("CAST(NULL AS ").append(dialect.getBooleanDataType()).append(")"); - getStringBuilder().append(B ? dialect.getBooleanTRUE() : dialect.getBooleanFALSE()); - return this; - } - - public SQLFragment appendValue(Integer I) - { - if (null == I) - return appendNull(); - getStringBuilder().append(I.intValue()); - return this; - } - - public SQLFragment appendValue(int i) - { - getStringBuilder().append(i); - return this; - } - - - public SQLFragment appendValue(Long L) - { - if (null == L) - return appendNull(); - getStringBuilder().append((long)L); - return this; - } - - public SQLFragment appendValue(long l) - { - getStringBuilder().append(l); - return this; - } - - public SQLFragment appendValue(Float F) - { - if (null == F) - return appendNull(); - return appendValue(F.floatValue()); - } - - public SQLFragment appendValue(float f) - { - if (Float.isFinite(f)) - { - getStringBuilder().append(f); - } - else - { - getStringBuilder().append("?"); - add(f); - } - return this; - } - - public SQLFragment appendValue(Double D) - { - if (null == D) - return appendNull(); - else - return appendValue(D.doubleValue()); - } - - public SQLFragment appendValue(double d) - { - if (Double.isFinite(d)) - { - getStringBuilder().append(d); - } - else - { - getStringBuilder().append("?"); - add(d); - } - return this; - } - - public SQLFragment appendValue(Number N) - { - if (null == N) - return appendNull(); - - if (N instanceof Quantity q) - N = q.value(); - - if (N instanceof BigDecimal || N instanceof BigInteger || N instanceof Long) - { - getStringBuilder().append(N); - } - else if (Double.isFinite(N.doubleValue())) - { - getStringBuilder().append(N); - } - else - { - getStringBuilder().append(" ? "); - add(N); - } - return this; - } - - public final SQLFragment appendNowTimestamp() - { - return appendValue(new NowTimestamp()); - } - - // Issue 27534: Stop using {fn now()} in function declarations - // Issue 48864: Query Table's use of web server time can cause discrepancies in created/modified timestamps - public final SQLFragment appendValue(NowTimestamp now) - { - if (null == now) - return appendNull(); - getStringBuilder().append("CURRENT_TIMESTAMP"); - return this; - } - - public final SQLFragment appendValue(java.util.Date d) - { - if (null == d) - return appendNull(); - if (d.getClass() == java.util.Date.class) - getStringBuilder().append("{ts '").append(new Timestamp(d.getTime())).append("'}"); - else if (d.getClass() == java.sql.Timestamp.class) - getStringBuilder().append("{ts '").append(d).append("'}"); - else if (d.getClass() == java.sql.Date.class) - getStringBuilder().append("{d '").append(d).append("'}"); - else - throw new IllegalStateException("Unexpected date type: " + d.getClass().getName()); - return this; - } - - public SQLFragment appendValue(GUID g) - { - return appendValue(g, null); - } - - public SQLFragment appendValue(GUID g, SqlDialect d) - { - if (null == g) - return appendNull(); - // doesn't need StringHandler, just hex and hyphen - String sqlGUID = "'" + g + "'"; - // I'm testing dialect type, because some dialects do not support getGuidType(), and postgers uses VARCHAR anyway - if (null != d && d.isSqlServer()) - getStringBuilder().append("CAST(").append(sqlGUID).append(" AS UNIQUEIDENTIFIER)"); - else - getStringBuilder().append(sqlGUID); - return this; - } - - public SQLFragment appendValue(Enum e) - { - if (null == e) - return appendNull(); - String name = e.name(); - // Enum.name() returns a legal Java identifier per JLS, so none of these characters can appear - // in practice. Defense in depth: reject anything SQL-active rather than only the apostrophe. - if (StringUtils.containsAny(name, "'\"\\;\r\n")) - throw new IllegalStateException("Unexpected character in Enum.name(): " + name); - getStringBuilder().append("'").append(name).append("'"); - return this; - } - - public SQLFragment append(FieldKey fk) - { - if (null == fk) - return appendNull(); - append(String.valueOf(fk)); - return this; - } - - - /** Adds the object as a JDBC parameter value */ - public SQLFragment add(Object p) - { - getMutableParams().add(p); - return this; - } - - public SQLFragment add(Object p, JdbcType type) - { - getMutableParams().add(new Parameter.TypedValue(p, type)); - return this; - } - - /** Adds the objects as JDBC parameter values */ - public SQLFragment addAll(Collection l) - { - getMutableParams().addAll(l); - return this; - } - - - /** Adds the objects as JDBC parameter values */ - public SQLFragment addAll(Object... values) - { - if (values == null) - return this; - addAll(Arrays.asList(values)); - return this; - } - - - /** Sets the parameter at the index to the object's value */ - public void set(int i, Object p) - { - getMutableParams().set(i,p); - } - - /** Append both the SQL and the parameters from the other SQLFragment to this SQLFragment */ - public SQLFragment append(SQLFragment f) - { - if (null != f.sb) - getStringBuilder().append(f.sb); - else - getStringBuilder().append(f.sql); - if (null != f.params) - addAll(f.params); - mergeCommonTableExpressions(f); - tempTokens.addAll(f.tempTokens); - return this; - } - - public SQLFragment append(@NotNull Iterable fragments, @NotNull String separator) - { - String s = ""; - for (SQLFragment fragment : fragments) - { - append(s); - s = separator; - append(fragment); - } - return this; - } - - // return boolean so this can be used in an assert. passing in a dialect is not ideal, but parsing comments out - // before submitting the fragment is not reliable and holding statements & comments separately (to eliminate the - // need to parse them) isn't particularly easy... so punt for now. - public boolean appendComment(String comment, SqlDialect dialect) - { - if (dialect.supportsComments()) - { - StringBuilder sb = getStringBuilder(); - int len = sb.length(); - if (len > 0 && sb.charAt(len-1) != '\n') - sb.append('\n'); - sb.append("\n-- "); - boolean truncated = comment.length() > 1000; - if (truncated) - comment = StringUtilsLabKey.leftSurrogatePairFriendly(comment, 1000); - // Strip CR/LF so an embedded newline can't terminate the `--` comment and turn the rest - // of the payload into live SQL. - comment = comment.replace('\r', ' ').replace('\n', ' '); - sb.append(comment); - if (StringUtils.countMatches(comment, "'")%2==1) - sb.append("'"); - if (truncated) - sb.append("..."); - sb.append('\n'); - } - return true; - } - - - /** see also append(TableInfo, String alias) */ - public SQLFragment append(TableInfo table) - { - SQLFragment s = table.getSQLName(); - if (s != null) - return append(s); - - // GitHub 1432: Cannot delete a multi-choice option on a source type - String alias = AliasManager.makeLegalName(table.getName(), table.getSchema().getSqlDialect()); - return append(table.getFromSQL(alias)); - } - - /** Add a table/query to the SQL with an alias, as used in a FROM clause */ - public SQLFragment append(TableInfo table, String alias) - { - return append(table.getFromSQL(alias)); - } - - /** Add to the SQL */ - @Override - public SQLFragment append(char ch) - { - getStringBuilder().append(ch); - return this; - } - - /** This is like appendValue(CharSequence s), but force use of literal syntax - * CAUTIONARY NOTE: String literals in PostgresSQL are tricky because of overloaded functions - * array_agg('string') fails array_agg('string'::VARCHAR) works - * json_object('{}) works json_object('string'::VARCHAR) fails - * In the case of json_object() it expects TEXT. Postgres will promote 'json' to TEXT, but not 'json'::VARCHAR - */ - public SQLFragment appendStringLiteral(CharSequence s, @NotNull SqlDialect d) - { - if (null==s) - return appendNull(); - getStringBuilder().append(d.getStringHandler().quoteStringLiteral(s.toString())); - return this; - } - - /** Add to the SQL as either an in-line string literal or as a JDBC parameter depending on whether it would need escaping */ - public SQLFragment appendValue(CharSequence s) - { - return appendValue(s, null); - } - - public SQLFragment appendValue(CharSequence s, SqlDialect d) - { - if (null==s) - return appendNull(); - if (null==d || s.length() > 200) - return append("?").add(s.toString()); - appendStringLiteral(s, d); - return this; - } - - public SQLFragment appendInClause(@NotNull Collection params, SqlDialect dialect) - { - dialect.appendInClauseSql(this, params); - return this; - } - - public CharSequence getSqlCharSequence() - { - if (null != sb) - { - return sb; - } - return sql; - } - - public void insert(int index, SQLFragment sql) - { - if (!sql.getParams().isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with parameters - they must be inserted/merged separately"); - } - if (sql.commonTableExpressionsMap != null && !sql.commonTableExpressionsMap.isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with CTEs - they must be inserted/merged separately"); - } - if (!tempTokens.isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with temp tokens - they must be inserted/merged separately"); - } - getStringBuilder().insert(index, sql.getRawSQL()); - } - - /** Insert into the SQL */ - public void insert(int index, String str) - { - if ((StringUtils.countMatches(str, '\'') % 2) != 0 || - (StringUtils.countMatches(str, '\"') % 2) != 0 || - StringUtils.contains(str, ';')) - { - throw new IllegalArgumentException("SQLFragment.insert(int,String) does not allow semicolons or unmatched quotes"); - } - - getStringBuilder().insert(index, str); - } - - /** Insert this SQLFragment's SQL and parameters at the start of the existing SQL and parameters */ - public void prepend(SQLFragment sql) - { - getStringBuilder().insert(0, sql.getSqlCharSequence().toString()); - if (null != sql.params) - getMutableParams().addAll(0, sql.params); - mergeCommonTableExpressions(sql); - } - - - public int indexOf(String str) - { - return getStringBuilder().indexOf(str); - } - - - // Display query in "English" (display SQL with params substituted) - // with a little more work could probably be made to be SQL legal - public String getFilterText() - { - String sql = getSQL().replaceFirst("WHERE ", ""); - List params = getParams(); - for (Object param1 : params) - { - String param = param1.toString(); - param = param.replaceAll("\\\\", "\\\\\\\\"); - param = param.replaceAll("\\$", "\\\\\\$"); - sql = sql.replaceFirst("\\?", param); - } - return sql.replace("\"", ""); - } - - - @Override - public char charAt(int index) - { - return getSqlCharSequence().charAt(index); - } - - @Override - public int length() - { - return getSqlCharSequence().length(); - } - - @Override - public @NotNull CharSequence subSequence(int start, int end) - { - return getSqlCharSequence().subSequence(start, end); - } - - /** - * KEY is used as a faster way to look for equivalent CTE expressions. - * returning a name here allows us to potentially merge CTE at add time - * - * if you don't have a key you can just use sqlf.toString() - */ - public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf) - { - return addCommonTableExpression(dialect, key, proposedName, sqlf, false); - } - - public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf, boolean recursive) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - CTE prev = commonTableExpressionsMap.get(key); - if (null != prev) - return prev.token(); - CTE cte = new CTE(dialect, proposedName, sqlf, recursive); - commonTableExpressionsMap.put(key, cte); - return cte.token(); - } - - public String createCommonTableExpressionToken(SqlDialect dialect, Object key, String proposedName) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - CTE prev = commonTableExpressionsMap.get(key); - if (null != prev) - throw new IllegalStateException("Cannot create CTE token from already used key."); - CTE cte = new CTE(dialect ,proposedName); - commonTableExpressionsMap.put(key, cte); - return cte.token(); - } - - public void setCommonTableExpressionSql(Object key, SQLFragment sqlf, boolean recursive) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - - if (null != sqlf.commonTableExpressionsMap && !sqlf.commonTableExpressionsMap.isEmpty()) - { - // Need to merge CTEs up; this.cte depends on newSql.ctes, so they need to come first - SQLFragment newSql = new SQLFragment(sqlf); - LinkedHashMap toMap = new LinkedHashMap<>(newSql.commonTableExpressionsMap); - for (Map.Entry e : commonTableExpressionsMap.entrySet()) - { - CTE from = e.getValue(); - CTE to = toMap.get(e.getKey()); - if (null != to) - to.tokens.addAll(from.tokens); - else - toMap.put(e.getKey(), from.copy(false)); - } - - commonTableExpressionsMap = toMap; - newSql.commonTableExpressionsMap = null; - sqlf = newSql; - } - - CTE cte = commonTableExpressionsMap == null ? null : commonTableExpressionsMap.get(key); - if (null == cte) - throw new IllegalStateException("CTE not found."); - cte.sqlf = sqlf; - cte.recursive = recursive; - } - - - private void mergeCommonTableExpressions(SQLFragment sqlFrom) - { - if (null == sqlFrom.commonTableExpressionsMap || sqlFrom.commonTableExpressionsMap.isEmpty()) - return; - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - for (Map.Entry e : sqlFrom.commonTableExpressionsMap.entrySet()) - { - CTE from = e.getValue(); - CTE to = commonTableExpressionsMap.get(e.getKey()); - if (null != to) - to.tokens.addAll(from.tokens); - else - commonTableExpressionsMap.put(e.getKey(), from.copy(false)); - } - } - - - public void addTempToken(Object tempToken) - { - tempTokens.add(tempToken); - } - - public void addTempTokens(SQLFragment other) - { - tempTokens.add(other.tempTokens); - } - - public static SQLFragment prettyPrint(SQLFragment from) - { - SQLFragment sqlf = new SQLFragment(from); - - String s = from.getSqlCharSequence().toString(); - StringBuilder sb = new StringBuilder(s.length() + 200); - String[] lines = StringUtils.split(s, '\n'); - int indent = 0; - - for (String line : lines) - { - String t = line.trim(); - - if (t.isEmpty()) - continue; - - if (t.startsWith("-- params = b.getParams(); - assertEquals(2,params.size()); - assertEquals(5, params.get(0)); - assertEquals("xxyzzy", params.get(1)); - - - SQLFragment c = new SQLFragment(b); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT a FROM b WHERE x=?) - SELECT * FROM CTE WHERE y=?""", - c.getSQL()); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT a FROM b WHERE x=5) - SELECT * FROM CTE WHERE y='xxyzzy'""", - filterDebugString(c.toDebugString())); - params = c.getParams(); - assertEquals(2,params.size()); - assertEquals(5, params.get(0)); - assertEquals("xxyzzy", params.get(1)); - - - // combining - - SQLFragment sqlf = new SQLFragment(); - String token = sqlf.addCommonTableExpression(dialect, "KEY_A", "cte1", new SQLFragment("SELECT * FROM a")); - sqlf.append("SELECT * FROM ").append(token).append(" _1"); - - assertEquals(""" - WITH - /*CTE*/ - \tcte1 AS (SELECT * FROM a) - SELECT * FROM cte1 _1""", - sqlf.getSQL()); - - SQLFragment sqlf2 = new SQLFragment(); - String token2 = sqlf2.addCommonTableExpression(dialect, "KEY_A", "cte2", new SQLFragment("SELECT * FROM a")); - sqlf2.append("SELECT * FROM ").append(token2).append(" _2"); - assertEquals(""" - WITH - /*CTE*/ - \tcte2 AS (SELECT * FROM a) - SELECT * FROM cte2 _2""", - sqlf2.getSQL()); - - SQLFragment sqlf3 = new SQLFragment(); - String token3 = sqlf3.addCommonTableExpression(dialect, "KEY_B", "cte3", new SQLFragment("SELECT * FROM b")); - sqlf3.append("SELECT * FROM ").append(token3).append(" _3"); - assertEquals(""" - WITH - /*CTE*/ - \tcte3 AS (SELECT * FROM b) - SELECT * FROM cte3 _3""", - sqlf3.getSQL()); - - SQLFragment union = new SQLFragment(); - union.append(sqlf); - union.append("\nUNION\n"); - union.append(sqlf2); - union.append("\nUNION\n"); - union.append(sqlf3); - assertEquals(""" - WITH - /*CTE*/ - \tcte1 AS (SELECT * FROM a) - ,/*CTE*/ - \tcte3 AS (SELECT * FROM b) - SELECT * FROM cte1 _1 - UNION - SELECT * FROM cte1 _2 - UNION - SELECT * FROM cte3 _3""", - union.getSQL()); - } - - @Test - public void nested_cte() - { - // one-level cte using cteToken (CTE fragment 'a' does not contain a CTE) - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); - assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); - SQLFragment b = new SQLFragment(); - String cteToken = b.addCommonTableExpression(dialect, new Object(), "CTE", a); - b.append("SELECT * FROM ").append(cteToken).append(" WHERE p=?").add("parameterTWO"); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) - SELECT * FROM CTE WHERE p='parameterTWO'""", - filterDebugString(b.toDebugString())); - assertEquals("parameterONE", b.getParams().getFirst()); - } - - // two-level cte using cteTokens (CTE fragment 'b' contains a CTE of fragment a) - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); - assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterTWO"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - c.append("SELECT * FROM ").append(cteTokenB).append(" WHERE i=?").add(3); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterTWO') - SELECT * FROM B_ WHERE i=3""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(3, params.size()); - assertEquals("parameterONE", params.get(0)); - assertEquals("parameterTWO", params.get(1)); - assertEquals(3, params.get(2)); - } - - // Same as previous but top-level query has both a nested and non-nested CTE - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); - SQLFragment a2 = new SQLFragment("SELECT 2 as i, 'Atwo' as s, CAST(? AS VARCHAR) as p", "parameterAtwo"); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - String cteTokenA2 = c.addCommonTableExpression(dialect, new Object(), "A2_", a2); - c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') - ,/*CTE*/ - \tA2_ AS (SELECT 2 as i, 'Atwo' as s, CAST('parameterAtwo' AS VARCHAR) as p) - SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(4, params.size()); - assertEquals("parameterAone", params.get(0)); - assertEquals("parameterB", params.get(1)); - assertEquals("parameterAtwo", params.get(2)); - assertEquals(4, params.get(3)); - } - - // Same as previous but two of the CTEs are the same and should be collapsed (e.g. imagine a container filter implemented with a CTE) - // TODO, we only collapse CTEs that are siblings - { - SQLFragment cf = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, "CTE_KEY_CF", "A_", cf); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - String cteTokenA2 = c.addCommonTableExpression(dialect, "CTE_KEY_CF", "A2_", cf); - c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') - ,/*CTE*/ - \tA2_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(4, params.size()); - assertEquals("parameterAone", params.get(0)); - assertEquals("parameterB", params.get(1)); - assertEquals("parameterAone", params.get(2)); - assertEquals(4, params.get(3)); - } - } - - - private void shouldFail(Runnable r) - { - try - { - r.run(); - fail("Expected IllegalArgumentException"); - } - catch (IllegalArgumentException e) - { - // expected - } - } - - - @Test - public void testIllegalArgument() - { - shouldFail(() -> new SQLFragment(";")); - shouldFail(() -> new SQLFragment().append(";")); - shouldFail(() -> new SQLFragment("AND name='")); - shouldFail(() -> new SQLFragment().append("AND name = '")); - shouldFail(() -> new SQLFragment().append("AND name = 'Robert'); DROP TABLE Students; --")); - - shouldFail(() -> new SQLFragment().appendIdentifier("column name")); - shouldFail(() -> new SQLFragment().appendIdentifier("?")); - shouldFail(() -> new SQLFragment().appendIdentifier(";")); - shouldFail(() -> new SQLFragment().appendIdentifier("\"column\"name\"")); - } - - - String mysqlQuoteIdentifier(String id) - { - return "`" + id.replace("`", "``") + "`"; - } - - @Test - public void testMysql() - { - // OK - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("mysql")); - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my`sql")); - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my\"sql")); - - // not OK - shouldFail(() -> new SQLFragment().appendIdentifier("`")); - shouldFail(() -> new SQLFragment().appendIdentifier("`a`a`")); - } - - @Test - public void testAppendCommentStripsNewlines() - { - // PR1-P3: an embedded newline in a comment payload must not terminate the `-- ` comment - // line and expose the trailing text as live SQL. - if (!dialect.supportsComments()) - return; - - SQLFragment sqlf = new SQLFragment(); - sqlf.appendComment("hello\nDROP TABLE x;--", dialect); - String sql = sqlf.getSQL(); - - assertFalse("appendComment leaked an embedded newline into emitted SQL: " + sql, sql.contains("hello\nDROP")); - assertTrue("appendComment should keep the payload on one comment line: " + sql, sql.contains("hello DROP TABLE x;--")); - - // CR is stripped too - SQLFragment sqlf2 = new SQLFragment(); - sqlf2.appendComment("hi\rDROP TABLE x;--", dialect); - assertFalse("appendComment leaked an embedded CR: " + sqlf2.getSQL(), sqlf2.getSQL().contains("hi\rDROP")); - } - - @Test - public void testAppendIdentifierPreQuotedDottedAccepted() - { - // PR2-G1: a dotted sequence of individually well-formed quoted identifiers is legitimate -- - // e.g. a fully-qualified "schema"."table" returned by TableInfo.getSelectName(). The strict - // check validates each quoted segment rather than blanket-rejecting any interior dot. - new SQLFragment().appendIdentifier("\"schema\".\"table\""); - new SQLFragment().appendIdentifier("\"a\".\"b\".\"c\""); - } - - @Test - public void testAppendIdentifierPreQuotedDotInsideQuotesAccepted() - { - // PR2-G1: a `.` *inside* the quotes is part of a single identifier's name, not a separator. - // This is the metadata-name case (e.g. a calculated column's generated class name) that the - // earlier strict check wrongly rejected. - new SQLFragment().appendIdentifier("\"name.with.dots\""); - new SQLFragment().appendIdentifier("\"org.labkey.query.sql.CalculatedExpressionColumn6b8f\""); - } - - @Test - public void testAppendIdentifierPreQuotedInteriorQuoteRejected() - { - // PR2-G1: an interior `"` that isn't part of the standard `""` doubling is a breakout. - // Even quote count alone is insufficient. - shouldFail(() -> new SQLFragment().appendIdentifier("\"foo\"bar\"baz\"")); - // A classic breakout: close the identifier early, then inject. The lone interior quote - // (after `foo`) is followed by non-separator text, so it must be rejected. - shouldFail(() -> new SQLFragment().appendIdentifier("\"foo\"; DROP TABLE x; --\"")); - // Stray text after a valid segment (not a `.` separator). - shouldFail(() -> new SQLFragment().appendIdentifier("\"a\" \"b\"")); - // Trailing separator with no following segment. - shouldFail(() -> new SQLFragment().appendIdentifier("\"a\".\"")); - // Empty segment between separators. - shouldFail(() -> new SQLFragment().appendIdentifier("\"a\"..\"b\"")); - } - - @Test - public void testAppendIdentifierPreQuotedValidCases() - { - // PR2-G1: legitimate pre-quoted identifiers still pass. - new SQLFragment().appendIdentifier("\"my_table\""); // simple quoted - new SQLFragment().appendIdentifier("\"with a space\""); // whitespace inside quotes is fine - new SQLFragment().appendIdentifier("\"foo\"\"bar\""); // embedded literal " via "" doubling - new SQLFragment().appendIdentifier("\"contains\"\"more\"\"doubles\""); - // Doubled interior quotes around a dot decode to a single identifier named weird"."name -- - // distinct from the two-segment "weird"."name" -- and cannot break out, so it's accepted. - new SQLFragment().appendIdentifier("\"weird\"\".\"\"name\""); - } - } - - @Override - public boolean equals(Object obj) - { - if (!(obj instanceof SQLFragment other)) - { - return false; - } - return getSQL().equals(other.getSQL()) && getParams().equals(other.getParams()); - } - - @Override - public int hashCode() - { - return Objects.hash(getSQL(), getParams()); - } - - /** - * Joins the SQLFragments in the provided {@code Iterable} into a single SQLFragment. The SQL is joined by string - * concatenation using the provided separator. The parameters are combined to form the new parameter list. - * - * @param fragments SQLFragments to join together - * @param separator Separator to use - * @return A new SQLFragment that joins all the SQLFragments - */ - public static SQLFragment join(Iterable fragments, SQLFragment separator) - { - SQLFragment join = new SQLFragment(); - boolean first = true; - - for (SQLFragment fragment : fragments) - { - if (first) - first = false; - else - join.append(separator); - join.append(fragment); - } - - return join; - } -} +/* + * Copyright (c) 2008-2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.data; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.query.AliasManager; +import org.labkey.api.query.FieldKey; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JdbcUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.labkey.api.query.ExprColumn.STR_TABLE_ALIAS; + +/// A composable SQL builder that pairs SQL text with its JDBC parameter values, ensuring +/// they travel together through query construction. Implements [Appendable] and +/// [CharSequence] for fluent assembly of SQL statements. +/// +/// Provides type-safe `appendValue()` methods for inlining literals of common +/// types (integers, strings, dates, GUIDs, etc.) and `add()` methods for binding +/// JDBC `?` parameters. Fragments can be composed via `append(SQLFragment)` to +/// merge both SQL text and parameter lists. +/// +/// Supports Common Table Expressions (CTEs) through +/// [#addCommonTableExpression(SqlDialect, Object, String, SQLFragment)], which +/// manages deduplication, token substitution, and correct ordering of WITH clauses +/// across nested and combined fragments. +/// +/// Enforces basic SQL injection safeguards by rejecting unmatched quotes and +/// semicolons in appended text. +public class SQLFragment implements Appendable, CharSequence +{ + private static final Logger LOG = LogHelper.getLogger(SQLFragment.class, "SQL injection safety net diagnostics"); + + public static final String FEATUREFLAG_DISABLE_STRICT_CHECKS = "SQLFragmentDisableStrictChecks"; + + private String sql; + private StringBuilder sb = null; + private List params; // TODO: Should be List + + private final List tempTokens = new ArrayList<>(); // Hold refs to ensure they're not GC'd + + // use ordered map to make sql generation more deterministic (see collectCommonTableExpressions()) + private LinkedHashMap commonTableExpressionsMap = null; + + private static class CTE + { + CTE(@NotNull SqlDialect dialect, @NotNull String name) + { + this.dialect = dialect; + this.preferredName = name; + tokens.add("/*$*/" + GUID.makeGUID() + ":" + name + "/*$*/"); + } + + CTE(@NotNull SqlDialect dialect, @NotNull String name, SQLFragment sqlf, boolean recursive) + { + this(dialect, name); + this.sqlf = sqlf; + this.recursive = recursive; + } + + CTE(CTE from) + { + this.dialect = from.dialect; + this.preferredName = from.preferredName; + this.tokens.addAll(from.tokens); + this.sqlf = from.sqlf; + this.recursive = from.recursive; + } + + public CTE copy(boolean deep) + { + CTE copy = new CTE(this); + if (deep) + copy.sqlf = new SQLFragment().append(copy.sqlf); + return copy; + } + + private String token() + { + return tokens.iterator().next(); + } + + private final @NotNull SqlDialect dialect; + final String preferredName; + boolean recursive = false; // NOTE this is dialect dependant (getSql() does not take a dialect) + final Set tokens = new TreeSet<>(); + SQLFragment sqlf = null; + } + + public SQLFragment() + { + sql = ""; + } + + public SQLFragment(CharSequence charseq, @Nullable List params) + { + if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || + (StringUtils.countMatches(charseq, '\"') % 2) != 0 || + StringUtils.contains(charseq, ';')) + { + throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); + } + + this.sql = charseq.toString(); + if (null != params) + this.params = new ArrayList<>(params); + } + + + public SQLFragment(CharSequence sql, Object... params) + { + this(sql, Arrays.asList(params)); + } + + + public SQLFragment(SQLFragment other) + { + this(other,false); + } + + + public SQLFragment(SQLFragment other, boolean deep) + { + sql = other.getSqlCharSequence().toString(); + if (null != other.params) + addAll(other.params); + if (null != other.commonTableExpressionsMap && !other.commonTableExpressionsMap.isEmpty()) + { + if (null == this.commonTableExpressionsMap) + this.commonTableExpressionsMap = new LinkedHashMap<>(); + for (Map.Entry e : other.commonTableExpressionsMap.entrySet()) + { + CTE cte = e.getValue().copy(deep); + this.commonTableExpressionsMap.put(e.getKey(),cte); + } + } + this.tempTokens.addAll(other.tempTokens); + } + + + @Override + public boolean isEmpty() + { + return (null == sb || sb.isEmpty()) && (sql == null || sql.isEmpty()); + } + + + /* same as getSQL() but without CTE handling */ + public String getRawSQL() + { + return null != sb ? sb.toString() : null != sql ? sql : ""; + } + + /* + * Directly set the current SQL. + * + * This is useful for wrapping existing SQL, for instance adding a cast + * Obviously parameter number and order must remain unchanged + * + * This can also be used for processing sql scripts (e.g. module .sql update scripts) + */ + public SQLFragment setSqlUnsafe(String unsafe) + { + this.sql = unsafe; + this.sb = null; + return this; + } + + public static SQLFragment unsafe(String unsafe) + { + return new SQLFragment().setSqlUnsafe(unsafe); + } + + + private String replaceCteTokens(String self, String select, List> ctes) + { + for (Pair pair : ctes) + { + String alias = pair.first; + CTE cte = pair.second; + for (String token : cte.tokens) + { + select = Strings.CS.replace(select, token, alias); + } + } + if (null != self) + select = Strings.CS.replace(select, "$SELF$", self); + return select; + } + + + private List collectCommonTableExpressions() + { + List list = new ArrayList<>(); + _collectCommonTableExpressions(list); + return list; + } + + private void _collectCommonTableExpressions(List list) + { + if (null != commonTableExpressionsMap) + { + commonTableExpressionsMap.values().forEach(cte -> cte.sqlf._collectCommonTableExpressions(list)); + list.addAll(commonTableExpressionsMap.values()); + } + } + + + public String getSQL() + { + if (null == commonTableExpressionsMap || commonTableExpressionsMap.isEmpty()) + return null != sb ? sb.toString() : null != sql ? sql : ""; + + List commonTableExpressions = collectCommonTableExpressions(); + assert !commonTableExpressions.isEmpty(); + + boolean recursive = commonTableExpressions.stream() + .anyMatch(cte -> cte.recursive); + StringBuilder ret = new StringBuilder("WITH" + (recursive ? " RECURSIVE" : "")); + + // generate final aliases for each CTE */ + SqlDialect dialect = Objects.requireNonNull(commonTableExpressions.getFirst().dialect); + AliasManager am = new AliasManager(dialect); + List> ctes = commonTableExpressions.stream() + .map(cte -> new Pair<>(am.decideAlias(cte.preferredName),cte)) + .collect(Collectors.toList()); + + String comma = "\n/*CTE*/\n\t"; + for (Pair p : ctes) + { + String alias = p.first; + CTE cte = p.second; + SQLFragment expr = cte.sqlf; + String sql = expr._getOwnSql(alias, ctes); + ret.append(comma).append(alias).append(" AS (").append(sql).append(")"); + comma = "\n,/*CTE*/\n\t"; + } + ret.append("\n"); + + String select = _getOwnSql( null, ctes ); + ret.append(replaceCteTokens(null, select, ctes)); + return ret.toString(); + } + + + private String _getOwnSql(String alias, List> ctes) + { + String ownSql = null != sb ? sb.toString() : null != this.sql ? this.sql : ""; + return replaceCteTokens(alias, ownSql, ctes); + } + + + static Pattern markerPattern = Pattern.compile("/\\*\\$\\*/.*/\\*\\$\\*/"); + + /* This is not an exhaustive .equals() test, but it give pretty good confidence that these statements are the same */ + static boolean debugCompareSQL(SQLFragment sql1, SQLFragment sql2) + { + String select1 = sql1.getRawSQL(); + String select2 = sql2.getRawSQL(); + + if ((null == sql1.commonTableExpressionsMap || sql1.commonTableExpressionsMap.isEmpty()) && + (null == sql2.commonTableExpressionsMap || sql2.commonTableExpressionsMap.isEmpty())) + return select1.equals(select2); + + select1 = markerPattern.matcher(select1).replaceAll("CTE"); + select2 = markerPattern.matcher(select2).replaceAll("CTE"); + if (!select1.equals(select2)) + return false; + + Set ctes1 = sql1.commonTableExpressionsMap.values().stream() + .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) + .collect(Collectors.toSet()); + Set ctes2 = sql2.commonTableExpressionsMap.values().stream() + .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) + .collect(Collectors.toSet()); + return ctes1.equals(ctes2); + } + + + // It is a little confusing that getString() does not return the same charsequence that this object purports to + // represent. However, this is a good "display value" for this object. + // see getSqlCharSequence() + @NotNull + public String toString() + { + return "SQLFragment@" + System.identityHashCode(this) + "\n" + toDebugString(); + } + + // Not recommended -- this uses the LabKey scope to dictate parsing of identifiers and string literals... which + // might not be correct for the incoming SQL. + @Deprecated // Use the variant below that takes a SqlDialect + public String toDebugString() + { + return toDebugString(DbScope.getLabKeyScope().getSqlDialect()); + } + + public String toDebugString(SqlDialect dialect) + { + return JdbcUtil.format(this, dialect); + } + + public List getParams() + { + var ctes = collectCommonTableExpressions(); + List ret = new ArrayList<>(); + + for (var cte : ctes) + ret.addAll(cte.sqlf.getParamsNoCTEs()); + ret.addAll(getParamsNoCTEs()); + return Collections.unmodifiableList(ret); + } + + + public List> getParamsWithFragments() + { + var ctes = collectCommonTableExpressions(); + List> ret = new ArrayList<>(); + + for (CTE cte : ctes) + { + if (null != cte.sqlf && null != cte.sqlf.params) + { + for (int i = 0; i < cte.sqlf.params.size(); i++) + { + ret.add(new Pair<>(cte.sqlf, i)); + } + } + } + + if (null != params) + { + for (int i = 0; i < params.size(); i++) + { + ret.add(new Pair<>(this, i)); + } + } + return ret; + } + + private final static Object[] EMPTY_ARRAY = new Object[0]; + + public Object[] getParamsArray() + { + return null == params ? EMPTY_ARRAY : params.toArray(); + } + + public List getParamsNoCTEs() + { + return params == null ? Collections.emptyList() : Collections.unmodifiableList(params); + } + + private List getMutableParams() + { + if (!(params instanceof ArrayList)) + { + List t = new ArrayList<>(); + if (params != null) + t.addAll(params); + params = t; + } + return params; + } + + + private StringBuilder getStringBuilder() + { + if (null == sb) + sb = new StringBuilder(null==sql?"":sql); + return sb; + } + + + @Override + public SQLFragment append(CharSequence charseq) + { + if (null == charseq) + return this; + + if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || + (StringUtils.countMatches(charseq, '\"') % 2) != 0 || + StringUtils.contains(charseq, ';')) + { + throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); + } + + getStringBuilder().append(charseq); + return this; + } + + public SQLFragment appendIdentifier(DatabaseIdentifier id) + { + return append(id.getSql()); + } + + /** Functionally the same as append(CharSequence). This method just has different asserts */ + public SQLFragment appendIdentifier(CharSequence charseq) + { + if (null == charseq) + return this; + if (charseq instanceof SQLFragment sqlf) + { + if (0 != sqlf.getParamsArray().length) + throw new IllegalStateException("Unexpected SQL in appendIdentifier()"); + charseq = sqlf.getRawSQL(); + } + + String identifier = charseq.toString().strip(); + + if (STR_TABLE_ALIAS.equals(identifier)) + { + getStringBuilder().append(identifier); + return this; + } + + boolean malformed; + boolean quoteWrapped = identifier.length() >= 2 && identifier.startsWith("\"") && identifier.endsWith("\""); + if (quoteWrapped) + malformed = (StringUtils.countMatches(identifier, '\"') % 2) != 0; + else if (identifier.length() >= 2 && identifier.startsWith("`") && identifier.endsWith("`")) + malformed = (StringUtils.countMatches(identifier, '`') % 2) != 0; + else + malformed = StringUtils.containsAny(identifier, "*/\\'\"`?;- \t\n"); + if (malformed) + throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) value appears to be incorrectly formatted: " + identifier); + + // A quote-wrapped value must be a well-formed quoted identifier, or dotted sequence of them. + if (quoteWrapped && !isQuotedIdentifierSequence(identifier)) + { + if (AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + LOG.warn("appendIdentifier strict pre-quoted check would have rejected (flag-on, allowed): {}", identifier); + else + { + LOG.warn("appendIdentifier strict pre-quoted check rejected (flag-off): {}", identifier); + throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) pre-quoted value is not a well-formed quoted identifier (or dotted sequence of them): " + identifier); + } + } + + getStringBuilder().append(charseq); + return this; + } + + // True iff the value is one or more double-quote-delimited identifiers joined by single dots, + // e.g. "a", "a"."b", "schema"."table". A literal quote within a segment must be escaped as "". + // A '.' inside quotes is part of the name; a '.' between segments is a separator. Anything else + // outside the quotes -- stray text, a lone (breakout) quote, an unterminated segment -- is rejected. + private static boolean isQuotedIdentifierSequence(String s) + { + int i = 0; + int n = s.length(); + while (i < n) + { + if (s.charAt(i) != '"') // each segment must open with a quote + return false; + i++; + boolean closed = false; + while (i < n) + { + if (s.charAt(i) == '"') + { + if (i + 1 < n && s.charAt(i + 1) == '"') // escaped "" -- part of the name + { + i += 2; + continue; + } + i++; // closing quote + closed = true; + break; + } + i++; // any other char is legal inside quotes + } + if (!closed) // ran off the end without a closing quote + return false; + if (i == n) // end of a valid final segment + return true; + if (s.charAt(i) != '.') // segments must be separated by a single dot + return false; + i++; // consume the separator and require another segment + } + return false; // trailing dot with no following segment + } + + // just to save some typing + public SQLFragment appendDottedIdentifiers(CharSequence table, DatabaseIdentifier col) + { + return appendIdentifier(table).append(".").appendIdentifier(col); + } + + // just to save some typing + public SQLFragment appendDottedIdentifiers(CharSequence... ids) + { + var dot = ""; + for (var id : ids) + { + append(dot).appendIdentifier(id); + dot = "."; + } + return this; + } + + /** append End Of Statement */ + public SQLFragment appendEOS() + { + getStringBuilder().append(";\n"); + return this; + } + + + @Override + public SQLFragment append(CharSequence csq, int start, int end) + { + append(csq.subSequence(start, end)); + return this; + } + + /** Adds the container's ID as an in-line string constant to the SQL */ + public SQLFragment appendValue(Container c) + { + if (null == c) + return appendNull(); + return appendValue(c, null); + } + + public SQLFragment appendValue(@NotNull Container c, SqlDialect dialect) + { + appendValue(c.getEntityId(), dialect); + String name = c.getName(); + if (!StringUtils.containsAny(name,"*/\\'\"?")) + append("/* ").append(name).append(" */"); + return this; + } + + public SQLFragment appendNull() + { + getStringBuilder().append("NULL"); + return this; + } + + public SQLFragment appendValue(Boolean B, @NotNull SqlDialect dialect) + { + if (null == B) + return append("CAST(NULL AS ").append(dialect.getBooleanDataType()).append(")"); + getStringBuilder().append(B ? dialect.getBooleanTRUE() : dialect.getBooleanFALSE()); + return this; + } + + public SQLFragment appendValue(Integer I) + { + if (null == I) + return appendNull(); + getStringBuilder().append(I.intValue()); + return this; + } + + public SQLFragment appendValue(int i) + { + getStringBuilder().append(i); + return this; + } + + + public SQLFragment appendValue(Long L) + { + if (null == L) + return appendNull(); + getStringBuilder().append((long)L); + return this; + } + + public SQLFragment appendValue(long l) + { + getStringBuilder().append(l); + return this; + } + + public SQLFragment appendValue(Float F) + { + if (null == F) + return appendNull(); + return appendValue(F.floatValue()); + } + + public SQLFragment appendValue(float f) + { + if (Float.isFinite(f)) + { + getStringBuilder().append(f); + } + else + { + getStringBuilder().append("?"); + add(f); + } + return this; + } + + public SQLFragment appendValue(Double D) + { + if (null == D) + return appendNull(); + else + return appendValue(D.doubleValue()); + } + + public SQLFragment appendValue(double d) + { + if (Double.isFinite(d)) + { + getStringBuilder().append(d); + } + else + { + getStringBuilder().append("?"); + add(d); + } + return this; + } + + public SQLFragment appendValue(Number N) + { + if (null == N) + return appendNull(); + + if (N instanceof Quantity q) + N = q.value(); + + if (N instanceof BigDecimal || N instanceof BigInteger || N instanceof Long) + { + getStringBuilder().append(N); + } + else if (Double.isFinite(N.doubleValue())) + { + getStringBuilder().append(N); + } + else + { + getStringBuilder().append(" ? "); + add(N); + } + return this; + } + + public final SQLFragment appendNowTimestamp() + { + return appendValue(new NowTimestamp()); + } + + // Issue 27534: Stop using {fn now()} in function declarations + // Issue 48864: Query Table's use of web server time can cause discrepancies in created/modified timestamps + public final SQLFragment appendValue(NowTimestamp now) + { + if (null == now) + return appendNull(); + getStringBuilder().append("CURRENT_TIMESTAMP"); + return this; + } + + public final SQLFragment appendValue(java.util.Date d) + { + if (null == d) + return appendNull(); + if (d.getClass() == java.util.Date.class) + getStringBuilder().append("{ts '").append(new Timestamp(d.getTime())).append("'}"); + else if (d.getClass() == java.sql.Timestamp.class) + getStringBuilder().append("{ts '").append(d).append("'}"); + else if (d.getClass() == java.sql.Date.class) + getStringBuilder().append("{d '").append(d).append("'}"); + else + throw new IllegalStateException("Unexpected date type: " + d.getClass().getName()); + return this; + } + + public SQLFragment appendValue(GUID g) + { + return appendValue(g, null); + } + + public SQLFragment appendValue(GUID g, SqlDialect d) + { + if (null == g) + return appendNull(); + // doesn't need StringHandler, just hex and hyphen + String sqlGUID = "'" + g + "'"; + // I'm testing dialect type, because some dialects do not support getGuidType(), and postgers uses VARCHAR anyway + if (null != d && d.isSqlServer()) + getStringBuilder().append("CAST(").append(sqlGUID).append(" AS UNIQUEIDENTIFIER)"); + else + getStringBuilder().append(sqlGUID); + return this; + } + + public SQLFragment appendValue(Enum e) + { + if (null == e) + return appendNull(); + String name = e.name(); + // Enum.name() returns a legal Java identifier per JLS, so none of these characters can appear + // in practice. Defense in depth: reject anything SQL-active rather than only the apostrophe. + if (StringUtils.containsAny(name, "'\"\\;\r\n")) + throw new IllegalStateException("Unexpected character in Enum.name(): " + name); + getStringBuilder().append("'").append(name).append("'"); + return this; + } + + public SQLFragment append(FieldKey fk) + { + if (null == fk) + return appendNull(); + append(String.valueOf(fk)); + return this; + } + + + /** Adds the object as a JDBC parameter value */ + public SQLFragment add(Object p) + { + getMutableParams().add(p); + return this; + } + + public SQLFragment add(Object p, JdbcType type) + { + getMutableParams().add(new Parameter.TypedValue(p, type)); + return this; + } + + /** Adds the objects as JDBC parameter values */ + public SQLFragment addAll(Collection l) + { + getMutableParams().addAll(l); + return this; + } + + + /** Adds the objects as JDBC parameter values */ + public SQLFragment addAll(Object... values) + { + if (values == null) + return this; + addAll(Arrays.asList(values)); + return this; + } + + + /** Sets the parameter at the index to the object's value */ + public void set(int i, Object p) + { + getMutableParams().set(i,p); + } + + /** Append both the SQL and the parameters from the other SQLFragment to this SQLFragment */ + public SQLFragment append(SQLFragment f) + { + if (null != f.sb) + getStringBuilder().append(f.sb); + else + getStringBuilder().append(f.sql); + if (null != f.params) + addAll(f.params); + mergeCommonTableExpressions(f); + tempTokens.addAll(f.tempTokens); + return this; + } + + public SQLFragment append(@NotNull Iterable fragments, @NotNull String separator) + { + String s = ""; + for (SQLFragment fragment : fragments) + { + append(s); + s = separator; + append(fragment); + } + return this; + } + + // return boolean so this can be used in an assert. passing in a dialect is not ideal, but parsing comments out + // before submitting the fragment is not reliable and holding statements & comments separately (to eliminate the + // need to parse them) isn't particularly easy... so punt for now. + public boolean appendComment(String comment, SqlDialect dialect) + { + if (dialect.supportsComments()) + { + StringBuilder sb = getStringBuilder(); + int len = sb.length(); + if (len > 0 && sb.charAt(len-1) != '\n') + sb.append('\n'); + sb.append("\n-- "); + boolean truncated = comment.length() > 1000; + if (truncated) + comment = StringUtilsLabKey.leftSurrogatePairFriendly(comment, 1000); + // Strip CR/LF so an embedded newline can't terminate the `--` comment and turn the rest + // of the payload into live SQL. + comment = comment.replace('\r', ' ').replace('\n', ' '); + sb.append(comment); + if (StringUtils.countMatches(comment, "'")%2==1) + sb.append("'"); + if (truncated) + sb.append("..."); + sb.append('\n'); + } + return true; + } + + + /** see also append(TableInfo, String alias) */ + public SQLFragment append(TableInfo table) + { + SQLFragment s = table.getSQLName(); + if (s != null) + return append(s); + + // GitHub 1432: Cannot delete a multi-choice option on a source type + String alias = AliasManager.makeLegalName(table.getName(), table.getSchema().getSqlDialect()); + return append(table.getFromSQL(alias)); + } + + /** Add a table/query to the SQL with an alias, as used in a FROM clause */ + public SQLFragment append(TableInfo table, String alias) + { + return append(table.getFromSQL(alias)); + } + + /** Add to the SQL */ + @Override + public SQLFragment append(char ch) + { + getStringBuilder().append(ch); + return this; + } + + /** This is like appendValue(CharSequence s), but force use of literal syntax + * CAUTIONARY NOTE: String literals in PostgresSQL are tricky because of overloaded functions + * array_agg('string') fails array_agg('string'::VARCHAR) works + * json_object('{}) works json_object('string'::VARCHAR) fails + * In the case of json_object() it expects TEXT. Postgres will promote 'json' to TEXT, but not 'json'::VARCHAR + */ + public SQLFragment appendStringLiteral(CharSequence s, @NotNull SqlDialect d) + { + if (null==s) + return appendNull(); + getStringBuilder().append(d.getStringHandler().quoteStringLiteral(s.toString())); + return this; + } + + /** Add to the SQL as either an in-line string literal or as a JDBC parameter depending on whether it would need escaping */ + public SQLFragment appendValue(CharSequence s) + { + return appendValue(s, null); + } + + public SQLFragment appendValue(CharSequence s, SqlDialect d) + { + if (null==s) + return appendNull(); + if (null==d || s.length() > 200) + return append("?").add(s.toString()); + appendStringLiteral(s, d); + return this; + } + + public SQLFragment appendInClause(@NotNull Collection params, SqlDialect dialect) + { + dialect.appendInClauseSql(this, params); + return this; + } + + public CharSequence getSqlCharSequence() + { + if (null != sb) + { + return sb; + } + return sql; + } + + public void insert(int index, SQLFragment sql) + { + if (!sql.getParams().isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with parameters - they must be inserted/merged separately"); + } + if (sql.commonTableExpressionsMap != null && !sql.commonTableExpressionsMap.isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with CTEs - they must be inserted/merged separately"); + } + if (!tempTokens.isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with temp tokens - they must be inserted/merged separately"); + } + getStringBuilder().insert(index, sql.getRawSQL()); + } + + /** Insert into the SQL */ + public void insert(int index, String str) + { + if ((StringUtils.countMatches(str, '\'') % 2) != 0 || + (StringUtils.countMatches(str, '\"') % 2) != 0 || + StringUtils.contains(str, ';')) + { + throw new IllegalArgumentException("SQLFragment.insert(int,String) does not allow semicolons or unmatched quotes"); + } + + getStringBuilder().insert(index, str); + } + + /** Insert this SQLFragment's SQL and parameters at the start of the existing SQL and parameters */ + public void prepend(SQLFragment sql) + { + getStringBuilder().insert(0, sql.getSqlCharSequence().toString()); + if (null != sql.params) + getMutableParams().addAll(0, sql.params); + mergeCommonTableExpressions(sql); + } + + + public int indexOf(String str) + { + return getStringBuilder().indexOf(str); + } + + + // Display query in "English" (display SQL with params substituted) + // with a little more work could probably be made to be SQL legal + public String getFilterText() + { + String sql = getSQL().replaceFirst("WHERE ", ""); + List params = getParams(); + for (Object param1 : params) + { + String param = param1.toString(); + param = param.replaceAll("\\\\", "\\\\\\\\"); + param = param.replaceAll("\\$", "\\\\\\$"); + sql = sql.replaceFirst("\\?", param); + } + return sql.replace("\"", ""); + } + + + @Override + public char charAt(int index) + { + return getSqlCharSequence().charAt(index); + } + + @Override + public int length() + { + return getSqlCharSequence().length(); + } + + @Override + public @NotNull CharSequence subSequence(int start, int end) + { + return getSqlCharSequence().subSequence(start, end); + } + + /** + * KEY is used as a faster way to look for equivalent CTE expressions. + * returning a name here allows us to potentially merge CTE at add time + * + * if you don't have a key you can just use sqlf.toString() + */ + public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf) + { + return addCommonTableExpression(dialect, key, proposedName, sqlf, false); + } + + public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf, boolean recursive) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + CTE prev = commonTableExpressionsMap.get(key); + if (null != prev) + return prev.token(); + CTE cte = new CTE(dialect, proposedName, sqlf, recursive); + commonTableExpressionsMap.put(key, cte); + return cte.token(); + } + + public String createCommonTableExpressionToken(SqlDialect dialect, Object key, String proposedName) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + CTE prev = commonTableExpressionsMap.get(key); + if (null != prev) + throw new IllegalStateException("Cannot create CTE token from already used key."); + CTE cte = new CTE(dialect ,proposedName); + commonTableExpressionsMap.put(key, cte); + return cte.token(); + } + + public void setCommonTableExpressionSql(Object key, SQLFragment sqlf, boolean recursive) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + + if (null != sqlf.commonTableExpressionsMap && !sqlf.commonTableExpressionsMap.isEmpty()) + { + // Need to merge CTEs up; this.cte depends on newSql.ctes, so they need to come first + SQLFragment newSql = new SQLFragment(sqlf); + LinkedHashMap toMap = new LinkedHashMap<>(newSql.commonTableExpressionsMap); + for (Map.Entry e : commonTableExpressionsMap.entrySet()) + { + CTE from = e.getValue(); + CTE to = toMap.get(e.getKey()); + if (null != to) + to.tokens.addAll(from.tokens); + else + toMap.put(e.getKey(), from.copy(false)); + } + + commonTableExpressionsMap = toMap; + newSql.commonTableExpressionsMap = null; + sqlf = newSql; + } + + CTE cte = commonTableExpressionsMap == null ? null : commonTableExpressionsMap.get(key); + if (null == cte) + throw new IllegalStateException("CTE not found."); + cte.sqlf = sqlf; + cte.recursive = recursive; + } + + + private void mergeCommonTableExpressions(SQLFragment sqlFrom) + { + if (null == sqlFrom.commonTableExpressionsMap || sqlFrom.commonTableExpressionsMap.isEmpty()) + return; + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + for (Map.Entry e : sqlFrom.commonTableExpressionsMap.entrySet()) + { + CTE from = e.getValue(); + CTE to = commonTableExpressionsMap.get(e.getKey()); + if (null != to) + to.tokens.addAll(from.tokens); + else + commonTableExpressionsMap.put(e.getKey(), from.copy(false)); + } + } + + + public void addTempToken(Object tempToken) + { + tempTokens.add(tempToken); + } + + public void addTempTokens(SQLFragment other) + { + tempTokens.add(other.tempTokens); + } + + public static SQLFragment prettyPrint(SQLFragment from) + { + SQLFragment sqlf = new SQLFragment(from); + + String s = from.getSqlCharSequence().toString(); + StringBuilder sb = new StringBuilder(s.length() + 200); + String[] lines = StringUtils.split(s, '\n'); + int indent = 0; + + for (String line : lines) + { + String t = line.trim(); + + if (t.isEmpty()) + continue; + + if (t.startsWith("-- params = b.getParams(); + assertEquals(2,params.size()); + assertEquals(5, params.get(0)); + assertEquals("xxyzzy", params.get(1)); + + + SQLFragment c = new SQLFragment(b); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT a FROM b WHERE x=?) + SELECT * FROM CTE WHERE y=?""", + c.getSQL()); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT a FROM b WHERE x=5) + SELECT * FROM CTE WHERE y='xxyzzy'""", + filterDebugString(c.toDebugString())); + params = c.getParams(); + assertEquals(2,params.size()); + assertEquals(5, params.get(0)); + assertEquals("xxyzzy", params.get(1)); + + + // combining + + SQLFragment sqlf = new SQLFragment(); + String token = sqlf.addCommonTableExpression(dialect, "KEY_A", "cte1", new SQLFragment("SELECT * FROM a")); + sqlf.append("SELECT * FROM ").append(token).append(" _1"); + + assertEquals(""" + WITH + /*CTE*/ + \tcte1 AS (SELECT * FROM a) + SELECT * FROM cte1 _1""", + sqlf.getSQL()); + + SQLFragment sqlf2 = new SQLFragment(); + String token2 = sqlf2.addCommonTableExpression(dialect, "KEY_A", "cte2", new SQLFragment("SELECT * FROM a")); + sqlf2.append("SELECT * FROM ").append(token2).append(" _2"); + assertEquals(""" + WITH + /*CTE*/ + \tcte2 AS (SELECT * FROM a) + SELECT * FROM cte2 _2""", + sqlf2.getSQL()); + + SQLFragment sqlf3 = new SQLFragment(); + String token3 = sqlf3.addCommonTableExpression(dialect, "KEY_B", "cte3", new SQLFragment("SELECT * FROM b")); + sqlf3.append("SELECT * FROM ").append(token3).append(" _3"); + assertEquals(""" + WITH + /*CTE*/ + \tcte3 AS (SELECT * FROM b) + SELECT * FROM cte3 _3""", + sqlf3.getSQL()); + + SQLFragment union = new SQLFragment(); + union.append(sqlf); + union.append("\nUNION\n"); + union.append(sqlf2); + union.append("\nUNION\n"); + union.append(sqlf3); + assertEquals(""" + WITH + /*CTE*/ + \tcte1 AS (SELECT * FROM a) + ,/*CTE*/ + \tcte3 AS (SELECT * FROM b) + SELECT * FROM cte1 _1 + UNION + SELECT * FROM cte1 _2 + UNION + SELECT * FROM cte3 _3""", + union.getSQL()); + } + + @Test + public void nested_cte() + { + // one-level cte using cteToken (CTE fragment 'a' does not contain a CTE) + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); + assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); + SQLFragment b = new SQLFragment(); + String cteToken = b.addCommonTableExpression(dialect, new Object(), "CTE", a); + b.append("SELECT * FROM ").append(cteToken).append(" WHERE p=?").add("parameterTWO"); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) + SELECT * FROM CTE WHERE p='parameterTWO'""", + filterDebugString(b.toDebugString())); + assertEquals("parameterONE", b.getParams().getFirst()); + } + + // two-level cte using cteTokens (CTE fragment 'b' contains a CTE of fragment a) + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); + assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterTWO"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + c.append("SELECT * FROM ").append(cteTokenB).append(" WHERE i=?").add(3); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterTWO') + SELECT * FROM B_ WHERE i=3""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(3, params.size()); + assertEquals("parameterONE", params.get(0)); + assertEquals("parameterTWO", params.get(1)); + assertEquals(3, params.get(2)); + } + + // Same as previous but top-level query has both a nested and non-nested CTE + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); + SQLFragment a2 = new SQLFragment("SELECT 2 as i, 'Atwo' as s, CAST(? AS VARCHAR) as p", "parameterAtwo"); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + String cteTokenA2 = c.addCommonTableExpression(dialect, new Object(), "A2_", a2); + c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') + ,/*CTE*/ + \tA2_ AS (SELECT 2 as i, 'Atwo' as s, CAST('parameterAtwo' AS VARCHAR) as p) + SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(4, params.size()); + assertEquals("parameterAone", params.get(0)); + assertEquals("parameterB", params.get(1)); + assertEquals("parameterAtwo", params.get(2)); + assertEquals(4, params.get(3)); + } + + // Same as previous but two of the CTEs are the same and should be collapsed (e.g. imagine a container filter implemented with a CTE) + // TODO, we only collapse CTEs that are siblings + { + SQLFragment cf = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, "CTE_KEY_CF", "A_", cf); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + String cteTokenA2 = c.addCommonTableExpression(dialect, "CTE_KEY_CF", "A2_", cf); + c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') + ,/*CTE*/ + \tA2_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(4, params.size()); + assertEquals("parameterAone", params.get(0)); + assertEquals("parameterB", params.get(1)); + assertEquals("parameterAone", params.get(2)); + assertEquals(4, params.get(3)); + } + } + + + private void shouldFail(Runnable r) + { + try + { + r.run(); + fail("Expected IllegalArgumentException"); + } + catch (IllegalArgumentException e) + { + // expected + } + } + + + @Test + public void testIllegalArgument() + { + shouldFail(() -> new SQLFragment(";")); + shouldFail(() -> new SQLFragment().append(";")); + shouldFail(() -> new SQLFragment("AND name='")); + shouldFail(() -> new SQLFragment().append("AND name = '")); + shouldFail(() -> new SQLFragment().append("AND name = 'Robert'); DROP TABLE Students; --")); + + shouldFail(() -> new SQLFragment().appendIdentifier("column name")); + shouldFail(() -> new SQLFragment().appendIdentifier("?")); + shouldFail(() -> new SQLFragment().appendIdentifier(";")); + shouldFail(() -> new SQLFragment().appendIdentifier("\"column\"name\"")); + } + + + String mysqlQuoteIdentifier(String id) + { + return "`" + id.replace("`", "``") + "`"; + } + + @Test + public void testMysql() + { + // OK + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("mysql")); + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my`sql")); + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my\"sql")); + + // not OK + shouldFail(() -> new SQLFragment().appendIdentifier("`")); + shouldFail(() -> new SQLFragment().appendIdentifier("`a`a`")); + } + + @Test + public void testAppendCommentStripsNewlines() + { + // PR1-P3: an embedded newline in a comment payload must not terminate the `-- ` comment + // line and expose the trailing text as live SQL. + if (!dialect.supportsComments()) + return; + + SQLFragment sqlf = new SQLFragment(); + sqlf.appendComment("hello\nDROP TABLE x;--", dialect); + String sql = sqlf.getSQL(); + + assertFalse("appendComment leaked an embedded newline into emitted SQL: " + sql, sql.contains("hello\nDROP")); + assertTrue("appendComment should keep the payload on one comment line: " + sql, sql.contains("hello DROP TABLE x;--")); + + // CR is stripped too + SQLFragment sqlf2 = new SQLFragment(); + sqlf2.appendComment("hi\rDROP TABLE x;--", dialect); + assertFalse("appendComment leaked an embedded CR: " + sqlf2.getSQL(), sqlf2.getSQL().contains("hi\rDROP")); + } + + @Test + public void testAppendIdentifierPreQuotedDottedAccepted() + { + // PR2-G1: a dotted sequence of individually well-formed quoted identifiers is legitimate -- + // e.g. a fully-qualified "schema"."table" returned by TableInfo.getSelectName(). The strict + // check validates each quoted segment rather than blanket-rejecting any interior dot. + new SQLFragment().appendIdentifier("\"schema\".\"table\""); + new SQLFragment().appendIdentifier("\"a\".\"b\".\"c\""); + } + + @Test + public void testAppendIdentifierPreQuotedDotInsideQuotesAccepted() + { + // PR2-G1: a `.` *inside* the quotes is part of a single identifier's name, not a separator. + // This is the metadata-name case (e.g. a calculated column's generated class name) that the + // earlier strict check wrongly rejected. + new SQLFragment().appendIdentifier("\"name.with.dots\""); + new SQLFragment().appendIdentifier("\"org.labkey.query.sql.CalculatedExpressionColumn6b8f\""); + } + + @Test + public void testAppendIdentifierPreQuotedInteriorQuoteRejected() + { + // PR2-G1: an interior `"` that isn't part of the standard `""` doubling is a breakout. + // Even quote count alone is insufficient. + shouldFail(() -> new SQLFragment().appendIdentifier("\"foo\"bar\"baz\"")); + // A classic breakout: close the identifier early, then inject. The lone interior quote + // (after `foo`) is followed by non-separator text, so it must be rejected. + shouldFail(() -> new SQLFragment().appendIdentifier("\"foo\"; DROP TABLE x; --\"")); + // Stray text after a valid segment (not a `.` separator). + shouldFail(() -> new SQLFragment().appendIdentifier("\"a\" \"b\"")); + // Trailing separator with no following segment. + shouldFail(() -> new SQLFragment().appendIdentifier("\"a\".\"")); + // Empty segment between separators. + shouldFail(() -> new SQLFragment().appendIdentifier("\"a\"..\"b\"")); + } + + @Test + public void testAppendIdentifierPreQuotedValidCases() + { + // PR2-G1: legitimate pre-quoted identifiers still pass. + new SQLFragment().appendIdentifier("\"my_table\""); // simple quoted + new SQLFragment().appendIdentifier("\"with a space\""); // whitespace inside quotes is fine + new SQLFragment().appendIdentifier("\"foo\"\"bar\""); // embedded literal " via "" doubling + new SQLFragment().appendIdentifier("\"contains\"\"more\"\"doubles\""); + // Doubled interior quotes around a dot decode to a single identifier named weird"."name -- + // distinct from the two-segment "weird"."name" -- and cannot break out, so it's accepted. + new SQLFragment().appendIdentifier("\"weird\"\".\"\"name\""); + } + } + + @Override + public boolean equals(Object obj) + { + if (!(obj instanceof SQLFragment other)) + { + return false; + } + return getSQL().equals(other.getSQL()) && getParams().equals(other.getParams()); + } + + @Override + public int hashCode() + { + return Objects.hash(getSQL(), getParams()); + } + + /** + * Joins the SQLFragments in the provided {@code Iterable} into a single SQLFragment. The SQL is joined by string + * concatenation using the provided separator. The parameters are combined to form the new parameter list. + * + * @param fragments SQLFragments to join together + * @param separator Separator to use + * @return A new SQLFragment that joins all the SQLFragments + */ + public static SQLFragment join(Iterable fragments, SQLFragment separator) + { + SQLFragment join = new SQLFragment(); + boolean first = true; + + for (SQLFragment fragment : fragments) + { + if (first) + first = false; + else + join.append(separator); + join.append(fragment); + } + + return join; + } +}