diff --git a/benchmarks/queries/clickbench/README.md b/benchmarks/queries/clickbench/README.md index 8b3d08b128866..2fd31c0b4c630 100644 --- a/benchmarks/queries/clickbench/README.md +++ b/benchmarks/queries/clickbench/README.md @@ -261,6 +261,44 @@ WHERE "URL" < 'zzzz'; ``` +### Q14: Grouped `COUNT(DISTINCT )` beside a non-distinct `COUNT(*)` + +**Question**: "For the top 10 search phrases by hit count, how many distinct +mobile phone models were used?" + +**Important Query Properties**: A single `COUNT(DISTINCT)` over a high +cardinality string, next to a non-distinct `COUNT(*)`, grouped by a high +cardinality string. + +`COUNT(DISTINCT)` has a specialized `GroupsAccumulator` for the integer types +and for no other type. Every other type falls back to +`GroupsAccumulatorAdapter`, which holds one boxed `Accumulator`, and therefore +one hash table, for each group. The cost of that fallback is per group, so it is +the group cardinality that decides how much it costs. + +Extended Q2 above is the only other query that puts a `COUNT(DISTINCT)` on a +non-integer column, and it groups by `BrowserCountry`. This query groups by +`SearchPhrase` instead, which has far more distinct values, so it exercises the +adapter where its cost actually scales. + +The shape is also the one `SingleDistinctToGroupBy` reasons about. That rule +rewrites a lone distinct aggregate into a two phase group by, which is what +keeps it off the adapter. Standard Q8, Q10, Q11 and Q13 are already rewritten +because their distinct aggregate stands alone, and Q9 is not because it carries +an `AVG`. Q22 is the only standard query that pairs a lone distinct aggregate +with a non-distinct count, and its distinct argument is an `Int64`, which has a +specialized accumulator and never reaches the adapter. So no query in either +suite covers a lone `COUNT(DISTINCT )` next to a non-distinct count. + +```sql +SELECT "SearchPhrase", COUNT(*) AS c, COUNT(DISTINCT "MobilePhoneModel") AS models +FROM hits +WHERE "SearchPhrase" <> '' +GROUP BY "SearchPhrase" +ORDER BY c DESC +LIMIT 10; +``` + ## Data Notes diff --git a/benchmarks/queries/clickbench/extended/q14.sql b/benchmarks/queries/clickbench/extended/q14.sql new file mode 100644 index 0000000000000..4ecc85bd22564 --- /dev/null +++ b/benchmarks/queries/clickbench/extended/q14.sql @@ -0,0 +1,9 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +-- A grouped COUNT(DISTINCT ) beside a non-distinct COUNT(*). No query in +-- the standard suite has this shape: q22 is the only one that pairs a lone +-- distinct aggregate with a non-distinct count, and its distinct argument is an +-- Int64, which has a specialized GroupsAccumulator. A string argument has none, +-- so one boxed Accumulator, and one hash table, is built for every group. +SELECT "SearchPhrase", COUNT(*) AS c, COUNT(DISTINCT "MobilePhoneModel") AS models FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/sql_benchmarks/clickbench_extended/benchmarks/q14.benchmark b/benchmarks/sql_benchmarks/clickbench_extended/benchmarks/q14.benchmark new file mode 100644 index 0000000000000..203bd1fe7a789 --- /dev/null +++ b/benchmarks/sql_benchmarks/clickbench_extended/benchmarks/q14.benchmark @@ -0,0 +1,22 @@ +name Q14 +group clickbench_extended +subgroup ${CLICKBENCH_TYPE:-single} + +init sql_benchmarks/clickbench/init/set_config.sql + +load sql_benchmarks/clickbench/init/load-${CLICKBENCH_TYPE:-single}.sql + +assert I +SELECT COUNT(*) > 0 from hits; +---- +true + +run +SELECT "SearchPhrase", COUNT(*) AS c, COUNT(DISTINCT "MobilePhoneModel") AS models +FROM hits +WHERE "SearchPhrase" <> '' +GROUP BY "SearchPhrase" +ORDER BY c DESC +LIMIT 10; + +result sql_benchmarks/clickbench_extended/results/q14.csv diff --git a/datafusion/sqllogictest/test_files/clickbench_extended.slt b/datafusion/sqllogictest/test_files/clickbench_extended.slt index c410c98fe8d74..ccc4ef67ed5eb 100644 --- a/datafusion/sqllogictest/test_files/clickbench_extended.slt +++ b/datafusion/sqllogictest/test_files/clickbench_extended.slt @@ -61,6 +61,76 @@ SELECT COUNT(*) AS ShareCount FROM hits WHERE "IsMobile" = 1 AND "MobilePhoneMod ---- 0 +query IIII +SELECT "WatchID", MIN("ResolutionWidth") as wmin, MAX("ResolutionWidth") as wmax, SUM("IsRefresh") as srefresh FROM hits GROUP BY "WatchID" ORDER BY "WatchID" DESC LIMIT 10; +---- +9110818468285196899 0 0 0 +8924809397503602651 0 0 0 +8740403056911509777 0 0 0 +8156744413230856864 0 0 0 +8120543446287442873 0 0 0 +6864353419233967042 0 0 0 +6635790769678439148 0 0 0 +6308646140879811077 0 0 0 +5206346422301499756 0 0 0 +4894690465724379622 0 0 0 + +query III?? +SELECT "RegionID", "UserAgent", "OS", AVG(to_timestamp("ResponseEndTiming")-to_timestamp("ResponseStartTiming")) as avg_response_time, AVG(to_timestamp("ResponseEndTiming")-to_timestamp("ConnectTiming")) as avg_latency FROM hits GROUP BY "RegionID", "UserAgent", "OS" ORDER BY avg_latency DESC limit 10; +---- +229 3 44 0 days 0 hours 0 mins 0.000000000 secs 0 days 0 hours 0 mins 0.000000000 secs +197 7 39 0 days 0 hours 0 mins 0.000000000 secs 0 days 0 hours 0 mins 0.000000000 secs +839 3 2 0 days 0 hours 0 mins 0.000000000 secs 0 days 0 hours 0 mins 0.000000000 secs +839 0 0 0 days 0 hours 0 mins 0.000000000 secs 0 days 0 hours 0 mins 0.000000000 secs +39 7 2 0 days 0 hours 0 mins 0.000000000 secs 0 days 0 hours 0 mins 0.000000000 secs + +query I +SELECT MAX(len) FROM ( + SELECT LENGTH(FIRST_VALUE("URL" ORDER BY "EventTime")) as len + FROM hits + GROUP BY "UserID" +); +---- +72 + +query I +SELECT MAX(len) FROM ( + SELECT LENGTH(FIRST_VALUE("URL" ORDER BY "EventTime")) as len + FROM hits + GROUP BY "OS" +); +---- +57 + +query I +SELECT MAX(fv) FROM ( + SELECT FIRST_VALUE("WatchID" ORDER BY "EventTime") as fv + FROM hits + GROUP BY "UserID" +); +---- +9110818468285196899 + +query I +SELECT MAX(fv) FROM ( + SELECT FIRST_VALUE("WatchID" ORDER BY "EventTime") as fv + FROM hits + GROUP BY "OS" +); +---- +9110818468285196899 + +query I +SELECT SUM("CounterID") AS counter_id_sum +FROM hits +WHERE "URL" < 'zzzz'; +---- +173 + +query TII +SELECT "SearchPhrase", COUNT(*) AS c, COUNT(DISTINCT "MobilePhoneModel") AS models FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; +---- + statement ok drop table hits;