From f073eeb63fec341d1d81daf2ae559a41e04810b1 Mon Sep 17 00:00:00 2001 From: TimelordUK Date: Wed, 9 Sep 2026 18:59:41 +0100 Subject: [PATCH 1/2] fix(output): truncate table cells to the column width MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `-o table` renderer measures each column, caps the width at --max-col-width (default 50, not unlimited), draws the border from the capped widths — and then printed the cell values untruncated. Any value longer than the cap overflowed its cell, so that row ran past the border and the table came out ragged. A TeamCity project path like "xTrader2 / Trading Services / Pricing / Analytics / master" is 58 chars, giving a 75-column row inside a 67-column frame. Headers had the same hole: `{:^width$}` pads but never truncates, and it pads by byte length, so a wide or accented header misaligned too. The comfy-table styles did truncate, but by byte slicing (`&s[..max-3]`), which panics on a multi-byte boundary. Add `truncate_to_width` to string_utils: ANSI-aware (escape codes pass through uncharged, so colours survive), unicode-width-aware (never splits a multi-byte char or a wide glyph), appending "..." within the budget. Use it for cells and headers in both copies of the renderer (non_interactive::output_table_old_style and its duplicate main::output_table_helper), centre headers by display width, and replace the byte-slicing in the comfy path. `--max-col-width 0` remains the way to get untruncated values; it was already aligned and stays so. Tests: 6 unit tests plus a doctest for the helper, including a loop asserting the result never exceeds the budget for widths 0..=60; and tests/python_tests/test_table_output_alignment.py, which asserts every line of a rendered table has identical display width across caps 1/2/3/4/5/12/30/57/58/59, long headers, accented and wide characters, uncapped, and markdown style. 11 of its 15 cases fail against the pre-fix binary. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01NBuWNEbq5TNWsF2YR9wYVH --- src/main.rs | 24 ++- src/non_interactive.rs | 48 +++--- src/utils/string_utils.rs | 131 +++++++++++++++- .../test_table_output_alignment.py | 147 ++++++++++++++++++ 4 files changed, 316 insertions(+), 34 deletions(-) create mode 100644 tests/python_tests/test_table_output_alignment.py diff --git a/src/main.rs b/src/main.rs index ed5d51f..ae5f728 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,7 +8,7 @@ use sql_cli::data::data_view::DataView; use sql_cli::data::datatable::DataValue; use sql_cli::non_interactive::{OutputFormat, TableStyle}; use sql_cli::utils::app_paths::AppPaths; -use sql_cli::utils::string_utils::display_width; +use sql_cli::utils::string_utils::{display_width, truncate_to_width}; use std::io::Write; use std::{borrow::Cow, io}; @@ -1328,7 +1328,16 @@ fn output_table_helper( write!(writer, "|")?; for (i, col) in columns.iter().enumerate() { - write!(writer, " {:^width$} |", col, width = widths[i])?; + let header = truncate_to_width(col, widths[i]); + let padding = widths[i].saturating_sub(display_width(&header)); + let left = padding / 2; + write!( + writer, + " {}{}{} |", + " ".repeat(left), + header, + " ".repeat(padding - left) + )?; } writeln!(writer)?; @@ -1344,17 +1353,16 @@ fn output_table_helper( write!(writer, "|")?; for (i, value) in row.values.iter().enumerate() { if i < widths.len() { - let value_str = format_datavalue(value); + // Truncate to the column width: without this, a value longer + // than --max-col-width overflows its cell and the row no + // longer lines up with the border. + let value_str = truncate_to_width(&format_datavalue(value), widths[i]); let display_len = display_width(&value_str); // For ANSI-colored strings, manual padding is needed // because format! uses byte length, not display width write!(writer, " {}", value_str)?; - let padding_needed = if display_len < widths[i] { - widths[i] - display_len - } else { - 0 - }; + let padding_needed = widths[i].saturating_sub(display_len); write!(writer, "{} |", " ".repeat(padding_needed))?; } } diff --git a/src/non_interactive.rs b/src/non_interactive.rs index ac6052d..6d7efe2 100644 --- a/src/non_interactive.rs +++ b/src/non_interactive.rs @@ -17,7 +17,7 @@ use crate::services::query_execution_service::QueryExecutionService; use crate::sql::parser::ast::{CTEType, TableSource, CTE}; use crate::sql::recursive_parser::Parser; use crate::sql::script_parser::{ScriptParser, ScriptResult}; -use crate::utils::string_utils::display_width; +use crate::utils::string_utils::{display_width, truncate_to_width}; /// Check if a query references temporary tables (starting with #) /// Temporary tables are only valid in script mode @@ -1483,10 +1483,19 @@ fn output_table_old_style( } writeln!(writer)?; - // Print headers + // Print headers, centred by display width (so wide glyphs line up too) write!(writer, "|")?; for (i, col) in columns.iter().enumerate() { - write!(writer, " {:^width$} |", col, width = widths[i])?; + let header = truncate_to_width(col, widths[i]); + let padding = widths[i].saturating_sub(display_width(&header)); + let left = padding / 2; + write!( + writer, + " {}{}{} |", + " ".repeat(left), + header, + " ".repeat(padding - left) + )?; } writeln!(writer)?; @@ -1504,17 +1513,16 @@ fn output_table_old_style( write!(writer, "|")?; for (i, value) in row.values.iter().enumerate() { if i < widths.len() { - let value_str = format_value(value); + // Truncate to the column width: without this, a value longer + // than --max-col-width overflows its cell and the row no + // longer lines up with the border. + let value_str = truncate_to_width(&format_value(value), widths[i]); let display_len = display_width(&value_str); // For ANSI-colored strings, manual padding is needed // because format! uses byte length, not display width write!(writer, " {}", value_str)?; - let padding_needed = if display_len < widths[i] { - widths[i] - display_len - } else { - 0 - }; + let padding_needed = widths[i].saturating_sub(display_len); write!(writer, "{} |", " ".repeat(padding_needed))?; } } @@ -1621,14 +1629,9 @@ fn output_table( .map(|v| { let s = format_value(v); // Apply max width truncation if specified - if let Some(max_width) = max_col_width { - if s.len() > max_width { - format!("{}...", &s[..max_width.saturating_sub(3)]) - } else { - s - } - } else { - s + match max_col_width { + Some(max_width) => truncate_to_width(&s, max_width), + None => s, } }) .collect() @@ -1653,14 +1656,9 @@ fn output_table( .map(|v| { let s = format_value(v); // Apply max width truncation if specified - if let Some(max_width) = max_col_width { - if s.len() > max_width { - format!("{}...", &s[..max_width.saturating_sub(3)]) - } else { - s - } - } else { - s + match max_col_width { + Some(max_width) => truncate_to_width(&s, max_width), + None => s, } }) .collect(); diff --git a/src/utils/string_utils.rs b/src/utils/string_utils.rs index d181f46..57ded8d 100644 --- a/src/utils/string_utils.rs +++ b/src/utils/string_utils.rs @@ -1,4 +1,4 @@ -use unicode_width::UnicodeWidthStr; +use unicode_width::{UnicodeWidthChar, UnicodeWidthStr}; /// Strip ANSI escape codes from a string and return the display width /// This handles ANSI SGR (Select Graphic Rendition) codes like colors and styles @@ -52,6 +52,80 @@ pub fn display_width(s: &str) -> usize { result.width() } +/// Truncate a string so that its display width is at most `max_width` columns, +/// appending an ASCII ellipsis (`...`) when anything was cut. +/// +/// Like [`display_width`], this skips ANSI SGR escape sequences when measuring +/// (they are copied through, so colours survive truncation) and uses Unicode +/// display width, so it never splits a multi-byte character or a wide glyph. +/// The returned string is guaranteed to satisfy +/// `display_width(&result) <= max_width`, which is what table renderers rely on +/// to keep every row the same width as the border. +/// +/// # Examples +/// +/// ``` +/// use sql_cli::utils::string_utils::{display_width, truncate_to_width}; +/// +/// assert_eq!(truncate_to_width("hello", 10), "hello"); +/// assert_eq!(truncate_to_width("hello world", 8), "hello..."); +/// assert_eq!(display_width(&truncate_to_width("a very long value", 6)), 6); +/// +/// // Never splits a wide character +/// assert_eq!(truncate_to_width("⚡⚡⚡", 3), "⚡"); +/// ``` +pub fn truncate_to_width(s: &str, max_width: usize) -> String { + if display_width(s) <= max_width { + return s.to_string(); + } + if max_width == 0 { + return String::new(); + } + + // Reserve room for the ellipsis, unless the budget is too small to bother. + const ELLIPSIS: &str = "..."; + let (budget, ellipsis) = if max_width > ELLIPSIS.len() { + (max_width - ELLIPSIS.len(), ELLIPSIS) + } else { + (max_width, "") + }; + + let mut out = String::new(); + let mut used = 0usize; + let mut had_ansi = false; + let mut chars = s.chars().peekable(); + + while let Some(ch) = chars.next() { + if ch == '\x1b' && chars.peek() == Some(&'[') { + // Copy the escape sequence through without charging it to the budget + had_ansi = true; + out.push(ch); + out.push('['); + chars.next(); + for next_ch in chars.by_ref() { + out.push(next_ch); + if next_ch.is_ascii_alphabetic() { + break; + } + } + continue; + } + + let ch_width = ch.width().unwrap_or(0); + if used + ch_width > budget { + break; + } + out.push(ch); + used += ch_width; + } + + if had_ansi { + out.push_str("\x1b[0m"); + } + out.push_str(ellipsis); + out +} + #[cfg(test)] mod tests { use super::*; @@ -96,4 +170,59 @@ mod tests { 9 // 2 (bolt) + 1 (space) + 6 (REPEAT) ); } + + #[test] + fn test_truncate_shorter_than_max_is_unchanged() { + assert_eq!(truncate_to_width("hello", 10), "hello"); + assert_eq!(truncate_to_width("hello", 5), "hello"); + assert_eq!(truncate_to_width("", 5), ""); + } + + #[test] + fn test_truncate_appends_ellipsis() { + assert_eq!(truncate_to_width("hello world", 8), "hello..."); + assert_eq!( + truncate_to_width("xTrader2 / Trading Services / Pricing", 20), + "xTrader2 / Tradin..." + ); + } + + #[test] + fn test_truncate_never_exceeds_max_width() { + let long = "xTrader2 / Trading Services / Pricing / Analytics / master"; + for max in 0..=60 { + let truncated = truncate_to_width(long, max); + assert!( + display_width(&truncated) <= max, + "width {} exceeded max {} for {:?}", + display_width(&truncated), + max, + truncated + ); + } + } + + #[test] + fn test_truncate_tiny_budgets_drop_the_ellipsis() { + assert_eq!(truncate_to_width("hello", 0), ""); + assert_eq!(truncate_to_width("hello", 1), "h"); + assert_eq!(truncate_to_width("hello", 3), "hel"); + assert_eq!(truncate_to_width("hello", 4), "h..."); + } + + #[test] + fn test_truncate_respects_unicode_boundaries() { + // Wide characters are never split in half + assert_eq!(truncate_to_width("\u{26a1}\u{26a1}\u{26a1}", 3), "\u{26a1}"); + assert_eq!(display_width(&truncate_to_width("caf\u{e9} au lait", 6)), 6); + } + + #[test] + fn test_truncate_preserves_ansi_and_ignores_it_for_width() { + let colored = "\x1b[31mhello world\x1b[0m"; + let truncated = truncate_to_width(colored, 8); + assert_eq!(display_width(&truncated), 8); + assert!(truncated.starts_with("\x1b[31m")); + assert!(truncated.ends_with("...")); + } } diff --git a/tests/python_tests/test_table_output_alignment.py b/tests/python_tests/test_table_output_alignment.py new file mode 100644 index 0000000..18a89f7 --- /dev/null +++ b/tests/python_tests/test_table_output_alignment.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Alignment guarantees for `-o table` output. + +The renderer sizes every column, caps that size at --max-col-width (default +50), then draws a border from the capped widths. It used to print the cell +values untruncated, so any value longer than the cap overflowed its cell and +that row ran past the border — a table of long "/"-delimited names (TeamCity +project paths, say) came out ragged and unusable for pasting elsewhere. + +Every line of a rendered table must therefore have the same display width. +""" + +import os +import subprocess +import sys +import unicodedata +from pathlib import Path + +import pytest + + +def _sql_cli_binary(): + """Path to the release binary, with the .exe suffix Windows needs.""" + base_dir = Path(__file__).parent.parent.parent + suffix = ".exe" if sys.platform == "win32" else "" + return base_dir / "target" / "release" / f"sql-cli{suffix}" + + +def _display_width(text): + """Column count of a string, matching the Rust side's unicode-width use.""" + return sum(2 if unicodedata.east_asian_width(ch) in ("W", "F") else 1 for ch in text) + + +def render_table(csv_text, query, extra_args=(), tmp_path=None): + """Run a query with -o table and return the table's lines.""" + sql_cli = _sql_cli_binary() + if not sql_cli.exists(): + pytest.skip(f"sql-cli not built at {sql_cli}") + + data_file = tmp_path / "projects.csv" + data_file.write_text(csv_text, encoding="utf-8") + + cmd = [str(sql_cli), str(data_file), "-q", query, "-o", "table"] + cmd.extend(str(a) for a in extra_args) + + env = dict(os.environ, PYTHONIOENCODING="utf-8") + result = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", env=env) + assert result.returncode == 0, f"query failed: {result.stderr}" + + # Drop the trailing "# Query completed" note and any blank lines + return [ + line + for line in result.stdout.splitlines() + if line.strip() and not line.startswith("#") + ] + + +def assert_lines_aligned(lines): + widths = {_display_width(line) for line in lines} + assert len(widths) == 1, ( + "table rows have differing widths " + f"{sorted(widths)}:\n" + "\n".join(lines) + ) + + +# A TeamCity-style project path: long, "/"-delimited, well past the 50-char default cap +LONG_PATH = "xTrader2 / Trading Services / Pricing / Analytics / master" + +PROJECTS_CSV = ( + "Project,DurationSecs\n" + f'"{LONG_PATH}",120\n' + f'"{LONG_PATH}",240\n' + '"xTrader2 / Risk / Limits / nightly",900\n' + "Core,50\n" + "Core,70\n" +) + + +def test_long_values_do_not_overflow_the_default_cap(tmp_path): + lines = render_table( + PROJECTS_CSV, + "SELECT Project, sum(DurationSecs) as total_secs, count(*) as n " + "FROM projects GROUP BY Project ORDER BY total_secs DESC", + tmp_path=tmp_path, + ) + assert_lines_aligned(lines) + # The over-long value is truncated with an ellipsis rather than spilling out + assert any("..." in line for line in lines) + + +def test_unlimited_width_keeps_values_intact(tmp_path): + lines = render_table( + PROJECTS_CSV, + "SELECT Project, sum(DurationSecs) as total_secs FROM projects GROUP BY Project", + extra_args=["--max-col-width", "0"], + tmp_path=tmp_path, + ) + assert_lines_aligned(lines) + assert any(LONG_PATH in line for line in lines), "full path should survive uncapped" + + +@pytest.mark.parametrize("max_width", [1, 2, 3, 4, 5, 12, 30, 57, 58, 59]) +def test_alignment_holds_at_every_cap(tmp_path, max_width): + lines = render_table( + PROJECTS_CSV, + "SELECT Project, sum(DurationSecs) as total_secs FROM projects GROUP BY Project", + extra_args=["--max-col-width", max_width], + tmp_path=tmp_path, + ) + assert_lines_aligned(lines) + + +def test_long_header_is_truncated_too(tmp_path): + lines = render_table( + PROJECTS_CSV, + "SELECT Project, sum(DurationSecs) as total_duration_seconds_for_this_project " + "FROM projects GROUP BY Project", + extra_args=["--max-col-width", "20"], + tmp_path=tmp_path, + ) + assert_lines_aligned(lines) + + +def test_alignment_survives_wide_and_accented_characters(tmp_path): + csv_text = ( + "Project,DurationSecs\n" + '"café ⚡ / Trading Services / Pricing / Analytics / master",120\n' + "Core,50\n" + ) + lines = render_table( + csv_text, + "SELECT Project, sum(DurationSecs) as total_secs FROM projects GROUP BY Project", + extra_args=["--max-col-width", "20"], + tmp_path=tmp_path, + ) + assert_lines_aligned(lines) + + +def test_markdown_style_is_also_aligned(tmp_path): + lines = render_table( + PROJECTS_CSV, + "SELECT Project, sum(DurationSecs) as total_secs, count(*) as n " + "FROM projects GROUP BY Project ORDER BY total_secs DESC", + extra_args=["--table-style", "markdown"], + tmp_path=tmp_path, + ) + assert_lines_aligned(lines) From a371efaf5c8fc5032be75eceab8bb1072be84d94 Mon Sep 17 00:00:00 2001 From: TimelordUK Date: Wed, 9 Sep 2026 19:17:38 +0100 Subject: [PATCH 2/2] docs+examples: TeamCity agent capacity skeleton, and P44 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a skeleton for the question "would more build agents actually help?", correlating TeamCity queue time with Metricbeat host CPU. examples/teamcity_agent_capacity.sql live TeamCity + Elasticsearch, marked [TEST:SKIP] (credentials) examples/teamcity_agent_capacity_demo.sql same logic over sample data, runs today, passes as a smoke test data/teamcity_{builds,cpu}_sample.csv 6 hosts x 5 agents, built to produce all three verdicts The analysis classifies each queued build rather than reporting queue time, because queue time alone does not say agents are short: hardware_bound agents busy AND CPU pegged -> more agents make it worse add_agents agents busy AND CPU idle -> more agents per server help pool_mismatch agents were free -> not a capacity problem Two things worth recording from building it: sql-cli has no ASOF JOIN (kdb `aj`); join types are Cross/Full/Inner/Left/Right. It is expressible by hand — inequality join to every earlier sample, then ROW_NUMBER() ... ORDER BY ts DESC filtered to rn = 1 — and the examples carry that pattern with the operand-order constraint spelled out (the planner requires the left table's column as the left operand). P44: PARSE_DATETIME's explicit-format path parses `%z` and then discards the offset, so `+0000`, `+0100` and `-0500` all yield the same instant, while the one-arg auto-detect path on ISO-8601 input honours it. DuckDB applies the offset in all three cases. This is the shape of bug that does not announce itself: it would have shifted every TeamCity timestamp an hour off UTC through BST while Metricbeat stayed correct, quietly scrambling the correlation for half the year. Logged in docs/SQL_PARITY.md; the skeleton works around it by rebuilding the stamp as ISO-8601 for the one-arg parser. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01NBuWNEbq5TNWsF2YR9wYVH --- data/teamcity_builds_sample.csv | 47 +++ data/teamcity_cpu_sample.csv | 367 ++++++++++++++++++++++ docs/SQL_PARITY.md | 36 +++ examples/teamcity_agent_capacity.sql | 232 ++++++++++++++ examples/teamcity_agent_capacity_demo.sql | 126 ++++++++ 5 files changed, 808 insertions(+) create mode 100644 data/teamcity_builds_sample.csv create mode 100644 data/teamcity_cpu_sample.csv create mode 100644 examples/teamcity_agent_capacity.sql create mode 100644 examples/teamcity_agent_capacity_demo.sql diff --git a/data/teamcity_builds_sample.csv b/data/teamcity_builds_sample.csv new file mode 100644 index 0000000..767ea1e --- /dev/null +++ b/data/teamcity_builds_sample.csv @@ -0,0 +1,47 @@ +build_id,project,agent,queued_epoch,started_epoch,finished_epoch +10001,xTrader2 / Trading Services / Pricing / Analytics / master,tcbuild21-agent1,1788940500,1788940740,1788942000 +10002,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild21-agent2,1788940500,1788940740,1788942000 +10003,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild21-agent3,1788940500,1788940740,1788942000 +10004,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild21-agent4,1788940500,1788940740,1788942000 +10005,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild21-agent5,1788940500,1788940740,1788942000 +10006,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild21-agent1,1788940920,1788942000,1788942300 +10007,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild21-agent1,1788941100,1788942000,1788942300 +10008,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild21-agent1,1788941280,1788942000,1788942300 +10009,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild21-agent1,1788941460,1788942000,1788942300 +10010,xTrader2 / Trading Services / Pricing / Analytics / master,tcbuild22-agent1,1788940500,1788940740,1788942000 +10011,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild22-agent2,1788940500,1788940740,1788942000 +10012,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild22-agent3,1788940500,1788940740,1788942000 +10013,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild22-agent4,1788940500,1788940740,1788942000 +10014,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild22-agent5,1788940500,1788940740,1788942000 +10015,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild22-agent1,1788940920,1788942000,1788942300 +10016,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild22-agent1,1788941100,1788942000,1788942300 +10017,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild22-agent1,1788941280,1788942000,1788942300 +10018,xTrader2 / Trading Services / Pricing / UnitTests / master,tcbuild22-agent1,1788941460,1788942000,1788942300 +10019,xTrader2 / Tools / Packaging / release,tcbuild23-agent1,1788941900,1788942000,1788943200 +10020,xTrader2 / Tools / Packaging / release,tcbuild23-agent2,1788941900,1788942000,1788943200 +10021,xTrader2 / Tools / Packaging / release,tcbuild23-agent3,1788941900,1788942000,1788943200 +10022,xTrader2 / Tools / Packaging / release,tcbuild23-agent4,1788941900,1788942000,1788943200 +10023,xTrader2 / Tools / Packaging / release,tcbuild23-agent5,1788941900,1788942000,1788943200 +10024,xTrader2 / Tools / Packaging / release,tcbuild23-agent1,1788942120,1788943200,1788943500 +10025,xTrader2 / Tools / Packaging / release,tcbuild23-agent1,1788942300,1788943200,1788943500 +10026,xTrader2 / Tools / Packaging / release,tcbuild23-agent1,1788942480,1788943200,1788943500 +10027,xTrader2 / Tools / Packaging / release,tcbuild23-agent1,1788942660,1788943200,1788943500 +10028,xTrader2 / Tools / Packaging / release,tcbuild24-agent1,1788941900,1788942000,1788943200 +10029,xTrader2 / Tools / Packaging / release,tcbuild24-agent2,1788941900,1788942000,1788943200 +10030,xTrader2 / Tools / Packaging / release,tcbuild24-agent3,1788941900,1788942000,1788943200 +10031,xTrader2 / Tools / Packaging / release,tcbuild24-agent4,1788941900,1788942000,1788943200 +10032,xTrader2 / Tools / Packaging / release,tcbuild24-agent5,1788941900,1788942000,1788943200 +10033,xTrader2 / Tools / Packaging / release,tcbuild24-agent1,1788942120,1788943200,1788943500 +10034,xTrader2 / Tools / Packaging / release,tcbuild24-agent1,1788942300,1788943200,1788943500 +10035,xTrader2 / Tools / Packaging / release,tcbuild24-agent1,1788942480,1788943200,1788943500 +10036,xTrader2 / Tools / Packaging / release,tcbuild24-agent1,1788942660,1788943200,1788943500 +10037,xTrader2 / Risk / Limits / nightly,tcbuild25-agent1,1788943100,1788943200,1788944400 +10038,xTrader2 / Risk / Limits / nightly,tcbuild25-agent2,1788943100,1788943200,1788944400 +10039,xTrader2 / Risk / Limits / nightly,tcbuild25-agent1,1788943300,1788944400,1788944700 +10040,xTrader2 / Risk / Limits / nightly,tcbuild25-agent1,1788943500,1788944400,1788944700 +10041,xTrader2 / Risk / Limits / nightly,tcbuild25-agent1,1788943700,1788944400,1788944700 +10042,xTrader2 / Risk / Limits / nightly,tcbuild26-agent1,1788943100,1788943200,1788944400 +10043,xTrader2 / Risk / Limits / nightly,tcbuild26-agent2,1788943100,1788943200,1788944400 +10044,xTrader2 / Risk / Limits / nightly,tcbuild26-agent1,1788943300,1788944400,1788944700 +10045,xTrader2 / Risk / Limits / nightly,tcbuild26-agent1,1788943500,1788944400,1788944700 +10046,xTrader2 / Risk / Limits / nightly,tcbuild26-agent1,1788943700,1788944400,1788944700 diff --git a/data/teamcity_cpu_sample.csv b/data/teamcity_cpu_sample.csv new file mode 100644 index 0000000..09df7b1 --- /dev/null +++ b/data/teamcity_cpu_sample.csv @@ -0,0 +1,367 @@ +host,ts_epoch,cpu_pct +tcbuild21,1788940800,0.97 +tcbuild21,1788940860,0.97 +tcbuild21,1788940920,0.97 +tcbuild21,1788940980,0.97 +tcbuild21,1788941040,0.97 +tcbuild21,1788941100,0.97 +tcbuild21,1788941160,0.97 +tcbuild21,1788941220,0.97 +tcbuild21,1788941280,0.97 +tcbuild21,1788941340,0.97 +tcbuild21,1788941400,0.97 +tcbuild21,1788941460,0.97 +tcbuild21,1788941520,0.97 +tcbuild21,1788941580,0.97 +tcbuild21,1788941640,0.97 +tcbuild21,1788941700,0.97 +tcbuild21,1788941760,0.97 +tcbuild21,1788941820,0.97 +tcbuild21,1788941880,0.97 +tcbuild21,1788941940,0.97 +tcbuild21,1788942000,0.97 +tcbuild21,1788942060,0.97 +tcbuild21,1788942120,0.97 +tcbuild21,1788942180,0.35 +tcbuild21,1788942240,0.35 +tcbuild21,1788942300,0.35 +tcbuild21,1788942360,0.35 +tcbuild21,1788942420,0.35 +tcbuild21,1788942480,0.35 +tcbuild21,1788942540,0.35 +tcbuild21,1788942600,0.35 +tcbuild21,1788942660,0.35 +tcbuild21,1788942720,0.35 +tcbuild21,1788942780,0.35 +tcbuild21,1788942840,0.35 +tcbuild21,1788942900,0.35 +tcbuild21,1788942960,0.35 +tcbuild21,1788943020,0.35 +tcbuild21,1788943080,0.35 +tcbuild21,1788943140,0.35 +tcbuild21,1788943200,0.35 +tcbuild21,1788943260,0.35 +tcbuild21,1788943320,0.35 +tcbuild21,1788943380,0.35 +tcbuild21,1788943440,0.35 +tcbuild21,1788943500,0.35 +tcbuild21,1788943560,0.35 +tcbuild21,1788943620,0.35 +tcbuild21,1788943680,0.35 +tcbuild21,1788943740,0.35 +tcbuild21,1788943800,0.35 +tcbuild21,1788943860,0.35 +tcbuild21,1788943920,0.35 +tcbuild21,1788943980,0.35 +tcbuild21,1788944040,0.35 +tcbuild21,1788944100,0.35 +tcbuild21,1788944160,0.35 +tcbuild21,1788944220,0.35 +tcbuild21,1788944280,0.35 +tcbuild21,1788944340,0.35 +tcbuild21,1788944400,0.35 +tcbuild22,1788940800,0.97 +tcbuild22,1788940860,0.97 +tcbuild22,1788940920,0.97 +tcbuild22,1788940980,0.97 +tcbuild22,1788941040,0.97 +tcbuild22,1788941100,0.97 +tcbuild22,1788941160,0.97 +tcbuild22,1788941220,0.97 +tcbuild22,1788941280,0.97 +tcbuild22,1788941340,0.97 +tcbuild22,1788941400,0.97 +tcbuild22,1788941460,0.97 +tcbuild22,1788941520,0.97 +tcbuild22,1788941580,0.97 +tcbuild22,1788941640,0.97 +tcbuild22,1788941700,0.97 +tcbuild22,1788941760,0.97 +tcbuild22,1788941820,0.97 +tcbuild22,1788941880,0.97 +tcbuild22,1788941940,0.97 +tcbuild22,1788942000,0.97 +tcbuild22,1788942060,0.97 +tcbuild22,1788942120,0.97 +tcbuild22,1788942180,0.35 +tcbuild22,1788942240,0.35 +tcbuild22,1788942300,0.35 +tcbuild22,1788942360,0.35 +tcbuild22,1788942420,0.35 +tcbuild22,1788942480,0.35 +tcbuild22,1788942540,0.35 +tcbuild22,1788942600,0.35 +tcbuild22,1788942660,0.35 +tcbuild22,1788942720,0.35 +tcbuild22,1788942780,0.35 +tcbuild22,1788942840,0.35 +tcbuild22,1788942900,0.35 +tcbuild22,1788942960,0.35 +tcbuild22,1788943020,0.35 +tcbuild22,1788943080,0.35 +tcbuild22,1788943140,0.35 +tcbuild22,1788943200,0.35 +tcbuild22,1788943260,0.35 +tcbuild22,1788943320,0.35 +tcbuild22,1788943380,0.35 +tcbuild22,1788943440,0.35 +tcbuild22,1788943500,0.35 +tcbuild22,1788943560,0.35 +tcbuild22,1788943620,0.35 +tcbuild22,1788943680,0.35 +tcbuild22,1788943740,0.35 +tcbuild22,1788943800,0.35 +tcbuild22,1788943860,0.35 +tcbuild22,1788943920,0.35 +tcbuild22,1788943980,0.35 +tcbuild22,1788944040,0.35 +tcbuild22,1788944100,0.35 +tcbuild22,1788944160,0.35 +tcbuild22,1788944220,0.35 +tcbuild22,1788944280,0.35 +tcbuild22,1788944340,0.35 +tcbuild22,1788944400,0.35 +tcbuild23,1788940800,0.30 +tcbuild23,1788940860,0.30 +tcbuild23,1788940920,0.30 +tcbuild23,1788940980,0.30 +tcbuild23,1788941040,0.30 +tcbuild23,1788941100,0.30 +tcbuild23,1788941160,0.30 +tcbuild23,1788941220,0.30 +tcbuild23,1788941280,0.30 +tcbuild23,1788941340,0.30 +tcbuild23,1788941400,0.30 +tcbuild23,1788941460,0.30 +tcbuild23,1788941520,0.30 +tcbuild23,1788941580,0.30 +tcbuild23,1788941640,0.30 +tcbuild23,1788941700,0.30 +tcbuild23,1788941760,0.30 +tcbuild23,1788941820,0.30 +tcbuild23,1788941880,0.12 +tcbuild23,1788941940,0.12 +tcbuild23,1788942000,0.12 +tcbuild23,1788942060,0.12 +tcbuild23,1788942120,0.12 +tcbuild23,1788942180,0.12 +tcbuild23,1788942240,0.12 +tcbuild23,1788942300,0.12 +tcbuild23,1788942360,0.12 +tcbuild23,1788942420,0.12 +tcbuild23,1788942480,0.12 +tcbuild23,1788942540,0.12 +tcbuild23,1788942600,0.12 +tcbuild23,1788942660,0.12 +tcbuild23,1788942720,0.12 +tcbuild23,1788942780,0.12 +tcbuild23,1788942840,0.12 +tcbuild23,1788942900,0.12 +tcbuild23,1788942960,0.12 +tcbuild23,1788943020,0.12 +tcbuild23,1788943080,0.12 +tcbuild23,1788943140,0.12 +tcbuild23,1788943200,0.12 +tcbuild23,1788943260,0.12 +tcbuild23,1788943320,0.12 +tcbuild23,1788943380,0.30 +tcbuild23,1788943440,0.30 +tcbuild23,1788943500,0.30 +tcbuild23,1788943560,0.30 +tcbuild23,1788943620,0.30 +tcbuild23,1788943680,0.30 +tcbuild23,1788943740,0.30 +tcbuild23,1788943800,0.30 +tcbuild23,1788943860,0.30 +tcbuild23,1788943920,0.30 +tcbuild23,1788943980,0.30 +tcbuild23,1788944040,0.30 +tcbuild23,1788944100,0.30 +tcbuild23,1788944160,0.30 +tcbuild23,1788944220,0.30 +tcbuild23,1788944280,0.30 +tcbuild23,1788944340,0.30 +tcbuild23,1788944400,0.30 +tcbuild24,1788940800,0.30 +tcbuild24,1788940860,0.30 +tcbuild24,1788940920,0.30 +tcbuild24,1788940980,0.30 +tcbuild24,1788941040,0.30 +tcbuild24,1788941100,0.30 +tcbuild24,1788941160,0.30 +tcbuild24,1788941220,0.30 +tcbuild24,1788941280,0.30 +tcbuild24,1788941340,0.30 +tcbuild24,1788941400,0.30 +tcbuild24,1788941460,0.30 +tcbuild24,1788941520,0.30 +tcbuild24,1788941580,0.30 +tcbuild24,1788941640,0.30 +tcbuild24,1788941700,0.30 +tcbuild24,1788941760,0.30 +tcbuild24,1788941820,0.30 +tcbuild24,1788941880,0.12 +tcbuild24,1788941940,0.12 +tcbuild24,1788942000,0.12 +tcbuild24,1788942060,0.12 +tcbuild24,1788942120,0.12 +tcbuild24,1788942180,0.12 +tcbuild24,1788942240,0.12 +tcbuild24,1788942300,0.12 +tcbuild24,1788942360,0.12 +tcbuild24,1788942420,0.12 +tcbuild24,1788942480,0.12 +tcbuild24,1788942540,0.12 +tcbuild24,1788942600,0.12 +tcbuild24,1788942660,0.12 +tcbuild24,1788942720,0.12 +tcbuild24,1788942780,0.12 +tcbuild24,1788942840,0.12 +tcbuild24,1788942900,0.12 +tcbuild24,1788942960,0.12 +tcbuild24,1788943020,0.12 +tcbuild24,1788943080,0.12 +tcbuild24,1788943140,0.12 +tcbuild24,1788943200,0.12 +tcbuild24,1788943260,0.12 +tcbuild24,1788943320,0.12 +tcbuild24,1788943380,0.30 +tcbuild24,1788943440,0.30 +tcbuild24,1788943500,0.30 +tcbuild24,1788943560,0.30 +tcbuild24,1788943620,0.30 +tcbuild24,1788943680,0.30 +tcbuild24,1788943740,0.30 +tcbuild24,1788943800,0.30 +tcbuild24,1788943860,0.30 +tcbuild24,1788943920,0.30 +tcbuild24,1788943980,0.30 +tcbuild24,1788944040,0.30 +tcbuild24,1788944100,0.30 +tcbuild24,1788944160,0.30 +tcbuild24,1788944220,0.30 +tcbuild24,1788944280,0.30 +tcbuild24,1788944340,0.30 +tcbuild24,1788944400,0.30 +tcbuild25,1788940800,0.22 +tcbuild25,1788940860,0.22 +tcbuild25,1788940920,0.22 +tcbuild25,1788940980,0.22 +tcbuild25,1788941040,0.22 +tcbuild25,1788941100,0.22 +tcbuild25,1788941160,0.22 +tcbuild25,1788941220,0.22 +tcbuild25,1788941280,0.22 +tcbuild25,1788941340,0.22 +tcbuild25,1788941400,0.22 +tcbuild25,1788941460,0.22 +tcbuild25,1788941520,0.22 +tcbuild25,1788941580,0.22 +tcbuild25,1788941640,0.22 +tcbuild25,1788941700,0.22 +tcbuild25,1788941760,0.22 +tcbuild25,1788941820,0.22 +tcbuild25,1788941880,0.22 +tcbuild25,1788941940,0.22 +tcbuild25,1788942000,0.22 +tcbuild25,1788942060,0.22 +tcbuild25,1788942120,0.22 +tcbuild25,1788942180,0.22 +tcbuild25,1788942240,0.22 +tcbuild25,1788942300,0.22 +tcbuild25,1788942360,0.22 +tcbuild25,1788942420,0.22 +tcbuild25,1788942480,0.22 +tcbuild25,1788942540,0.22 +tcbuild25,1788942600,0.22 +tcbuild25,1788942660,0.22 +tcbuild25,1788942720,0.22 +tcbuild25,1788942780,0.22 +tcbuild25,1788942840,0.22 +tcbuild25,1788942900,0.22 +tcbuild25,1788942960,0.22 +tcbuild25,1788943020,0.22 +tcbuild25,1788943080,0.22 +tcbuild25,1788943140,0.22 +tcbuild25,1788943200,0.22 +tcbuild25,1788943260,0.22 +tcbuild25,1788943320,0.22 +tcbuild25,1788943380,0.22 +tcbuild25,1788943440,0.22 +tcbuild25,1788943500,0.22 +tcbuild25,1788943560,0.22 +tcbuild25,1788943620,0.22 +tcbuild25,1788943680,0.22 +tcbuild25,1788943740,0.22 +tcbuild25,1788943800,0.22 +tcbuild25,1788943860,0.22 +tcbuild25,1788943920,0.22 +tcbuild25,1788943980,0.22 +tcbuild25,1788944040,0.22 +tcbuild25,1788944100,0.22 +tcbuild25,1788944160,0.22 +tcbuild25,1788944220,0.22 +tcbuild25,1788944280,0.22 +tcbuild25,1788944340,0.22 +tcbuild25,1788944400,0.22 +tcbuild26,1788940800,0.22 +tcbuild26,1788940860,0.22 +tcbuild26,1788940920,0.22 +tcbuild26,1788940980,0.22 +tcbuild26,1788941040,0.22 +tcbuild26,1788941100,0.22 +tcbuild26,1788941160,0.22 +tcbuild26,1788941220,0.22 +tcbuild26,1788941280,0.22 +tcbuild26,1788941340,0.22 +tcbuild26,1788941400,0.22 +tcbuild26,1788941460,0.22 +tcbuild26,1788941520,0.22 +tcbuild26,1788941580,0.22 +tcbuild26,1788941640,0.22 +tcbuild26,1788941700,0.22 +tcbuild26,1788941760,0.22 +tcbuild26,1788941820,0.22 +tcbuild26,1788941880,0.22 +tcbuild26,1788941940,0.22 +tcbuild26,1788942000,0.22 +tcbuild26,1788942060,0.22 +tcbuild26,1788942120,0.22 +tcbuild26,1788942180,0.22 +tcbuild26,1788942240,0.22 +tcbuild26,1788942300,0.22 +tcbuild26,1788942360,0.22 +tcbuild26,1788942420,0.22 +tcbuild26,1788942480,0.22 +tcbuild26,1788942540,0.22 +tcbuild26,1788942600,0.22 +tcbuild26,1788942660,0.22 +tcbuild26,1788942720,0.22 +tcbuild26,1788942780,0.22 +tcbuild26,1788942840,0.22 +tcbuild26,1788942900,0.22 +tcbuild26,1788942960,0.22 +tcbuild26,1788943020,0.22 +tcbuild26,1788943080,0.22 +tcbuild26,1788943140,0.22 +tcbuild26,1788943200,0.22 +tcbuild26,1788943260,0.22 +tcbuild26,1788943320,0.22 +tcbuild26,1788943380,0.22 +tcbuild26,1788943440,0.22 +tcbuild26,1788943500,0.22 +tcbuild26,1788943560,0.22 +tcbuild26,1788943620,0.22 +tcbuild26,1788943680,0.22 +tcbuild26,1788943740,0.22 +tcbuild26,1788943800,0.22 +tcbuild26,1788943860,0.22 +tcbuild26,1788943920,0.22 +tcbuild26,1788943980,0.22 +tcbuild26,1788944040,0.22 +tcbuild26,1788944100,0.22 +tcbuild26,1788944160,0.22 +tcbuild26,1788944220,0.22 +tcbuild26,1788944280,0.22 +tcbuild26,1788944340,0.22 +tcbuild26,1788944400,0.22 diff --git a/docs/SQL_PARITY.md b/docs/SQL_PARITY.md index 0b479ac..8cc9d87 100644 --- a/docs/SQL_PARITY.md +++ b/docs/SQL_PARITY.md @@ -1922,6 +1922,42 @@ out of date. --- +### P44 — `PARSE_DATETIME`'s explicit format parses `%z` but throws the offset away +- **Status:** 🔴 OPEN — silent, and wrong by whole hours +- **Corpus:** none yet — no corpus case parses an offset-bearing timestamp. +- **Observed:** with an explicit format string the UTC offset is consumed by the + parser and then discarded, so every offset collapses to `+0000`. The one-arg + auto-detect path on the same value gets it right, so the two spellings of the + "same" parse disagree: + + | Query | Ours | DuckDB `strptime` | + |---|---|---| + | `PARSE_DATETIME_UTC('20260909T143000+0000','%Y%m%dT%H%M%S%z')` | 1788964200 | 1788964200 ✅ | + | `PARSE_DATETIME_UTC('20260909T143000+0100','%Y%m%dT%H%M%S%z')` | **1788964200** 🚫 | 1788960600 | + | `PARSE_DATETIME_UTC('20260909T143000-0500','%Y%m%dT%H%M%S%z')` | **1788964200** 🚫 | 1788982200 | + | `PARSE_DATETIME_UTC('2026-09-09T14:30:00+01:00')` (auto-detect) | 1788960600 ✅ | — | + + All three explicit-format spellings return the same instant. `+0100` should be + an hour *earlier* in UTC, `-0500` five hours later. +- **Found:** 2026-09-09, sketching a TeamCity/Metricbeat correlation. TeamCity's + REST API stamps builds as `yyyyMMdd'T'HHmmssZ` **with a real offset** + (`+0100` through BST), while Metricbeat ships UTC. Joining the two on time is + the entire point of that analysis, and this bug shifts one side by an hour for + half the year — silently, with every row still present and plausible. It is a + good example of the class of divergence that does not announce itself: nothing + errors, nothing looks empty, the correlation is just quietly wrong. +- **Why it bites harder than most:** the failure is seasonal. A query built and + checked in winter (`+0000`) is correct; the same query in summer is an hour + out. That is close to the worst possible shape for a data bug. +- **Decision:** fix — apply the parsed offset, and make the one-arg and two-arg + paths agree. Worth pinning corpus cases for `%z` with positive, negative and + zero offsets at the same time, since none exist today. +- **Workaround until then:** prefer the one-arg auto-detect on ISO-8601 input, + or normalise the source to UTC before parsing. +- **Related:** same silent-wrong-answer family as [P39](#p39) and [P43](#p43). + +--- + ## Deferred / won't fix (intentional) ### D1 — Recursive CTEs (`WITH RECURSIVE`) diff --git a/examples/teamcity_agent_capacity.sql b/examples/teamcity_agent_capacity.sql new file mode 100644 index 0000000..b1b13f7 --- /dev/null +++ b/examples/teamcity_agent_capacity.sql @@ -0,0 +1,232 @@ +-- [TEST:SKIP] Needs live TeamCity + Elasticsearch credentials +-- +-- Would more build agents actually help? (live endpoints) +-- +-- Runnable twin with sample data: teamcity_agent_capacity_demo.sql — start there +-- to see the output shape, then swap the two source CTEs below for these. +-- +-- Environment expected: +-- TC_HOST https://teamcity.example.com +-- TC_TOKEN TeamCity bearer token +-- ES_HOST https://elastic.example.com:9200 +-- ES_KEY Elasticsearch API key +-- +-- --------------------------------------------------------------------------- +-- THINGS TO FIX BEFORE THIS RUNS (all marked <> below) +-- 1. Server names. Assumed `tcbuildNN` for NN in 21..26, and agent names +-- shaped `-agentN` so the host is the part before the first '-'. +-- If agents are named differently, change the LEFT(...INDEXOF...) split +-- and the host list in the ES query. This is the #1 thing to get right: +-- every join in here keys on host. +-- 2. AGENTS_PER_HOST is hardcoded to 5 in the CASE below. +-- 3. The 0.85 "CPU is pegged" line, and the 60s "this build actually waited" +-- floor, are both judgement calls — tune once you see the distribution. +-- 4. The time window is inlined in BOTH queries and must match, or the asof +-- join silently has nothing to match against. +-- --------------------------------------------------------------------------- + +WITH + -- ==== SOURCE 1: TeamCity builds ======================================== + -- Two handles on the same data: the occupancy self-join needs the table on + -- both sides, and one CTE is one source. Yes, this fetches twice. + WEB builds_raw AS ( + URL '${TC_HOST}/app/rest/builds?locator=sinceDate:20260909T000000%2B0000,count:5000&fields=build(id,queuedDate,startDate,finishDate,agent(name),buildType(projectName))' + METHOD GET + HEADERS ( + 'Authorization': 'Bearer ${TC_TOKEN}', + 'Accept': 'application/json' + ) + FORMAT JSON + JSON_PATH 'build' + ), + WEB builds_raw2 AS ( + URL '${TC_HOST}/app/rest/builds?locator=sinceDate:20260909T000000%2B0000,count:5000&fields=build(id,queuedDate,startDate,finishDate,agent(name),buildType(projectName))' + METHOD GET + HEADERS ( + 'Authorization': 'Bearer ${TC_TOKEN}', + 'Accept': 'application/json' + ) + FORMAT JSON + JSON_PATH 'build' + ), + + -- ==== SOURCE 2: Metricbeat host CPU ==================================== + -- A `composite` aggregation, not a plain search: it returns ONE FLAT LIST of + -- {host, minute} buckets, which is much easier to read than nested + -- per_host -> per_minute buckets, or than raw `_source` hits. + -- + -- Use system.cpu.total.norm.pct (0..1 regardless of core count), NOT + -- system.cpu.total.pct (which goes to N for an N-core box). "Is the box + -- flat out" is a normalised question. + -- + -- <> the host list, and mind the composite `size` — beyond 1000 + -- host/minute buckets you need to page it with `after_key`. + WEB cpu_raw AS ( + URL '${ES_HOST}/metricbeat-*/_search' + METHOD POST + HEADERS ( + 'Authorization': 'ApiKey ${ES_KEY}', + 'Content-Type': 'application/json' + ) + BODY '{ + "size": 0, + "query": { "bool": { "filter": [ + { "range": { "@timestamp": { "gte": "2026-09-09T00:00:00Z", "lte": "2026-09-09T23:59:59Z" } } }, + { "terms": { "host.name": ["tcbuild21","tcbuild22","tcbuild23","tcbuild24","tcbuild25","tcbuild26"] } } + ] } }, + "aggs": { + "flat": { + "composite": { + "size": 1000, + "sources": [ + { "host": { "terms": { "field": "host.name" } } }, + { "ts": { "date_histogram": { "field": "@timestamp", "fixed_interval": "1m" } } } + ] + }, + "aggs": { "cpu": { "avg": { "field": "system.cpu.total.norm.pct" } } } + } + } + }' + FORMAT JSON + JSON_PATH 'aggregations.flat.buckets' + ), + + -- ==== NORMALISE ======================================================== + -- TeamCity stamps builds `20260909T143000+0100` — compact, WITH a real + -- offset. Two traps, both live: + -- * the one-arg auto-detect parser does not recognise the compact form + -- * the two-arg explicit-format parser DOES parse '%z' and then THROWS THE + -- OFFSET AWAY (docs/SQL_PARITY.md P44) — so through BST every TeamCity + -- timestamp lands an hour off UTC while Metricbeat stays correct, + -- silently scrambling the correlation for half the year. + -- So: rebuild the stamp as ISO-8601 and use the one-arg parser, which does + -- honour the offset. Delete this dance once P44 is fixed. + builds_iso AS ( + SELECT + id as build_id, + buildType_projectName as project, + agent_name as agent, + SUBSTRING(queuedDate,1,4) || '-' || SUBSTRING(queuedDate,5,2) || '-' || + SUBSTRING(queuedDate,7,2) || 'T' || SUBSTRING(queuedDate,10,2) || ':' || + SUBSTRING(queuedDate,12,2) || ':' || SUBSTRING(queuedDate,14,2) || + SUBSTRING(queuedDate,16,3) || ':' || SUBSTRING(queuedDate,19,2) as queued_iso, + SUBSTRING(startDate,1,4) || '-' || SUBSTRING(startDate,5,2) || '-' || + SUBSTRING(startDate,7,2) || 'T' || SUBSTRING(startDate,10,2) || ':' || + SUBSTRING(startDate,12,2) || ':' || SUBSTRING(startDate,14,2) || + SUBSTRING(startDate,16,3) || ':' || SUBSTRING(startDate,19,2) as started_iso + FROM builds_raw + ), + waiting AS ( + SELECT + build_id, + project, + LEFT(agent, INDEXOF(agent, '-')) as host, -- <> host split + UNIX_TIMESTAMP(PARSE_DATETIME_UTC(queued_iso)) as queued_epoch, + UNIX_TIMESTAMP(PARSE_DATETIME_UTC(started_iso)) as started_epoch + FROM builds_iso + ), + waited AS ( + SELECT build_id, project, host, queued_epoch, + started_epoch - queued_epoch as queue_secs + FROM waiting + WHERE started_epoch - queued_epoch > 60 -- <> jitter floor + ), + + -- Same normalisation on the second handle, as occupancy intervals. + running AS ( + SELECT + LEFT(agent_name, INDEXOF(agent_name, '-')) as r_host, + UNIX_TIMESTAMP(PARSE_DATETIME_UTC( + SUBSTRING(startDate,1,4) || '-' || SUBSTRING(startDate,5,2) || '-' || + SUBSTRING(startDate,7,2) || 'T' || SUBSTRING(startDate,10,2) || ':' || + SUBSTRING(startDate,12,2) || ':' || SUBSTRING(startDate,14,2) || + SUBSTRING(startDate,16,3) || ':' || SUBSTRING(startDate,19,2))) as r_start, + UNIX_TIMESTAMP(PARSE_DATETIME_UTC( + SUBSTRING(finishDate,1,4) || '-' || SUBSTRING(finishDate,5,2) || '-' || + SUBSTRING(finishDate,7,2) || 'T' || SUBSTRING(finishDate,10,2) || ':' || + SUBSTRING(finishDate,12,2) || ':' || SUBSTRING(finishDate,14,2) || + SUBSTRING(finishDate,16,3) || ':' || SUBSTRING(finishDate,19,2))) as r_end + FROM builds_raw2 + ), + + -- Metricbeat composite buckets -> flat columns. + -- <> Bucket keys arrive nested as key.host / key.ts, and the metric as + -- cpu.value. Reading nested columns is a known rough edge here; if these + -- names do not resolve, dump the ES response to a file and READ_JSON it, or + -- flatten with a one-line jq first. This is the one part I could not test + -- without a live cluster. + cpu_norm AS ( + SELECT + key_host as host, + key_ts / 1000 as ts_epoch, -- composite date key is epoch MILLIS + cpu_value as cpu_pct + FROM cpu_raw + ), + + -- ==== HOW BUSY WAS THE HOST WHEN THIS BUILD QUEUED? ==================== + occupancy AS ( + SELECT + waited.build_id, waited.project, waited.host, + waited.queued_epoch, waited.queue_secs, + COUNT(*) as busy_agents + FROM waited + JOIN running + ON waited.host = running.r_host + AND waited.queued_epoch >= running.r_start + AND waited.queued_epoch < running.r_end + GROUP BY waited.build_id, waited.project, waited.host, + waited.queued_epoch, waited.queue_secs + ), + + -- ==== ASOF JOIN, BY HAND =============================================== + -- There is no ASOF JOIN / kdb `aj` here. Pair each build with every CPU + -- sample at or before it, then keep the latest — same semantics as `aj`. + -- The join condition needs the LEFT table's column as the LEFT operand + -- (`a.t >= b.t`, never `b.t <= a.t`) or the planner rejects it outright. + cpu_candidates AS ( + SELECT occupancy.build_id, cpu_norm.ts_epoch, cpu_norm.cpu_pct + FROM occupancy + JOIN cpu_norm + ON occupancy.host = cpu_norm.host + AND occupancy.queued_epoch >= cpu_norm.ts_epoch + ), + cpu_ranked AS ( + SELECT build_id, ts_epoch, cpu_pct, + ROW_NUMBER() OVER (PARTITION BY build_id ORDER BY ts_epoch DESC) as rn + FROM cpu_candidates + ), + cpu_at_queue AS ( + SELECT build_id as c_build_id, ts_epoch as c_ts, cpu_pct as c_cpu + FROM cpu_ranked WHERE rn = 1 + ), + + -- ==== VERDICT ========================================================== + classified AS ( + SELECT + occupancy.build_id, occupancy.project, occupancy.host, + occupancy.queue_secs, occupancy.busy_agents, + cpu_at_queue.c_cpu as cpu_at_queue, + -- Guard: if the nearest CPU sample is minutes stale, the verdict on + -- that row is worth little. Watch this column. + occupancy.queued_epoch - cpu_at_queue.c_ts as cpu_staleness_secs, + CASE + WHEN occupancy.busy_agents < 5 THEN 'pool_mismatch' -- <> agents/host + WHEN cpu_at_queue.c_cpu >= 0.85 THEN 'hardware_bound' -- <> pegged line + ELSE 'add_agents' + END as verdict + FROM occupancy + JOIN cpu_at_queue ON occupancy.build_id = cpu_at_queue.c_build_id + ) + +SELECT + verdict, + COUNT(*) as builds_waiting, + SUM(queue_secs) as total_queue_secs, + ROUND(AVG(queue_secs), 0) as avg_queue_secs, + ROUND(AVG(cpu_at_queue), 2) as avg_cpu, + ROUND(AVG(busy_agents), 1) as avg_busy_agents, + MAX(cpu_staleness_secs) as worst_cpu_staleness +FROM classified +GROUP BY verdict +ORDER BY total_queue_secs DESC; +GO diff --git a/examples/teamcity_agent_capacity_demo.sql b/examples/teamcity_agent_capacity_demo.sql new file mode 100644 index 0000000..74fc9a9 --- /dev/null +++ b/examples/teamcity_agent_capacity_demo.sql @@ -0,0 +1,126 @@ +-- #! ../data/teamcity_builds_sample.csv +-- +-- Would more build agents actually help? (runnable demo, sample data) +-- +-- This is the same logic as teamcity_agent_capacity.sql, but against checked-in +-- sample data so it runs with no credentials. See that file for the versions of +-- the two source CTEs that hit the real TeamCity and Elasticsearch endpoints. +-- +-- The point of the analysis: queue time on its own does NOT tell you agents are +-- short. Each queued build is classified into one of three verdicts, and only +-- one of them is fixed by adding agents: +-- +-- hardware_bound agents all busy AND host CPU pegged -> more agents on these +-- boxes make things WORSE (ninja already owns every core) +-- add_agents agents all busy AND host CPU idle -> more agents per +-- server is a free win +-- pool_mismatch agents were FREE while this build waited -> not a capacity +-- problem at all; check agent pool / build requirements +-- +-- The sample data is built to produce all three, two hosts each. + +WITH + -- Two handles on the same build data: the self-join that computes agent + -- occupancy needs the table on both sides, and each CTE is one source. + WEB builds_raw AS (URL 'file://data/teamcity_builds_sample.csv' FORMAT CSV), + WEB builds_raw2 AS (URL 'file://data/teamcity_builds_sample.csv' FORMAT CSV), + WEB cpu_raw AS (URL 'file://data/teamcity_cpu_sample.csv' FORMAT CSV), + + -- Builds that actually had to wait. 60s filter drops normal scheduling jitter. + waiting AS ( + SELECT + build_id, + project, + LEFT(agent, INDEXOF(agent, '-')) as host, + queued_epoch, + started_epoch - queued_epoch as queue_secs + FROM builds_raw + WHERE started_epoch - queued_epoch > 60 + ), + + -- Every build, as an occupancy interval [start, end) on its host. + running AS ( + SELECT + LEFT(agent, INDEXOF(agent, '-')) as r_host, + started_epoch as r_start, + finished_epoch as r_end + FROM builds_raw2 + ), + + -- How many agents on that host were busy at the moment this build queued? + occupancy AS ( + SELECT + waiting.build_id, + waiting.project, + waiting.host, + waiting.queued_epoch, + waiting.queue_secs, + COUNT(*) as busy_agents + FROM waiting + JOIN running + ON waiting.host = running.r_host + AND waiting.queued_epoch >= running.r_start + AND waiting.queued_epoch < running.r_end + GROUP BY waiting.build_id, waiting.project, waiting.host, + waiting.queued_epoch, waiting.queue_secs + ), + + -- ASOF JOIN, done by hand: sql-cli has no ASOF/kdb-style `aj`, so pair every + -- build with every CPU sample at or before it, then keep the latest one. + -- NOTE: the join condition needs the LEFT table's column as the LEFT operand + -- (`a.t >= b.t`, never `b.t <= a.t`) or the planner rejects it. + cpu_candidates AS ( + SELECT + occupancy.build_id, + cpu_raw.ts_epoch, + cpu_raw.cpu_pct + FROM occupancy + JOIN cpu_raw + ON occupancy.host = cpu_raw.host + AND occupancy.queued_epoch >= cpu_raw.ts_epoch + ), + cpu_ranked AS ( + SELECT + build_id, + ts_epoch, + cpu_pct, + ROW_NUMBER() OVER (PARTITION BY build_id ORDER BY ts_epoch DESC) as rn + FROM cpu_candidates + ), + cpu_at_queue AS ( + SELECT build_id as c_build_id, ts_epoch as c_ts, cpu_pct as c_cpu + FROM cpu_ranked + WHERE rn = 1 + ), + + -- Join the two together and apply the verdict. + -- AGENTS_PER_HOST = 5 and the 0.85 CPU line are the two knobs to tune. + classified AS ( + SELECT + occupancy.build_id, + occupancy.project, + occupancy.host, + occupancy.queue_secs, + occupancy.busy_agents, + cpu_at_queue.c_cpu as cpu_at_queue, + occupancy.queued_epoch - cpu_at_queue.c_ts as cpu_staleness_secs, + CASE + WHEN occupancy.busy_agents < 5 THEN 'pool_mismatch' + WHEN cpu_at_queue.c_cpu >= 0.85 THEN 'hardware_bound' + ELSE 'add_agents' + END as verdict + FROM occupancy + JOIN cpu_at_queue ON occupancy.build_id = cpu_at_queue.c_build_id + ) + +SELECT + verdict, + COUNT(*) as builds_waiting, + SUM(queue_secs) as total_queue_secs, + ROUND(AVG(queue_secs), 0) as avg_queue_secs, + ROUND(AVG(cpu_at_queue), 2) as avg_cpu, + ROUND(AVG(busy_agents), 1) as avg_busy_agents +FROM classified +GROUP BY verdict +ORDER BY total_queue_secs DESC; +GO