') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); fix(builder/parser): correct JSON filtering and deterministic filter order by pcfreak30 · Pull Request #40 · LumeWeb/queryutil · GitHub
Skip to content

fix(builder/parser): correct JSON filtering and deterministic filter order - #40

Merged
pcfreak30 merged 1 commit into
developfrom
fix/json-filtering-and-parser-order
Aug 29, 2026
Merged

fix(builder/parser): correct JSON filtering and deterministic filter order#40
pcfreak30 merged 1 commit into
developfrom
fix/json-filtering-and-parser-order

Conversation

@pcfreak30

@pcfreak30pcfreak30 commented Aug 29, 2026

Copy link
Copy Markdown
Member

Fixes JSON pattern matching on MySQL by unquoting extracted string values and applying an explicit case-insensitive collation, aligning behavior with SQLite.

Makes QueryParamParser emit filters in deterministic key order, removing an intermittent serializer failure caused by randomized Go map iteration.

Resolves pre-existing failures in the json_startswith_filter, json_not_contains_case-sensitive_filter, and json_field_with_explicit_null_value tests while preserving backwards-compatible OpNull semantics.


Based on the code changes, this pull request addresses two main issues:

Summary

This PR fixes JSON filtering behavior across different database dialects and ensures deterministic filter ordering in query parameter parsing.

Key Changes

1. Fixed JSON Filtering for MySQL/MariaDB Pattern Matching

Problem: MySQL's JSON_EXTRACT() function preserves surrounding quotes on string values, which breaks pattern matching operations (e.g., LIKE 'dar%' would fail against "dark" instead of matching dark).

Solution:

  • Added a new jsonExtractExpr() helper method that wraps MySQL/MariaDB JSON_EXTRACT() in JSON_UNQUOTE() to strip quotes and match SQLite's json_extract() behavior
  • For MySQL/MariaDB, added explicit case-insensitive collation (COLLATE utf8mb4_0900_ai_ci) to LIKE operations to match SQLite's default case-insensitive behavior
  • Refactored the JSON pattern matching clause builder to use consistent SQL construction across all operators (contains, not contains, starts with, ends with, and their case-sensitive/negative variants)

2. Fixed Non-Deterministic Filter Ordering

Problem: Go map iteration order is randomized, causing inconsistent filter application order when parsing query parameters.

Solution: Added sorting of map keys before building filters to ensure deterministic, reproducible filter ordering based on alphabetical key order.

3. Corrected NULL JSON Path Filtering Test

Updated test expectations to correctly reflect that OpNull matches missing JSON paths (treating them as NULL), consistent with documented JSON null filtering behavior.

4. Code Cleanup

  • Extracted dialect name constants for better maintainability
  • Removed empty error handling blocks in the parser code
  • Added documentation comments for the new JSON extraction helper

@kody-ai

This comment has been minimized.

@kody-ai

kody-aiBot commented Aug 29, 2026

Copy link
Copy Markdown

kody code-reviewKody Rulesmedium

The new buildJSONClause method constructs raw SQL clauses with LIKE/NOT LIKE operators via conditionBuilderDB.Where(...) without a context timeout, allowing queries to run indefinitely. Add .WithContext(ctx) before the .Where() call or wrap query construction in a db.RetryableComponentLock.

Kody rule violation: Disallow GORM queries without timeout

Comment threadfilter/builder/gorm_builder.go Outdated
Comment threadfilter/builder/gorm_builder.go Outdated
Comment threadfilter/builder/gorm_builder.go
@kody-ai

This comment has been minimized.

Comment threadfilter/builder/gorm_builder.go Outdated
Comment threadfilter/builder/gorm_builder.go Outdated
@kody-ai

This comment has been minimized.

Comment threadfilter/builder/gorm_builder.go Outdated
@kody-ai

This comment has been minimized.

kody-ai[bot]
kody-aiBot approved these changes Aug 29, 2026
@github-actions

github-actionsBot commented Aug 29, 2026

Copy link
Copy Markdown

Code Coverage Report

Total Coverage: 88.8%

Generated from commit: 569e0e7
Repository: LumeWeb/queryutil

@kody-ai

This comment has been minimized.

@kody-ai

kody-aiBot commented Aug 29, 2026

Copy link
Copy Markdown

kody code-reviewKody Rulesmedium

The added line at 184 uses b.baseTx.Raw("SELECT VERSION()").Scan(&version) without a context with timeout, which can hang indefinitely if the database is unresponsive. Per the rule, all GORM operations must use .WithContext(ctx). Although this is a raw SQL call via GORM, it still performs a database operation and should be given a timeout context to prevent blocking the query builder. Consider adding a context with a reasonable timeout (e.g., context.WithTimeout(...)) and passing it via .WithContext(ctx) before .Raw(...). This aligns with the rule's intent to prevent unbounded database queries.

Kody rule violation: Disallow GORM queries without timeout

Comment threadfilter/builder/gorm_builder.go Outdated
@kody-ai

This comment has been minimized.

Comment threadfilter/builder/gorm_builder.go
@pcfreak30
pcfreak30force-pushed the fix/json-filtering-and-parser-order branch from 02c9e84 to d430849CompareAugust 29, 2026 02:25
@kody-ai

This comment has been minimized.

Comment threadfilter/builder/gorm_builder.go Outdated
@pcfreak30
pcfreak30force-pushed the fix/json-filtering-and-parser-order branch from d430849 to ca70e06CompareAugust 29, 2026 02:28
@kody-ai

This comment has been minimized.

@pcfreak30
pcfreak30force-pushed the fix/json-filtering-and-parser-order branch from ca70e06 to a010be8CompareAugust 29, 2026 02:31
Comment threadfilter/builder/gorm_builder.go
Comment threadfilter/builder/gorm_builder.go
Comment threadfilter/builder/gorm_builder.go
@pcfreak30
pcfreak30force-pushed the fix/json-filtering-and-parser-order branch from a010be8 to 2d22c00CompareAugust 29, 2026 02:34
@kody-ai

This comment has been minimized.

@kody-ai

kody-aiBot commented Aug 29, 2026

Copy link
Copy Markdown

kody code-reviewKody Rulesmedium

The new detectMySQLFlavor function executes SELECT VERSION() without a context, which can block indefinitely if the database is unresponsive. Use db.WithContext(ctx) with a caller-provided timeout context to bound the query execution.

Kody rule violation: Disallow GORM queries without timeout

Comment threadfilter/builder/gorm_builder.go Outdated
@pcfreak30
pcfreak30force-pushed the fix/json-filtering-and-parser-order branch from 2d22c00 to 9e2b0f0CompareAugust 29, 2026 02:38
@kody-ai

This comment has been minimized.

@kody-ai

kody-aiBot commented Aug 29, 2026

Copy link
Copy Markdown

kody code-reviewKody Rulesmedium

In detectMySQLFlavor, the db.Raw("SELECT VERSION()").Scan(&version) call lacks a timeout context, executing without a timeout and potentially hanging if the database is unresponsive, which could block the builder indefinitely. Add a context with a timeout via db.WithContext(ctx) before calling Raw to prevent unbounded query execution. Note that this call is not inside a db.RetryableComponentLock, so the exception does not apply.

Kody rule violation: Disallow GORM queries without timeout

Comment threadfilter/builder/gorm_builder.go
Comment threadfilter/builder/gorm_builder.go
@pcfreak30
pcfreak30force-pushed the fix/json-filtering-and-parser-order branch from 9e2b0f0 to 4001828CompareAugust 29, 2026 02:42
@kody-ai

This comment has been minimized.

@pcfreak30
pcfreak30force-pushed the fix/json-filtering-and-parser-order branch from 4001828 to 8ecdee8CompareAugust 29, 2026 02:43
Comment threadfilter/builder/gorm_builder.go
Comment threadfilter/builder/gorm_builder.go
Comment threadfilter/builder/gorm_builder.go Outdated
- Fix MySQL JSON_EXTRACT returning quoted strings by wrapping in JSON_UNQUOTE
- Quote non-identifier JSON path segments for MySQL compatibility
- DRY operator maps into dialectPattern descriptors with precomputed lookups
- Detect MariaDB vs MySQL via SELECT VERSION(), cache on root *sql.DB
- Make dialect resolution lazy so ApplySort pays no DB round-trip
- ApplySort no longer constructs a GORMBuilder
- Sort filter map keys for deterministic serialization
- Add dialect detection and operator map unit tests
@pcfreak30
pcfreak30force-pushed the fix/json-filtering-and-parser-order branch from 8ecdee8 to 011dda2CompareAugust 29, 2026 02:46
@kody-ai

kody-aiBot commented Aug 29, 2026

Copy link
Copy Markdown

Kody Review Complete

Great news! 🎉
No issues were found that match your current review configurations.

Keep up the excellent work! 🚀

Kody Guide: Usage and Configuration
Interacting with Kody
  • Request a Review: Ask Kody to review your PR manually by adding a comment with the @kody start-review command at the root of your PR.

  • Validate Business Logic: Ask Kody to validate your code against business rules by adding a comment with the @kody -v business-logic command.

  • Provide Feedback: Help Kody learn and improve by reacting to its comments with a 👍 for helpful suggestions or a 👎 if improvements are needed.

Current Kody Configuration
Review Options

The following review options are enabled or disabled:

OptionsEnabled
Bug
Performance
Security
Business Logic

Access your configuration settings here.

kody-ai[bot]
kody-aiBot approved these changes Aug 29, 2026
@pcfreak30
pcfreak30 merged commit ad18be4 into developAug 29, 2026
4 checks passed
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

1 participant

@pcfreak30