From 09e3ba357bab25a4dae41651e80b79a56f22a868 Mon Sep 17 00:00:00 2001 From: Sarthak Agarwal Date: Wed, 4 Feb 2026 13:59:19 +0530 Subject: [PATCH] xml parsing fix and refactor to support directory --- codeflash/languages/javascript/parse.py | 419 ++++++++++++++++++ codeflash/verification/parse_test_output.py | 370 +--------------- .../languages/javascript/test_vitest_junit.py | 214 +++++++++ 3 files changed, 647 insertions(+), 356 deletions(-) create mode 100644 codeflash/languages/javascript/parse.py diff --git a/codeflash/languages/javascript/parse.py b/codeflash/languages/javascript/parse.py new file mode 100644 index 000000000..bc24013d9 --- /dev/null +++ b/codeflash/languages/javascript/parse.py @@ -0,0 +1,419 @@ +"""Jest/Vitest JUnit XML parsing for JavaScript/TypeScript tests. + +This module handles parsing of JUnit XML test results produced by Jest and Vitest +test runners. It extracts test results, timing information, and maps them back +to instrumented test files. +""" + +from __future__ import annotations + +import contextlib +import json +import re +from pathlib import Path +from typing import TYPE_CHECKING + +from junitparser.xunit2 import JUnitXml + +from codeflash.cli_cmds.console import logger +from codeflash.models.models import ( + FunctionTestInvocation, + InvocationId, + TestResults, + TestType, +) + +if TYPE_CHECKING: + import subprocess + + from codeflash.models.models import TestFiles + from codeflash.verification.verification_utils import TestConfig + + +# Jest timing marker patterns (from codeflash-jest-helper.js console.log output) +# Format: !$######testName:testName:funcName:loopIndex:lineId######$! (start) +# Format: !######testName:testName:funcName:loopIndex:lineId:durationNs######! (end) +jest_start_pattern = re.compile(r"!\$######([^:]+):([^:]+):([^:]+):([^:]+):([^#]+)######\$!") +jest_end_pattern = re.compile(r"!######([^:]+):([^:]+):([^:]+):([^:]+):([^:]+):(\d+)######!") + + +def _extract_jest_console_output(suite_elem) -> str: + """Extract console output from Jest's JUnit XML system-out element. + + Jest-junit writes console.log output as a JSON array in the testsuite's system-out. + Each entry has: {"message": "...", "origin": "...", "type": "log"} + + Args: + suite_elem: The testsuite lxml element + + Returns: + Concatenated message content from all log entries + + """ + system_out_elem = suite_elem.find("system-out") + if system_out_elem is None or system_out_elem.text is None: + return "" + + raw_content = system_out_elem.text.strip() + if not raw_content: + return "" + + # Jest-junit wraps console output in a JSON array + # Try to parse as JSON first + try: + log_entries = json.loads(raw_content) + if isinstance(log_entries, list): + # Extract message field from each log entry + messages = [] + for entry in log_entries: + if isinstance(entry, dict) and "message" in entry: + messages.append(entry["message"]) + return "\n".join(messages) + except (json.JSONDecodeError, TypeError): + # Not JSON - return as plain text (fallback for pytest-style output) + pass + + return raw_content + + +def parse_jest_test_xml( + test_xml_file_path: Path, + test_files: TestFiles, + test_config: TestConfig, + run_result: subprocess.CompletedProcess | None = None, + parse_func=None, + resolve_test_file_from_class_path=None, +) -> TestResults: + """Parse Jest JUnit XML test results. + + Jest-junit has a different structure than pytest: + - system-out is at the testsuite level (not testcase) + - system-out contains a JSON array of log entries + - Timing markers are in the message field of log entries + + Args: + test_xml_file_path: Path to the Jest JUnit XML file + test_files: TestFiles object with test file information + test_config: Test configuration + run_result: Optional subprocess result for logging + parse_func: XML parser function (injected to avoid circular imports) + resolve_test_file_from_class_path: Function to resolve test file paths (injected) + + Returns: + TestResults containing parsed test invocations + + """ + test_results = TestResults() + + if not test_xml_file_path.exists(): + logger.warning(f"No JavaScript test results for {test_xml_file_path} found.") + return test_results + + # Log file size for debugging + file_size = test_xml_file_path.stat().st_size + logger.debug(f"Jest XML file size: {file_size} bytes at {test_xml_file_path}") + + try: + xml = JUnitXml.fromfile(str(test_xml_file_path), parse_func=parse_func) + logger.debug(f"Successfully parsed Jest JUnit XML from {test_xml_file_path}") + except Exception as e: + logger.warning(f"Failed to parse {test_xml_file_path} as JUnitXml. Exception: {e}") + return test_results + + base_dir = test_config.tests_project_rootdir + logger.debug(f"Jest XML parsing: base_dir={base_dir}, num_test_files={len(test_files.test_files)}") + + # Build lookup from instrumented file path to TestFile for direct matching + # This handles cases where instrumented files are in temp directories + instrumented_path_lookup: dict[str, tuple[Path, TestType]] = {} + for test_file in test_files.test_files: + if test_file.instrumented_behavior_file_path: + # Store both the absolute path and resolved path as keys + abs_path = str(test_file.instrumented_behavior_file_path.resolve()) + instrumented_path_lookup[abs_path] = (test_file.instrumented_behavior_file_path, test_file.test_type) + # Also store the string representation in case of minor path differences + instrumented_path_lookup[str(test_file.instrumented_behavior_file_path)] = ( + test_file.instrumented_behavior_file_path, + test_file.test_type, + ) + logger.debug(f"Jest XML lookup: registered {abs_path}") + + # Also build a filename-only lookup for fallback matching + # This handles cases where JUnit XML has relative paths that don't match absolute paths + # e.g., JUnit has "test/utils__perfinstrumented.test.ts" but lookup has absolute paths + filename_lookup: dict[str, tuple[Path, TestType]] = {} + for test_file in test_files.test_files: + if test_file.instrumented_behavior_file_path: + filename = test_file.instrumented_behavior_file_path.name + # Only add if not already present (avoid overwrites in case of duplicate filenames) + if filename not in filename_lookup: + filename_lookup[filename] = (test_file.instrumented_behavior_file_path, test_file.test_type) + logger.debug(f"Jest XML filename lookup: registered {filename}") + + # Fallback: if JUnit XML doesn't have system-out, use subprocess stdout directly + global_stdout = "" + if run_result is not None: + try: + global_stdout = run_result.stdout if isinstance(run_result.stdout, str) else run_result.stdout.decode() + # Debug: log if timing markers are found in stdout + if global_stdout: + marker_count = len(jest_start_pattern.findall(global_stdout)) + if marker_count > 0: + logger.debug(f"Found {marker_count} timing start markers in Jest stdout") + else: + logger.debug(f"No timing start markers found in Jest stdout (len={len(global_stdout)})") + except (AttributeError, UnicodeDecodeError): + global_stdout = "" + + suite_count = 0 + testcase_count = 0 + for suite in xml: + suite_count += 1 + # Extract console output from suite-level system-out (Jest specific) + suite_stdout = _extract_jest_console_output(suite._elem) # noqa: SLF001 + + # Fallback: use subprocess stdout if XML system-out is empty + if not suite_stdout and global_stdout: + suite_stdout = global_stdout + + # Parse timing markers from the suite's console output + start_matches = list(jest_start_pattern.finditer(suite_stdout)) + end_matches_dict = {} + for match in jest_end_pattern.finditer(suite_stdout): + # Key: (testName, testName2, funcName, loopIndex, lineId) + key = match.groups()[:5] + end_matches_dict[key] = match + + for testcase in suite: + testcase_count += 1 + test_class_path = testcase.classname # For Jest, this is the file path + test_name = testcase.name + + if test_name is None: + logger.debug(f"testcase.name is None in Jest XML {test_xml_file_path}, skipping") + continue + + logger.debug(f"Jest XML: processing testcase name={test_name}, classname={test_class_path}") + + # First, try direct lookup in instrumented file paths + # This handles cases where instrumented files are in temp directories + test_file_path = None + test_type = None + + if test_class_path: + # Try exact match with classname (which should be the filepath from jest-junit) + if test_class_path in instrumented_path_lookup: + test_file_path, test_type = instrumented_path_lookup[test_class_path] + else: + # Try resolving the path and matching + try: + resolved_path = str(Path(test_class_path).resolve()) + if resolved_path in instrumented_path_lookup: + test_file_path, test_type = instrumented_path_lookup[resolved_path] + except Exception: + pass + + # If direct lookup failed, try the file attribute + if test_file_path is None: + test_file_name = suite._elem.attrib.get("file") or testcase._elem.attrib.get("file") # noqa: SLF001 + if test_file_name: + if test_file_name in instrumented_path_lookup: + test_file_path, test_type = instrumented_path_lookup[test_file_name] + else: + try: + resolved_path = str(Path(test_file_name).resolve()) + if resolved_path in instrumented_path_lookup: + test_file_path, test_type = instrumented_path_lookup[resolved_path] + except Exception: + pass + + # Fall back to traditional path resolution if direct lookup failed + if test_file_path is None and resolve_test_file_from_class_path is not None: + test_file_path = resolve_test_file_from_class_path(test_class_path, base_dir) + if test_file_path is None: + test_file_name = suite._elem.attrib.get("file") or testcase._elem.attrib.get("file") # noqa: SLF001 + if test_file_name: + test_file_path = base_dir.parent / test_file_name + if not test_file_path.exists(): + test_file_path = base_dir / test_file_name + + # Fallback: try matching by filename only + # This handles when JUnit XML has relative paths like "test/utils__perfinstrumented.test.ts" + # that can't be resolved to absolute paths because they're relative to Jest's CWD, not parse CWD + if test_file_path is None and test_class_path: + # Extract filename from the path (handles both forward and back slashes) + path_filename = Path(test_class_path).name + if path_filename in filename_lookup: + test_file_path, test_type = filename_lookup[path_filename] + logger.debug(f"Jest XML: matched by filename {path_filename}") + + # Also try filename matching on the file attribute if classname matching failed + if test_file_path is None: + test_file_name = suite._elem.attrib.get("file") or testcase._elem.attrib.get("file") # noqa: SLF001 + if test_file_name: + file_attr_filename = Path(test_file_name).name + if file_attr_filename in filename_lookup: + test_file_path, test_type = filename_lookup[file_attr_filename] + logger.debug(f"Jest XML: matched by file attr filename {file_attr_filename}") + + # For Jest tests in monorepos, test files may not exist after cleanup + # but we can still parse results and infer test type from the path + if test_file_path is None: + logger.warning(f"Could not resolve test file for Jest test: {test_class_path}") + continue + + # Get test type if not already set from lookup + if test_type is None and test_file_path.exists(): + test_type = test_files.get_test_type_by_instrumented_file_path(test_file_path) + if test_type is None: + # Infer test type from filename pattern + filename = test_file_path.name + if "__perf_test_" in filename or "_perf_test_" in filename: + test_type = TestType.GENERATED_PERFORMANCE + elif "__unit_test_" in filename or "_unit_test_" in filename: + test_type = TestType.GENERATED_REGRESSION + else: + # Default to GENERATED_REGRESSION for Jest tests + test_type = TestType.GENERATED_REGRESSION + + # For Jest tests, keep the relative file path with extension intact + # (Python uses module_name_from_file_path which strips extensions) + try: + test_module_path = str(test_file_path.relative_to(test_config.tests_project_rootdir)) + except ValueError: + test_module_path = test_file_path.name + result = testcase.is_passed + + # Check for timeout + timed_out = False + if len(testcase.result) >= 1: + message = (testcase.result[0].message or "").lower() + if "timeout" in message or "timed out" in message: + timed_out = True + + # Find matching timing markers for this test + # Jest test names in markers are sanitized by codeflash-jest-helper's sanitizeTestId() + # which replaces: !#: (space) ()[]{}|\/*?^$.+- with underscores + # IMPORTANT: Must match Jest helper's sanitization exactly for marker matching to work + # Pattern from capture.js: /[!#: ()\[\]{}|\\/*?^$.+\-]/g + sanitized_test_name = re.sub(r"[!#: ()\[\]{}|\\/*?^$.+\-]", "_", test_name) + matching_starts = [m for m in start_matches if sanitized_test_name in m.group(2)] + + # For performance tests (capturePerf), there are no START markers - only END markers with duration + # Check for END markers directly if no START markers found + matching_ends_direct = [] + if not matching_starts: + # Look for END markers that match this test (performance test format) + # END marker format: !######module:testName:funcName:loopIndex:invocationId:durationNs######! + for end_key, end_match in end_matches_dict.items(): + # end_key is (module, testName, funcName, loopIndex, invocationId) + if len(end_key) >= 2 and sanitized_test_name in end_key[1]: + matching_ends_direct.append(end_match) + + if not matching_starts and not matching_ends_direct: + # No timing markers found - add basic result + test_results.add( + FunctionTestInvocation( + loop_index=1, + id=InvocationId( + test_module_path=test_module_path, + test_class_name=None, + test_function_name=test_name, + function_getting_tested="", + iteration_id="", + ), + file_name=test_file_path, + runtime=None, + test_framework=test_config.test_framework, + did_pass=result, + test_type=test_type, + return_value=None, + timed_out=timed_out, + stdout="", + ) + ) + elif matching_ends_direct: + # Performance test format: process END markers directly (no START markers) + for end_match in matching_ends_direct: + groups = end_match.groups() + # groups: (module, testName, funcName, loopIndex, invocationId, durationNs) + func_name = groups[2] + loop_index = int(groups[3]) if groups[3].isdigit() else 1 + line_id = groups[4] + try: + runtime = int(groups[5]) + except (ValueError, IndexError): + runtime = None + test_results.add( + FunctionTestInvocation( + loop_index=loop_index, + id=InvocationId( + test_module_path=test_module_path, + test_class_name=None, + test_function_name=test_name, + function_getting_tested=func_name, + iteration_id=line_id, + ), + file_name=test_file_path, + runtime=runtime, + test_framework=test_config.test_framework, + did_pass=result, + test_type=test_type, + return_value=None, + timed_out=timed_out, + stdout="", + ) + ) + else: + # Process each timing marker + for match in matching_starts: + groups = match.groups() + # groups: (testName, testName2, funcName, loopIndex, lineId) + func_name = groups[2] + loop_index = int(groups[3]) if groups[3].isdigit() else 1 + line_id = groups[4] + + # Find matching end marker + end_key = groups[:5] + end_match = end_matches_dict.get(end_key) + + runtime = None + if end_match: + # Duration is in the 6th group (index 5) + with contextlib.suppress(ValueError, IndexError): + runtime = int(end_match.group(6)) + test_results.add( + FunctionTestInvocation( + loop_index=loop_index, + id=InvocationId( + test_module_path=test_module_path, + test_class_name=None, + test_function_name=test_name, + function_getting_tested=func_name, + iteration_id=line_id, + ), + file_name=test_file_path, + runtime=runtime, + test_framework=test_config.test_framework, + did_pass=result, + test_type=test_type, + return_value=None, + timed_out=timed_out, + stdout="", + ) + ) + + if not test_results: + logger.info( + f"No Jest test results parsed from {test_xml_file_path} " + f"(found {suite_count} suites, {testcase_count} testcases)" + ) + if run_result is not None: + logger.debug(f"Jest stdout: {run_result.stdout[:1000] if run_result.stdout else 'empty'}") + else: + logger.debug( + f"Jest XML parsing complete: {len(test_results.test_results)} results " + f"from {suite_count} suites, {testcase_count} testcases" + ) + + return test_results diff --git a/codeflash/verification/parse_test_output.py b/codeflash/verification/parse_test_output.py index 59b4f0acc..c80a287e5 100644 --- a/codeflash/verification/parse_test_output.py +++ b/codeflash/verification/parse_test_output.py @@ -32,6 +32,10 @@ ) from codeflash.verification.coverage_utils import CoverageUtils, JestCoverageUtils +# Import Jest-specific parsing from the JavaScript language module +from codeflash.languages.javascript.parse import jest_end_pattern, jest_start_pattern +from codeflash.languages.javascript.parse import parse_jest_test_xml as _parse_jest_test_xml + if TYPE_CHECKING: import subprocess @@ -52,11 +56,8 @@ def parse_func(file_path: Path) -> XMLParser: start_pattern = re.compile(r"!\$######([^:]*):([^:]*):([^:]*):([^:]*):([^:]+)######\$!") end_pattern = re.compile(r"!######([^:]*):([^:]*):([^:]*):([^:]*):([^:]+):([^:]+)######!") -# Jest timing marker patterns (from codeflash-jest-helper.js console.log output) -# Format: !$######testName:testName:funcName:loopIndex:lineId######$! (start) -# Format: !######testName:testName:funcName:loopIndex:lineId:durationNs######! (end) -jest_start_pattern = re.compile(r"!\$######([^:]+):([^:]+):([^:]+):([^:]+):([^#]+)######\$!") -jest_end_pattern = re.compile(r"!######([^:]+):([^:]+):([^:]+):([^:]+):([^:]+):(\d+)######!") +# Jest timing marker patterns are imported from codeflash.languages.javascript.parse +# and re-exported here for backwards compatibility def calculate_function_throughput_from_test_results(test_results: TestResults, function_name: str) -> int: @@ -556,356 +557,6 @@ def parse_sqlite_test_results(sqlite_file_path: Path, test_files: TestFiles, tes return test_results -def _extract_jest_console_output(suite_elem) -> str: - """Extract console output from Jest's JUnit XML system-out element. - - Jest-junit writes console.log output as a JSON array in the testsuite's system-out. - Each entry has: {"message": "...", "origin": "...", "type": "log"} - - Args: - suite_elem: The testsuite lxml element - - Returns: - Concatenated message content from all log entries - - """ - import json - - system_out_elem = suite_elem.find("system-out") - if system_out_elem is None or system_out_elem.text is None: - return "" - - raw_content = system_out_elem.text.strip() - if not raw_content: - return "" - - # Jest-junit wraps console output in a JSON array - # Try to parse as JSON first - try: - log_entries = json.loads(raw_content) - if isinstance(log_entries, list): - # Extract message field from each log entry - messages = [] - for entry in log_entries: - if isinstance(entry, dict) and "message" in entry: - messages.append(entry["message"]) - return "\n".join(messages) - except (json.JSONDecodeError, TypeError): - # Not JSON - return as plain text (fallback for pytest-style output) - pass - - return raw_content - - -# TODO: {Claude} we need to move to the support directory. -def parse_jest_test_xml( - test_xml_file_path: Path, - test_files: TestFiles, - test_config: TestConfig, - run_result: subprocess.CompletedProcess | None = None, -) -> TestResults: - """Parse Jest JUnit XML test results. - - Jest-junit has a different structure than pytest: - - system-out is at the testsuite level (not testcase) - - system-out contains a JSON array of log entries - - Timing markers are in the message field of log entries - - Args: - test_xml_file_path: Path to the Jest JUnit XML file - test_files: TestFiles object with test file information - test_config: Test configuration - run_result: Optional subprocess result for logging - - Returns: - TestResults containing parsed test invocations - - """ - test_results = TestResults() - - if not test_xml_file_path.exists(): - logger.warning(f"No JavaScript test results for {test_xml_file_path} found.") - return test_results - - # Log file size for debugging - file_size = test_xml_file_path.stat().st_size - logger.debug(f"Jest XML file size: {file_size} bytes at {test_xml_file_path}") - - try: - xml = JUnitXml.fromfile(str(test_xml_file_path), parse_func=parse_func) - logger.debug(f"Successfully parsed Jest JUnit XML from {test_xml_file_path}") - except Exception as e: - logger.warning(f"Failed to parse {test_xml_file_path} as JUnitXml. Exception: {e}") - return test_results - - base_dir = test_config.tests_project_rootdir - logger.debug(f"Jest XML parsing: base_dir={base_dir}, num_test_files={len(test_files.test_files)}") - - # Build lookup from instrumented file path to TestFile for direct matching - # This handles cases where instrumented files are in temp directories - instrumented_path_lookup: dict[str, tuple[Path, TestType]] = {} - for test_file in test_files.test_files: - if test_file.instrumented_behavior_file_path: - # Store both the absolute path and resolved path as keys - abs_path = str(test_file.instrumented_behavior_file_path.resolve()) - instrumented_path_lookup[abs_path] = (test_file.instrumented_behavior_file_path, test_file.test_type) - # Also store the string representation in case of minor path differences - instrumented_path_lookup[str(test_file.instrumented_behavior_file_path)] = ( - test_file.instrumented_behavior_file_path, - test_file.test_type, - ) - logger.debug(f"Jest XML lookup: registered {abs_path}") - - # Fallback: if JUnit XML doesn't have system-out, use subprocess stdout directly - global_stdout = "" - if run_result is not None: - try: - global_stdout = run_result.stdout if isinstance(run_result.stdout, str) else run_result.stdout.decode() - # Debug: log if timing markers are found in stdout - if global_stdout: - marker_count = len(jest_start_pattern.findall(global_stdout)) - if marker_count > 0: - logger.debug(f"Found {marker_count} timing start markers in Jest stdout") - else: - logger.debug(f"No timing start markers found in Jest stdout (len={len(global_stdout)})") - except (AttributeError, UnicodeDecodeError): - global_stdout = "" - - suite_count = 0 - testcase_count = 0 - for suite in xml: - suite_count += 1 - # Extract console output from suite-level system-out (Jest specific) - suite_stdout = _extract_jest_console_output(suite._elem) # noqa: SLF001 - - # Fallback: use subprocess stdout if XML system-out is empty - if not suite_stdout and global_stdout: - suite_stdout = global_stdout - - # Parse timing markers from the suite's console output - start_matches = list(jest_start_pattern.finditer(suite_stdout)) - end_matches_dict = {} - for match in jest_end_pattern.finditer(suite_stdout): - # Key: (testName, testName2, funcName, loopIndex, lineId) - key = match.groups()[:5] - end_matches_dict[key] = match - - for testcase in suite: - testcase_count += 1 - test_class_path = testcase.classname # For Jest, this is the file path - test_name = testcase.name - - if test_name is None: - logger.debug(f"testcase.name is None in Jest XML {test_xml_file_path}, skipping") - continue - - logger.debug(f"Jest XML: processing testcase name={test_name}, classname={test_class_path}") - - # First, try direct lookup in instrumented file paths - # This handles cases where instrumented files are in temp directories - test_file_path = None - test_type = None - - if test_class_path: - # Try exact match with classname (which should be the filepath from jest-junit) - if test_class_path in instrumented_path_lookup: - test_file_path, test_type = instrumented_path_lookup[test_class_path] - else: - # Try resolving the path and matching - try: - resolved_path = str(Path(test_class_path).resolve()) - if resolved_path in instrumented_path_lookup: - test_file_path, test_type = instrumented_path_lookup[resolved_path] - except Exception: - pass - - # If direct lookup failed, try the file attribute - if test_file_path is None: - test_file_name = suite._elem.attrib.get("file") or testcase._elem.attrib.get("file") # noqa: SLF001 - if test_file_name: - if test_file_name in instrumented_path_lookup: - test_file_path, test_type = instrumented_path_lookup[test_file_name] - else: - try: - resolved_path = str(Path(test_file_name).resolve()) - if resolved_path in instrumented_path_lookup: - test_file_path, test_type = instrumented_path_lookup[resolved_path] - except Exception: - pass - - # Fall back to traditional path resolution if direct lookup failed - if test_file_path is None: - test_file_path = resolve_test_file_from_class_path(test_class_path, base_dir) - if test_file_path is None: - test_file_name = suite._elem.attrib.get("file") or testcase._elem.attrib.get("file") # noqa: SLF001 - if test_file_name: - test_file_path = base_dir.parent / test_file_name - if not test_file_path.exists(): - test_file_path = base_dir / test_file_name - - # For Jest tests in monorepos, test files may not exist after cleanup - # but we can still parse results and infer test type from the path - if test_file_path is None: - logger.warning(f"Could not resolve test file for Jest test: {test_class_path}") - continue - - # Get test type if not already set from lookup - if test_type is None and test_file_path.exists(): - test_type = test_files.get_test_type_by_instrumented_file_path(test_file_path) - if test_type is None: - # Infer test type from filename pattern - filename = test_file_path.name - if "__perf_test_" in filename or "_perf_test_" in filename: - test_type = TestType.GENERATED_PERFORMANCE - elif "__unit_test_" in filename or "_unit_test_" in filename: - test_type = TestType.GENERATED_REGRESSION - else: - # Default to GENERATED_REGRESSION for Jest tests - test_type = TestType.GENERATED_REGRESSION - - # For Jest tests, keep the relative file path with extension intact - # (Python uses module_name_from_file_path which strips extensions) - try: - test_module_path = str(test_file_path.relative_to(test_config.tests_project_rootdir)) - except ValueError: - test_module_path = test_file_path.name - result = testcase.is_passed - - # Check for timeout - timed_out = False - if len(testcase.result) >= 1: - message = (testcase.result[0].message or "").lower() - if "timeout" in message or "timed out" in message: - timed_out = True - - # Find matching timing markers for this test - # Jest test names in markers are sanitized by codeflash-jest-helper's sanitizeTestId() - # which replaces: !#: (space) ()[]{}|\/*?^$.+- with underscores - # IMPORTANT: Must match Jest helper's sanitization exactly for marker matching to work - # Pattern from capture.js: /[!#: ()\[\]{}|\\/*?^$.+\-]/g - sanitized_test_name = re.sub(r"[!#: ()\[\]{}|\\/*?^$.+\-]", "_", test_name) - matching_starts = [m for m in start_matches if sanitized_test_name in m.group(2)] - - # For performance tests (capturePerf), there are no START markers - only END markers with duration - # Check for END markers directly if no START markers found - matching_ends_direct = [] - if not matching_starts: - # Look for END markers that match this test (performance test format) - # END marker format: !######module:testName:funcName:loopIndex:invocationId:durationNs######! - for end_key, end_match in end_matches_dict.items(): - # end_key is (module, testName, funcName, loopIndex, invocationId) - if len(end_key) >= 2 and sanitized_test_name in end_key[1]: - matching_ends_direct.append(end_match) - - if not matching_starts and not matching_ends_direct: - # No timing markers found - add basic result - test_results.add( - FunctionTestInvocation( - loop_index=1, - id=InvocationId( - test_module_path=test_module_path, - test_class_name=None, - test_function_name=test_name, - function_getting_tested="", - iteration_id="", - ), - file_name=test_file_path, - runtime=None, - test_framework=test_config.test_framework, - did_pass=result, - test_type=test_type, - return_value=None, - timed_out=timed_out, - stdout="", - ) - ) - elif matching_ends_direct: - # Performance test format: process END markers directly (no START markers) - for end_match in matching_ends_direct: - groups = end_match.groups() - # groups: (module, testName, funcName, loopIndex, invocationId, durationNs) - func_name = groups[2] - loop_index = int(groups[3]) if groups[3].isdigit() else 1 - line_id = groups[4] - try: - runtime = int(groups[5]) - except (ValueError, IndexError): - runtime = None - test_results.add( - FunctionTestInvocation( - loop_index=loop_index, - id=InvocationId( - test_module_path=test_module_path, - test_class_name=None, - test_function_name=test_name, - function_getting_tested=func_name, - iteration_id=line_id, - ), - file_name=test_file_path, - runtime=runtime, - test_framework=test_config.test_framework, - did_pass=result, - test_type=test_type, - return_value=None, - timed_out=timed_out, - stdout="", - ) - ) - else: - # Process each timing marker - for match in matching_starts: - groups = match.groups() - # groups: (testName, testName2, funcName, loopIndex, lineId) - func_name = groups[2] - loop_index = int(groups[3]) if groups[3].isdigit() else 1 - line_id = groups[4] - - # Find matching end marker - end_key = groups[:5] - end_match = end_matches_dict.get(end_key) - - runtime = None - if end_match: - # Duration is in the 6th group (index 5) - with contextlib.suppress(ValueError, IndexError): - runtime = int(end_match.group(6)) - test_results.add( - FunctionTestInvocation( - loop_index=loop_index, - id=InvocationId( - test_module_path=test_module_path, - test_class_name=None, - test_function_name=test_name, - function_getting_tested=func_name, - iteration_id=line_id, - ), - file_name=test_file_path, - runtime=runtime, - test_framework=test_config.test_framework, - did_pass=result, - test_type=test_type, - return_value=None, - timed_out=timed_out, - stdout="", - ) - ) - - if not test_results: - logger.info( - f"No Jest test results parsed from {test_xml_file_path} " - f"(found {suite_count} suites, {testcase_count} testcases)" - ) - if run_result is not None: - logger.debug(f"Jest stdout: {run_result.stdout[:1000] if run_result.stdout else 'empty'}") - else: - logger.debug( - f"Jest XML parsing complete: {len(test_results.test_results)} results " - f"from {suite_count} suites, {testcase_count} testcases" - ) - - return test_results - - def parse_test_xml( test_xml_file_path: Path, test_files: TestFiles, @@ -914,7 +565,14 @@ def parse_test_xml( ) -> TestResults: # Route to Jest-specific parser for JavaScript/TypeScript tests if is_javascript(): - return parse_jest_test_xml(test_xml_file_path, test_files, test_config, run_result) + return _parse_jest_test_xml( + test_xml_file_path, + test_files, + test_config, + run_result, + parse_func=parse_func, + resolve_test_file_from_class_path=resolve_test_file_from_class_path, + ) test_results = TestResults() # Parse unittest output diff --git a/tests/languages/javascript/test_vitest_junit.py b/tests/languages/javascript/test_vitest_junit.py index 28a24ba08..ac52ffe3e 100644 --- a/tests/languages/javascript/test_vitest_junit.py +++ b/tests/languages/javascript/test_vitest_junit.py @@ -245,3 +245,217 @@ def test_timing_marker_with_special_characters_in_test_name(self) -> None: assert len(matches) == 1 assert matches[0][1] == "handles_n=0_correctly" + + +class TestFilenameBasedLookupFallback: + """Tests for filename-based lookup fallback in Jest/Vitest XML parsing. + + When JUnit XML has relative paths that can't be resolved to absolute paths + (because they're relative to Jest's CWD, not the parse-time CWD), the parser + should fall back to matching by filename only. + """ + + def test_filename_lookup_matches_relative_path(self) -> None: + """Should match test file by filename when classname has unresolvable relative path.""" + from unittest.mock import MagicMock + + from codeflash.languages.javascript.parse import parse_jest_test_xml + from codeflash.models.models import TestFile, TestFiles, TestType + + # Create a temporary XML file with a relative path that won't resolve + xml_content = """ + + + + +""" + + with tempfile.NamedTemporaryFile(suffix=".xml", mode="w", delete=False) as f: + f.write(xml_content) + f.flush() + junit_file = Path(f.name) + + # Create a mock test file with an absolute instrumented path + # The filename should match even though the full path differs + with tempfile.TemporaryDirectory() as tmpdir: + instrumented_path = Path(tmpdir) / "utils__perfinstrumented.test.ts" + instrumented_path.touch() + + test_file = TestFile( + original_file_path=Path(tmpdir) / "utils.test.ts", + test_type=TestType.GENERATED_REGRESSION, + instrumented_behavior_file_path=instrumented_path, + ) + test_files = TestFiles(test_files=[test_file]) + + test_config = MagicMock() + test_config.tests_project_rootdir = Path(tmpdir) + test_config.test_framework = "jest" + + # Parse the XML - should use filename fallback + results = parse_jest_test_xml( + junit_file, + test_files, + test_config, + parse_func=None, # Will use default + resolve_test_file_from_class_path=lambda x, y: None, # Force fallback + ) + + # Should have found 1 test result via filename matching + assert len(results.test_results) == 1 + assert results.test_results[0].file_name == instrumented_path + assert results.test_results[0].test_type == TestType.GENERATED_REGRESSION + + def test_filename_lookup_with_duplicate_filenames_uses_first(self) -> None: + """When multiple test files have same filename, use the first one registered.""" + from unittest.mock import MagicMock + + from codeflash.languages.javascript.parse import parse_jest_test_xml + from codeflash.models.models import TestFile, TestFiles, TestType + + xml_content = """ + + + + +""" + + with tempfile.NamedTemporaryFile(suffix=".xml", mode="w", delete=False) as f: + f.write(xml_content) + f.flush() + junit_file = Path(f.name) + + with tempfile.TemporaryDirectory() as tmpdir: + # Create two test files with the same filename in different directories + dir1 = Path(tmpdir) / "pkg1" + dir2 = Path(tmpdir) / "pkg2" + dir1.mkdir() + dir2.mkdir() + + path1 = dir1 / "same_name.test.ts" + path2 = dir2 / "same_name.test.ts" + path1.touch() + path2.touch() + + test_file1 = TestFile( + original_file_path=path1, + test_type=TestType.GENERATED_REGRESSION, + instrumented_behavior_file_path=path1, + ) + test_file2 = TestFile( + original_file_path=path2, + test_type=TestType.REPLAY_TEST, # Different type + instrumented_behavior_file_path=path2, + ) + # First file should win in filename lookup + test_files = TestFiles(test_files=[test_file1, test_file2]) + + test_config = MagicMock() + test_config.tests_project_rootdir = Path(tmpdir) + test_config.test_framework = "jest" + + results = parse_jest_test_xml( + junit_file, + test_files, + test_config, + parse_func=None, + resolve_test_file_from_class_path=lambda x, y: None, + ) + + assert len(results.test_results) == 1 + # Should use first registered file + assert results.test_results[0].file_name == path1 + assert results.test_results[0].test_type == TestType.GENERATED_REGRESSION + + def test_filename_lookup_extracts_filename_from_nested_path(self) -> None: + """Should extract filename correctly from deeply nested relative paths.""" + from unittest.mock import MagicMock + + from codeflash.languages.javascript.parse import parse_jest_test_xml + from codeflash.models.models import TestFile, TestFiles, TestType + + xml_content = """ + + + + +""" + + with tempfile.NamedTemporaryFile(suffix=".xml", mode="w", delete=False) as f: + f.write(xml_content) + f.flush() + junit_file = Path(f.name) + + with tempfile.TemporaryDirectory() as tmpdir: + instrumented_path = Path(tmpdir) / "utils__perfinstrumented.test.ts" + instrumented_path.touch() + + test_file = TestFile( + original_file_path=Path(tmpdir) / "utils.test.ts", + test_type=TestType.GENERATED_REGRESSION, + instrumented_behavior_file_path=instrumented_path, + ) + test_files = TestFiles(test_files=[test_file]) + + test_config = MagicMock() + test_config.tests_project_rootdir = Path(tmpdir) + test_config.test_framework = "jest" + + results = parse_jest_test_xml( + junit_file, + test_files, + test_config, + parse_func=None, + resolve_test_file_from_class_path=lambda x, y: None, + ) + + # Should match despite deeply nested path in XML + assert len(results.test_results) == 1 + assert results.test_results[0].file_name == instrumented_path + + def test_no_match_when_filename_not_in_lookup(self) -> None: + """Should skip test case when filename doesn't match any registered test file.""" + from unittest.mock import MagicMock + + from codeflash.languages.javascript.parse import parse_jest_test_xml + from codeflash.models.models import TestFile, TestFiles, TestType + + # XML with a filename that doesn't match any registered test file + xml_content = """ + + + + +""" + + with tempfile.NamedTemporaryFile(suffix=".xml", mode="w", delete=False) as f: + f.write(xml_content) + f.flush() + junit_file = Path(f.name) + + with tempfile.TemporaryDirectory() as tmpdir: + # Register a test file with a DIFFERENT filename + instrumented_path = Path(tmpdir) / "different_file.test.ts" + instrumented_path.touch() + + test_file = TestFile( + original_file_path=Path(tmpdir) / "different.test.ts", + test_type=TestType.GENERATED_REGRESSION, + instrumented_behavior_file_path=instrumented_path, + ) + test_files = TestFiles(test_files=[test_file]) + + test_config = MagicMock() + test_config.tests_project_rootdir = Path(tmpdir) + test_config.test_framework = "jest" + + results = parse_jest_test_xml( + junit_file, + test_files, + test_config, + parse_func=None, + resolve_test_file_from_class_path=lambda x, y: None, + ) + + # Should have no results since filename doesn't match + assert len(results.test_results) == 0