diff --git a/application/single_app/config.py b/application/single_app/config.py index 5c5db025b..a89ae20cb 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -95,7 +95,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.242.066" +VERSION = "0.242.068" SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') SESSION_COOKIE_HTTPONLY = os.getenv('SESSION_COOKIE_HTTPONLY', 'true').lower() != 'false' diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index 10a658fb5..3cad573a5 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -41,6 +41,7 @@ import queue import re import requests +import time import traceback from urllib.parse import urlparse import threading @@ -1098,6 +1099,8 @@ def _strip_agent_citation_artifact_refs(agent_citations): TABULAR_RELATED_DOCUMENT_MAX_MATCHES_PER_ROW = 3 TABULAR_RELATED_DOCUMENT_MAX_SUMMARY_ROWS = 8 TABULAR_RELATED_DOCUMENT_MAX_EXCERPT_CHARS = 500 +TABULAR_SK_ANALYSIS_MAX_CHARS = 100000 +TABULAR_COMPUTED_RESULTS_HANDOFF_MAX_CHARS = 100000 TABULAR_GENERATED_OUTPUT_INTERNAL_ROW_FIELDS = { '_matched_columns', '_matched_values', @@ -3661,8 +3664,13 @@ def build_tabular_related_document_evidence_summary(invocations): def build_tabular_computed_results_system_message(source_label, tabular_analysis, related_document_evidence_summary=''): """Build the outer-model handoff message for successful tabular analysis.""" rendered_analysis = str(tabular_analysis or '').strip() - max_handoff_chars = 24000 + max_handoff_chars = TABULAR_COMPUTED_RESULTS_HANDOFF_MAX_CHARS if len(rendered_analysis) > max_handoff_chars: + original_length = len(rendered_analysis) + log_event( + f"[Tabular SK Analysis] Computed results handoff truncated from {original_length} to {max_handoff_chars} chars", + level=logging.WARNING, + ) rendered_analysis = ( rendered_analysis[:max_handoff_chars] + "\n[Computed results handoff truncated for prompt budget.]" @@ -9376,6 +9384,7 @@ def build_system_prompt(force_tool_use=False, tool_error_messages=None, return None for attempt_number in range(1, 4): + attempt_started_at = time.monotonic() force_tool_use = attempt_number > 1 or (attempt_number == 1 and analysis_requires_immediate_tool_choice) if callable(thought_callback) and attempt_number > 1: await emit_tabular_analysis_lifecycle_thought( @@ -9494,13 +9503,24 @@ def build_system_prompt(force_tool_use=False, tool_error_messages=None, if result and result[0].content: analysis = result[0].content.strip() - if len(analysis) > 20000: - analysis = analysis[:20000] + "\n[Analysis truncated]" + if len(analysis) > TABULAR_SK_ANALYSIS_MAX_CHARS: + original_analysis_length = len(analysis) + log_event( + f"[Tabular SK Analysis] Attempt {attempt_number} analysis text truncated from {original_analysis_length} to {TABULAR_SK_ANALYSIS_MAX_CHARS} chars", + level=logging.WARNING, + ) + analysis = analysis[:TABULAR_SK_ANALYSIS_MAX_CHARS] + "\n[Analysis truncated]" + attempt_elapsed_ms = int((time.monotonic() - attempt_started_at) * 1000) if schema_summary_mode: if successful_schema_summary_invocations: log_event( f"[Tabular SK Analysis] Schema summary complete via {len(successful_schema_summary_invocations)} workbook tool call(s) on attempt {attempt_number}", + extra={ + 'attempt_number': attempt_number, + 'elapsed_ms': attempt_elapsed_ms, + 'analysis_length': len(analysis), + }, level=logging.INFO, ) return analysis @@ -9601,6 +9621,11 @@ def build_system_prompt(force_tool_use=False, tool_error_messages=None, previous_execution_gap_messages = [] log_event( f"[Tabular SK Analysis] Analysis complete via {len(successful_analytical_invocations)} analytical tool call(s) on attempt {attempt_number}", + extra={ + 'attempt_number': attempt_number, + 'elapsed_ms': attempt_elapsed_ms, + 'analysis_length': len(analysis), + }, level=logging.INFO ) return analysis diff --git a/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py b/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py index c1ebe8f85..99ce381e2 100644 --- a/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py +++ b/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py @@ -110,6 +110,15 @@ class TabularProcessingPlugin: RELATIONSHIP_VALUE_SAMPLE_LIMIT = 500 RELATIONSHIP_SHARED_VALUE_LIMIT = 5 SOURCE_VALUE_MATCH_COUNT_LIMIT = 100 + ROW_OUTPUT_SAFE_CHAR_LIMIT = 50000 + ROW_OUTPUT_PROTECTED_COLUMNS = ( + '_sheet', + '_matched_columns', + '_matched_values', + '_matched_on', + '_matched_source_values', + '_related_document_reference_values', + ) def __init__(self): self._df_cache = {} # Per-instance cache: (container, blob_name, sheet_name) -> DataFrame @@ -515,6 +524,204 @@ def _match_workbook_sheet_name(self, requested_sheet_name: Optional[str], availa return None + def _parse_row_page_arguments(self, start_row=None, max_rows=None, default_max_rows=100) -> tuple: + """Normalize row pagination arguments for tool-call row payloads.""" + try: + normalized_start_row = int(start_row or 0) + except (TypeError, ValueError): + normalized_start_row = 0 + + try: + normalized_max_rows = int(max_rows or default_max_rows) + except (TypeError, ValueError): + normalized_max_rows = int(default_max_rows) + + return max(0, normalized_start_row), max(1, normalized_max_rows) + + def _get_protected_row_output_columns(self, columns, additional_protected_columns=None) -> list: + """Return protected metadata columns that should survive projection and trimming.""" + protected_column_names = { + str(column_name) + for column_name in self.ROW_OUTPUT_PROTECTED_COLUMNS + } + protected_column_names.update( + str(column_name) + for column_name in (additional_protected_columns or []) + if str(column_name or '').strip() + ) + + return [ + column_name for column_name in columns + if str(column_name) in protected_column_names + ] + + def _build_row_output_records(self, row_frame, selected_columns): + """Build JSON-ready row records and preserve hidden document references.""" + selected_column_names = [ + column_name for column_name in selected_columns + if column_name in row_frame.columns + ] + output_records = [] + + for _, row in row_frame.iterrows(): + row_payload = { + str(column_name): row.get(column_name) + for column_name in selected_column_names + } + related_document_reference_values = self._build_related_document_reference_values( + row, + excluded_columns=selected_column_names, + ) + if related_document_reference_values: + existing_reference_values = row_payload.get('_related_document_reference_values') + if isinstance(existing_reference_values, dict): + merged_reference_values = dict(existing_reference_values) + merged_reference_values.update(related_document_reference_values) + row_payload['_related_document_reference_values'] = merged_reference_values + else: + row_payload['_related_document_reference_values'] = related_document_reference_values + output_records.append(row_payload) + + return output_records + + def _estimate_row_output_chars(self, row_records) -> int: + """Estimate serialized JSON character size for a row payload.""" + return len(json.dumps(row_records, default=str)) + + def _select_auto_trimmed_columns(self, row_frame, protected_columns, max_chars): + """Drop heavy non-protected columns until the row payload fits the safe budget.""" + selected_columns = list(row_frame.columns) + protected_column_set = {str(column_name) for column_name in protected_columns} + excluded_columns = [] + + if not selected_columns: + return selected_columns, excluded_columns + + row_records = self._build_row_output_records(row_frame, selected_columns) + if self._estimate_row_output_chars(row_records) <= max_chars: + return selected_columns, excluded_columns + + sample_size = min(20, len(row_frame)) or 1 + sample_frame = row_frame.head(sample_size) + column_average_lengths = { + column_name: sample_frame[column_name].astype(str).str.len().mean() + for column_name in selected_columns + } + removable_columns = [ + column_name for column_name in selected_columns + if str(column_name) not in protected_column_set + ] + + for column_to_drop in sorted( + removable_columns, + key=lambda column_name: column_average_lengths.get(column_name, 0), + reverse=True, + ): + if len(selected_columns) <= len(protected_columns) + 1: + break + selected_columns.remove(column_to_drop) + excluded_columns.append(str(column_to_drop)) + + row_records = self._build_row_output_records(row_frame, selected_columns) + if self._estimate_row_output_chars(row_records) <= max_chars: + break + + return selected_columns, excluded_columns + + def _build_tabular_row_page_payload( + self, + rows, + total_row_count, + start_row=0, + max_rows=100, + return_columns=None, + protected_columns=None, + max_chars=None, + ) -> dict: + """Shape row data with pagination, projection, and safe output trimming.""" + normalized_start_row, normalized_max_rows = self._parse_row_page_arguments(start_row, max_rows) + safe_max_chars = int(max_chars or self.ROW_OUTPUT_SAFE_CHAR_LIMIT) + requested_return_columns = self._parse_optional_column_list_argument(return_columns) + return_columns_requested = return_columns is not None and str(return_columns).strip().casefold() not in {'', '*', 'all', 'all_columns', 'all columns'} + row_frame = pandas.DataFrame(list(rows or [])) + total_count = max(0, int(total_row_count or 0)) + + if row_frame.empty: + return { + 'start_row': normalized_start_row, + 'page_size': normalized_max_rows, + 'returned_rows': 0, + 'has_more': False, + 'next_start_row': None, + 'data': [], + } + + protected_output_columns = self._get_protected_row_output_columns( + list(row_frame.columns), + additional_protected_columns=protected_columns, + ) + resolved_return_columns = [ + column_name for column_name in (requested_return_columns or []) + if column_name in row_frame.columns + ] + + if resolved_return_columns: + selected_columns = list(resolved_return_columns) + for protected_column in protected_output_columns: + if protected_column not in selected_columns: + selected_columns.append(protected_column) + auto_excluded_columns = [] + else: + selected_columns, auto_excluded_columns = self._select_auto_trimmed_columns( + row_frame, + protected_output_columns, + safe_max_chars, + ) + + working_frame = row_frame + row_records = self._build_row_output_records(working_frame, selected_columns) + row_trimmed_for_budget = False + + while len(row_records) > 1 and self._estimate_row_output_chars(row_records) > safe_max_chars: + current_size = self._estimate_row_output_chars(row_records) + estimated_target_rows = max(1, int(len(row_records) * safe_max_chars / max(current_size, 1))) + if estimated_target_rows >= len(row_records): + estimated_target_rows = len(row_records) - 1 + working_frame = working_frame.head(estimated_target_rows) + row_records = self._build_row_output_records(working_frame, selected_columns) + row_trimmed_for_budget = True + + returned_row_count = len(row_records) + next_start_row = normalized_start_row + returned_row_count + has_more = next_start_row < total_count + payload = { + 'start_row': normalized_start_row, + 'page_size': normalized_max_rows, + 'returned_rows': returned_row_count, + 'has_more': has_more, + 'next_start_row': next_start_row if has_more else None, + 'data': row_records, + } + + if return_columns_requested: + payload['return_columns'] = resolved_return_columns + if auto_excluded_columns: + payload['auto_excluded_columns'] = auto_excluded_columns + payload['output_trimmed'] = True + payload['note'] = ( + f"Columns {auto_excluded_columns!r} were automatically excluded because the row payload " + "would exceed the safe output size. Use return_columns to request specific columns, " + "or use start_row/max_rows pagination to retrieve smaller pages." + ) + if row_trimmed_for_budget: + payload['output_trimmed'] = True + payload['note'] = ( + f"Result page was automatically reduced to {returned_row_count} row(s) to stay within " + "the safe output size. Use next_start_row with start_row to continue paging." + ) + + return payload + def _filter_rows_across_sheets( self, container_name: str, @@ -528,6 +735,8 @@ def _filter_rows_across_sheets( additional_filter_value=None, normalize_match: bool = False, max_rows: int = 100, + start_row: int = 0, + return_columns=None, ) -> Optional[str]: """Search for matching rows across all sheets that contain the requested column. @@ -546,6 +755,9 @@ def _filter_rows_across_sheets( sheets_searched = [] sheets_matched = [] total_matches = 0 + applied_filters = [] + skip_remaining = max(0, int(start_row or 0)) + requested_max_rows = max(1, int(max_rows or 100)) for sheet in available_sheets: df = self._read_tabular_blob_to_dataframe( @@ -559,7 +771,7 @@ def _filter_rows_across_sheets( continue try: - filtered_df, applied_filters = self._apply_optional_dataframe_filters( + filtered_df, sheet_filters = self._apply_optional_dataframe_filters( df, filter_column=column, filter_operator=operator_str, @@ -573,6 +785,7 @@ def _filter_rows_across_sheets( continue sheets_searched.append(sheet) + applied_filters = applied_filters or sheet_filters sheet_matches = len(filtered_df) if sheet_matches == 0: @@ -580,12 +793,19 @@ def _filter_rows_across_sheets( sheets_matched.append(sheet) total_matches += sheet_matches - remaining_capacity = max(0, max_rows - len(combined_results)) + if skip_remaining >= sheet_matches: + skip_remaining -= sheet_matches + continue + + remaining_capacity = max(0, requested_max_rows - len(combined_results)) if remaining_capacity > 0: - filtered = filtered_df.head(remaining_capacity) + filtered = filtered_df.iloc[skip_remaining:skip_remaining + remaining_capacity] + skip_remaining = 0 for row in filtered.to_dict(orient='records'): row['_sheet'] = sheet combined_results.append(row) + else: + skip_remaining = 0 if not sheets_searched: return None @@ -598,16 +818,25 @@ def _filter_rows_across_sheets( level=logging.INFO, ) - return json.dumps({ + row_page_payload = self._build_tabular_row_page_payload( + combined_results, + total_matches, + start_row=start_row, + max_rows=max_rows, + return_columns=return_columns, + protected_columns=['_sheet'], + ) + + response_payload = { "filename": filename, "selected_sheet": "ALL (cross-sheet search)", "sheets_searched": sheets_searched, "sheets_matched": sheets_matched, "filter_applied": applied_filters, "total_matches": total_matches, - "returned_rows": len(combined_results), - "data": combined_results, - }, indent=2, default=str) + } + response_payload.update(row_page_payload) + return json.dumps(response_payload, indent=2, default=str) def _search_rows_across_sheets( self, @@ -627,6 +856,7 @@ def _search_rows_across_sheets( additional_filter_value=None, normalize_match: bool = False, max_rows: int = 100, + start_row: int = 0, ) -> Optional[str]: """Search rows across worksheets when the relevant text column is unknown or broad.""" workbook_metadata = self._get_workbook_metadata(container_name, blob_name) @@ -648,6 +878,8 @@ def _search_rows_across_sheets( seen_searched_columns = set() matched_columns = [] seen_matched_columns = set() + skip_remaining = max(0, int(start_row or 0)) + requested_max_rows = max(1, int(max_rows or 100)) for sheet in available_sheets: df = self._read_tabular_blob_to_dataframe( @@ -678,11 +910,8 @@ def _search_rows_across_sheets( 'selected_sheet': 'ALL (cross-sheet search)', }, indent=2, default=str) - remaining_capacity = max(0, max_rows - len(combined_results)) - if remaining_capacity <= 0: - break - try: + remaining_capacity = max(0, requested_max_rows - len(combined_results)) search_result = self._search_dataframe_rows( filtered_df, search_value=search_value, @@ -690,6 +919,7 @@ def _search_rows_across_sheets( search_operator=search_operator, return_columns=requested_return_columns, normalize_match=normalize_match, + start_row=skip_remaining, max_rows=remaining_capacity, ) except KeyError: @@ -715,6 +945,11 @@ def _search_rows_across_sheets( if sheet_match_count > 0: sheets_matched.append(sheet) + if skip_remaining >= sheet_match_count: + skip_remaining -= sheet_match_count + else: + skip_remaining = 0 + for column_name in search_result['matched_columns']: lowered_name = str(column_name).lower() if lowered_name in seen_matched_columns: @@ -744,7 +979,16 @@ def _search_rows_across_sheets( level=logging.INFO, ) - return json.dumps({ + row_page_payload = self._build_tabular_row_page_payload( + combined_results, + total_matches, + start_row=start_row, + max_rows=max_rows, + return_columns=requested_return_columns, + protected_columns=['_sheet', '_matched_columns', '_matched_values', '_related_document_reference_values'], + ) + + response_payload = { 'filename': filename, 'selected_sheet': 'ALL (cross-sheet search)', 'search_value': search_value, @@ -757,9 +1001,9 @@ def _search_rows_across_sheets( 'filter_applied': applied_filters, 'normalize_match': normalize_match, 'total_matches': total_matches, - 'returned_rows': len(combined_results), - 'data': combined_results, - }, indent=2, default=str) + } + response_payload.update(row_page_payload) + return json.dumps(response_payload, indent=2, default=str) def _lookup_value_across_sheets( self, @@ -772,6 +1016,7 @@ def _lookup_value_across_sheets( match_operator: str = "equals", normalize_match: bool = False, max_rows: int = 25, + start_row: int = 0, ) -> Optional[str]: """Look up matching rows across all sheets that contain the lookup column. @@ -792,6 +1037,8 @@ def _lookup_value_across_sheets( total_matches = 0 operator = (match_operator or 'equals').strip().lower() normalized_lookup_value = str(lookup_value_str) + skip_remaining = max(0, int(start_row or 0)) + requested_max_rows = max(1, int(max_rows or 25)) for sheet in available_sheets: df = self._read_tabular_blob_to_dataframe( @@ -826,9 +1073,14 @@ def _lookup_value_across_sheets( sheets_matched.append(sheet) total_matches += sheet_matches - remaining_capacity = max(0, max_rows - len(combined_results)) + if skip_remaining >= sheet_matches: + skip_remaining -= sheet_matches + continue + + remaining_capacity = max(0, requested_max_rows - len(combined_results)) if remaining_capacity > 0: - matched_df = df[mask].head(remaining_capacity) + matched_df = df[mask].iloc[skip_remaining:skip_remaining + remaining_capacity] + skip_remaining = 0 if target_column and target_column in df.columns: for _, row in matched_df.iterrows(): combined_results.append({ @@ -841,6 +1093,8 @@ def _lookup_value_across_sheets( for row in matched_df.to_dict(orient='records'): row['_sheet'] = sheet combined_results.append(row) + else: + skip_remaining = 0 if not sheets_searched: return None @@ -853,15 +1107,23 @@ def _lookup_value_across_sheets( level=logging.INFO, ) - return json.dumps({ + row_page_payload = self._build_tabular_row_page_payload( + combined_results, + total_matches, + start_row=start_row, + max_rows=max_rows, + protected_columns=['_sheet'], + ) + + response_payload = { "filename": filename, "selected_sheet": "ALL (cross-sheet search)", "sheets_searched": sheets_searched, "sheets_matched": sheets_matched, "total_matches": total_matches, - "returned_rows": len(combined_results), - "data": combined_results, - }, indent=2, default=str) + } + response_payload.update(row_page_payload) + return json.dumps(response_payload, indent=2, default=str) def _query_tabular_data_across_sheets( self, @@ -870,6 +1132,8 @@ def _query_tabular_data_across_sheets( filename: str, query_expression: str, max_rows: int = 100, + start_row: int = 0, + return_columns=None, ) -> Optional[str]: """Execute a pandas query expression across all sheets of a multi-sheet workbook. @@ -889,6 +1153,8 @@ def _query_tabular_data_across_sheets( sheets_matched = [] total_matches = 0 query_errors = [] + skip_remaining = max(0, int(start_row or 0)) + requested_max_rows = max(1, int(max_rows or 100)) for sheet in available_sheets: df = self._read_tabular_blob_to_dataframe( @@ -918,11 +1184,18 @@ def _query_tabular_data_across_sheets( sheets_matched.append(sheet) total_matches += sheet_matches - remaining_capacity = max(0, max_rows - len(combined_results)) + if skip_remaining >= sheet_matches: + skip_remaining -= sheet_matches + continue + + remaining_capacity = max(0, requested_max_rows - len(combined_results)) if remaining_capacity > 0: - for row in result_df.head(remaining_capacity).to_dict(orient='records'): + for row in result_df.iloc[skip_remaining:skip_remaining + remaining_capacity].to_dict(orient='records'): row['_sheet'] = sheet combined_results.append(row) + skip_remaining = 0 + else: + skip_remaining = 0 if not sheets_searched: if query_errors: @@ -958,15 +1231,24 @@ def _query_tabular_data_across_sheets( level=logging.INFO, ) - return json.dumps({ + row_page_payload = self._build_tabular_row_page_payload( + combined_results, + total_matches, + start_row=start_row, + max_rows=max_rows, + return_columns=return_columns, + protected_columns=['_sheet'], + ) + + response_payload = { "filename": filename, "selected_sheet": "ALL (cross-sheet search)", "sheets_searched": sheets_searched, "sheets_matched": sheets_matched, "total_matches": total_matches, - "returned_rows": len(combined_results), - "data": combined_results, - }, indent=2, default=str) + } + response_payload.update(row_page_payload) + return json.dumps(response_payload, indent=2, default=str) def _count_rows_across_sheets( self, @@ -1222,6 +1504,8 @@ def _evaluate_related_value_membership( target_alias_column: Optional[str] = None, normalize_match: bool = True, max_rows: int = 100, + start_row: int = 0, + return_columns=None, ) -> dict: """Evaluate a semi-join between a source cohort and a target fact worksheet.""" source_sheet, workbook_metadata = self._resolve_sheet_selection( @@ -1447,7 +1731,17 @@ def _evaluate_related_value_membership( source_value_match_count_limit = self.SOURCE_VALUE_MATCH_COUNT_LIMIT matched_target_row_count = len(matched_target_rows) - return { + start, limit = self._parse_row_page_arguments(start_row, max_rows) + paged_matched_target_rows = matched_target_rows[start:start + limit] + row_page_payload = self._build_tabular_row_page_payload( + paged_matched_target_rows, + matched_target_row_count, + start_row=start, + max_rows=limit, + return_columns=return_columns, + protected_columns=['_matched_on', '_matched_source_values', '_related_document_reference_values'], + ) + response_payload = { 'filename': filename, 'selected_sheet': target_sheet if workbook_metadata.get('is_workbook') else None, 'relationship_type': 'set_membership', @@ -1470,10 +1764,10 @@ def _evaluate_related_value_membership( 'source_value_match_counts': source_value_match_counts[:source_value_match_count_limit], 'target_rows_scanned': len(filtered_target_df), 'matched_target_row_count': matched_target_row_count, - 'returned_rows': min(matched_target_row_count, max_rows), - 'rows_limited': matched_target_row_count > max_rows, - 'data': matched_target_rows[:max_rows], + 'rows_limited': row_page_payload.get('has_more', False), } + response_payload.update(row_page_payload) + return response_payload def _format_datetime_column_label(self, value) -> str: """Render date-like Excel header labels into stable analysis-friendly strings.""" @@ -1901,6 +2195,7 @@ def _search_dataframe_rows( search_operator: str = 'contains', return_columns=None, normalize_match: bool = False, + start_row: int = 0, max_rows: int = 100, ) -> dict: """Search one or more columns in a DataFrame and return row-context results.""" @@ -1939,7 +2234,15 @@ def _search_dataframe_rows( seen_matched_columns = set() result_rows = [] - for row_index, row in matched_df.head(int(max_rows)).iterrows(): + normalized_start_row, normalized_max_rows = self._parse_row_page_arguments(start_row, max_rows) + try: + if int(max_rows or 0) <= 0: + normalized_max_rows = 0 + except (TypeError, ValueError): + pass + + paged_matched_df = matched_df.iloc[normalized_start_row:normalized_start_row + normalized_max_rows] + for row_index, row in paged_matched_df.iterrows(): row_matched_columns = [] for column_name in resolved_search_columns: if not bool(column_masks[column_name].loc[row_index]): @@ -1979,6 +2282,10 @@ def _search_dataframe_rows( 'return_columns': resolved_return_columns or None, 'total_matches': len(matched_df), 'returned_rows': len(result_rows), + 'start_row': normalized_start_row, + 'page_size': normalized_max_rows, + 'has_more': normalized_start_row + len(result_rows) < len(matched_df), + 'next_start_row': normalized_start_row + len(result_rows) if normalized_start_row + len(result_rows) < len(matched_df) else None, 'data': result_rows, } @@ -3091,8 +3398,8 @@ async def list_tabular_files( self, user_id: Annotated[str, "The user ID (from Scope ID in Conversation Metadata)"], conversation_id: Annotated[str, "The conversation ID (from Conversation Metadata)"], - group_id: Annotated[Optional[str], "Group ID (for group workspace documents)"] = None, - public_workspace_id: Annotated[Optional[str], "Public workspace ID (for public workspace documents)"] = None, + group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, + public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, ) -> Annotated[str, "JSON list of available tabular files"]: """List all tabular files available for the user across all accessible containers.""" try: @@ -3226,11 +3533,11 @@ async def describe_tabular_file( user_id: Annotated[str, "The user ID (from Scope ID in Conversation Metadata)"], conversation_id: Annotated[str, "The conversation ID (from Conversation Metadata)"], filename: Annotated[str, "The filename of the tabular file"], - sheet_name: Annotated[Optional[str], "Optional worksheet name for Excel files. When omitted on multi-sheet workbooks, the response returns workbook-level sheet schemas."] = None, - sheet_index: Annotated[Optional[str], "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, + sheet_name: Annotated[str, "Optional worksheet name for Excel files. When omitted on multi-sheet workbooks, the response returns workbook-level sheet schemas."] = None, + sheet_index: Annotated[str, "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", - group_id: Annotated[Optional[str], "Group ID (for group workspace documents)"] = None, - public_workspace_id: Annotated[Optional[str], "Public workspace ID (for public workspace documents)"] = None, + group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, + public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, ) -> Annotated[str, "JSON summary of the tabular file"]: """Get schema and preview of a tabular file.""" def _sync_work(): @@ -3294,12 +3601,13 @@ async def lookup_value( target_column: Annotated[str, "The target column containing the desired value, such as Nov-25"], match_operator: Annotated[str, "Match operator: equals, contains, startswith, endswith"] = "equals", normalize_match: Annotated[str, "Whether to normalize string/entity matching for text comparisons (true/false)"] = "false", - sheet_name: Annotated[Optional[str], "Optional worksheet name for Excel files. Required for analytical calls on multi-sheet workbooks unless sheet_index is provided."] = None, - sheet_index: Annotated[Optional[str], "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, + sheet_name: Annotated[str, "Optional worksheet name for Excel files. Required for analytical calls on multi-sheet workbooks unless sheet_index is provided."] = None, + sheet_index: Annotated[str, "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", + start_row: Annotated[str, "Zero-based row offset for pagination. Use next_start_row from a previous result to continue."] = "0", max_rows: Annotated[str, "Maximum matching rows to return"] = "25", - group_id: Annotated[Optional[str], "Group ID (for group workspace documents)"] = None, - public_workspace_id: Annotated[Optional[str], "Public workspace ID (for public workspace documents)"] = None, + group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, + public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, ) -> Annotated[str, "JSON result containing matching rows and target-column values"]: """Look up values from a target column for matching rows.""" def _sync_work(): @@ -3319,6 +3627,7 @@ def _sync_work(): match_operator=match_operator, normalize_match=normalize_match_flag, max_rows=int(max_rows), + start_row=int(start_row), ) if cross_sheet_result is not None: return cross_sheet_result @@ -3377,8 +3686,9 @@ def _sync_work(): except ValueError: return json.dumps({"error": f"Unsupported match_operator: {match_operator}"}) - limit = int(max_rows) - matches = df[mask].head(limit) + start, limit = self._parse_row_page_arguments(start_row, max_rows, default_max_rows=25) + matched_df = df[mask] + matches = matched_df.iloc[start:start + limit] response = { "filename": filename, "selected_sheet": selected_sheet if workbook_metadata.get('is_workbook') else None, @@ -3387,10 +3697,14 @@ def _sync_work(): "target_column": target_column, "match_operator": operator, "normalize_match": normalize_match_flag, - "total_matches": int(mask.sum()), - "returned_rows": len(matches), - "data": matches.to_dict(orient='records'), + "total_matches": len(matched_df), } + response.update(self._build_tabular_row_page_payload( + matches.to_dict(orient='records'), + len(matched_df), + start_row=start, + max_rows=limit, + )) if len(matches) == 1: response["value"] = matches.iloc[0][target_column] @@ -3415,23 +3729,23 @@ async def get_distinct_values( conversation_id: Annotated[str, "The conversation ID (from Conversation Metadata)"], filename: Annotated[str, "The filename of the tabular file"], column: Annotated[str, "The column from which to return distinct values"], - query_expression: Annotated[Optional[str], "Optional pandas DataFrame.query() expression to apply before collecting distinct values"] = None, - filter_column: Annotated[Optional[str], "Optional column to filter on before collecting distinct values"] = None, + query_expression: Annotated[str, "Optional pandas DataFrame.query() expression to apply before collecting distinct values"] = None, + filter_column: Annotated[str, "Optional column to filter on before collecting distinct values"] = None, filter_operator: Annotated[str, "Optional filter operator when filter_column is provided"] = "equals", - filter_value: Annotated[Optional[str], "Optional filter value when filter_column is provided"] = None, - additional_filter_column: Annotated[Optional[str], "Optional second column to filter on before collecting distinct values"] = None, + filter_value: Annotated[str, "Optional filter value when filter_column is provided"] = None, + additional_filter_column: Annotated[str, "Optional second column to filter on before collecting distinct values"] = None, additional_filter_operator: Annotated[str, "Optional filter operator when additional_filter_column is provided"] = "equals", - additional_filter_value: Annotated[Optional[str], "Optional filter value when additional_filter_column is provided"] = None, - extract_mode: Annotated[Optional[str], "Optional embedded extraction mode: 'url' or 'regex'"] = None, - extract_pattern: Annotated[Optional[str], "Optional regex pattern when extract_mode is 'regex'"] = None, - url_path_segments: Annotated[Optional[str], "Optional number of URL path segments to keep when extract_mode is 'url'"] = None, + additional_filter_value: Annotated[str, "Optional filter value when additional_filter_column is provided"] = None, + extract_mode: Annotated[str, "Optional embedded extraction mode: 'url' or 'regex'"] = None, + extract_pattern: Annotated[str, "Optional regex pattern when extract_mode is 'regex'"] = None, + url_path_segments: Annotated[str, "Optional number of URL path segments to keep when extract_mode is 'url'"] = None, normalize_match: Annotated[str, "Whether to normalize string/entity matching and deduplication (true/false)"] = "true", - sheet_name: Annotated[Optional[str], "Optional worksheet name for Excel files. When omitted, the plugin may perform a cross-sheet distinct-value search."] = None, - sheet_index: Annotated[Optional[str], "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, + sheet_name: Annotated[str, "Optional worksheet name for Excel files. When omitted, the plugin may perform a cross-sheet distinct-value search."] = None, + sheet_index: Annotated[str, "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", max_values: Annotated[str, "Maximum distinct values to return"] = "100", - group_id: Annotated[Optional[str], "Group ID (for group workspace documents)"] = None, - public_workspace_id: Annotated[Optional[str], "Public workspace ID (for public workspace documents)"] = None, + group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, + public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, ) -> Annotated[str, "JSON result containing deterministic distinct values and counts"]: """Return deterministic distinct values from a worksheet or across worksheets.""" def _sync_work(): @@ -3587,19 +3901,19 @@ async def count_rows( user_id: Annotated[str, "The user ID (from Scope ID in Conversation Metadata)"], conversation_id: Annotated[str, "The conversation ID (from Conversation Metadata)"], filename: Annotated[str, "The filename of the tabular file"], - query_expression: Annotated[Optional[str], "Optional pandas DataFrame.query() expression to apply before counting rows"] = None, - filter_column: Annotated[Optional[str], "Optional column to filter on before counting rows"] = None, + query_expression: Annotated[str, "Optional pandas DataFrame.query() expression to apply before counting rows"] = None, + filter_column: Annotated[str, "Optional column to filter on before counting rows"] = None, filter_operator: Annotated[str, "Optional filter operator when filter_column is provided"] = "equals", - filter_value: Annotated[Optional[str], "Optional filter value when filter_column is provided"] = None, - additional_filter_column: Annotated[Optional[str], "Optional second column to filter on before counting rows"] = None, + filter_value: Annotated[str, "Optional filter value when filter_column is provided"] = None, + additional_filter_column: Annotated[str, "Optional second column to filter on before counting rows"] = None, additional_filter_operator: Annotated[str, "Optional filter operator when additional_filter_column is provided"] = "equals", - additional_filter_value: Annotated[Optional[str], "Optional filter value when additional_filter_column is provided"] = None, + additional_filter_value: Annotated[str, "Optional filter value when additional_filter_column is provided"] = None, normalize_match: Annotated[str, "Whether to normalize string/entity matching for text comparisons (true/false)"] = "false", - sheet_name: Annotated[Optional[str], "Optional worksheet name for Excel files. When omitted, the plugin may perform a cross-sheet row count."] = None, - sheet_index: Annotated[Optional[str], "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, + sheet_name: Annotated[str, "Optional worksheet name for Excel files. When omitted, the plugin may perform a cross-sheet row count."] = None, + sheet_index: Annotated[str, "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", - group_id: Annotated[Optional[str], "Group ID (for group workspace documents)"] = None, - public_workspace_id: Annotated[Optional[str], "Public workspace ID (for public workspace documents)"] = None, + group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, + public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, ) -> Annotated[str, "JSON result containing a deterministic row count"]: """Count rows deterministically after optional filters or queries.""" def _sync_work(): @@ -3709,11 +4023,11 @@ async def aggregate_column( filename: Annotated[str, "The filename of the tabular file"], column: Annotated[str, "The column name to aggregate"], operation: Annotated[str, "Aggregation: sum, mean, count, min, max, median, std, nunique, value_counts"], - sheet_name: Annotated[Optional[str], "Optional worksheet name for Excel files. Required for analytical calls on multi-sheet workbooks unless sheet_index is provided."] = None, - sheet_index: Annotated[Optional[str], "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, + sheet_name: Annotated[str, "Optional worksheet name for Excel files. Required for analytical calls on multi-sheet workbooks unless sheet_index is provided."] = None, + sheet_index: Annotated[str, "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", - group_id: Annotated[Optional[str], "Group ID (for group workspace documents)"] = None, - public_workspace_id: Annotated[Optional[str], "Public workspace ID (for public workspace documents)"] = None, + group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, + public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, ) -> Annotated[str, "JSON result of the aggregation"]: """Execute an aggregation operation on a column.""" def _sync_work(): @@ -3803,21 +4117,24 @@ async def filter_rows( column: Annotated[str, "The column to filter on"], operator: Annotated[str, "Operator: ==, !=, >, <, >=, <=, contains, startswith, endswith"], value: Annotated[str, "The value to compare against"], - additional_filter_column: Annotated[Optional[str], "Optional second column to filter on"] = None, + additional_filter_column: Annotated[str, "Optional second column to filter on"] = None, additional_filter_operator: Annotated[str, "Optional filter operator when additional_filter_column is provided"] = "equals", - additional_filter_value: Annotated[Optional[str], "Optional filter value when additional_filter_column is provided"] = None, + additional_filter_value: Annotated[str, "Optional filter value when additional_filter_column is provided"] = None, normalize_match: Annotated[str, "Whether to normalize string/entity matching for text comparisons (true/false)"] = "false", - sheet_name: Annotated[Optional[str], "Optional worksheet name for Excel files. Required for analytical calls on multi-sheet workbooks unless sheet_index is provided."] = None, - sheet_index: Annotated[Optional[str], "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, + sheet_name: Annotated[str, "Optional worksheet name for Excel files. Required for analytical calls on multi-sheet workbooks unless sheet_index is provided."] = None, + sheet_index: Annotated[str, "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", - max_rows: Annotated[str, "Maximum rows to return"] = "100", - group_id: Annotated[Optional[str], "Group ID (for group workspace documents)"] = None, - public_workspace_id: Annotated[Optional[str], "Public workspace ID (for public workspace documents)"] = None, + return_columns: Annotated[str, "Optional comma-separated columns to include in each result row. Omit to return all columns."] = None, + start_row: Annotated[str, "Zero-based row offset for pagination. Use next_start_row from a previous result to continue."] = "0", + max_rows: Annotated[str, "Maximum rows to return per page"] = "100", + group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, + public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, ) -> Annotated[str, "JSON list of matching rows"]: """Filter rows based on a condition.""" def _sync_work(): try: normalize_match_flag = self._parse_boolean_argument(normalize_match, default=False) + parsed_return_columns = self._parse_optional_column_list_argument(return_columns) container, blob_path = self._resolve_blob_location_with_fallback( user_id, conversation_id, filename, source, group_id=group_id, public_workspace_id=public_workspace_id @@ -3838,6 +4155,8 @@ def _sync_work(): additional_filter_value=additional_filter_value, normalize_match=normalize_match_flag, max_rows=int(max_rows), + start_row=int(start_row), + return_columns=parsed_return_columns, ) if cross_sheet_result is not None: return cross_sheet_result @@ -3898,17 +4217,23 @@ def _sync_work(): except ValueError as filter_error: return json.dumps({"error": str(filter_error)}) - limit = int(max_rows) - filtered = filtered_df.head(limit) - return json.dumps({ + start, limit = self._parse_row_page_arguments(start_row, max_rows) + filtered = filtered_df.iloc[start:start + limit] + response_payload = { "filename": filename, "selected_sheet": selected_sheet if workbook_metadata.get('is_workbook') else None, "filter_applied": applied_filters, "normalize_match": normalize_match_flag, "total_matches": len(filtered_df), - "returned_rows": len(filtered), - "data": filtered.to_dict(orient='records') - }, indent=2, default=str) + } + response_payload.update(self._build_tabular_row_page_payload( + filtered.to_dict(orient='records'), + len(filtered_df), + start_row=start, + max_rows=limit, + return_columns=parsed_return_columns, + )) + return json.dumps(response_payload, indent=2, default=str) except Exception as e: log_event(f"[TabularProcessingPlugin] Error filtering rows: {e}", level=logging.WARNING) return json.dumps({"error": str(e)}) @@ -3928,23 +4253,24 @@ async def search_rows( conversation_id: Annotated[str, "The conversation ID (from Conversation Metadata)"], filename: Annotated[str, "The filename of the tabular file"], search_value: Annotated[str, "The text or value to search for"], - search_columns: Annotated[Optional[str], "Optional comma-separated columns to search. Omit to search all columns."] = None, + search_columns: Annotated[str, "Optional comma-separated columns to search. Omit to search all columns."] = None, search_operator: Annotated[str, "Search operator: equals, contains, startswith, endswith"] = "contains", - return_columns: Annotated[Optional[str], "Optional comma-separated columns to include in each result row. Omit to return the full row."] = None, - query_expression: Annotated[Optional[str], "Optional pandas DataFrame.query() expression to apply before searching"] = None, - filter_column: Annotated[Optional[str], "Optional first column filter to narrow the search cohort"] = None, + return_columns: Annotated[str, "Optional comma-separated columns to include in each result row. Omit to return the full row."] = None, + query_expression: Annotated[str, "Optional pandas DataFrame.query() expression to apply before searching"] = None, + filter_column: Annotated[str, "Optional first column filter to narrow the search cohort"] = None, filter_operator: Annotated[str, "Optional filter operator when filter_column is provided"] = "equals", - filter_value: Annotated[Optional[str], "Optional filter value when filter_column is provided"] = None, - additional_filter_column: Annotated[Optional[str], "Optional second column filter to narrow the search cohort"] = None, + filter_value: Annotated[str, "Optional filter value when filter_column is provided"] = None, + additional_filter_column: Annotated[str, "Optional second column filter to narrow the search cohort"] = None, additional_filter_operator: Annotated[str, "Optional filter operator when additional_filter_column is provided"] = "equals", - additional_filter_value: Annotated[Optional[str], "Optional filter value when additional_filter_column is provided"] = None, + additional_filter_value: Annotated[str, "Optional filter value when additional_filter_column is provided"] = None, normalize_match: Annotated[str, "Whether to normalize string/entity matching for text comparisons (true/false)"] = "false", - sheet_name: Annotated[Optional[str], "Optional worksheet name for Excel files. When omitted, the plugin may perform a cross-sheet search."] = None, - sheet_index: Annotated[Optional[str], "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, + sheet_name: Annotated[str, "Optional worksheet name for Excel files. When omitted, the plugin may perform a cross-sheet search."] = None, + sheet_index: Annotated[str, "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", - max_rows: Annotated[str, "Maximum matching rows to return"] = "100", - group_id: Annotated[Optional[str], "Group ID (for group workspace documents)"] = None, - public_workspace_id: Annotated[Optional[str], "Public workspace ID (for public workspace documents)"] = None, + start_row: Annotated[str, "Zero-based row offset for pagination. Use next_start_row from a previous result to continue."] = "0", + max_rows: Annotated[str, "Maximum matching rows to return per page"] = "100", + group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, + public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, ) -> Annotated[str, "JSON result containing matching rows, matched columns, and search metadata"]: """Search rows across one or more columns while preserving row context.""" def _sync_work(): @@ -3977,6 +4303,7 @@ def _sync_work(): additional_filter_value=additional_filter_value, normalize_match=normalize_match_flag, max_rows=int(max_rows), + start_row=int(start_row), ) if cross_sheet_result is not None: return cross_sheet_result @@ -4046,6 +4373,7 @@ def _sync_work(): search_operator=search_operator, return_columns=parsed_return_columns, normalize_match=normalize_match_flag, + start_row=int(start_row), max_rows=int(max_rows), ) except KeyError as missing_column_error: @@ -4090,6 +4418,10 @@ def _sync_work(): 'normalize_match': normalize_match_flag, 'total_matches': search_result['total_matches'], 'returned_rows': search_result['returned_rows'], + 'start_row': search_result['start_row'], + 'page_size': search_result['page_size'], + 'has_more': search_result['has_more'], + 'next_start_row': search_result['next_start_row'], 'data': search_result['data'], }, indent=2, default=str) except Exception as e: @@ -4113,16 +4445,19 @@ async def query_tabular_data( conversation_id: Annotated[str, "The conversation ID (from Conversation Metadata)"], filename: Annotated[str, "The filename of the tabular file"], query_expression: Annotated[str, "Pandas query expression (e.g. 'Age > 30 and State == \"CA\"')"], - sheet_name: Annotated[Optional[str], "Optional worksheet name for Excel files. Required for analytical calls on multi-sheet workbooks unless sheet_index is provided."] = None, - sheet_index: Annotated[Optional[str], "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, + sheet_name: Annotated[str, "Optional worksheet name for Excel files. Required for analytical calls on multi-sheet workbooks unless sheet_index is provided."] = None, + sheet_index: Annotated[str, "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", - max_rows: Annotated[str, "Maximum rows to return"] = "100", - group_id: Annotated[Optional[str], "Group ID (for group workspace documents)"] = None, - public_workspace_id: Annotated[Optional[str], "Public workspace ID (for public workspace documents)"] = None, + return_columns: Annotated[str, "Optional comma-separated columns to include in each result row. Omit to return all columns."] = None, + start_row: Annotated[str, "Zero-based row offset for pagination. Use next_start_row from a previous result to continue."] = "0", + max_rows: Annotated[str, "Maximum rows to return per page"] = "100", + group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, + public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, ) -> Annotated[str, "JSON result of the query"]: """Execute a pandas query expression against a tabular file.""" def _sync_work(): try: + parsed_return_columns = self._parse_optional_column_list_argument(return_columns) container, blob_path = self._resolve_blob_location_with_fallback( user_id, conversation_id, filename, source, group_id=group_id, public_workspace_id=public_workspace_id @@ -4134,6 +4469,8 @@ def _sync_work(): cross_sheet_result = self._query_tabular_data_across_sheets( container, blob_path, filename, query_expression, max_rows=int(max_rows), + start_row=int(start_row), + return_columns=parsed_return_columns, ) if cross_sheet_result is not None: return cross_sheet_result @@ -4157,16 +4494,23 @@ def _sync_work(): query_expression=query_expression, normalize_match=False, ) - limit = int(max_rows) - return json.dumps({ + start, limit = self._parse_row_page_arguments(start_row, max_rows) + sliced_result_df = result_df.iloc[start:start + limit] + response_payload = { "filename": filename, "selected_sheet": selected_sheet if workbook_metadata.get('is_workbook') else None, "query_expression": query_expression, "query_expression_fallback": used_reviewer_style_fallback, "total_matches": len(result_df), - "returned_rows": min(len(result_df), limit), - "data": result_df.head(limit).to_dict(orient='records') - }, indent=2, default=str) + } + response_payload.update(self._build_tabular_row_page_payload( + sliced_result_df.to_dict(orient='records'), + len(result_df), + start_row=start, + max_rows=limit, + return_columns=parsed_return_columns, + )) + return json.dumps(response_payload, indent=2, default=str) except Exception as e: log_event(f"[TabularProcessingPlugin] Error querying data: {e}", level=logging.WARNING) return json.dumps({"error": f"Query error: {str(e)}. Ensure column names and values are correct."}) @@ -4189,23 +4533,25 @@ async def filter_rows_by_related_values( source_value_column: Annotated[str, "Column on the source worksheet that contains the canonical cohort values"], target_sheet_name: Annotated[str, "Worksheet containing the fact rows to filter"], target_match_column: Annotated[str, "Column on the target worksheet that should match the source cohort values"], - source_query_expression: Annotated[Optional[str], "Optional pandas DataFrame.query() expression to narrow the source cohort"] = None, - source_filter_column: Annotated[Optional[str], "Optional source-sheet filter column"] = None, + source_query_expression: Annotated[str, "Optional pandas DataFrame.query() expression to narrow the source cohort"] = None, + source_filter_column: Annotated[str, "Optional source-sheet filter column"] = None, source_filter_operator: Annotated[str, "Optional source-sheet filter operator"] = "equals", - source_filter_value: Annotated[Optional[str], "Optional source-sheet filter value"] = None, - target_query_expression: Annotated[Optional[str], "Optional pandas DataFrame.query() expression to narrow target rows before matching"] = None, - target_filter_column: Annotated[Optional[str], "Optional target-sheet filter column"] = None, + source_filter_value: Annotated[str, "Optional source-sheet filter value"] = None, + target_query_expression: Annotated[str, "Optional pandas DataFrame.query() expression to narrow target rows before matching"] = None, + target_filter_column: Annotated[str, "Optional target-sheet filter column"] = None, target_filter_operator: Annotated[str, "Optional target-sheet filter operator"] = "equals", - target_filter_value: Annotated[Optional[str], "Optional target-sheet filter value"] = None, - source_alias_column: Annotated[Optional[str], "Optional alternate or alias source column used for normalized matching"] = None, - target_alias_column: Annotated[Optional[str], "Optional alternate or alias target column used for normalized matching"] = None, + target_filter_value: Annotated[str, "Optional target-sheet filter value"] = None, + source_alias_column: Annotated[str, "Optional alternate or alias source column used for normalized matching"] = None, + target_alias_column: Annotated[str, "Optional alternate or alias target column used for normalized matching"] = None, normalize_match: Annotated[str, "Whether to normalize entity-style text matching across worksheets (true/false)"] = "true", - source_sheet_index: Annotated[Optional[str], "Optional zero-based source worksheet index if sheet name is not used"] = None, - target_sheet_index: Annotated[Optional[str], "Optional zero-based target worksheet index if sheet name is not used"] = None, + source_sheet_index: Annotated[str, "Optional zero-based source worksheet index if sheet name is not used"] = None, + target_sheet_index: Annotated[str, "Optional zero-based target worksheet index if sheet name is not used"] = None, source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", - max_rows: Annotated[str, "Maximum related target rows to return"] = "100", - group_id: Annotated[Optional[str], "Group ID (for group workspace documents)"] = None, - public_workspace_id: Annotated[Optional[str], "Public workspace ID (for public workspace documents)"] = None, + return_columns: Annotated[str, "Optional comma-separated target-row columns to include. Omit to return all target-row columns."] = None, + start_row: Annotated[str, "Zero-based target-row offset for pagination. Use next_start_row from a previous result to continue."] = "0", + max_rows: Annotated[str, "Maximum related target rows to return per page"] = "100", + group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, + public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, ) -> Annotated[str, "JSON result containing explainable set-membership filtering output"]: """Filter target rows by membership in a source-sheet cohort.""" def _sync_work(): @@ -4237,6 +4583,8 @@ def _sync_work(): target_alias_column=target_alias_column, normalize_match=normalize_match_flag, max_rows=int(max_rows), + start_row=int(start_row), + return_columns=return_columns, ) return json.dumps(result_payload, indent=2, default=str) except Exception as e: @@ -4262,22 +4610,22 @@ async def count_rows_by_related_values( source_value_column: Annotated[str, "Column on the source worksheet that contains the canonical cohort values"], target_sheet_name: Annotated[str, "Worksheet containing the fact rows to count"], target_match_column: Annotated[str, "Column on the target worksheet that should match the source cohort values"], - source_query_expression: Annotated[Optional[str], "Optional pandas DataFrame.query() expression to narrow the source cohort"] = None, - source_filter_column: Annotated[Optional[str], "Optional source-sheet filter column"] = None, + source_query_expression: Annotated[str, "Optional pandas DataFrame.query() expression to narrow the source cohort"] = None, + source_filter_column: Annotated[str, "Optional source-sheet filter column"] = None, source_filter_operator: Annotated[str, "Optional source-sheet filter operator"] = "equals", - source_filter_value: Annotated[Optional[str], "Optional source-sheet filter value"] = None, - target_query_expression: Annotated[Optional[str], "Optional pandas DataFrame.query() expression to narrow target rows before matching"] = None, - target_filter_column: Annotated[Optional[str], "Optional target-sheet filter column"] = None, + source_filter_value: Annotated[str, "Optional source-sheet filter value"] = None, + target_query_expression: Annotated[str, "Optional pandas DataFrame.query() expression to narrow target rows before matching"] = None, + target_filter_column: Annotated[str, "Optional target-sheet filter column"] = None, target_filter_operator: Annotated[str, "Optional target-sheet filter operator"] = "equals", - target_filter_value: Annotated[Optional[str], "Optional target-sheet filter value"] = None, - source_alias_column: Annotated[Optional[str], "Optional alternate or alias source column used for normalized matching"] = None, - target_alias_column: Annotated[Optional[str], "Optional alternate or alias target column used for normalized matching"] = None, + target_filter_value: Annotated[str, "Optional target-sheet filter value"] = None, + source_alias_column: Annotated[str, "Optional alternate or alias source column used for normalized matching"] = None, + target_alias_column: Annotated[str, "Optional alternate or alias target column used for normalized matching"] = None, normalize_match: Annotated[str, "Whether to normalize entity-style text matching across worksheets (true/false)"] = "true", - source_sheet_index: Annotated[Optional[str], "Optional zero-based source worksheet index if sheet name is not used"] = None, - target_sheet_index: Annotated[Optional[str], "Optional zero-based target worksheet index if sheet name is not used"] = None, + source_sheet_index: Annotated[str, "Optional zero-based source worksheet index if sheet name is not used"] = None, + target_sheet_index: Annotated[str, "Optional zero-based target worksheet index if sheet name is not used"] = None, source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", - group_id: Annotated[Optional[str], "Group ID (for group workspace documents)"] = None, - public_workspace_id: Annotated[Optional[str], "Public workspace ID (for public workspace documents)"] = None, + group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, + public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, ) -> Annotated[str, "JSON result containing an explainable relational row count"]: """Count target rows by membership in a source-sheet cohort.""" def _sync_work(): @@ -4340,13 +4688,13 @@ async def group_by_aggregate( group_by_column: Annotated[str, "The column to group by"], aggregate_column: Annotated[str, "The column to aggregate"], operation: Annotated[str, "Aggregation operation: sum, mean, count, min, max, median, std"], - sheet_name: Annotated[Optional[str], "Optional worksheet name for Excel files. Required for analytical calls on multi-sheet workbooks unless sheet_index is provided."] = None, - sheet_index: Annotated[Optional[str], "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, + sheet_name: Annotated[str, "Optional worksheet name for Excel files. Required for analytical calls on multi-sheet workbooks unless sheet_index is provided."] = None, + sheet_index: Annotated[str, "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", top_n: Annotated[str, "How many top groups to return in descending or ascending order"] = "10", sort_descending: Annotated[str, "Whether top_results should be sorted descending (true/false)"] = "true", - group_id: Annotated[Optional[str], "Group ID (for group workspace documents)"] = None, - public_workspace_id: Annotated[Optional[str], "Public workspace ID (for public workspace documents)"] = None, + group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, + public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, ) -> Annotated[str, "JSON result of the group-by aggregation"]: """Group by one column and aggregate another.""" def _sync_work(): @@ -4438,16 +4786,16 @@ async def group_by_datetime_component( filename: Annotated[str, "The filename of the tabular file"], datetime_column: Annotated[str, "The datetime-like column to extract a component from"], datetime_component: Annotated[str, "Component: year, month, month_name, day, date, hour, minute, day_name, weekday_number, quarter, or week"], - aggregate_column: Annotated[Optional[str], "The numeric column to aggregate. Leave empty and use operation='count' to count rows."] = "", + aggregate_column: Annotated[str, "The numeric column to aggregate. Leave empty and use operation='count' to count rows."] = "", operation: Annotated[str, "Aggregation operation: count, sum, mean, min, max, median, std"] = "count", - sheet_name: Annotated[Optional[str], "Optional worksheet name for Excel files. Required for analytical calls on multi-sheet workbooks unless sheet_index is provided."] = None, - sheet_index: Annotated[Optional[str], "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, + sheet_name: Annotated[str, "Optional worksheet name for Excel files. Required for analytical calls on multi-sheet workbooks unless sheet_index is provided."] = None, + sheet_index: Annotated[str, "Optional zero-based worksheet index for Excel files. Ignored when sheet_name is provided."] = None, source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", - filter_expression: Annotated[Optional[str], "Optional pandas query filter applied before grouping"] = "", + filter_expression: Annotated[str, "Optional pandas query filter applied before grouping"] = "", top_n: Annotated[str, "How many top groups to return in descending order"] = "10", sort_descending: Annotated[str, "Whether top_results should be sorted descending (true/false)"] = "true", - group_id: Annotated[Optional[str], "Group ID (for group workspace documents)"] = None, - public_workspace_id: Annotated[Optional[str], "Public workspace ID (for public workspace documents)"] = None, + group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, + public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, ) -> Annotated[str, "JSON result of the datetime component grouping analysis"]: """Group data by a datetime component and aggregate a metric.""" def _sync_work(): diff --git a/docs/explanation/features/index.md b/docs/explanation/features/index.md index 5452ec8b6..adc6ae922 100644 --- a/docs/explanation/features/index.md +++ b/docs/explanation/features/index.md @@ -34,6 +34,7 @@ category: Version History ## Versioned Features +- [Tabular SK Large Result Pagination](v0.242.067/TABULAR_SK_LARGE_RESULT_PAGINATION.md) - [Model Endpoint Model Icon Picker](v0.242.060/MODEL_ENDPOINT_MODEL_ICON_PICKER.md) - [Deployer Capacity Defaults](v0.241.085/DEPLOYER_CAPACITY_DEFAULTS.md) - [Chat Inline Export Action Progress Labels](v0.241.107/CHAT_INLINE_EXPORT_ACTION_PROGRESS.md) diff --git a/docs/explanation/features/v0.242.067/TABULAR_SK_LARGE_RESULT_PAGINATION.md b/docs/explanation/features/v0.242.067/TABULAR_SK_LARGE_RESULT_PAGINATION.md new file mode 100644 index 000000000..8017cd415 --- /dev/null +++ b/docs/explanation/features/v0.242.067/TABULAR_SK_LARGE_RESULT_PAGINATION.md @@ -0,0 +1,60 @@ +# Tabular SK Large Result Pagination + +Implemented in version: **0.242.067** + +## Overview + +Tabular Semantic Kernel analysis now supports safer large-result handling for row-returning tools. The feature adds explicit pagination metadata, preserves caller-requested projections with `return_columns`, trims oversized row payloads when projection is not provided, and raises the computed-results handoff guardrail from 24K to 100K characters. + +## Technical Specifications + +### Architecture + +The tabular processing plugin centralizes row payload shaping through a shared helper that: + +- Normalizes `start_row` and `max_rows`. +- Returns `has_more` and `next_start_row` for continuation. +- Applies `return_columns` projection when requested. +- Preserves protected row metadata such as `_sheet`, `_matched_columns`, `_matched_values`, `_matched_on`, `_matched_source_values`, and `_related_document_reference_values`. +- Estimates serialized JSON size and auto-excludes heavy non-protected columns when the row payload would exceed the safe output budget. +- Reduces rows only when column trimming is insufficient, advancing `next_start_row` by the number of rows actually returned. + +### Tools Updated + +- `lookup_value` +- `filter_rows` +- `search_rows` +- `query_tabular_data` +- `filter_rows_by_related_values` + +Count and aggregation tools remain compact summary tools and do not expose row pagination. + +### Handoff Limits + +`route_backend_chats.py` now uses a 100K-character guardrail for tabular SK analysis text and computed-results handoff messages. Truncation emits warning logs with the original and configured limit details. + +## Usage Instructions + +Call row-returning tabular tools with `max_rows` to limit page size. If the tool response includes `has_more: true`, call the same tool again with `start_row` set to `next_start_row`. + +Use `return_columns` when the answer only needs specific fields. This bypasses automatic heavy-column exclusion and keeps returned rows focused. + +## Testing and Validation + +Functional tests: + +- `functional_tests/test_tabular_large_result_pagination.py` +- `functional_tests/test_tabular_large_result_handoff.py` + +These tests validate row continuation, auto-trim behavior, `return_columns` projection, cross-sheet pagination, attachment-reference preservation, and the 100K handoff guardrail. + +## Attribution + +This feature was inspired by and adapted from the design direction in PR #894 by @vivche, which proposed tabular SK pagination, `return_columns` forwarding, automatic large-result trimming, and a larger computed-results handoff guardrail. + +The implementation in this branch was rebuilt against the current Development tabular pipeline to preserve newer model-context routing, related-document evidence, generated tabular outputs, and thought tracking. + +## Related Version Updates + +- `application/single_app/config.py` updated to `0.242.067` for the initial feature. +- `application/single_app/config.py` updated to `0.242.068` for the Python 3.13 Semantic Kernel parameter compatibility follow-up. diff --git a/docs/explanation/fixes/index.md b/docs/explanation/fixes/index.md index f8402fa9a..255236935 100644 --- a/docs/explanation/fixes/index.md +++ b/docs/explanation/fixes/index.md @@ -6,6 +6,7 @@ order: 120 category: Version History --- +- [Tabular SK Python 3.13 Kernel Parameter Fix](v0.242.068/TABULAR_SK_PY313_KERNEL_PARAMETER_FIX.md) - [Python 3.12 CI and XSS Guardrail Fix](PYTHON_312_CI_AND_XSS_GUARDRAIL_FIX.md) - [PR Readiness Guardrail Cleanup Fix](PR_READINESS_GUARDRAIL_CLEANUP_FIX.md) - [CSRF State-Changing Route Guard Fix](CSRF_STATE_CHANGING_ROUTE_GUARD_FIX.md) diff --git a/docs/explanation/fixes/v0.242.068/TABULAR_SK_PY313_KERNEL_PARAMETER_FIX.md b/docs/explanation/fixes/v0.242.068/TABULAR_SK_PY313_KERNEL_PARAMETER_FIX.md new file mode 100644 index 000000000..425a52e26 --- /dev/null +++ b/docs/explanation/fixes/v0.242.068/TABULAR_SK_PY313_KERNEL_PARAMETER_FIX.md @@ -0,0 +1,52 @@ +# Tabular SK Python 3.13 Kernel Parameter Fix + +Fixed in version: **0.242.068** + +## Issue Description + +Semantic Kernel tool-call parsing can fail on Python 3.13 when public plugin parameters are annotated as `Annotated[Optional[str], ...]`. During argument coercion, Semantic Kernel may try to instantiate `typing.Optional[str]`, which is a `Union[str, None]` and is not callable. + +The failure appears as a `TypeError` similar to `Cannot instantiate typing.Union` and can prevent tabular workbook tools from running even when the same tool signatures worked under older runtime behavior. + +## Root Cause Analysis + +The affected surface is the public `@kernel_function` method signatures in `application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py`. These signatures are parsed by Semantic Kernel, unlike normal internal helper annotations. + +Internal helpers can safely keep `Optional[str]` annotations. Public Semantic Kernel tool parameters should use concrete parseable types, while defaults such as `= None` continue to represent omitted optional arguments. + +## Technical Details + +### Files Modified + +- `application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py` +- `functional_tests/test_tabular_kernel_parameter_annotations.py` +- `application/single_app/config.py` + +### Code Changes Summary + +- Replaced public `@kernel_function` parameter annotations from `Annotated[Optional[str], ...]` to `Annotated[str, ...]`. +- Kept `None` defaults for optional tool parameters so omitted values remain supported. +- Preserved internal helper type hints that are not parsed by Semantic Kernel. +- Added a regression test that parses the tabular plugin AST and fails if any `@kernel_function` parameter reintroduces `Annotated[Optional[str], ...]`. + +## Validation + +Validation includes: + +- Python compile checks for the tabular plugin and new functional test. +- `functional_tests/test_tabular_kernel_parameter_annotations.py`. +- Existing tabular pagination and relational helper tests to ensure the annotation cleanup does not change runtime behavior. + +## Impact Analysis + +This change supports both Python 3.12 and Python 3.13. Tool-call behavior remains the same for callers: optional arguments can still be omitted, and plugin code still treats `None` and empty strings as not provided where appropriate. + +## Attribution + +This fix was inspired by and adapted from PR #892 by @vivche, which identified the Python 3.13 Semantic Kernel `Optional[str]` parameter parsing issue while investigating tabular SK multi-endpoint workbook analysis. + +The multi-endpoint route changes from PR #892 were not merged directly because current Development already supersedes them through the shared model-context runtime, but the Python 3.13 compatibility insight is preserved here with a focused regression test. + +## Related Version Updates + +- `application/single_app/config.py` updated to `0.242.068`. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 3aa347546..002e80e32 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -1,9 +1,29 @@ -This page tracks notable Simple Chat releases and organizes the detailed change log by version. The timeline below provides a quick visual overview of the current release progression through v0.242.066, and the per-version entries continue immediately after it. +This page tracks notable Simple Chat releases and organizes the detailed change log by version. The timeline below provides a quick visual overview of the current release progression through v0.242.068, and the per-version entries continue immediately after it. For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.242.068)** + +#### New Features + +* **Tabular SK Large Result Pagination** + * Added continuation metadata for row-returning tabular Semantic Kernel tools, including `start_row`, `page_size`, `has_more`, and `next_start_row`. + * Added safe row payload trimming for oversized tool results, while preserving explicit `return_columns` projection and protected row metadata used for sheet context, matched values, and row-linked document evidence. + * Raised tabular computed-results handoff guardrails to 100K characters with warning logs when truncation is still required. + * Inspired by and adapted from PR #894 by @vivche. + * (Ref: tabular SK pagination, `return_columns`, large-result handoff, `tabular_processing_plugin.py`, `route_backend_chats.py`) + +#### Bug Fixes + +* **Tabular SK Python 3.13 Kernel Parameter Compatibility** + * Updated public tabular `@kernel_function` parameters to avoid `Annotated[Optional[str], ...]` so Semantic Kernel argument parsing works on both Python 3.12 and Python 3.13. + * Added a guardrail test that fails if optional string annotations are reintroduced on public tabular tool parameters. + * Preserved current Development model-context routing instead of reintroducing older endpoint-specific route wiring. + * Inspired by and adapted from PR #892 by @vivche. + * (Ref: Python 3.13, Semantic Kernel tool parsing, tabular SK parameters, `test_tabular_kernel_parameter_annotations.py`) + ### **(v0.242.066)** #### Bug Fixes diff --git a/functional_tests/test_agents_catalog_feature.py b/functional_tests/test_agents_catalog_feature.py index 45c0fed4c..fc93cbf5d 100644 --- a/functional_tests/test_agents_catalog_feature.py +++ b/functional_tests/test_agents_catalog_feature.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ Functional test for the Agents catalog page and agent icon/tag metadata. -Version: 0.242.066 +Version: 0.242.068 Implemented in: 0.242.061; updated in 0.242.064; 0.242.065; 0.242.066 This test ensures the global Agents page, shared catalog APIs, safe agent diff --git a/functional_tests/test_azd_managed_identity_preflight.py b/functional_tests/test_azd_managed_identity_preflight.py index 71b0175fe..e043d0391 100644 --- a/functional_tests/test_azd_managed_identity_preflight.py +++ b/functional_tests/test_azd_managed_identity_preflight.py @@ -2,7 +2,7 @@ # test_azd_managed_identity_preflight.py """ Functional test for AZD managed identity RBAC preflight. -Version: 0.242.066 +Version: 0.242.068 Implemented in: 0.242.057 This test ensures managed identity deployments fail before provisioning when diff --git a/functional_tests/test_broken_access_control_findings_fix.py b/functional_tests/test_broken_access_control_findings_fix.py index 432fa89ce..46911d6d4 100644 --- a/functional_tests/test_broken_access_control_findings_fix.py +++ b/functional_tests/test_broken_access_control_findings_fix.py @@ -2,7 +2,7 @@ #!/usr/bin/env python3 """ Functional test for Broken Access Control audit findings. -Version: 0.242.066 +Version: 0.242.068 Implemented in: 0.242.049 This test ensures workflow conversation IDs, SimpleChat plugin conversation @@ -327,7 +327,7 @@ def test_group_document_delete_uses_scoped_lookup_before_delete(): def test_version_bumped_for_access_control_fixes(): """Config version should identify the Broken Access Control fix release.""" print('Testing fix version...') - assert read_config_version() == '0.242.066' + assert read_config_version() == '0.242.068' print('Fix version verified.') diff --git a/functional_tests/test_broken_access_control_guardrails_checker.py b/functional_tests/test_broken_access_control_guardrails_checker.py index ce8797718..6270d2897 100644 --- a/functional_tests/test_broken_access_control_guardrails_checker.py +++ b/functional_tests/test_broken_access_control_guardrails_checker.py @@ -2,7 +2,7 @@ # test_broken_access_control_guardrails_checker.py """ Functional test for Broken Access Control PR guardrail checker. -Version: 0.242.066 +Version: 0.242.068 Implemented in: 0.241.022 This test ensures the changed-file BAC checker flags the repo's target @@ -221,7 +221,7 @@ def test_checker_assets_and_version_are_wired_into_repo() -> None: assert ROUTE_AUTH_PROMPT_FILE.exists(), f'Expected route auth audit prompt at {ROUTE_AUTH_PROMPT_FILE}' assert FEATURE_DOC.exists(), f'Expected feature document at {FEATURE_DOC}' assert FULL_SCAN_FEATURE_DOC.exists(), f'Expected full-scan feature document at {FULL_SCAN_FEATURE_DOC}' - assert read_config_version() == '0.242.066' + assert read_config_version() == '0.242.068' workflow_source = read_text(WORKFLOW_FILE) assert 'scripts/check_broken_access_control.py' in workflow_source diff --git a/functional_tests/test_governance_enforcement_logic.py b/functional_tests/test_governance_enforcement_logic.py index 9d5a169cc..bebfecbb7 100644 --- a/functional_tests/test_governance_enforcement_logic.py +++ b/functional_tests/test_governance_enforcement_logic.py @@ -2,7 +2,7 @@ #!/usr/bin/env python3 """ Functional test for governance enforcement logic. -Version: 0.242.066 +Version: 0.242.068 Implemented in: 0.241.010; updated in 0.242.022; 0.242.063; 0.242.064; 0.242.065; 0.242.066 This test ensures ensure_governance_access correctly allows and denies access diff --git a/functional_tests/test_group_manage_settings_tab_visibility.py b/functional_tests/test_group_manage_settings_tab_visibility.py index d4d90dc7d..61f295a0c 100644 --- a/functional_tests/test_group_manage_settings_tab_visibility.py +++ b/functional_tests/test_group_manage_settings_tab_visibility.py @@ -1,7 +1,7 @@ # test_group_manage_settings_tab_visibility.py """ Functional test for group manage settings tab visibility. -Version: 0.242.066 +Version: 0.242.068 Implemented in: 0.241.204 This test ensures the group manage Settings pane is unhidden for group owners @@ -18,7 +18,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] APP_ROOT = REPO_ROOT / "application" / "single_app" -CURRENT_VERSION = "0.242.066" +CURRENT_VERSION = "0.242.068" FIX_DOC = REPO_ROOT / "docs" / "explanation" / "fixes" / "GROUP_PUBLIC_WORKSPACE_DOWNLOAD_SETTINGS_VISIBILITY_FIX.md" diff --git a/functional_tests/test_msg_file_upload_support.py b/functional_tests/test_msg_file_upload_support.py index 16b3b9c2d..75ea59d81 100644 --- a/functional_tests/test_msg_file_upload_support.py +++ b/functional_tests/test_msg_file_upload_support.py @@ -2,7 +2,7 @@ #!/usr/bin/env python3 """ Functional test for Outlook MSG file upload support. -Version: 0.242.066 +Version: 0.242.068 Implemented in: 0.242.063 This test ensures Outlook .msg files are accepted for workspace and chat uploads, @@ -28,7 +28,7 @@ GROUP_ROUTE_PATH = SINGLE_APP_DIR / "route_frontend_group_workspaces.py" CHAT_TEMPLATE_PATH = SINGLE_APP_DIR / "templates" / "chats.html" FEATURE_DOC_PATH = ROOT_DIR / "docs" / "explanation" / "features" / "v0.242.063" / "MSG_FILE_INGESTION.md" -EXPECTED_CONFIG_VERSION = "0.242.066" +EXPECTED_CONFIG_VERSION = "0.242.068" EXPECTED_FEATURE_VERSION = "0.242.063" diff --git a/functional_tests/test_plugin_tool_agent_security_audit.py b/functional_tests/test_plugin_tool_agent_security_audit.py index 01709edb9..4204a25fd 100644 --- a/functional_tests/test_plugin_tool_agent_security_audit.py +++ b/functional_tests/test_plugin_tool_agent_security_audit.py @@ -2,7 +2,7 @@ #!/usr/bin/env python3 """ Functional test for plugin, tool, and agent security audit fixes. -Version: 0.242.066 +Version: 0.242.068 Implemented in: 0.242.055 This test ensures plugin invocation records and OpenAPI diagnostics redact @@ -214,7 +214,7 @@ def test_simplechat_active_group_fallback_uses_authorized_helper(): def main(): - expected_version = '0.242.066' + expected_version = '0.242.068' actual_version = read_config_version() assert actual_version == expected_version, f'Expected version {expected_version}, found {actual_version}' diff --git a/functional_tests/test_privacy_logging_telemetry_audit.py b/functional_tests/test_privacy_logging_telemetry_audit.py index 7d39cf746..32fae30a5 100644 --- a/functional_tests/test_privacy_logging_telemetry_audit.py +++ b/functional_tests/test_privacy_logging_telemetry_audit.py @@ -2,7 +2,7 @@ #!/usr/bin/env python3 """ Functional test for privacy logging and telemetry audit fixes. -Version: 0.242.066 +Version: 0.242.068 Implemented in: 0.242.058 This test ensures logging, telemetry, and document-processing diagnostics redact @@ -171,7 +171,7 @@ def test_document_processing_logs_avoid_raw_document_text(): def main(): - expected_version = '0.242.066' + expected_version = '0.242.068' actual_version = read_config_version() assert actual_version == expected_version, f'Expected version {expected_version}, found {actual_version}' diff --git a/functional_tests/test_public_workspace_manage_asset_versioning.py b/functional_tests/test_public_workspace_manage_asset_versioning.py index ce5f54e46..3e420652b 100644 --- a/functional_tests/test_public_workspace_manage_asset_versioning.py +++ b/functional_tests/test_public_workspace_manage_asset_versioning.py @@ -2,7 +2,7 @@ # test_public_workspace_manage_asset_versioning.py """ Functional test for public workspace manage asset versioning. -Version: 0.242.066 +Version: 0.242.068 Implemented in: 0.242.058 This test ensures the manage public workspace page references its management @@ -104,7 +104,7 @@ def test_fix_artifacts_are_in_sync(): print("Testing public workspace manage asset versioning artifact alignment...") version = read_config_version() - assert version == "0.242.066", f"Expected config version 0.242.066, saw {version}." + assert version == "0.242.068", f"Expected config version 0.242.068, saw {version}." assert FIX_DOC.exists(), f"Expected fix documentation at {FIX_DOC}" fix_doc_source = read_file_text(FIX_DOC) diff --git a/functional_tests/test_tabular_kernel_parameter_annotations.py b/functional_tests/test_tabular_kernel_parameter_annotations.py new file mode 100644 index 000000000..a8ef68fab --- /dev/null +++ b/functional_tests/test_tabular_kernel_parameter_annotations.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# test_tabular_kernel_parameter_annotations.py +""" +Functional test for tabular SK Python 3.13 kernel parameter annotations. +Version: 0.242.068 +Implemented in: 0.242.068 + +This test ensures public Semantic Kernel tabular tool parameters avoid +Annotated[Optional[str], ...] so tool-call argument parsing remains compatible +with both Python 3.12 and Python 3.13. +""" + +import ast +import os +import sys + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PLUGIN_FILE = os.path.join( + ROOT_DIR, + 'application', + 'single_app', + 'semantic_kernel_plugins', + 'tabular_processing_plugin.py', +) +CONFIG_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'config.py') + + +def read_text(path): + """Read a UTF-8 text file.""" + with open(path, 'r', encoding='utf-8') as file_handle: + return file_handle.read() + + +def read_config_version(): + """Read the current application version from config.py.""" + for line in read_text(CONFIG_FILE).splitlines(): + if line.strip().startswith('VERSION = '): + return line.split('=', 1)[1].strip().strip('"\'') + raise AssertionError('VERSION assignment not found in config.py') + + +def decorator_is_kernel_function(decorator): + """Return True when an AST decorator is @kernel_function(...).""" + if isinstance(decorator, ast.Call): + decorator = decorator.func + return isinstance(decorator, ast.Name) and decorator.id == 'kernel_function' + + +def annotation_is_annotated_optional_str(annotation): + """Return True for Annotated[Optional[str], ...] annotations.""" + if not isinstance(annotation, ast.Subscript): + return False + if not isinstance(annotation.value, ast.Name) or annotation.value.id != 'Annotated': + return False + + annotation_slice = annotation.slice + if isinstance(annotation_slice, ast.Tuple): + first_argument = annotation_slice.elts[0] + else: + first_argument = annotation_slice + + if not isinstance(first_argument, ast.Subscript): + return False + if not isinstance(first_argument.value, ast.Name) or first_argument.value.id != 'Optional': + return False + + optional_slice = first_argument.slice + return isinstance(optional_slice, ast.Name) and optional_slice.id == 'str' + + +def test_kernel_function_parameters_do_not_use_optional_str_annotations(): + """Validate public SK tool parameters use concrete str annotations.""" + print('๐Ÿ” Testing tabular kernel parameter annotations...') + + try: + parsed = ast.parse(read_text(PLUGIN_FILE), filename=PLUGIN_FILE) + violations = [] + + for node in ast.walk(parsed): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + if not any(decorator_is_kernel_function(decorator) for decorator in node.decorator_list): + continue + + for argument in node.args.args: + if argument.arg == 'self': + continue + if annotation_is_annotated_optional_str(argument.annotation): + violations.append(f'{node.name}.{argument.arg}') + + assert not violations, f'Kernel function parameters cannot use Annotated[Optional[str], ...]: {violations}' + assert read_config_version() == '0.242.068' + + print('โœ… Tabular kernel parameter annotations passed') + return True + except Exception as exc: + print(f'โŒ Test failed: {exc}') + import traceback + traceback.print_exc() + return False + + +if __name__ == '__main__': + tests = [ + test_kernel_function_parameters_do_not_use_optional_str_annotations, + ] + + results = [] + for test in tests: + print(f'\n๐Ÿงช Running {test.__name__}...') + results.append(test()) + + success = all(results) + print(f'\n๐Ÿ“Š Results: {sum(results)}/{len(results)} tests passed') + sys.exit(0 if success else 1) diff --git a/functional_tests/test_tabular_large_result_handoff.py b/functional_tests/test_tabular_large_result_handoff.py new file mode 100644 index 000000000..f7592cecd --- /dev/null +++ b/functional_tests/test_tabular_large_result_handoff.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# test_tabular_large_result_handoff.py +""" +Functional test for tabular computed-results handoff size handling. +Version: 0.242.068 +Implemented in: 0.242.067 + +This test ensures the tabular SK analysis handoff preserves computed results +above the previous 24K limit while still truncating pathological payloads at +the 100K handoff guardrail. +""" + +import ast +import os +import sys + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ROUTE_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'route_backend_chats.py') + +TARGET_ASSIGNMENTS = { + 'TABULAR_COMPUTED_RESULTS_HANDOFF_MAX_CHARS', +} +TARGET_FUNCTIONS = { + 'build_tabular_computed_results_system_message', +} + + +def load_handoff_helper(): + """Load the handoff helper and constants without importing the full route module.""" + with open(ROUTE_FILE, 'r', encoding='utf-8') as file_handle: + route_content = file_handle.read() + + parsed = ast.parse(route_content, filename=ROUTE_FILE) + selected_nodes = [] + for node in parsed.body: + if isinstance(node, ast.Assign): + target_names = [target.id for target in node.targets if isinstance(target, ast.Name)] + if any(target_name in TARGET_ASSIGNMENTS for target_name in target_names): + selected_nodes.append(node) + elif isinstance(node, ast.FunctionDef) and node.name in TARGET_FUNCTIONS: + selected_nodes.append(node) + + namespace = { + 'log_event': lambda *args, **kwargs: None, + 'logging': type('LoggingStub', (), {'WARNING': 'WARNING'}), + } + module = ast.Module(body=selected_nodes, type_ignores=[]) + exec(compile(module, ROUTE_FILE, 'exec'), namespace) + return namespace + + +def test_tabular_handoff_preserves_results_above_previous_limit(): + """Verify analysis larger than 24K but under 100K is not truncated.""" + print('๐Ÿ” Testing tabular handoff above previous limit...') + + try: + helpers = load_handoff_helper() + build_handoff = helpers['build_tabular_computed_results_system_message'] + analysis = 'A' * 50000 + message = build_handoff('test workbook', analysis) + + assert 'A' * 50000 in message, 'Expected the 50K analysis payload to remain intact.' + assert '[Computed results handoff truncated for prompt budget.]' not in message, message[-200:] + print('โœ… Tabular handoff above previous limit passed') + return True + except Exception as exc: + print(f'โŒ Test failed: {exc}') + import traceback + traceback.print_exc() + return False + + +def test_tabular_handoff_truncates_at_100k_guardrail(): + """Verify pathological computed-result payloads still get bounded.""" + print('๐Ÿ” Testing tabular handoff 100K guardrail...') + + try: + helpers = load_handoff_helper() + build_handoff = helpers['build_tabular_computed_results_system_message'] + limit = helpers['TABULAR_COMPUTED_RESULTS_HANDOFF_MAX_CHARS'] + message = build_handoff('test workbook', 'B' * (limit + 5000)) + + assert limit == 100000, limit + assert '[Computed results handoff truncated for prompt budget.]' in message, message[-200:] + assert 'B' * 1000 in message, 'Expected truncated payload prefix to remain present.' + assert len(message) < limit + 1000, len(message) + print('โœ… Tabular handoff 100K guardrail passed') + return True + except Exception as exc: + print(f'โŒ Test failed: {exc}') + import traceback + traceback.print_exc() + return False + + +if __name__ == '__main__': + tests = [ + test_tabular_handoff_preserves_results_above_previous_limit, + test_tabular_handoff_truncates_at_100k_guardrail, + ] + + results = [] + for test in tests: + print(f'\n๐Ÿงช Running {test.__name__}...') + results.append(test()) + + success = all(results) + print(f'\n๐Ÿ“Š Results: {sum(results)}/{len(results)} tests passed') + sys.exit(0 if success else 1) \ No newline at end of file diff --git a/functional_tests/test_tabular_large_result_pagination.py b/functional_tests/test_tabular_large_result_pagination.py new file mode 100644 index 000000000..3770f33a9 --- /dev/null +++ b/functional_tests/test_tabular_large_result_pagination.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +# test_tabular_large_result_pagination.py +""" +Functional test for tabular SK large-result pagination and output trimming. +Version: 0.242.068 +Implemented in: 0.242.067 + +This test ensures row-returning tabular processing tools support start_row/max_rows +pagination, avoid skipped rows after auto-trimming oversized output, honor +return_columns projection, and preserve hidden attachment references used by +row-linked document evidence enrichment. +""" + +import asyncio +import importlib.util +import json +import os +import sys + +import pandas as pd + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(ROOT_DIR) +sys.path.append(os.path.join(ROOT_DIR, 'application', 'single_app')) + +PLUGIN_FILE = os.path.join( + ROOT_DIR, + 'application', + 'single_app', + 'semantic_kernel_plugins', + 'tabular_processing_plugin.py', +) + +PLUGIN_SPEC = importlib.util.spec_from_file_location('tabular_processing_plugin', PLUGIN_FILE) +PLUGIN_MODULE = importlib.util.module_from_spec(PLUGIN_SPEC) +PLUGIN_SPEC.loader.exec_module(PLUGIN_MODULE) +TabularProcessingPlugin = PLUGIN_MODULE.TabularProcessingPlugin + + +def build_workbook_plugin(workbook_frames): + """Create a TabularProcessingPlugin backed by in-memory workbook frames.""" + plugin = TabularProcessingPlugin() + container_name = 'mock-container' + blob_name = 'large-results.xlsx' + sheet_names = list(workbook_frames.keys()) + workbook_metadata = { + 'is_workbook': True, + 'sheet_names': sheet_names, + 'sheet_count': len(sheet_names), + 'default_sheet': sheet_names[0], + } + + plugin._resolve_blob_location_with_fallback = lambda *args, **kwargs: (container_name, blob_name) + plugin._get_workbook_metadata = lambda *args, **kwargs: workbook_metadata.copy() + + def read_dataframe(container, blob, sheet_name=None, sheet_index=None, require_explicit_sheet=False): + selected_sheet, _ = plugin._resolve_sheet_selection( + container, + blob, + sheet_name=sheet_name, + sheet_index=sheet_index, + require_explicit_sheet=require_explicit_sheet, + ) + return workbook_frames[selected_sheet].copy() + + plugin._read_tabular_blob_to_dataframe = read_dataframe + return plugin + + +def test_filter_rows_paginates_without_skipping_after_row_trim(): + """Verify oversized one-column pages advance by returned rows, not requested rows.""" + print('๐Ÿ” Testing row-trim pagination cursor...') + + try: + long_text = 'match ' + ('x' * 25000) + plugin = build_workbook_plugin({ + 'Data': pd.DataFrame([ + {'Notes': f'{long_text} {row_index}'} + for row_index in range(8) + ]), + }) + + payload = json.loads(asyncio.run(plugin.filter_rows( + user_id='test-user', + conversation_id='test-conversation', + filename='large-results.xlsx', + sheet_name='Data', + column='Notes', + operator='contains', + value='match', + source='workspace', + max_rows='8', + ))) + + assert payload['total_matches'] == 8, payload + assert payload['output_trimmed'] is True, payload + assert payload['returned_rows'] < payload['page_size'], payload + assert payload['has_more'] is True, payload + assert payload['next_start_row'] == payload['returned_rows'], payload + + print('โœ… Row-trim pagination cursor passed') + return True + except Exception as exc: + print(f'โŒ Test failed: {exc}') + import traceback + traceback.print_exc() + return False + + +def test_filter_rows_auto_excludes_heavy_columns_and_return_columns_skips_trim(): + """Verify heavy columns are excluded unless the caller explicitly projects columns.""" + print('๐Ÿ” Testing auto-trim and return_columns projection...') + + try: + plugin = build_workbook_plugin({ + 'Data': pd.DataFrame([ + { + 'ID': row_index, + 'Status': 'Open', + 'LargeNarrative': 'details ' + ('z' * 9000), + } + for row_index in range(12) + ]), + }) + + trimmed_payload = json.loads(asyncio.run(plugin.filter_rows( + user_id='test-user', + conversation_id='test-conversation', + filename='large-results.xlsx', + sheet_name='Data', + column='Status', + operator='equals', + value='Open', + source='workspace', + max_rows='10', + ))) + + assert trimmed_payload['total_matches'] == 12, trimmed_payload + assert trimmed_payload['returned_rows'] == 10, trimmed_payload + assert trimmed_payload['has_more'] is True, trimmed_payload + assert trimmed_payload['next_start_row'] == 10, trimmed_payload + assert 'LargeNarrative' in trimmed_payload['auto_excluded_columns'], trimmed_payload + assert 'LargeNarrative' not in trimmed_payload['data'][0], trimmed_payload + + projected_payload = json.loads(asyncio.run(plugin.filter_rows( + user_id='test-user', + conversation_id='test-conversation', + filename='large-results.xlsx', + sheet_name='Data', + column='Status', + operator='equals', + value='Open', + source='workspace', + return_columns='ID,Status', + max_rows='5', + ))) + + assert projected_payload['return_columns'] == ['ID', 'Status'], projected_payload + assert 'auto_excluded_columns' not in projected_payload, projected_payload + assert projected_payload['data'][0] == {'ID': 0, 'Status': 'Open'}, projected_payload + assert projected_payload['has_more'] is True, projected_payload + assert projected_payload['next_start_row'] == 5, projected_payload + + print('โœ… Auto-trim and return_columns projection passed') + return True + except Exception as exc: + print(f'โŒ Test failed: {exc}') + import traceback + traceback.print_exc() + return False + + +def test_cross_sheet_filter_rows_paginates_across_sheet_boundary(): + """Verify cross-sheet pagination continues without losing boundary rows.""" + print('๐Ÿ” Testing cross-sheet pagination...') + + try: + plugin = build_workbook_plugin({ + 'SheetA': pd.DataFrame([ + {'ID': f'A-{row_index}', 'Status': 'Open'} + for row_index in range(3) + ]), + 'SheetB': pd.DataFrame([ + {'ID': f'B-{row_index}', 'Status': 'Open'} + for row_index in range(3) + ]), + }) + + first_page = json.loads(asyncio.run(plugin.filter_rows( + user_id='test-user', + conversation_id='test-conversation', + filename='large-results.xlsx', + column='Status', + operator='equals', + value='Open', + source='workspace', + max_rows='4', + ))) + + second_page = json.loads(asyncio.run(plugin.filter_rows( + user_id='test-user', + conversation_id='test-conversation', + filename='large-results.xlsx', + column='Status', + operator='equals', + value='Open', + source='workspace', + start_row=str(first_page['next_start_row']), + max_rows='4', + ))) + + assert first_page['selected_sheet'] == 'ALL (cross-sheet search)', first_page + assert first_page['total_matches'] == 6, first_page + assert [row['ID'] for row in first_page['data']] == ['A-0', 'A-1', 'A-2', 'B-0'], first_page + assert first_page['next_start_row'] == 4, first_page + assert [row['ID'] for row in second_page['data']] == ['B-1', 'B-2'], second_page + assert second_page['has_more'] is False, second_page + + print('โœ… Cross-sheet pagination passed') + return True + except Exception as exc: + print(f'โŒ Test failed: {exc}') + import traceback + traceback.print_exc() + return False + + +def test_search_rows_preserves_attachment_references_with_return_columns(): + """Verify projected search results keep hidden attachment columns for enrichment.""" + print('๐Ÿ” Testing attachment reference preservation with projection...') + + try: + plugin = build_workbook_plugin({ + 'Data': pd.DataFrame([ + { + 'Summary': 'urgent review needed', + 'AttachmentFile': 'case-notes.pdf', + 'Owner': 'Analyst', + }, + ]), + }) + + payload = json.loads(asyncio.run(plugin.search_rows( + user_id='test-user', + conversation_id='test-conversation', + filename='large-results.xlsx', + sheet_name='Data', + search_value='urgent', + return_columns='Summary', + source='workspace', + max_rows='5', + ))) + + assert payload['returned_rows'] == 1, payload + row = payload['data'][0] + assert row['Summary'] == 'urgent review needed', payload + assert 'AttachmentFile' not in row, payload + assert row['_related_document_reference_values']['AttachmentFile'] == 'case-notes.pdf', payload + assert row['_matched_columns'] == ['Summary'], payload + + print('โœ… Attachment reference preservation passed') + return True + except Exception as exc: + print(f'โŒ Test failed: {exc}') + import traceback + traceback.print_exc() + return False + + +def test_query_tabular_data_supports_return_columns_and_pagination(): + """Verify query results support explicit projection and continuation metadata.""" + print('๐Ÿ” Testing query pagination with return_columns...') + + try: + plugin = build_workbook_plugin({ + 'Data': pd.DataFrame([ + {'ID': row_index, 'Status': 'Open', 'Payload': 'large ' + ('q' * 1000)} + for row_index in range(6) + ]), + }) + + payload = json.loads(asyncio.run(plugin.query_tabular_data( + user_id='test-user', + conversation_id='test-conversation', + filename='large-results.xlsx', + sheet_name='Data', + query_expression='Status == "Open"', + return_columns='ID,Status', + start_row='2', + max_rows='3', + source='workspace', + ))) + + assert payload['total_matches'] == 6, payload + assert payload['start_row'] == 2, payload + assert payload['returned_rows'] == 3, payload + assert payload['has_more'] is True, payload + assert payload['next_start_row'] == 5, payload + assert payload['return_columns'] == ['ID', 'Status'], payload + assert [row['ID'] for row in payload['data']] == [2, 3, 4], payload + assert 'Payload' not in payload['data'][0], payload + + print('โœ… Query pagination with return_columns passed') + return True + except Exception as exc: + print(f'โŒ Test failed: {exc}') + import traceback + traceback.print_exc() + return False + + +if __name__ == '__main__': + tests = [ + test_filter_rows_paginates_without_skipping_after_row_trim, + test_filter_rows_auto_excludes_heavy_columns_and_return_columns_skips_trim, + test_cross_sheet_filter_rows_paginates_across_sheet_boundary, + test_search_rows_preserves_attachment_references_with_return_columns, + test_query_tabular_data_supports_return_columns_and_pagination, + ] + + results = [] + for test in tests: + print(f'\n๐Ÿงช Running {test.__name__}...') + results.append(test()) + + success = all(results) + print(f'\n๐Ÿ“Š Results: {sum(results)}/{len(results)} tests passed') + sys.exit(0 if success else 1) \ No newline at end of file diff --git a/functional_tests/test_tabular_related_document_evidence.py b/functional_tests/test_tabular_related_document_evidence.py index 01d693265..4880b165e 100644 --- a/functional_tests/test_tabular_related_document_evidence.py +++ b/functional_tests/test_tabular_related_document_evidence.py @@ -2,8 +2,8 @@ # test_tabular_related_document_evidence.py """ Functional test for tabular related-document evidence. -Version: 0.241.141 -Implemented in: 0.241.140 +Version: 0.242.068 +Implemented in: 0.241.140; updated in 0.242.067 This test ensures that tabular rows can resolve explicit references to related workspace documents, summarize that evidence for prompt handoff, and preserve @@ -28,8 +28,10 @@ 'TABULAR_RELATED_DOCUMENT_MAX_MATCHES_PER_ROW', 'TABULAR_RELATED_DOCUMENT_MAX_SUMMARY_ROWS', 'TABULAR_RELATED_DOCUMENT_MAX_EXCERPT_CHARS', + 'TABULAR_COMPUTED_RESULTS_HANDOFF_MAX_CHARS', } TARGET_FUNCTIONS = { + '_normalize_requested_scope_ids', '_normalize_tabular_related_document_text', '_normalize_tabular_related_document_basename', '_is_tabular_related_document_candidate', diff --git a/functional_tests/test_xss_guardrails_checker.py b/functional_tests/test_xss_guardrails_checker.py index 78ffb4bc2..8158dbe2a 100644 --- a/functional_tests/test_xss_guardrails_checker.py +++ b/functional_tests/test_xss_guardrails_checker.py @@ -2,7 +2,7 @@ # test_xss_guardrails_checker.py """ Functional test for XSS PR guardrail checker. -Version: 0.242.066 +Version: 0.242.068 Implemented in: 0.241.021 This test ensures the changed-file XSS checker flags the repo's target sink @@ -128,7 +128,7 @@ def test_checker_assets_and_version_are_wired_into_repo() -> None: assert INSTRUCTION_FILE.exists(), f'Expected instruction file at {INSTRUCTION_FILE}' assert FULL_AUDIT_PROMPT_FILE.exists(), f'Expected full-audit prompt at {FULL_AUDIT_PROMPT_FILE}' assert FEATURE_DOC.exists(), f'Expected feature document at {FEATURE_DOC}' - assert read_config_version() == '0.242.066' + assert read_config_version() == '0.242.068' workflow_source = read_text(WORKFLOW_FILE) assert 'scripts/check_xss_sinks.py' in workflow_source