fix: pin tree-sitter below 0.26.0 to fix segfault

pyproject.toml

@@ -12,7 +12,7 @@ dependencies = [
  "numpy",
  "psutil",
  "httpx",
- "tree-sitter!=0.25.0",
+ "tree-sitter>=0.25.1,<0.26.0",
  "tree-sitter-language-pack",
  "pygments",
  "transformers>=4.36.0,!=4.51.0,!=4.51.1,!=4.51.2",

src/vectorcode/chunking.py

@@ -1,17 +1,18 @@
 import logging
 import os
 import re
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from functools import cache
 from io import TextIOWrapper
 from typing import Generator, Optional, cast
 
 from pygments.lexer import Lexer
 from pygments.lexers import get_lexer_for_filename
 from pygments.util import ClassNotFound
 from tree_sitter import Node, Point
 from tree_sitter_language_pack import SupportedLanguage, get_parser
+from tree_sitter_language_pack import Error as TreeSitterLanguagePackError
 
 from vectorcode.cli_utils import Config
 
@@ -63,7 +64,7 @@
 
 
 class ChunkerBase(ABC): # pragma: nocover
  def __init__(self, config: Optional[Config] = None) -> None:
  if config is None:
  config = Config()
  assert 0 <= config.overlap_ratio < 1, (
@@ -73,18 +74,18 @@
 
  @abstractmethod
  def chunk(
  self, data, opts: Optional[ChunkOpts] = None
  ) -> Generator[Chunk, None, None]:
  raise NotImplementedError
 
 
 class StringChunker(ChunkerBase):
  def __init__(self, config: Optional[Config] = None) -> None:
  if config is None:
  config = Config()
  super().__init__(config)
 
  def chunk(self, data: str, opts: Optional[ChunkOpts] = None):
  start_pos = Point(row=1, column=0)
  if opts is not None:
  start_pos = opts.start_pos
@@ -134,13 +135,13 @@
 
 
 class FileChunker(ChunkerBase):
  def __init__(self, config: Optional[Config] = None) -> None:
  if config is None:
  config = Config()
  super().__init__(config)
 
  def chunk(
  self, data: TextIOWrapper, opts: Optional[ChunkOpts] = None
  ) -> Generator[Chunk, None, None]:
  logger.info("Started chunking %s using FileChunker.", data.name)
  lines = data.readlines()
@@ -194,7 +195,7 @@
 
 
 class TreeSitterChunker(ChunkerBase):
  def __init__(self, config: Optional[Config] = None):
  if config is None:
  config = Config()
  super().__init__(config)
@@ -370,11 +371,10 @@
  f"\nInvalid regex pattern '{pattern}' for language '{language}' in filetype_map"
  )
  raise
- except LookupError as e:
- e.add_note(
- f"\nTreeSitter Parser for language '{language}' not found. Please check your filetype_map config."
- )
- raise
+ except (LookupError, TreeSitterLanguagePackError) as e:
+ raise LookupError(
+ f"TreeSitter Parser for language '{language}' not found. Please check your filetype_map config."
+ ) from e
 
  logger.debug(f"No matching filetype map entry found for {filename}.")
  return None
@@ -412,7 +412,7 @@
  language,
  )
  break
- except LookupError: # pragma: nocover
+ except (LookupError, TreeSitterLanguagePackError): # pragma: nocover
  pass
 
  if parser is None:

tests/test_cli_utils.py

@@ -210,9 +210,7 @@ async def test_load_config_file_invalid_json():
 @pytest.mark.asyncio
 async def test_load_from_default_config():
  for name in ("config.json5", "config.json"):
- with (
- tempfile.TemporaryDirectory() as fake_home,
- ):
+ with (tempfile.TemporaryDirectory() as fake_home,):
  os.environ.update({"HOME": fake_home})
  config_path = os.path.join(fake_home, ".config", "vectorcode", name)
  config_dir = os.path.join(fake_home, ".config", "vectorcode")
@@ -562,14 +560,23 @@ def test_cleanup_path():
 
 def test_shtab():
  for shell in ("bash", "zsh", "tcsh"):
- assert (
- subprocess.Popen(
- [sys.executable, "-m", "vectorcode.main", "-s", shell],
- stderr=subprocess.PIPE,
- )
- .stderr.read()
- .decode()
- ) == ""
+ result = subprocess.Popen(
+ [sys.executable, "-m", "vectorcode.main", "-s", shell],
+ stderr=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ )
+ assert result.stderr is not None
+
+ stderr_output = result.stderr.read().decode()
+
+ # Filter out ONNX Runtime warnings which are not test failures
+ filtered_stderr = "\n".join(
+ line
+ for line in stderr_output.split("\n")
+ if "onnxruntime" not in line.lower() and line.strip()
+ )
+
+ assert filtered_stderr == ""
 
 
 @pytest.mark.asyncio