mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-02-28 09:23:08 +08:00
Add comprehensive tests for ast-grep and tree-sitter relationship extraction
- Introduced test suite for AstGrepPythonProcessor covering pattern definitions, parsing, and relationship extraction. - Added comparison tests between tree-sitter and ast-grep for consistency in relationship extraction. - Implemented tests for ast-grep binding module to verify functionality and availability. - Ensured tests cover various scenarios including inheritance, function calls, and imports.
This commit is contained in:
1
codex-lens/tests/parsers/__init__.py
Normal file
1
codex-lens/tests/parsers/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Tests for codexlens.parsers modules."""
|
||||
444
codex-lens/tests/parsers/test_astgrep_extraction.py
Normal file
444
codex-lens/tests/parsers/test_astgrep_extraction.py
Normal file
@@ -0,0 +1,444 @@
|
||||
"""Tests for dedicated extraction methods: extract_inherits, extract_calls, extract_imports.
|
||||
|
||||
Tests pattern-based relationship extraction from Python source code
|
||||
using ast-grep-py bindings for INHERITS, CALL, and IMPORTS relationships.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from codexlens.parsers.astgrep_processor import (
|
||||
AstGrepPythonProcessor,
|
||||
is_astgrep_processor_available,
|
||||
)
|
||||
from codexlens.entities import RelationshipType
|
||||
|
||||
|
||||
# Check if ast-grep is available for conditional test skipping
|
||||
ASTGREP_AVAILABLE = is_astgrep_processor_available()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not ASTGREP_AVAILABLE, reason="ast-grep-py not installed")
|
||||
class TestExtractInherits:
|
||||
"""Tests for extract_inherits method - INHERITS relationship extraction."""
|
||||
|
||||
def test_single_inheritance(self):
|
||||
"""Test extraction of single inheritance relationship."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
class Animal:
|
||||
pass
|
||||
|
||||
class Dog(Animal):
|
||||
pass
|
||||
"""
|
||||
relationships = processor.extract_inherits(code, "test.py")
|
||||
|
||||
assert len(relationships) == 1
|
||||
rel = relationships[0]
|
||||
assert rel.source_symbol == "Dog"
|
||||
assert rel.target_symbol == "Animal"
|
||||
assert rel.relationship_type == RelationshipType.INHERITS
|
||||
|
||||
def test_multiple_inheritance(self):
|
||||
"""Test extraction of multiple inheritance relationships."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
class A:
|
||||
pass
|
||||
|
||||
class B:
|
||||
pass
|
||||
|
||||
class C(A, B):
|
||||
pass
|
||||
"""
|
||||
relationships = processor.extract_inherits(code, "test.py")
|
||||
|
||||
# Should have 2 relationships: C->A and C->B
|
||||
assert len(relationships) == 2
|
||||
targets = {r.target_symbol for r in relationships}
|
||||
assert "A" in targets
|
||||
assert "B" in targets
|
||||
for rel in relationships:
|
||||
assert rel.source_symbol == "C"
|
||||
|
||||
def test_no_inheritance(self):
|
||||
"""Test that classes without inheritance return empty list."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
class Standalone:
|
||||
pass
|
||||
"""
|
||||
relationships = processor.extract_inherits(code, "test.py")
|
||||
|
||||
assert len(relationships) == 0
|
||||
|
||||
def test_nested_class_inheritance(self):
|
||||
"""Test extraction of inheritance in nested classes."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
class Outer:
|
||||
class Inner(Base):
|
||||
pass
|
||||
"""
|
||||
relationships = processor.extract_inherits(code, "test.py")
|
||||
|
||||
assert len(relationships) == 1
|
||||
assert relationships[0].source_symbol == "Inner"
|
||||
assert relationships[0].target_symbol == "Base"
|
||||
|
||||
def test_inheritance_with_complex_bases(self):
|
||||
"""Test extraction with generic or complex base classes."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
class Service(BaseService, mixins.Loggable):
|
||||
pass
|
||||
"""
|
||||
relationships = processor.extract_inherits(code, "test.py")
|
||||
|
||||
assert len(relationships) == 2
|
||||
targets = {r.target_symbol for r in relationships}
|
||||
assert "BaseService" in targets
|
||||
assert "mixins.Loggable" in targets
|
||||
|
||||
|
||||
@pytest.mark.skipif(not ASTGREP_AVAILABLE, reason="ast-grep-py not installed")
|
||||
class TestExtractCalls:
|
||||
"""Tests for extract_calls method - CALL relationship extraction."""
|
||||
|
||||
def test_simple_function_call(self):
|
||||
"""Test extraction of simple function calls."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
def main():
|
||||
print("hello")
|
||||
len([1, 2, 3])
|
||||
"""
|
||||
relationships = processor.extract_calls(code, "test.py", "main")
|
||||
|
||||
targets = {r.target_symbol for r in relationships}
|
||||
assert "print" in targets
|
||||
assert "len" in targets
|
||||
|
||||
def test_method_call(self):
|
||||
"""Test extraction of method calls."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
def process():
|
||||
obj.method()
|
||||
items.append(1)
|
||||
"""
|
||||
relationships = processor.extract_calls(code, "test.py", "process")
|
||||
|
||||
targets = {r.target_symbol for r in relationships}
|
||||
assert "obj.method" in targets
|
||||
assert "items.append" in targets
|
||||
|
||||
def test_skips_self_calls(self):
|
||||
"""Test that self.method() calls are filtered."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
class Service:
|
||||
def process(self):
|
||||
self.internal()
|
||||
external_func()
|
||||
"""
|
||||
relationships = processor.extract_calls(code, "test.py", "Service")
|
||||
|
||||
targets = {r.target_symbol for r in relationships}
|
||||
# self.internal should be filtered
|
||||
assert "self.internal" not in targets
|
||||
assert "internal" not in targets
|
||||
assert "external_func" in targets
|
||||
|
||||
def test_skips_cls_calls(self):
|
||||
"""Test that cls.method() calls are filtered."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
class Factory:
|
||||
@classmethod
|
||||
def create(cls):
|
||||
cls.helper()
|
||||
other_func()
|
||||
"""
|
||||
relationships = processor.extract_calls(code, "test.py", "Factory")
|
||||
|
||||
targets = {r.target_symbol for r in relationships}
|
||||
assert "cls.helper" not in targets
|
||||
assert "other_func" in targets
|
||||
|
||||
def test_alias_resolution(self):
|
||||
"""Test call alias resolution using import map."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
def main():
|
||||
np.array([1, 2, 3])
|
||||
"""
|
||||
alias_map = {"np": "numpy"}
|
||||
relationships = processor.extract_calls(code, "test.py", "main", alias_map)
|
||||
|
||||
assert len(relationships) >= 1
|
||||
# Should resolve np.array to numpy.array
|
||||
assert any("numpy.array" in r.target_symbol for r in relationships)
|
||||
|
||||
def test_no_calls(self):
|
||||
"""Test that code without calls returns empty list."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
x = 1
|
||||
y = x + 2
|
||||
"""
|
||||
relationships = processor.extract_calls(code, "test.py")
|
||||
|
||||
assert len(relationships) == 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(not ASTGREP_AVAILABLE, reason="ast-grep-py not installed")
|
||||
class TestExtractImports:
|
||||
"""Tests for extract_imports method - IMPORTS relationship extraction."""
|
||||
|
||||
def test_simple_import(self):
|
||||
"""Test extraction of simple import statements."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = "import os"
|
||||
|
||||
relationships, alias_map = processor.extract_imports(code, "test.py")
|
||||
|
||||
assert len(relationships) == 1
|
||||
assert relationships[0].target_symbol == "os"
|
||||
assert relationships[0].relationship_type == RelationshipType.IMPORTS
|
||||
assert alias_map.get("os") == "os"
|
||||
|
||||
def test_import_with_alias(self):
|
||||
"""Test extraction of import with alias."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = "import numpy as np"
|
||||
|
||||
relationships, alias_map = processor.extract_imports(code, "test.py")
|
||||
|
||||
assert len(relationships) == 1
|
||||
assert relationships[0].target_symbol == "numpy"
|
||||
assert alias_map.get("np") == "numpy"
|
||||
|
||||
def test_from_import(self):
|
||||
"""Test extraction of from-import statements."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = "from typing import List, Dict"
|
||||
|
||||
relationships, alias_map = processor.extract_imports(code, "test.py")
|
||||
|
||||
assert len(relationships) == 1
|
||||
assert relationships[0].target_symbol == "typing"
|
||||
assert alias_map.get("List") == "typing.List"
|
||||
assert alias_map.get("Dict") == "typing.Dict"
|
||||
|
||||
def test_from_import_with_alias(self):
|
||||
"""Test extraction of from-import with alias."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = "from collections import defaultdict as dd"
|
||||
|
||||
relationships, alias_map = processor.extract_imports(code, "test.py")
|
||||
|
||||
assert len(relationships) == 1
|
||||
# The alias map should map dd to collections.defaultcount
|
||||
assert "dd" in alias_map
|
||||
assert "defaultdict" in alias_map.get("dd", "")
|
||||
|
||||
def test_star_import(self):
|
||||
"""Test extraction of star imports."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = "from module import *"
|
||||
|
||||
relationships, alias_map = processor.extract_imports(code, "test.py")
|
||||
|
||||
assert len(relationships) >= 1
|
||||
# Star import should be recorded
|
||||
star_imports = [r for r in relationships if "*" in r.target_symbol]
|
||||
assert len(star_imports) >= 1
|
||||
|
||||
def test_relative_import(self):
|
||||
"""Test extraction of relative imports."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = "from .utils import helper"
|
||||
|
||||
relationships, alias_map = processor.extract_imports(code, "test.py")
|
||||
|
||||
# Should capture the relative import
|
||||
assert len(relationships) >= 1
|
||||
rel_imports = [r for r in relationships if r.target_symbol.startswith(".")]
|
||||
assert len(rel_imports) >= 1
|
||||
|
||||
def test_multiple_imports(self):
|
||||
"""Test extraction of multiple import types."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
import os
|
||||
import sys
|
||||
from typing import List
|
||||
from collections import defaultdict as dd
|
||||
"""
|
||||
|
||||
relationships, alias_map = processor.extract_imports(code, "test.py")
|
||||
|
||||
assert len(relationships) >= 4
|
||||
targets = {r.target_symbol for r in relationships}
|
||||
assert "os" in targets
|
||||
assert "sys" in targets
|
||||
assert "typing" in targets
|
||||
assert "collections" in targets
|
||||
|
||||
def test_no_imports(self):
|
||||
"""Test that code without imports returns empty list."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
x = 1
|
||||
def foo():
|
||||
pass
|
||||
"""
|
||||
relationships, alias_map = processor.extract_imports(code, "test.py")
|
||||
|
||||
assert len(relationships) == 0
|
||||
assert len(alias_map) == 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(not ASTGREP_AVAILABLE, reason="ast-grep-py not installed")
|
||||
class TestExtractMethodsIntegration:
|
||||
"""Integration tests combining multiple extraction methods."""
|
||||
|
||||
def test_full_file_extraction(self):
|
||||
"""Test extracting all relationships from a complete file."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
import os
|
||||
from typing import List, Optional
|
||||
|
||||
class Base:
|
||||
pass
|
||||
|
||||
class Service(Base):
|
||||
def __init__(self):
|
||||
self.data = []
|
||||
|
||||
def process(self):
|
||||
result = os.path.join("a", "b")
|
||||
items = List([1, 2, 3])
|
||||
return result
|
||||
|
||||
def main():
|
||||
svc = Service()
|
||||
svc.process()
|
||||
"""
|
||||
source_file = "test.py"
|
||||
|
||||
# Extract all relationship types
|
||||
imports, alias_map = processor.extract_imports(code, source_file)
|
||||
inherits = processor.extract_inherits(code, source_file)
|
||||
calls = processor.extract_calls(code, source_file, alias_map=alias_map)
|
||||
|
||||
# Verify we got all expected relationships
|
||||
assert len(imports) >= 2 # os and typing
|
||||
assert len(inherits) == 1 # Service -> Base
|
||||
assert len(calls) >= 2 # os.path.join and others
|
||||
|
||||
# Verify inheritance
|
||||
assert any(r.source_symbol == "Service" and r.target_symbol == "Base"
|
||||
for r in inherits)
|
||||
|
||||
def test_alias_propagation(self):
|
||||
"""Test that import aliases propagate to call resolution."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
import numpy as np
|
||||
|
||||
def compute():
|
||||
arr = np.array([1, 2, 3])
|
||||
return np.sum(arr)
|
||||
"""
|
||||
source_file = "test.py"
|
||||
|
||||
imports, alias_map = processor.extract_imports(code, source_file)
|
||||
calls = processor.extract_calls(code, source_file, alias_map=alias_map)
|
||||
|
||||
# Alias map should have np -> numpy
|
||||
assert alias_map.get("np") == "numpy"
|
||||
|
||||
# Calls should resolve np.array and np.sum
|
||||
resolved_targets = {r.target_symbol for r in calls}
|
||||
# At minimum, np.array and np.sum should be captured
|
||||
np_calls = [t for t in resolved_targets if "np" in t or "numpy" in t]
|
||||
assert len(np_calls) >= 2
|
||||
|
||||
|
||||
class TestExtractMethodFallback:
|
||||
"""Tests for fallback behavior when ast-grep unavailable."""
|
||||
|
||||
def test_extract_inherits_empty_when_unavailable(self):
|
||||
"""Test extract_inherits returns empty list when unavailable."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
if not processor.is_available():
|
||||
code = "class Dog(Animal): pass"
|
||||
relationships = processor.extract_inherits(code, "test.py")
|
||||
assert relationships == []
|
||||
|
||||
def test_extract_calls_empty_when_unavailable(self):
|
||||
"""Test extract_calls returns empty list when unavailable."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
if not processor.is_available():
|
||||
code = "print('hello')"
|
||||
relationships = processor.extract_calls(code, "test.py")
|
||||
assert relationships == []
|
||||
|
||||
def test_extract_imports_empty_when_unavailable(self):
|
||||
"""Test extract_imports returns empty tuple when unavailable."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
if not processor.is_available():
|
||||
code = "import os"
|
||||
relationships, alias_map = processor.extract_imports(code, "test.py")
|
||||
assert relationships == []
|
||||
assert alias_map == {}
|
||||
|
||||
|
||||
class TestHelperMethods:
|
||||
"""Tests for internal helper methods."""
|
||||
|
||||
def test_parse_base_classes_single(self):
|
||||
"""Test _parse_base_classes with single base."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
result = processor._parse_base_classes("BaseClass")
|
||||
assert result == ["BaseClass"]
|
||||
|
||||
def test_parse_base_classes_multiple(self):
|
||||
"""Test _parse_base_classes with multiple bases."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
result = processor._parse_base_classes("A, B, C")
|
||||
assert result == ["A", "B", "C"]
|
||||
|
||||
def test_parse_base_classes_with_generics(self):
|
||||
"""Test _parse_base_classes with generic types."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
result = processor._parse_base_classes("Generic[T], Mixin")
|
||||
assert "Generic[T]" in result
|
||||
assert "Mixin" in result
|
||||
|
||||
def test_resolve_call_alias_simple(self):
|
||||
"""Test _resolve_call_alias with simple name."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
alias_map = {"np": "numpy"}
|
||||
result = processor._resolve_call_alias("np", alias_map)
|
||||
assert result == "numpy"
|
||||
|
||||
def test_resolve_call_alias_qualified(self):
|
||||
"""Test _resolve_call_alias with qualified name."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
alias_map = {"np": "numpy"}
|
||||
result = processor._resolve_call_alias("np.array", alias_map)
|
||||
assert result == "numpy.array"
|
||||
|
||||
def test_resolve_call_alias_no_match(self):
|
||||
"""Test _resolve_call_alias when no alias exists."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
alias_map = {}
|
||||
result = processor._resolve_call_alias("myfunc", alias_map)
|
||||
assert result == "myfunc"
|
||||
402
codex-lens/tests/parsers/test_astgrep_processor.py
Normal file
402
codex-lens/tests/parsers/test_astgrep_processor.py
Normal file
@@ -0,0 +1,402 @@
|
||||
"""Tests for AstGrepPythonProcessor.
|
||||
|
||||
Tests pattern-based relationship extraction from Python source code
|
||||
using ast-grep-py bindings.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from codexlens.parsers.astgrep_processor import (
|
||||
AstGrepPythonProcessor,
|
||||
BaseAstGrepProcessor,
|
||||
is_astgrep_processor_available,
|
||||
)
|
||||
from codexlens.parsers.patterns.python import (
|
||||
PYTHON_PATTERNS,
|
||||
METAVARS,
|
||||
RELATIONSHIP_PATTERNS,
|
||||
get_pattern,
|
||||
get_patterns_for_relationship,
|
||||
get_metavar,
|
||||
)
|
||||
|
||||
|
||||
# Check if ast-grep is available for conditional test skipping
|
||||
ASTGREP_AVAILABLE = is_astgrep_processor_available()
|
||||
|
||||
|
||||
class TestPatternDefinitions:
|
||||
"""Tests for Python pattern definitions."""
|
||||
|
||||
def test_python_patterns_exist(self):
|
||||
"""Verify all expected patterns are defined."""
|
||||
expected_patterns = [
|
||||
"class_def",
|
||||
"class_with_bases",
|
||||
"func_def",
|
||||
"async_func_def",
|
||||
"import_stmt",
|
||||
"import_from",
|
||||
"call",
|
||||
"method_call",
|
||||
]
|
||||
for pattern_name in expected_patterns:
|
||||
assert pattern_name in PYTHON_PATTERNS, f"Missing pattern: {pattern_name}"
|
||||
|
||||
def test_get_pattern_returns_correct_pattern(self):
|
||||
"""Test get_pattern returns expected pattern strings."""
|
||||
# Note: ast-grep-py 0.40+ uses $$$ for zero-or-more multi-match
|
||||
assert get_pattern("class_def") == "class $NAME $$$BODY"
|
||||
assert get_pattern("func_def") == "def $NAME($$$PARAMS): $$$BODY"
|
||||
assert get_pattern("import_stmt") == "import $MODULE"
|
||||
|
||||
def test_get_pattern_raises_for_unknown(self):
|
||||
"""Test get_pattern raises KeyError for unknown patterns."""
|
||||
with pytest.raises(KeyError):
|
||||
get_pattern("nonexistent_pattern")
|
||||
|
||||
def test_metavars_defined(self):
|
||||
"""Verify metavariable mappings are defined."""
|
||||
expected_metavars = [
|
||||
"class_name",
|
||||
"func_name",
|
||||
"import_module",
|
||||
"call_func",
|
||||
]
|
||||
for var in expected_metavars:
|
||||
assert var in METAVARS, f"Missing metavar: {var}"
|
||||
|
||||
def test_get_metavar(self):
|
||||
"""Test get_metavar returns correct values."""
|
||||
assert get_metavar("class_name") == "NAME"
|
||||
assert get_metavar("func_name") == "NAME"
|
||||
assert get_metavar("import_module") == "MODULE"
|
||||
|
||||
def test_relationship_patterns_mapping(self):
|
||||
"""Test relationship type to pattern mapping."""
|
||||
assert "class_with_bases" in get_patterns_for_relationship("inheritance")
|
||||
assert "import_stmt" in get_patterns_for_relationship("imports")
|
||||
assert "import_from" in get_patterns_for_relationship("imports")
|
||||
assert "call" in get_patterns_for_relationship("calls")
|
||||
|
||||
|
||||
class TestAstGrepPythonProcessorAvailability:
|
||||
"""Tests for processor availability."""
|
||||
|
||||
def test_is_available_returns_bool(self):
|
||||
"""Test is_available returns a boolean."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
assert isinstance(processor.is_available(), bool)
|
||||
|
||||
def test_is_available_matches_global_check(self):
|
||||
"""Test is_available matches is_astgrep_processor_available."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
assert processor.is_available() == is_astgrep_processor_available()
|
||||
|
||||
def test_module_level_check(self):
|
||||
"""Test module-level availability function."""
|
||||
assert isinstance(is_astgrep_processor_available(), bool)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not ASTGREP_AVAILABLE, reason="ast-grep-py not installed")
|
||||
class TestAstGrepPythonProcessorParsing:
|
||||
"""Tests for Python parsing with ast-grep."""
|
||||
|
||||
def test_parse_simple_function(self):
|
||||
"""Test parsing a simple function definition."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = "def hello():\n pass"
|
||||
result = processor.parse(code, Path("test.py"))
|
||||
|
||||
assert result is not None
|
||||
assert result.language == "python"
|
||||
assert len(result.symbols) == 1
|
||||
assert result.symbols[0].name == "hello"
|
||||
assert result.symbols[0].kind == "function"
|
||||
|
||||
def test_parse_class(self):
|
||||
"""Test parsing a class definition."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = "class MyClass:\n pass"
|
||||
result = processor.parse(code, Path("test.py"))
|
||||
|
||||
assert result is not None
|
||||
assert len(result.symbols) == 1
|
||||
assert result.symbols[0].name == "MyClass"
|
||||
assert result.symbols[0].kind == "class"
|
||||
|
||||
def test_parse_async_function(self):
|
||||
"""Test parsing an async function definition."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = "async def fetch_data():\n pass"
|
||||
result = processor.parse(code, Path("test.py"))
|
||||
|
||||
assert result is not None
|
||||
assert len(result.symbols) == 1
|
||||
assert result.symbols[0].name == "fetch_data"
|
||||
|
||||
def test_parse_class_with_inheritance(self):
|
||||
"""Test parsing class with inheritance."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
class Base:
|
||||
pass
|
||||
|
||||
class Child(Base):
|
||||
pass
|
||||
"""
|
||||
result = processor.parse(code, Path("test.py"))
|
||||
|
||||
assert result is not None
|
||||
names = [s.name for s in result.symbols]
|
||||
assert "Base" in names
|
||||
assert "Child" in names
|
||||
|
||||
# Check inheritance relationship
|
||||
inherits = [
|
||||
r for r in result.relationships
|
||||
if r.relationship_type.value == "inherits"
|
||||
]
|
||||
assert any(r.source_symbol == "Child" for r in inherits)
|
||||
|
||||
def test_parse_imports(self):
|
||||
"""Test parsing import statements."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
import os
|
||||
from sys import path
|
||||
"""
|
||||
result = processor.parse(code, Path("test.py"))
|
||||
|
||||
assert result is not None
|
||||
imports = [
|
||||
r for r in result.relationships
|
||||
if r.relationship_type.value == "imports"
|
||||
]
|
||||
assert len(imports) >= 1
|
||||
targets = {r.target_symbol for r in imports}
|
||||
assert "os" in targets
|
||||
|
||||
def test_parse_function_calls(self):
|
||||
"""Test parsing function calls."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
def main():
|
||||
print("hello")
|
||||
len([1, 2, 3])
|
||||
"""
|
||||
result = processor.parse(code, Path("test.py"))
|
||||
|
||||
assert result is not None
|
||||
calls = [
|
||||
r for r in result.relationships
|
||||
if r.relationship_type.value == "calls"
|
||||
]
|
||||
targets = {r.target_symbol for r in calls}
|
||||
assert "print" in targets
|
||||
assert "len" in targets
|
||||
|
||||
def test_parse_empty_file(self):
|
||||
"""Test parsing an empty file."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
result = processor.parse("", Path("test.py"))
|
||||
|
||||
assert result is not None
|
||||
assert len(result.symbols) == 0
|
||||
|
||||
def test_parse_returns_indexed_file(self):
|
||||
"""Test that parse returns proper IndexedFile structure."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = "def test():\n pass"
|
||||
result = processor.parse(code, Path("test.py"))
|
||||
|
||||
assert result is not None
|
||||
assert result.path.endswith("test.py")
|
||||
assert result.language == "python"
|
||||
assert isinstance(result.symbols, list)
|
||||
assert isinstance(result.chunks, list)
|
||||
assert isinstance(result.relationships, list)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not ASTGREP_AVAILABLE, reason="ast-grep-py not installed")
|
||||
class TestAstGrepPythonProcessorRelationships:
|
||||
"""Tests for relationship extraction."""
|
||||
|
||||
def test_inheritance_extraction(self):
|
||||
"""Test extraction of inheritance relationships."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
class Animal:
|
||||
pass
|
||||
|
||||
class Dog(Animal):
|
||||
pass
|
||||
|
||||
class Cat(Animal):
|
||||
pass
|
||||
"""
|
||||
result = processor.parse(code, Path("test.py"))
|
||||
|
||||
assert result is not None
|
||||
inherits = [
|
||||
r for r in result.relationships
|
||||
if r.relationship_type.value == "inherits"
|
||||
]
|
||||
# Should have 2 inheritance relationships
|
||||
assert len(inherits) >= 2
|
||||
sources = {r.source_symbol for r in inherits}
|
||||
assert "Dog" in sources
|
||||
assert "Cat" in sources
|
||||
|
||||
def test_call_extraction_skips_self(self):
|
||||
"""Test that self.method() calls are filtered."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
class Service:
|
||||
def process(self):
|
||||
self.internal()
|
||||
external_call()
|
||||
|
||||
def external_call():
|
||||
pass
|
||||
"""
|
||||
result = processor.parse(code, Path("test.py"))
|
||||
|
||||
assert result is not None
|
||||
calls = [
|
||||
r for r in result.relationships
|
||||
if r.relationship_type.value == "calls"
|
||||
]
|
||||
targets = {r.target_symbol for r in calls}
|
||||
# self.internal should be filtered
|
||||
assert "self.internal" not in targets
|
||||
assert "external_call" in targets
|
||||
|
||||
def test_import_with_alias_resolution(self):
|
||||
"""Test import alias resolution in calls."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = """
|
||||
import os.path as osp
|
||||
|
||||
def main():
|
||||
osp.join("a", "b")
|
||||
"""
|
||||
result = processor.parse(code, Path("test.py"))
|
||||
|
||||
assert result is not None
|
||||
calls = [
|
||||
r for r in result.relationships
|
||||
if r.relationship_type.value == "calls"
|
||||
]
|
||||
targets = {r.target_symbol for r in calls}
|
||||
# Should resolve osp to os.path
|
||||
assert any("os.path" in t for t in targets)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not ASTGREP_AVAILABLE, reason="ast-grep-py not installed")
|
||||
class TestAstGrepPythonProcessorRunAstGrep:
|
||||
"""Tests for run_ast_grep method."""
|
||||
|
||||
def test_run_ast_grep_returns_list(self):
|
||||
"""Test run_ast_grep returns a list."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = "def hello():\n pass"
|
||||
processor._binding.parse(code) if processor._binding else None
|
||||
|
||||
matches = processor.run_ast_grep(code, "def $NAME($$$PARAMS) $$$BODY")
|
||||
assert isinstance(matches, list)
|
||||
|
||||
def test_run_ast_grep_finds_matches(self):
|
||||
"""Test run_ast_grep finds expected matches."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = "def hello():\n pass"
|
||||
|
||||
matches = processor.run_ast_grep(code, "def $NAME($$$PARAMS) $$$BODY")
|
||||
assert len(matches) >= 1
|
||||
|
||||
def test_run_ast_grep_empty_code(self):
|
||||
"""Test run_ast_grep with empty code."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
matches = processor.run_ast_grep("", "def $NAME($$$PARAMS) $$$BODY")
|
||||
assert matches == []
|
||||
|
||||
def test_run_ast_grep_no_matches(self):
|
||||
"""Test run_ast_grep when pattern doesn't match."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
code = "x = 1"
|
||||
matches = processor.run_ast_grep(code, "class $NAME $$$BODY")
|
||||
assert matches == []
|
||||
|
||||
|
||||
class TestAstGrepPythonProcessorFallback:
|
||||
"""Tests for fallback behavior when ast-grep unavailable."""
|
||||
|
||||
def test_parse_returns_none_when_unavailable(self):
|
||||
"""Test parse returns None when ast-grep unavailable."""
|
||||
# This test runs regardless of availability
|
||||
# When unavailable, should gracefully return None
|
||||
processor = AstGrepPythonProcessor()
|
||||
if not processor.is_available():
|
||||
code = "def test():\n pass"
|
||||
result = processor.parse(code, Path("test.py"))
|
||||
assert result is None
|
||||
|
||||
def test_run_ast_grep_empty_when_unavailable(self):
|
||||
"""Test run_ast_grep returns empty list when unavailable."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
if not processor.is_available():
|
||||
matches = processor.run_ast_grep("code", "pattern")
|
||||
assert matches == []
|
||||
|
||||
|
||||
class TestBaseAstGrepProcessor:
|
||||
"""Tests for abstract base class."""
|
||||
|
||||
def test_cannot_instantiate_base_class(self):
|
||||
"""Test that BaseAstGrepProcessor cannot be instantiated directly."""
|
||||
with pytest.raises(TypeError):
|
||||
BaseAstGrepProcessor("python") # type: ignore[abstract]
|
||||
|
||||
def test_subclass_implements_abstract_methods(self):
|
||||
"""Test that AstGrepPythonProcessor implements all abstract methods."""
|
||||
processor = AstGrepPythonProcessor()
|
||||
# Should have process_matches method
|
||||
assert hasattr(processor, "process_matches")
|
||||
# Should have parse method
|
||||
assert hasattr(processor, "parse")
|
||||
# Check methods are callable
|
||||
assert callable(processor.process_matches)
|
||||
assert callable(processor.parse)
|
||||
|
||||
|
||||
class TestPatternIntegration:
|
||||
"""Tests for pattern module integration with processor."""
|
||||
|
||||
def test_processor_uses_pattern_module(self):
|
||||
"""Verify processor uses patterns from pattern module."""
|
||||
# The processor should import and use patterns from patterns/python/
|
||||
from codexlens.parsers.astgrep_processor import get_pattern
|
||||
|
||||
# Verify pattern access works
|
||||
assert get_pattern("class_def") is not None
|
||||
assert get_pattern("func_def") is not None
|
||||
|
||||
def test_pattern_consistency(self):
|
||||
"""Test pattern definitions are consistent."""
|
||||
# Patterns used by processor should exist in pattern module
|
||||
patterns_needed = [
|
||||
"class_def",
|
||||
"class_with_bases",
|
||||
"func_def",
|
||||
"async_func_def",
|
||||
"import_stmt",
|
||||
"import_from",
|
||||
"call",
|
||||
]
|
||||
for pattern_name in patterns_needed:
|
||||
# Should not raise KeyError
|
||||
pattern = get_pattern(pattern_name)
|
||||
assert pattern is not None
|
||||
assert len(pattern) > 0
|
||||
526
codex-lens/tests/parsers/test_comparison.py
Normal file
526
codex-lens/tests/parsers/test_comparison.py
Normal file
@@ -0,0 +1,526 @@
|
||||
"""Comparison tests for tree-sitter vs ast-grep Python relationship extraction.
|
||||
|
||||
Validates that both parsers produce consistent output for Python relationship
|
||||
extraction (INHERITS, CALL, IMPORTS).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List, Set, Tuple
|
||||
|
||||
import pytest
|
||||
|
||||
from codexlens.config import Config
|
||||
from codexlens.entities import CodeRelationship, RelationshipType
|
||||
from codexlens.parsers.treesitter_parser import TreeSitterSymbolParser
|
||||
|
||||
|
||||
# Sample Python code for testing relationship extraction
|
||||
SAMPLE_PYTHON_CODE = '''
|
||||
"""Module docstring."""
|
||||
import os
|
||||
import sys
|
||||
from typing import List, Dict, Optional
|
||||
from collections import defaultdict as dd
|
||||
from pathlib import Path as PPath
|
||||
|
||||
class BaseClass:
|
||||
"""Base class."""
|
||||
|
||||
def base_method(self):
|
||||
pass
|
||||
|
||||
def another_method(self):
|
||||
return self.base_method()
|
||||
|
||||
|
||||
class Mixin:
|
||||
"""Mixin class."""
|
||||
|
||||
def mixin_func(self):
|
||||
return "mixin"
|
||||
|
||||
|
||||
class ChildClass(BaseClass, Mixin):
|
||||
"""Child class with multiple inheritance."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.data = dd(list)
|
||||
|
||||
def process(self, items: List[str]) -> Dict[str, int]:
|
||||
result = {}
|
||||
for item in items:
|
||||
result[item] = len(item)
|
||||
return result
|
||||
|
||||
def call_external(self, path: str) -> Optional[str]:
|
||||
p = PPath(path)
|
||||
if p.exists():
|
||||
return str(p.read_text())
|
||||
return None
|
||||
|
||||
|
||||
def standalone_function():
|
||||
"""Standalone function."""
|
||||
data = [1, 2, 3]
|
||||
return sum(data)
|
||||
|
||||
|
||||
async def async_function():
|
||||
"""Async function."""
|
||||
import asyncio
|
||||
await asyncio.sleep(1)
|
||||
'''
|
||||
|
||||
|
||||
def relationship_to_tuple(rel: CodeRelationship) -> Tuple[str, str, str, int]:
|
||||
"""Convert relationship to a comparable tuple.
|
||||
|
||||
Returns:
|
||||
(source_symbol, target_symbol, relationship_type, source_line)
|
||||
"""
|
||||
return (
|
||||
rel.source_symbol,
|
||||
rel.target_symbol,
|
||||
rel.relationship_type.value,
|
||||
rel.source_line,
|
||||
)
|
||||
|
||||
|
||||
def extract_relationship_tuples(
|
||||
relationships: List[CodeRelationship],
|
||||
) -> Set[Tuple[str, str, str]]:
|
||||
"""Extract relationship tuples without line numbers for comparison.
|
||||
|
||||
Returns:
|
||||
Set of (source_symbol, target_symbol, relationship_type) tuples
|
||||
"""
|
||||
return {
|
||||
(rel.source_symbol, rel.target_symbol, rel.relationship_type.value)
|
||||
for rel in relationships
|
||||
}
|
||||
|
||||
|
||||
def filter_by_type(
|
||||
relationships: List[CodeRelationship],
|
||||
rel_type: RelationshipType,
|
||||
) -> List[CodeRelationship]:
|
||||
"""Filter relationships by type."""
|
||||
return [r for r in relationships if r.relationship_type == rel_type]
|
||||
|
||||
|
||||
class TestTreeSitterVsAstGrepComparison:
|
||||
"""Compare tree-sitter and ast-grep Python relationship extraction."""
|
||||
|
||||
@pytest.fixture
|
||||
def sample_path(self, tmp_path: Path) -> Path:
|
||||
"""Create a temporary Python file with sample code."""
|
||||
py_file = tmp_path / "sample.py"
|
||||
py_file.write_text(SAMPLE_PYTHON_CODE)
|
||||
return py_file
|
||||
|
||||
@pytest.fixture
|
||||
def ts_parser_default(self) -> TreeSitterSymbolParser:
|
||||
"""Create tree-sitter parser with default config (use_astgrep=False)."""
|
||||
config = Config()
|
||||
assert config.use_astgrep is False
|
||||
return TreeSitterSymbolParser("python", config=config)
|
||||
|
||||
@pytest.fixture
|
||||
def ts_parser_astgrep(self) -> TreeSitterSymbolParser:
|
||||
"""Create tree-sitter parser with ast-grep enabled."""
|
||||
config = Config()
|
||||
config.use_astgrep = True
|
||||
return TreeSitterSymbolParser("python", config=config)
|
||||
|
||||
def test_parser_availability(self, ts_parser_default: TreeSitterSymbolParser) -> None:
|
||||
"""Test that tree-sitter parser is available."""
|
||||
assert ts_parser_default.is_available()
|
||||
|
||||
def test_astgrep_processor_initialization(
|
||||
self, ts_parser_astgrep: TreeSitterSymbolParser
|
||||
) -> None:
|
||||
"""Test that ast-grep processor is initialized when config enables it."""
|
||||
# The processor should be initialized (may be None if ast-grep-py not installed)
|
||||
# This test just verifies the initialization path works
|
||||
assert ts_parser_astgrep._config is not None
|
||||
assert ts_parser_astgrep._config.use_astgrep is True
|
||||
|
||||
def _skip_if_astgrep_unavailable(
|
||||
self, ts_parser_astgrep: TreeSitterSymbolParser
|
||||
) -> None:
|
||||
"""Skip test if ast-grep is not available."""
|
||||
if ts_parser_astgrep._astgrep_processor is None:
|
||||
pytest.skip("ast-grep-py not installed")
|
||||
|
||||
def test_parse_returns_valid_result(
|
||||
self,
|
||||
ts_parser_default: TreeSitterSymbolParser,
|
||||
sample_path: Path,
|
||||
) -> None:
|
||||
"""Test that parsing returns a valid IndexedFile."""
|
||||
source_code = sample_path.read_text()
|
||||
result = ts_parser_default.parse(source_code, sample_path)
|
||||
|
||||
assert result is not None
|
||||
assert result.language == "python"
|
||||
assert len(result.symbols) > 0
|
||||
assert len(result.relationships) > 0
|
||||
|
||||
def test_extracted_symbols_match(
|
||||
self,
|
||||
ts_parser_default: TreeSitterSymbolParser,
|
||||
ts_parser_astgrep: TreeSitterSymbolParser,
|
||||
sample_path: Path,
|
||||
) -> None:
|
||||
"""Test that both parsers extract similar symbols."""
|
||||
self._skip_if_astgrep_unavailable(ts_parser_astgrep)
|
||||
|
||||
source_code = sample_path.read_text()
|
||||
|
||||
result_ts = ts_parser_default.parse(source_code, sample_path)
|
||||
result_astgrep = ts_parser_astgrep.parse(source_code, sample_path)
|
||||
|
||||
assert result_ts is not None
|
||||
assert result_astgrep is not None
|
||||
|
||||
# Compare symbol names
|
||||
ts_symbols = {s.name for s in result_ts.symbols}
|
||||
astgrep_symbols = {s.name for s in result_astgrep.symbols}
|
||||
|
||||
# Should have the same symbols (classes, functions, methods)
|
||||
assert ts_symbols == astgrep_symbols
|
||||
|
||||
def test_inheritance_relationships(
|
||||
self,
|
||||
ts_parser_default: TreeSitterSymbolParser,
|
||||
ts_parser_astgrep: TreeSitterSymbolParser,
|
||||
sample_path: Path,
|
||||
) -> None:
|
||||
"""Test INHERITS relationship extraction consistency."""
|
||||
self._skip_if_astgrep_unavailable(ts_parser_astgrep)
|
||||
|
||||
source_code = sample_path.read_text()
|
||||
|
||||
result_ts = ts_parser_default.parse(source_code, sample_path)
|
||||
result_astgrep = ts_parser_astgrep.parse(source_code, sample_path)
|
||||
|
||||
assert result_ts is not None
|
||||
assert result_astgrep is not None
|
||||
|
||||
# Extract inheritance relationships
|
||||
ts_inherits = filter_by_type(result_ts.relationships, RelationshipType.INHERITS)
|
||||
astgrep_inherits = filter_by_type(
|
||||
result_astgrep.relationships, RelationshipType.INHERITS
|
||||
)
|
||||
|
||||
ts_tuples = extract_relationship_tuples(ts_inherits)
|
||||
astgrep_tuples = extract_relationship_tuples(astgrep_inherits)
|
||||
|
||||
# Both should detect ChildClass(BaseClass, Mixin)
|
||||
assert ts_tuples == astgrep_tuples
|
||||
|
||||
# Verify specific inheritance relationships
|
||||
expected_inherits = {
|
||||
("ChildClass", "BaseClass", "inherits"),
|
||||
("ChildClass", "Mixin", "inherits"),
|
||||
}
|
||||
assert ts_tuples == expected_inherits
|
||||
|
||||
def test_import_relationships(
|
||||
self,
|
||||
ts_parser_default: TreeSitterSymbolParser,
|
||||
ts_parser_astgrep: TreeSitterSymbolParser,
|
||||
sample_path: Path,
|
||||
) -> None:
|
||||
"""Test IMPORTS relationship extraction consistency."""
|
||||
self._skip_if_astgrep_unavailable(ts_parser_astgrep)
|
||||
|
||||
source_code = sample_path.read_text()
|
||||
|
||||
result_ts = ts_parser_default.parse(source_code, sample_path)
|
||||
result_astgrep = ts_parser_astgrep.parse(source_code, sample_path)
|
||||
|
||||
assert result_ts is not None
|
||||
assert result_astgrep is not None
|
||||
|
||||
# Extract import relationships
|
||||
ts_imports = filter_by_type(result_ts.relationships, RelationshipType.IMPORTS)
|
||||
astgrep_imports = filter_by_type(
|
||||
result_astgrep.relationships, RelationshipType.IMPORTS
|
||||
)
|
||||
|
||||
ts_tuples = extract_relationship_tuples(ts_imports)
|
||||
astgrep_tuples = extract_relationship_tuples(astgrep_imports)
|
||||
|
||||
# Compare - should be similar (may differ in exact module representation)
|
||||
# At minimum, both should detect the top-level imports
|
||||
ts_modules = {t[1].split(".")[0] for t in ts_tuples}
|
||||
astgrep_modules = {t[1].split(".")[0] for t in astgrep_tuples}
|
||||
|
||||
# Should have imports from: os, sys, typing, collections, pathlib
|
||||
expected_modules = {"os", "sys", "typing", "collections", "pathlib", "asyncio"}
|
||||
assert ts_modules >= expected_modules or astgrep_modules >= expected_modules
|
||||
|
||||
def test_call_relationships(
|
||||
self,
|
||||
ts_parser_default: TreeSitterSymbolParser,
|
||||
ts_parser_astgrep: TreeSitterSymbolParser,
|
||||
sample_path: Path,
|
||||
) -> None:
|
||||
"""Test CALL relationship extraction consistency."""
|
||||
self._skip_if_astgrep_unavailable(ts_parser_astgrep)
|
||||
|
||||
source_code = sample_path.read_text()
|
||||
|
||||
result_ts = ts_parser_default.parse(source_code, sample_path)
|
||||
result_astgrep = ts_parser_astgrep.parse(source_code, sample_path)
|
||||
|
||||
assert result_ts is not None
|
||||
assert result_astgrep is not None
|
||||
|
||||
# Extract call relationships
|
||||
ts_calls = filter_by_type(result_ts.relationships, RelationshipType.CALL)
|
||||
astgrep_calls = filter_by_type(
|
||||
result_astgrep.relationships, RelationshipType.CALL
|
||||
)
|
||||
|
||||
# Calls may differ due to scope tracking differences
|
||||
# Just verify both parsers find call relationships
|
||||
assert len(ts_calls) > 0
|
||||
assert len(astgrep_calls) > 0
|
||||
|
||||
# Verify specific calls that should be detected
|
||||
ts_call_targets = {r.target_symbol for r in ts_calls}
|
||||
astgrep_call_targets = {r.target_symbol for r in astgrep_calls}
|
||||
|
||||
# Both should detect at least some common calls
|
||||
# (exact match not required due to scope tracking differences)
|
||||
common_targets = ts_call_targets & astgrep_call_targets
|
||||
assert len(common_targets) > 0
|
||||
|
||||
def test_relationship_count_similarity(
|
||||
self,
|
||||
ts_parser_default: TreeSitterSymbolParser,
|
||||
ts_parser_astgrep: TreeSitterSymbolParser,
|
||||
sample_path: Path,
|
||||
) -> None:
|
||||
"""Test that relationship counts are similar (>95% consistency)."""
|
||||
self._skip_if_astgrep_unavailable(ts_parser_astgrep)
|
||||
|
||||
source_code = sample_path.read_text()
|
||||
|
||||
result_ts = ts_parser_default.parse(source_code, sample_path)
|
||||
result_astgrep = ts_parser_astgrep.parse(source_code, sample_path)
|
||||
|
||||
assert result_ts is not None
|
||||
assert result_astgrep is not None
|
||||
|
||||
ts_count = len(result_ts.relationships)
|
||||
astgrep_count = len(result_astgrep.relationships)
|
||||
|
||||
# Calculate consistency percentage
|
||||
if max(ts_count, astgrep_count) == 0:
|
||||
consistency = 100.0
|
||||
else:
|
||||
consistency = (
|
||||
min(ts_count, astgrep_count) / max(ts_count, astgrep_count) * 100
|
||||
)
|
||||
|
||||
# Require >95% consistency
|
||||
assert consistency >= 95.0, (
|
||||
f"Relationship consistency {consistency:.1f}% below 95% threshold "
|
||||
f"(tree-sitter: {ts_count}, ast-grep: {astgrep_count})"
|
||||
)
|
||||
|
||||
def test_config_switch_affects_parser(
|
||||
self, sample_path: Path
|
||||
) -> None:
|
||||
"""Test that config.use_astgrep affects which parser is used."""
|
||||
config_default = Config()
|
||||
config_astgrep = Config()
|
||||
config_astgrep.use_astgrep = True
|
||||
|
||||
parser_default = TreeSitterSymbolParser("python", config=config_default)
|
||||
parser_astgrep = TreeSitterSymbolParser("python", config=config_astgrep)
|
||||
|
||||
# Default parser should not have ast-grep processor
|
||||
assert parser_default._astgrep_processor is None
|
||||
|
||||
# Ast-grep parser may have processor if ast-grep-py is installed
|
||||
# (could be None if not installed, which is fine)
|
||||
if parser_astgrep._astgrep_processor is not None:
|
||||
# If available, verify it's the right type
|
||||
from codexlens.parsers.astgrep_processor import AstGrepPythonProcessor
|
||||
|
||||
assert isinstance(
|
||||
parser_astgrep._astgrep_processor, AstGrepPythonProcessor
|
||||
)
|
||||
|
||||
def test_fallback_to_treesitter_on_astgrep_failure(
|
||||
self,
|
||||
ts_parser_astgrep: TreeSitterSymbolParser,
|
||||
sample_path: Path,
|
||||
) -> None:
|
||||
"""Test that parser falls back to tree-sitter if ast-grep fails."""
|
||||
source_code = sample_path.read_text()
|
||||
|
||||
# Even with use_astgrep=True, should get valid results
|
||||
result = ts_parser_astgrep.parse(source_code, sample_path)
|
||||
|
||||
# Should always return a valid result (either from ast-grep or tree-sitter fallback)
|
||||
assert result is not None
|
||||
assert result.language == "python"
|
||||
assert len(result.relationships) > 0
|
||||
|
||||
|
||||
class TestSimpleCodeSamples:
|
||||
"""Test with simple code samples for precise comparison."""
|
||||
|
||||
def test_simple_inheritance(self) -> None:
|
||||
"""Test simple single inheritance."""
|
||||
code = """
|
||||
class Parent:
|
||||
pass
|
||||
|
||||
class Child(Parent):
|
||||
pass
|
||||
"""
|
||||
self._compare_parsers(code, expected_inherits={("Child", "Parent")})
|
||||
|
||||
def test_multiple_inheritance(self) -> None:
|
||||
"""Test multiple inheritance."""
|
||||
code = """
|
||||
class A:
|
||||
pass
|
||||
|
||||
class B:
|
||||
pass
|
||||
|
||||
class C(A, B):
|
||||
pass
|
||||
"""
|
||||
self._compare_parsers(
|
||||
code, expected_inherits={("C", "A"), ("C", "B")}
|
||||
)
|
||||
|
||||
def test_simple_imports(self) -> None:
|
||||
"""Test simple import statements."""
|
||||
code = """
|
||||
import os
|
||||
import sys
|
||||
"""
|
||||
config_ts = Config()
|
||||
config_ag = Config()
|
||||
config_ag.use_astgrep = True
|
||||
|
||||
parser_ts = TreeSitterSymbolParser("python", config=config_ts)
|
||||
parser_ag = TreeSitterSymbolParser("python", config=config_ag)
|
||||
|
||||
tmp_path = Path("test.py")
|
||||
result_ts = parser_ts.parse(code, tmp_path)
|
||||
result_ag = parser_ag.parse(code, tmp_path)
|
||||
|
||||
assert result_ts is not None
|
||||
# ast-grep result may be None if not installed
|
||||
|
||||
if result_ag is not None:
|
||||
ts_imports = {
|
||||
r.target_symbol
|
||||
for r in result_ts.relationships
|
||||
if r.relationship_type == RelationshipType.IMPORTS
|
||||
}
|
||||
ag_imports = {
|
||||
r.target_symbol
|
||||
for r in result_ag.relationships
|
||||
if r.relationship_type == RelationshipType.IMPORTS
|
||||
}
|
||||
assert ts_imports == ag_imports
|
||||
|
||||
def test_imports_inside_function(self) -> None:
|
||||
"""Test simple import inside a function scope is recorded.
|
||||
|
||||
Note: tree-sitter parser requires a scope to record imports.
|
||||
Module-level imports without any function/class are not recorded
|
||||
because scope_stack is empty at module level.
|
||||
"""
|
||||
code = """
|
||||
def my_function():
|
||||
import collections
|
||||
return collections
|
||||
"""
|
||||
config_ts = Config()
|
||||
config_ag = Config()
|
||||
config_ag.use_astgrep = True
|
||||
|
||||
parser_ts = TreeSitterSymbolParser("python", config=config_ts)
|
||||
parser_ag = TreeSitterSymbolParser("python", config=config_ag)
|
||||
|
||||
tmp_path = Path("test.py")
|
||||
result_ts = parser_ts.parse(code, tmp_path)
|
||||
result_ag = parser_ag.parse(code, tmp_path)
|
||||
|
||||
assert result_ts is not None
|
||||
|
||||
# Get import relationship targets
|
||||
ts_imports = [
|
||||
r.target_symbol
|
||||
for r in result_ts.relationships
|
||||
if r.relationship_type == RelationshipType.IMPORTS
|
||||
]
|
||||
|
||||
# Should have collections
|
||||
ts_has_collections = any("collections" in t for t in ts_imports)
|
||||
assert ts_has_collections, f"Expected collections import, got: {ts_imports}"
|
||||
|
||||
# If ast-grep is available, verify it also finds the imports
|
||||
if result_ag is not None:
|
||||
ag_imports = [
|
||||
r.target_symbol
|
||||
for r in result_ag.relationships
|
||||
if r.relationship_type == RelationshipType.IMPORTS
|
||||
]
|
||||
ag_has_collections = any("collections" in t for t in ag_imports)
|
||||
assert ag_has_collections, f"Expected collections import in ast-grep, got: {ag_imports}"
|
||||
|
||||
def _compare_parsers(
|
||||
self,
|
||||
code: str,
|
||||
expected_inherits: Set[Tuple[str, str]],
|
||||
) -> None:
|
||||
"""Helper to compare parser outputs for inheritance."""
|
||||
config_ts = Config()
|
||||
config_ag = Config()
|
||||
config_ag.use_astgrep = True
|
||||
|
||||
parser_ts = TreeSitterSymbolParser("python", config=config_ts)
|
||||
parser_ag = TreeSitterSymbolParser("python", config=config_ag)
|
||||
|
||||
tmp_path = Path("test.py")
|
||||
result_ts = parser_ts.parse(code, tmp_path)
|
||||
|
||||
assert result_ts is not None
|
||||
|
||||
# Verify tree-sitter finds expected inheritance
|
||||
ts_inherits = {
|
||||
(r.source_symbol, r.target_symbol)
|
||||
for r in result_ts.relationships
|
||||
if r.relationship_type == RelationshipType.INHERITS
|
||||
}
|
||||
assert ts_inherits == expected_inherits
|
||||
|
||||
# If ast-grep is available, verify it matches
|
||||
result_ag = parser_ag.parse(code, tmp_path)
|
||||
if result_ag is not None:
|
||||
ag_inherits = {
|
||||
(r.source_symbol, r.target_symbol)
|
||||
for r in result_ag.relationships
|
||||
if r.relationship_type == RelationshipType.INHERITS
|
||||
}
|
||||
assert ag_inherits == expected_inherits
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
191
codex-lens/tests/test_astgrep_binding.py
Normal file
191
codex-lens/tests/test_astgrep_binding.py
Normal file
@@ -0,0 +1,191 @@
|
||||
"""Tests for ast-grep binding module.
|
||||
|
||||
Verifies basic import and functionality of AstGrepBinding.
|
||||
Run with: python -m pytest tests/test_astgrep_binding.py -v
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class TestAstGrepBindingAvailability:
|
||||
"""Test availability checks."""
|
||||
|
||||
def test_is_astgrep_available_function(self):
|
||||
"""Test is_astgrep_available function returns boolean."""
|
||||
from codexlens.parsers.astgrep_binding import is_astgrep_available
|
||||
result = is_astgrep_available()
|
||||
assert isinstance(result, bool)
|
||||
|
||||
def test_get_supported_languages(self):
|
||||
"""Test get_supported_languages returns expected languages."""
|
||||
from codexlens.parsers.astgrep_binding import get_supported_languages
|
||||
languages = get_supported_languages()
|
||||
assert isinstance(languages, list)
|
||||
assert "python" in languages
|
||||
assert "javascript" in languages
|
||||
assert "typescript" in languages
|
||||
|
||||
|
||||
class TestAstGrepBindingInit:
|
||||
"""Test AstGrepBinding initialization."""
|
||||
|
||||
def test_init_python(self):
|
||||
"""Test initialization with Python language."""
|
||||
from codexlens.parsers.astgrep_binding import AstGrepBinding
|
||||
binding = AstGrepBinding("python")
|
||||
assert binding.language_id == "python"
|
||||
|
||||
def test_init_typescript_with_tsx(self):
|
||||
"""Test TSX detection from file extension."""
|
||||
from codexlens.parsers.astgrep_binding import AstGrepBinding
|
||||
binding = AstGrepBinding("typescript", Path("component.tsx"))
|
||||
assert binding.language_id == "typescript"
|
||||
|
||||
def test_is_available_returns_boolean(self):
|
||||
"""Test is_available returns boolean."""
|
||||
from codexlens.parsers.astgrep_binding import AstGrepBinding
|
||||
binding = AstGrepBinding("python")
|
||||
result = binding.is_available()
|
||||
assert isinstance(result, bool)
|
||||
|
||||
|
||||
def _is_astgrep_installed():
|
||||
"""Check if ast-grep-py is installed."""
|
||||
try:
|
||||
import ast_grep_py # noqa: F401
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _is_astgrep_installed(),
|
||||
reason="ast-grep-py not installed"
|
||||
)
|
||||
class TestAstGrepBindingWithAstGrep:
|
||||
"""Tests that require ast-grep-py to be installed."""
|
||||
|
||||
def test_parse_simple_python(self):
|
||||
"""Test parsing simple Python code."""
|
||||
from codexlens.parsers.astgrep_binding import AstGrepBinding
|
||||
binding = AstGrepBinding("python")
|
||||
|
||||
if not binding.is_available():
|
||||
pytest.skip("ast-grep not available")
|
||||
|
||||
source = "x = 1"
|
||||
result = binding.parse(source)
|
||||
assert result is True
|
||||
|
||||
def test_find_inheritance(self):
|
||||
"""Test finding class inheritance."""
|
||||
from codexlens.parsers.astgrep_binding import AstGrepBinding
|
||||
binding = AstGrepBinding("python")
|
||||
|
||||
if not binding.is_available():
|
||||
pytest.skip("ast-grep not available")
|
||||
|
||||
source = """
|
||||
class MyClass(BaseClass):
|
||||
pass
|
||||
"""
|
||||
binding.parse(source)
|
||||
results = binding.find_inheritance()
|
||||
assert len(results) >= 0 # May or may not find depending on pattern match
|
||||
|
||||
def test_find_calls(self):
|
||||
"""Test finding function calls."""
|
||||
from codexlens.parsers.astgrep_binding import AstGrepBinding
|
||||
binding = AstGrepBinding("python")
|
||||
|
||||
if not binding.is_available():
|
||||
pytest.skip("ast-grep not available")
|
||||
|
||||
source = """
|
||||
def foo():
|
||||
bar()
|
||||
baz.qux()
|
||||
"""
|
||||
binding.parse(source)
|
||||
results = binding.find_calls()
|
||||
assert isinstance(results, list)
|
||||
|
||||
def test_find_imports(self):
|
||||
"""Test finding import statements."""
|
||||
from codexlens.parsers.astgrep_binding import AstGrepBinding
|
||||
binding = AstGrepBinding("python")
|
||||
|
||||
if not binding.is_available():
|
||||
pytest.skip("ast-grep not available")
|
||||
|
||||
source = """
|
||||
import os
|
||||
from typing import List
|
||||
"""
|
||||
binding.parse(source)
|
||||
results = binding.find_imports()
|
||||
assert isinstance(results, list)
|
||||
|
||||
|
||||
def test_basic_import():
|
||||
"""Test that the module can be imported."""
|
||||
try:
|
||||
from codexlens.parsers.astgrep_binding import (
|
||||
AstGrepBinding,
|
||||
is_astgrep_available,
|
||||
get_supported_languages,
|
||||
ASTGREP_AVAILABLE,
|
||||
)
|
||||
assert True
|
||||
except ImportError as e:
|
||||
pytest.fail(f"Failed to import astgrep_binding: {e}")
|
||||
|
||||
|
||||
def test_availability_flag():
|
||||
"""Test ASTGREP_AVAILABLE flag is defined."""
|
||||
from codexlens.parsers.astgrep_binding import ASTGREP_AVAILABLE
|
||||
assert isinstance(ASTGREP_AVAILABLE, bool)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Run basic verification
|
||||
print("Testing astgrep_binding module...")
|
||||
|
||||
from codexlens.parsers.astgrep_binding import (
|
||||
AstGrepBinding,
|
||||
is_astgrep_available,
|
||||
get_supported_languages,
|
||||
)
|
||||
|
||||
print(f"ast-grep available: {is_astgrep_available()}")
|
||||
print(f"Supported languages: {get_supported_languages()}")
|
||||
|
||||
binding = AstGrepBinding("python")
|
||||
print(f"Python binding available: {binding.is_available()}")
|
||||
|
||||
if binding.is_available():
|
||||
test_code = """
|
||||
import os
|
||||
from typing import List
|
||||
|
||||
class MyClass(BaseClass):
|
||||
def method(self):
|
||||
self.helper()
|
||||
external_func()
|
||||
|
||||
def helper():
|
||||
pass
|
||||
"""
|
||||
binding.parse(test_code)
|
||||
print(f"Inheritance found: {binding.find_inheritance()}")
|
||||
print(f"Calls found: {binding.find_calls()}")
|
||||
print(f"Imports found: {binding.find_imports()}")
|
||||
else:
|
||||
print("Note: ast-grep-py not installed. To install:")
|
||||
print(" pip install ast-grep-py")
|
||||
print(" Note: May have compatibility issues with Python 3.13")
|
||||
|
||||
print("Basic verification complete!")
|
||||
Reference in New Issue
Block a user