mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-02-11 02:33:51 +08:00
feat: Add support for Tree-Sitter parsing and enhance SQLite storage performance
This commit is contained in:
catlog22
@@ -1,8 +1,8 @@
|
||||
"""Parser factory for CodexLens.
|
||||
|
||||
The project currently ships lightweight regex-based parsers per language.
|
||||
They can be swapped for tree-sitter based parsers later without changing
|
||||
CLI or storage interfaces.
|
||||
Python and JavaScript/TypeScript parsing use Tree-Sitter grammars when
|
||||
available. Regex fallbacks are retained to preserve the existing parser
|
||||
interface and behavior in minimal environments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -10,7 +10,16 @@ from __future__ import annotations
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Protocol
|
||||
from typing import Dict, Iterable, List, Optional, Protocol
|
||||
|
||||
try:
|
||||
from tree_sitter import Language as TreeSitterLanguage
|
||||
from tree_sitter import Node as TreeSitterNode
|
||||
from tree_sitter import Parser as TreeSitterParser
|
||||
except Exception: # pragma: no cover
|
||||
TreeSitterLanguage = None # type: ignore[assignment]
|
||||
TreeSitterNode = None # type: ignore[assignment]
|
||||
TreeSitterParser = None # type: ignore[assignment]
|
||||
|
||||
from codexlens.config import Config
|
||||
from codexlens.entities import IndexedFile, Symbol
|
||||
@@ -25,11 +34,10 @@ class SimpleRegexParser:
|
||||
language_id: str
|
||||
|
||||
def parse(self, text: str, path: Path) -> IndexedFile:
|
||||
symbols: List[Symbol] = []
|
||||
if self.language_id == "python":
|
||||
symbols = _parse_python_symbols(text)
|
||||
elif self.language_id in {"javascript", "typescript"}:
|
||||
symbols = _parse_js_ts_symbols(text)
|
||||
symbols = _parse_js_ts_symbols(text, self.language_id, path)
|
||||
elif self.language_id == "java":
|
||||
symbols = _parse_java_symbols(text)
|
||||
elif self.language_id == "go":
|
||||
@@ -57,24 +65,135 @@ class ParserFactory:
|
||||
|
||||
|
||||
_PY_CLASS_RE = re.compile(r"^\s*class\s+([A-Za-z_]\w*)\b")
|
||||
_PY_DEF_RE = re.compile(r"^\s*def\s+([A-Za-z_]\w*)\s*\(")
|
||||
_PY_DEF_RE = re.compile(r"^\s*(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\(")
|
||||
|
||||
_TREE_SITTER_LANGUAGE_CACHE: Dict[str, TreeSitterLanguage] = {}
|
||||
|
||||
|
||||
def _parse_python_symbols(text: str) -> List[Symbol]:
|
||||
def _get_tree_sitter_language(language_id: str, path: Path | None = None) -> TreeSitterLanguage | None:
|
||||
if TreeSitterLanguage is None:
|
||||
return None
|
||||
|
||||
cache_key = language_id
|
||||
if language_id == "typescript" and path is not None and path.suffix.lower() == ".tsx":
|
||||
cache_key = "tsx"
|
||||
|
||||
cached = _TREE_SITTER_LANGUAGE_CACHE.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
try:
|
||||
if cache_key == "python":
|
||||
import tree_sitter_python # type: ignore[import-not-found]
|
||||
|
||||
language = TreeSitterLanguage(tree_sitter_python.language())
|
||||
elif cache_key == "javascript":
|
||||
import tree_sitter_javascript # type: ignore[import-not-found]
|
||||
|
||||
language = TreeSitterLanguage(tree_sitter_javascript.language())
|
||||
elif cache_key == "typescript":
|
||||
import tree_sitter_typescript # type: ignore[import-not-found]
|
||||
|
||||
language = TreeSitterLanguage(tree_sitter_typescript.language_typescript())
|
||||
elif cache_key == "tsx":
|
||||
import tree_sitter_typescript # type: ignore[import-not-found]
|
||||
|
||||
language = TreeSitterLanguage(tree_sitter_typescript.language_tsx())
|
||||
else:
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
_TREE_SITTER_LANGUAGE_CACHE[cache_key] = language
|
||||
return language
|
||||
|
||||
|
||||
def _iter_tree_sitter_nodes(root: TreeSitterNode) -> Iterable[TreeSitterNode]:
|
||||
stack: List[TreeSitterNode] = [root]
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
yield node
|
||||
for child in reversed(node.children):
|
||||
stack.append(child)
|
||||
|
||||
|
||||
def _node_text(source_bytes: bytes, node: TreeSitterNode) -> str:
|
||||
return source_bytes[node.start_byte:node.end_byte].decode("utf8")
|
||||
|
||||
|
||||
def _node_range(node: TreeSitterNode) -> tuple[int, int]:
|
||||
start_line = node.start_point[0] + 1
|
||||
end_line = node.end_point[0] + 1
|
||||
return (start_line, max(start_line, end_line))
|
||||
|
||||
|
||||
def _python_kind_for_function_node(node: TreeSitterNode) -> str:
|
||||
parent = node.parent
|
||||
while parent is not None:
|
||||
if parent.type in {"function_definition", "async_function_definition"}:
|
||||
return "function"
|
||||
if parent.type == "class_definition":
|
||||
return "method"
|
||||
parent = parent.parent
|
||||
return "function"
|
||||
|
||||
|
||||
def _parse_python_symbols_tree_sitter(text: str) -> List[Symbol] | None:
|
||||
if TreeSitterParser is None:
|
||||
return None
|
||||
|
||||
language = _get_tree_sitter_language("python")
|
||||
if language is None:
|
||||
return None
|
||||
|
||||
parser = TreeSitterParser()
|
||||
if hasattr(parser, "set_language"):
|
||||
parser.set_language(language) # type: ignore[attr-defined]
|
||||
else:
|
||||
parser.language = language # type: ignore[assignment]
|
||||
|
||||
source_bytes = text.encode("utf8")
|
||||
tree = parser.parse(source_bytes)
|
||||
root = tree.root_node
|
||||
|
||||
symbols: List[Symbol] = []
|
||||
for node in _iter_tree_sitter_nodes(root):
|
||||
if node.type == "class_definition":
|
||||
name_node = node.child_by_field_name("name")
|
||||
if name_node is None:
|
||||
continue
|
||||
symbols.append(Symbol(
|
||||
name=_node_text(source_bytes, name_node),
|
||||
kind="class",
|
||||
range=_node_range(node),
|
||||
))
|
||||
elif node.type in {"function_definition", "async_function_definition"}:
|
||||
name_node = node.child_by_field_name("name")
|
||||
if name_node is None:
|
||||
continue
|
||||
symbols.append(Symbol(
|
||||
name=_node_text(source_bytes, name_node),
|
||||
kind=_python_kind_for_function_node(node),
|
||||
range=_node_range(node),
|
||||
))
|
||||
|
||||
return symbols
|
||||
|
||||
|
||||
def _parse_python_symbols_regex(text: str) -> List[Symbol]:
|
||||
symbols: List[Symbol] = []
|
||||
current_class_indent: Optional[int] = None
|
||||
for i, line in enumerate(text.splitlines(), start=1):
|
||||
if _PY_CLASS_RE.match(line):
|
||||
name = _PY_CLASS_RE.match(line).group(1)
|
||||
class_match = _PY_CLASS_RE.match(line)
|
||||
if class_match:
|
||||
current_class_indent = len(line) - len(line.lstrip(" "))
|
||||
symbols.append(Symbol(name=name, kind="class", range=(i, i)))
|
||||
symbols.append(Symbol(name=class_match.group(1), kind="class", range=(i, i)))
|
||||
continue
|
||||
def_match = _PY_DEF_RE.match(line)
|
||||
if def_match:
|
||||
name = def_match.group(1)
|
||||
indent = len(line) - len(line.lstrip(" "))
|
||||
kind = "method" if current_class_indent is not None and indent > current_class_indent else "function"
|
||||
symbols.append(Symbol(name=name, kind=kind, range=(i, i)))
|
||||
symbols.append(Symbol(name=def_match.group(1), kind=kind, range=(i, i)))
|
||||
continue
|
||||
if current_class_indent is not None:
|
||||
indent = len(line) - len(line.lstrip(" "))
|
||||
@@ -83,23 +202,153 @@ def _parse_python_symbols(text: str) -> List[Symbol]:
|
||||
return symbols
|
||||
|
||||
|
||||
_JS_FUNC_RE = re.compile(r"^\s*(?:export\s+)?function\s+([A-Za-z_$][\w$]*)\s*\(")
|
||||
def _parse_python_symbols(text: str) -> List[Symbol]:
|
||||
symbols = _parse_python_symbols_tree_sitter(text)
|
||||
if symbols is not None:
|
||||
return symbols
|
||||
return _parse_python_symbols_regex(text)
|
||||
|
||||
|
||||
_JS_FUNC_RE = re.compile(r"^\s*(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\(")
|
||||
_JS_CLASS_RE = re.compile(r"^\s*(?:export\s+)?class\s+([A-Za-z_$][\w$]*)\b")
|
||||
_JS_ARROW_RE = re.compile(
|
||||
r"^\s*(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?\(?[^)]*\)?\s*=>"
|
||||
)
|
||||
_JS_METHOD_RE = re.compile(r"^\s+(?:async\s+)?([A-Za-z_$][\w$]*)\s*\([^)]*\)\s*\{")
|
||||
|
||||
|
||||
def _parse_js_ts_symbols(text: str) -> List[Symbol]:
|
||||
def _js_has_class_ancestor(node: TreeSitterNode) -> bool:
|
||||
parent = node.parent
|
||||
while parent is not None:
|
||||
if parent.type in {"class_declaration", "class"}:
|
||||
return True
|
||||
parent = parent.parent
|
||||
return False
|
||||
|
||||
|
||||
def _parse_js_ts_symbols_tree_sitter(
|
||||
text: str,
|
||||
language_id: str,
|
||||
path: Path | None = None,
|
||||
) -> List[Symbol] | None:
|
||||
if TreeSitterParser is None:
|
||||
return None
|
||||
|
||||
language = _get_tree_sitter_language(language_id, path)
|
||||
if language is None:
|
||||
return None
|
||||
|
||||
parser = TreeSitterParser()
|
||||
if hasattr(parser, "set_language"):
|
||||
parser.set_language(language) # type: ignore[attr-defined]
|
||||
else:
|
||||
parser.language = language # type: ignore[assignment]
|
||||
|
||||
source_bytes = text.encode("utf8")
|
||||
tree = parser.parse(source_bytes)
|
||||
root = tree.root_node
|
||||
|
||||
symbols: List[Symbol] = []
|
||||
for node in _iter_tree_sitter_nodes(root):
|
||||
if node.type in {"class_declaration", "class"}:
|
||||
name_node = node.child_by_field_name("name")
|
||||
if name_node is None:
|
||||
continue
|
||||
symbols.append(Symbol(
|
||||
name=_node_text(source_bytes, name_node),
|
||||
kind="class",
|
||||
range=_node_range(node),
|
||||
))
|
||||
elif node.type in {"function_declaration", "generator_function_declaration"}:
|
||||
name_node = node.child_by_field_name("name")
|
||||
if name_node is None:
|
||||
continue
|
||||
symbols.append(Symbol(
|
||||
name=_node_text(source_bytes, name_node),
|
||||
kind="function",
|
||||
range=_node_range(node),
|
||||
))
|
||||
elif node.type == "variable_declarator":
|
||||
name_node = node.child_by_field_name("name")
|
||||
value_node = node.child_by_field_name("value")
|
||||
if (
|
||||
name_node is None
|
||||
or value_node is None
|
||||
or name_node.type not in {"identifier", "property_identifier"}
|
||||
or value_node.type != "arrow_function"
|
||||
):
|
||||
continue
|
||||
symbols.append(Symbol(
|
||||
name=_node_text(source_bytes, name_node),
|
||||
kind="function",
|
||||
range=_node_range(node),
|
||||
))
|
||||
elif node.type == "method_definition" and _js_has_class_ancestor(node):
|
||||
name_node = node.child_by_field_name("name")
|
||||
if name_node is None:
|
||||
continue
|
||||
name = _node_text(source_bytes, name_node)
|
||||
if name == "constructor":
|
||||
continue
|
||||
symbols.append(Symbol(
|
||||
name=name,
|
||||
kind="method",
|
||||
range=_node_range(node),
|
||||
))
|
||||
|
||||
return symbols
|
||||
|
||||
|
||||
def _parse_js_ts_symbols_regex(text: str) -> List[Symbol]:
|
||||
symbols: List[Symbol] = []
|
||||
in_class = False
|
||||
class_brace_depth = 0
|
||||
brace_depth = 0
|
||||
|
||||
for i, line in enumerate(text.splitlines(), start=1):
|
||||
brace_depth += line.count("{") - line.count("}")
|
||||
|
||||
class_match = _JS_CLASS_RE.match(line)
|
||||
if class_match:
|
||||
symbols.append(Symbol(name=class_match.group(1), kind="class", range=(i, i)))
|
||||
in_class = True
|
||||
class_brace_depth = brace_depth
|
||||
continue
|
||||
|
||||
if in_class and brace_depth < class_brace_depth:
|
||||
in_class = False
|
||||
|
||||
func_match = _JS_FUNC_RE.match(line)
|
||||
if func_match:
|
||||
symbols.append(Symbol(name=func_match.group(1), kind="function", range=(i, i)))
|
||||
continue
|
||||
class_match = _JS_CLASS_RE.match(line)
|
||||
if class_match:
|
||||
symbols.append(Symbol(name=class_match.group(1), kind="class", range=(i, i)))
|
||||
|
||||
arrow_match = _JS_ARROW_RE.match(line)
|
||||
if arrow_match:
|
||||
symbols.append(Symbol(name=arrow_match.group(1), kind="function", range=(i, i)))
|
||||
continue
|
||||
|
||||
if in_class:
|
||||
method_match = _JS_METHOD_RE.match(line)
|
||||
if method_match:
|
||||
name = method_match.group(1)
|
||||
if name != "constructor":
|
||||
symbols.append(Symbol(name=name, kind="method", range=(i, i)))
|
||||
|
||||
return symbols
|
||||
|
||||
|
||||
def _parse_js_ts_symbols(
|
||||
text: str,
|
||||
language_id: str = "javascript",
|
||||
path: Path | None = None,
|
||||
) -> List[Symbol]:
|
||||
symbols = _parse_js_ts_symbols_tree_sitter(text, language_id, path)
|
||||
if symbols is not None:
|
||||
return symbols
|
||||
return _parse_js_ts_symbols_regex(text)
|
||||
|
||||
|
||||
_JAVA_CLASS_RE = re.compile(r"^\s*(?:public\s+)?class\s+([A-Za-z_]\w*)\b")
|
||||
_JAVA_METHOD_RE = re.compile(
|
||||
r"^\s*(?:public|private|protected|static|\s)+[\w<>\[\]]+\s+([A-Za-z_]\w*)\s*\("
|
||||
@@ -151,4 +400,3 @@ def _parse_generic_symbols(text: str) -> List[Symbol]:
|
||||
if def_match:
|
||||
symbols.append(Symbol(name=def_match.group(1), kind="function", range=(i, i)))
|
||||
return symbols
|
||||
|
||||
|
||||
Reference in New Issue
Block a user