mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-02-11 02:33:51 +08:00
Optimize SQLite FTS storage and pooling
This commit is contained in:
@@ -137,6 +137,7 @@ def init(
|
|||||||
languages = _parse_languages(language)
|
languages = _parse_languages(language)
|
||||||
base_path = path.expanduser().resolve()
|
base_path = path.expanduser().resolve()
|
||||||
|
|
||||||
|
store: SQLiteStore | None = None
|
||||||
try:
|
try:
|
||||||
# Determine database location
|
# Determine database location
|
||||||
if use_global:
|
if use_global:
|
||||||
@@ -197,6 +198,9 @@ def init(
|
|||||||
print_json(success=False, error=str(exc))
|
print_json(success=False, error=str(exc))
|
||||||
else:
|
else:
|
||||||
raise typer.Exit(code=1)
|
raise typer.Exit(code=1)
|
||||||
|
finally:
|
||||||
|
if store is not None:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
@app.command()
|
@app.command()
|
||||||
@@ -214,6 +218,7 @@ def search(
|
|||||||
"""
|
"""
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose)
|
||||||
|
|
||||||
|
store: SQLiteStore | None = None
|
||||||
try:
|
try:
|
||||||
store, db_path = _get_store_for_path(Path.cwd(), use_global)
|
store, db_path = _get_store_for_path(Path.cwd(), use_global)
|
||||||
store.initialize()
|
store.initialize()
|
||||||
@@ -229,6 +234,9 @@ def search(
|
|||||||
else:
|
else:
|
||||||
console.print(f"[red]Search failed:[/red] {exc}")
|
console.print(f"[red]Search failed:[/red] {exc}")
|
||||||
raise typer.Exit(code=1)
|
raise typer.Exit(code=1)
|
||||||
|
finally:
|
||||||
|
if store is not None:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
@app.command()
|
@app.command()
|
||||||
@@ -252,6 +260,7 @@ def symbol(
|
|||||||
"""
|
"""
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose)
|
||||||
|
|
||||||
|
store: SQLiteStore | None = None
|
||||||
try:
|
try:
|
||||||
store, db_path = _get_store_for_path(Path.cwd(), use_global)
|
store, db_path = _get_store_for_path(Path.cwd(), use_global)
|
||||||
store.initialize()
|
store.initialize()
|
||||||
@@ -267,6 +276,9 @@ def symbol(
|
|||||||
else:
|
else:
|
||||||
console.print(f"[red]Symbol lookup failed:[/red] {exc}")
|
console.print(f"[red]Symbol lookup failed:[/red] {exc}")
|
||||||
raise typer.Exit(code=1)
|
raise typer.Exit(code=1)
|
||||||
|
finally:
|
||||||
|
if store is not None:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
@app.command()
|
@app.command()
|
||||||
@@ -316,6 +328,7 @@ def status(
|
|||||||
"""
|
"""
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose)
|
||||||
|
|
||||||
|
store: SQLiteStore | None = None
|
||||||
try:
|
try:
|
||||||
store, db_path = _get_store_for_path(Path.cwd(), use_global)
|
store, db_path = _get_store_for_path(Path.cwd(), use_global)
|
||||||
store.initialize()
|
store.initialize()
|
||||||
@@ -330,6 +343,9 @@ def status(
|
|||||||
else:
|
else:
|
||||||
console.print(f"[red]Status failed:[/red] {exc}")
|
console.print(f"[red]Status failed:[/red] {exc}")
|
||||||
raise typer.Exit(code=1)
|
raise typer.Exit(code=1)
|
||||||
|
finally:
|
||||||
|
if store is not None:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
@app.command()
|
@app.command()
|
||||||
@@ -351,6 +367,7 @@ def update(
|
|||||||
config = Config()
|
config = Config()
|
||||||
factory = ParserFactory(config)
|
factory = ParserFactory(config)
|
||||||
|
|
||||||
|
store: SQLiteStore | None = None
|
||||||
try:
|
try:
|
||||||
store, db_path = _get_store_for_path(Path.cwd(), use_global)
|
store, db_path = _get_store_for_path(Path.cwd(), use_global)
|
||||||
store.initialize()
|
store.initialize()
|
||||||
@@ -427,6 +444,9 @@ def update(
|
|||||||
else:
|
else:
|
||||||
console.print(f"[red]Update failed:[/red] {exc}")
|
console.print(f"[red]Update failed:[/red] {exc}")
|
||||||
raise typer.Exit(code=1)
|
raise typer.Exit(code=1)
|
||||||
|
finally:
|
||||||
|
if store is not None:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
@app.command()
|
@app.command()
|
||||||
|
|||||||
@@ -14,201 +14,256 @@ from codexlens.errors import StorageError
|
|||||||
|
|
||||||
|
|
||||||
class SQLiteStore:
|
class SQLiteStore:
|
||||||
"""SQLiteStore providing FTS5 search and symbol lookup."""
|
"""SQLiteStore providing FTS5 search and symbol lookup.
|
||||||
|
|
||||||
|
Implements thread-local connection pooling for improved performance.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, db_path: str | Path) -> None:
|
def __init__(self, db_path: str | Path) -> None:
|
||||||
self.db_path = Path(db_path)
|
self.db_path = Path(db_path)
|
||||||
self._lock = threading.RLock()
|
self._lock = threading.RLock()
|
||||||
|
self._local = threading.local()
|
||||||
|
self._pool_lock = threading.Lock()
|
||||||
|
self._pool: Dict[int, sqlite3.Connection] = {}
|
||||||
|
self._pool_generation = 0
|
||||||
|
|
||||||
|
def _get_connection(self) -> sqlite3.Connection:
|
||||||
|
"""Get or create a thread-local database connection."""
|
||||||
|
thread_id = threading.get_ident()
|
||||||
|
if getattr(self._local, "generation", None) == self._pool_generation:
|
||||||
|
conn = getattr(self._local, "conn", None)
|
||||||
|
if conn is not None:
|
||||||
|
return conn
|
||||||
|
|
||||||
|
with self._pool_lock:
|
||||||
|
conn = self._pool.get(thread_id)
|
||||||
|
if conn is None:
|
||||||
|
conn = sqlite3.connect(self.db_path, check_same_thread=False)
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
conn.execute("PRAGMA journal_mode=WAL")
|
||||||
|
conn.execute("PRAGMA synchronous=NORMAL")
|
||||||
|
conn.execute("PRAGMA foreign_keys=ON")
|
||||||
|
self._pool[thread_id] = conn
|
||||||
|
|
||||||
|
self._local.conn = conn
|
||||||
|
self._local.generation = self._pool_generation
|
||||||
|
return conn
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
"""Close all pooled connections."""
|
||||||
|
with self._lock:
|
||||||
|
with self._pool_lock:
|
||||||
|
for conn in self._pool.values():
|
||||||
|
conn.close()
|
||||||
|
self._pool.clear()
|
||||||
|
self._pool_generation += 1
|
||||||
|
|
||||||
|
if hasattr(self._local, "conn"):
|
||||||
|
self._local.conn = None
|
||||||
|
if hasattr(self._local, "generation"):
|
||||||
|
self._local.generation = self._pool_generation
|
||||||
|
|
||||||
|
def __enter__(self) -> SQLiteStore:
|
||||||
|
self.initialize()
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, exc_type: object, exc: object, tb: object) -> None:
|
||||||
|
self.close()
|
||||||
|
|
||||||
def initialize(self) -> None:
|
def initialize(self) -> None:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
with self._connect() as conn:
|
conn = self._get_connection()
|
||||||
self._create_schema(conn)
|
self._create_schema(conn)
|
||||||
|
self._ensure_fts_external_content(conn)
|
||||||
|
|
||||||
|
|
||||||
def add_file(self, indexed_file: IndexedFile, content: str) -> None:
|
def add_file(self, indexed_file: IndexedFile, content: str) -> None:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
with self._connect() as conn:
|
conn = self._get_connection()
|
||||||
path = str(Path(indexed_file.path).resolve())
|
path = str(Path(indexed_file.path).resolve())
|
||||||
language = indexed_file.language
|
language = indexed_file.language
|
||||||
mtime = Path(path).stat().st_mtime if Path(path).exists() else None
|
mtime = Path(path).stat().st_mtime if Path(path).exists() else None
|
||||||
line_count = content.count("\n") + 1
|
line_count = content.count(chr(10)) + 1
|
||||||
|
|
||||||
conn.execute(
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO files(path, language, content, mtime, line_count)
|
||||||
|
VALUES(?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(path) DO UPDATE SET
|
||||||
|
language=excluded.language,
|
||||||
|
content=excluded.content,
|
||||||
|
mtime=excluded.mtime,
|
||||||
|
line_count=excluded.line_count
|
||||||
|
""",
|
||||||
|
(path, language, content, mtime, line_count),
|
||||||
|
)
|
||||||
|
|
||||||
|
row = conn.execute("SELECT id FROM files WHERE path=?", (path,)).fetchone()
|
||||||
|
if not row:
|
||||||
|
raise StorageError(f"Failed to read file id for {path}")
|
||||||
|
file_id = int(row["id"])
|
||||||
|
|
||||||
|
conn.execute("DELETE FROM symbols WHERE file_id=?", (file_id,))
|
||||||
|
if indexed_file.symbols:
|
||||||
|
conn.executemany(
|
||||||
"""
|
"""
|
||||||
INSERT INTO files(path, language, content, mtime, line_count)
|
INSERT INTO symbols(file_id, name, kind, start_line, end_line)
|
||||||
VALUES(?, ?, ?, ?, ?)
|
VALUES(?, ?, ?, ?, ?)
|
||||||
ON CONFLICT(path) DO UPDATE SET
|
|
||||||
language=excluded.language,
|
|
||||||
content=excluded.content,
|
|
||||||
mtime=excluded.mtime,
|
|
||||||
line_count=excluded.line_count
|
|
||||||
""",
|
""",
|
||||||
(path, language, content, mtime, line_count),
|
[
|
||||||
|
(file_id, s.name, s.kind, s.range[0], s.range[1])
|
||||||
|
for s in indexed_file.symbols
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
conn.commit()
|
||||||
row = conn.execute("SELECT id FROM files WHERE path=?", (path,)).fetchone()
|
|
||||||
if not row:
|
|
||||||
raise StorageError(f"Failed to read file id for {path}")
|
|
||||||
file_id = int(row["id"])
|
|
||||||
|
|
||||||
conn.execute(
|
|
||||||
"INSERT OR REPLACE INTO files_fts(rowid, path, language, content) VALUES(?, ?, ?, ?)",
|
|
||||||
(file_id, path, language, content),
|
|
||||||
)
|
|
||||||
|
|
||||||
conn.execute("DELETE FROM symbols WHERE file_id=?", (file_id,))
|
|
||||||
if indexed_file.symbols:
|
|
||||||
conn.executemany(
|
|
||||||
"""
|
|
||||||
INSERT INTO symbols(file_id, name, kind, start_line, end_line)
|
|
||||||
VALUES(?, ?, ?, ?, ?)
|
|
||||||
""",
|
|
||||||
[
|
|
||||||
(file_id, s.name, s.kind, s.range[0], s.range[1])
|
|
||||||
for s in indexed_file.symbols
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
def remove_file(self, path: str | Path) -> bool:
|
def remove_file(self, path: str | Path) -> bool:
|
||||||
"""Remove a file from the index.
|
"""Remove a file from the index."""
|
||||||
|
|
||||||
Returns True if the file was removed, False if it didn't exist.
|
|
||||||
"""
|
|
||||||
with self._lock:
|
with self._lock:
|
||||||
with self._connect() as conn:
|
conn = self._get_connection()
|
||||||
resolved_path = str(Path(path).resolve())
|
resolved_path = str(Path(path).resolve())
|
||||||
|
|
||||||
# Get file_id first
|
row = conn.execute(
|
||||||
row = conn.execute(
|
"SELECT id FROM files WHERE path=?", (resolved_path,)
|
||||||
"SELECT id FROM files WHERE path=?", (resolved_path,)
|
).fetchone()
|
||||||
).fetchone()
|
|
||||||
|
|
||||||
if not row:
|
if not row:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
file_id = int(row["id"])
|
file_id = int(row["id"])
|
||||||
|
conn.execute("DELETE FROM files WHERE id=?", (file_id,))
|
||||||
# Delete from FTS index
|
conn.commit()
|
||||||
conn.execute("DELETE FROM files_fts WHERE rowid=?", (file_id,))
|
return True
|
||||||
|
|
||||||
# Delete symbols (CASCADE should handle this, but be explicit)
|
|
||||||
conn.execute("DELETE FROM symbols WHERE file_id=?", (file_id,))
|
|
||||||
|
|
||||||
# Delete file record
|
|
||||||
conn.execute("DELETE FROM files WHERE id=?", (file_id,))
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
def file_exists(self, path: str | Path) -> bool:
|
def file_exists(self, path: str | Path) -> bool:
|
||||||
"""Check if a file exists in the index."""
|
"""Check if a file exists in the index."""
|
||||||
with self._lock:
|
with self._lock:
|
||||||
with self._connect() as conn:
|
conn = self._get_connection()
|
||||||
resolved_path = str(Path(path).resolve())
|
resolved_path = str(Path(path).resolve())
|
||||||
row = conn.execute(
|
row = conn.execute(
|
||||||
"SELECT 1 FROM files WHERE path=?", (resolved_path,)
|
"SELECT 1 FROM files WHERE path=?", (resolved_path,)
|
||||||
).fetchone()
|
).fetchone()
|
||||||
return row is not None
|
return row is not None
|
||||||
|
|
||||||
def get_file_mtime(self, path: str | Path) -> float | None:
|
def get_file_mtime(self, path: str | Path) -> float | None:
|
||||||
"""Get the stored mtime for a file, or None if not indexed."""
|
"""Get the stored mtime for a file."""
|
||||||
with self._lock:
|
with self._lock:
|
||||||
with self._connect() as conn:
|
conn = self._get_connection()
|
||||||
resolved_path = str(Path(path).resolve())
|
resolved_path = str(Path(path).resolve())
|
||||||
row = conn.execute(
|
row = conn.execute(
|
||||||
"SELECT mtime FROM files WHERE path=?", (resolved_path,)
|
"SELECT mtime FROM files WHERE path=?", (resolved_path,)
|
||||||
).fetchone()
|
).fetchone()
|
||||||
return float(row["mtime"]) if row and row["mtime"] else None
|
return float(row["mtime"]) if row and row["mtime"] else None
|
||||||
|
|
||||||
|
|
||||||
def search_fts(self, query: str, *, limit: int = 20, offset: int = 0) -> List[SearchResult]:
|
def search_fts(self, query: str, *, limit: int = 20, offset: int = 0) -> List[SearchResult]:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
with self._connect() as conn:
|
conn = self._get_connection()
|
||||||
try:
|
try:
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
"""
|
"""
|
||||||
SELECT rowid, path, bm25(files_fts) AS rank,
|
SELECT rowid, path, bm25(files_fts) AS rank,
|
||||||
snippet(files_fts, 2, '[bold red]', '[/bold red]', '…', 20) AS excerpt
|
snippet(files_fts, 2, '[bold red]', '[/bold red]', "...", 20) AS excerpt
|
||||||
FROM files_fts
|
FROM files_fts
|
||||||
WHERE files_fts MATCH ?
|
WHERE files_fts MATCH ?
|
||||||
ORDER BY rank
|
ORDER BY rank
|
||||||
LIMIT ? OFFSET ?
|
LIMIT ? OFFSET ?
|
||||||
""",
|
""",
|
||||||
(query, limit, offset),
|
(query, limit, offset),
|
||||||
).fetchall()
|
).fetchall()
|
||||||
except sqlite3.DatabaseError as exc:
|
except sqlite3.DatabaseError as exc:
|
||||||
raise StorageError(f"FTS search failed: {exc}") from exc
|
raise StorageError(f"FTS search failed: {exc}") from exc
|
||||||
|
|
||||||
results: List[SearchResult] = []
|
results: List[SearchResult] = []
|
||||||
for row in rows:
|
for row in rows:
|
||||||
# BM25 returns negative values where more negative = better match
|
rank = float(row["rank"]) if row["rank"] is not None else 0.0
|
||||||
# Convert to positive score where higher = better
|
score = max(0.0, -rank)
|
||||||
rank = float(row["rank"]) if row["rank"] is not None else 0.0
|
results.append(
|
||||||
score = max(0.0, -rank) # Negate to make positive, clamp at 0
|
SearchResult(
|
||||||
results.append(
|
path=row["path"],
|
||||||
SearchResult(
|
score=score,
|
||||||
path=row["path"],
|
excerpt=row["excerpt"],
|
||||||
score=score,
|
|
||||||
excerpt=row["excerpt"],
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
return results
|
)
|
||||||
|
return results
|
||||||
|
|
||||||
|
def search_files_only(
|
||||||
|
self, query: str, *, limit: int = 20, offset: int = 0
|
||||||
|
) -> List[str]:
|
||||||
|
"""Search indexed file contents and return only file paths."""
|
||||||
|
with self._lock:
|
||||||
|
conn = self._get_connection()
|
||||||
|
try:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT path
|
||||||
|
FROM files_fts
|
||||||
|
WHERE files_fts MATCH ?
|
||||||
|
ORDER BY bm25(files_fts)
|
||||||
|
LIMIT ? OFFSET ?
|
||||||
|
""",
|
||||||
|
(query, limit, offset),
|
||||||
|
).fetchall()
|
||||||
|
except sqlite3.DatabaseError as exc:
|
||||||
|
raise StorageError(f"FTS search failed: {exc}") from exc
|
||||||
|
|
||||||
|
return [row["path"] for row in rows]
|
||||||
|
|
||||||
def search_symbols(
|
def search_symbols(
|
||||||
self, name: str, *, kind: Optional[str] = None, limit: int = 50
|
self, name: str, *, kind: Optional[str] = None, limit: int = 50
|
||||||
) -> List[Symbol]:
|
) -> List[Symbol]:
|
||||||
pattern = f"%{name}%"
|
pattern = f"%{name}%"
|
||||||
with self._lock:
|
with self._lock:
|
||||||
with self._connect() as conn:
|
conn = self._get_connection()
|
||||||
if kind:
|
if kind:
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
"""
|
"""
|
||||||
SELECT name, kind, start_line, end_line
|
SELECT name, kind, start_line, end_line
|
||||||
FROM symbols
|
FROM symbols
|
||||||
WHERE name LIKE ? AND kind=?
|
WHERE name LIKE ? AND kind=?
|
||||||
ORDER BY name
|
ORDER BY name
|
||||||
LIMIT ?
|
LIMIT ?
|
||||||
""",
|
""",
|
||||||
(pattern, kind, limit),
|
(pattern, kind, limit),
|
||||||
).fetchall()
|
).fetchall()
|
||||||
else:
|
else:
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
"""
|
"""
|
||||||
SELECT name, kind, start_line, end_line
|
SELECT name, kind, start_line, end_line
|
||||||
FROM symbols
|
FROM symbols
|
||||||
WHERE name LIKE ?
|
WHERE name LIKE ?
|
||||||
ORDER BY name
|
ORDER BY name
|
||||||
LIMIT ?
|
LIMIT ?
|
||||||
""",
|
""",
|
||||||
(pattern, limit),
|
(pattern, limit),
|
||||||
).fetchall()
|
).fetchall()
|
||||||
|
|
||||||
|
return [
|
||||||
|
Symbol(name=row["name"], kind=row["kind"], range=(row["start_line"], row["end_line"]))
|
||||||
|
for row in rows
|
||||||
|
]
|
||||||
|
|
||||||
return [
|
|
||||||
Symbol(name=row["name"], kind=row["kind"], range=(row["start_line"], row["end_line"]))
|
|
||||||
for row in rows
|
|
||||||
]
|
|
||||||
|
|
||||||
def stats(self) -> Dict[str, Any]:
|
def stats(self) -> Dict[str, Any]:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
with self._connect() as conn:
|
conn = self._get_connection()
|
||||||
file_count = conn.execute("SELECT COUNT(*) AS c FROM files").fetchone()["c"]
|
file_count = conn.execute("SELECT COUNT(*) AS c FROM files").fetchone()["c"]
|
||||||
symbol_count = conn.execute("SELECT COUNT(*) AS c FROM symbols").fetchone()["c"]
|
symbol_count = conn.execute("SELECT COUNT(*) AS c FROM symbols").fetchone()["c"]
|
||||||
lang_rows = conn.execute(
|
lang_rows = conn.execute(
|
||||||
"SELECT language, COUNT(*) AS c FROM files GROUP BY language ORDER BY c DESC"
|
"SELECT language, COUNT(*) AS c FROM files GROUP BY language ORDER BY c DESC"
|
||||||
).fetchall()
|
).fetchall()
|
||||||
languages = {row["language"]: row["c"] for row in lang_rows}
|
languages = {row["language"]: row["c"] for row in lang_rows}
|
||||||
return {
|
return {
|
||||||
"files": int(file_count),
|
"files": int(file_count),
|
||||||
"symbols": int(symbol_count),
|
"symbols": int(symbol_count),
|
||||||
"languages": languages,
|
"languages": languages,
|
||||||
"db_path": str(self.db_path),
|
"db_path": str(self.db_path),
|
||||||
}
|
}
|
||||||
|
|
||||||
def _connect(self) -> sqlite3.Connection:
|
def _connect(self) -> sqlite3.Connection:
|
||||||
conn = sqlite3.connect(self.db_path, check_same_thread=False)
|
"""Legacy method for backward compatibility."""
|
||||||
conn.row_factory = sqlite3.Row
|
return self._get_connection()
|
||||||
conn.execute("PRAGMA journal_mode=WAL")
|
|
||||||
conn.execute("PRAGMA synchronous=NORMAL")
|
|
||||||
return conn
|
|
||||||
|
|
||||||
def _create_schema(self, conn: sqlite3.Connection) -> None:
|
def _create_schema(self, conn: sqlite3.Connection) -> None:
|
||||||
try:
|
try:
|
||||||
@@ -224,15 +279,6 @@ class SQLiteStore:
|
|||||||
)
|
)
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
conn.execute(
|
|
||||||
"""
|
|
||||||
CREATE VIRTUAL TABLE IF NOT EXISTS files_fts USING fts5(
|
|
||||||
path UNINDEXED,
|
|
||||||
language UNINDEXED,
|
|
||||||
content
|
|
||||||
)
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"""
|
"""
|
||||||
CREATE TABLE IF NOT EXISTS symbols (
|
CREATE TABLE IF NOT EXISTS symbols (
|
||||||
@@ -247,6 +293,110 @@ class SQLiteStore:
|
|||||||
)
|
)
|
||||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols(name)")
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols(name)")
|
||||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_symbols_kind ON symbols(kind)")
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_symbols_kind ON symbols(kind)")
|
||||||
|
conn.commit()
|
||||||
except sqlite3.DatabaseError as exc:
|
except sqlite3.DatabaseError as exc:
|
||||||
raise StorageError(f"Failed to initialize database schema: {exc}") from exc
|
raise StorageError(f"Failed to initialize database schema: {exc}") from exc
|
||||||
|
|
||||||
|
def _ensure_fts_external_content(self, conn: sqlite3.Connection) -> None:
|
||||||
|
"""Ensure files_fts is an FTS5 external-content table (no content duplication)."""
|
||||||
|
try:
|
||||||
|
sql_row = conn.execute(
|
||||||
|
"SELECT sql FROM sqlite_master WHERE type='table' AND name='files_fts'"
|
||||||
|
).fetchone()
|
||||||
|
sql = str(sql_row["sql"]) if sql_row and sql_row["sql"] else None
|
||||||
|
|
||||||
|
if sql is None:
|
||||||
|
self._create_external_fts(conn)
|
||||||
|
conn.commit()
|
||||||
|
return
|
||||||
|
|
||||||
|
if (
|
||||||
|
"content='files'" in sql
|
||||||
|
or 'content="files"' in sql
|
||||||
|
or "content=files" in sql
|
||||||
|
):
|
||||||
|
self._create_fts_triggers(conn)
|
||||||
|
conn.commit()
|
||||||
|
return
|
||||||
|
|
||||||
|
self._migrate_fts_to_external(conn)
|
||||||
|
except sqlite3.DatabaseError as exc:
|
||||||
|
raise StorageError(f"Failed to ensure FTS schema: {exc}") from exc
|
||||||
|
|
||||||
|
def _create_external_fts(self, conn: sqlite3.Connection) -> None:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
CREATE VIRTUAL TABLE files_fts USING fts5(
|
||||||
|
path UNINDEXED,
|
||||||
|
language UNINDEXED,
|
||||||
|
content,
|
||||||
|
content='files',
|
||||||
|
content_rowid='id'
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
self._create_fts_triggers(conn)
|
||||||
|
|
||||||
|
def _create_fts_triggers(self, conn: sqlite3.Connection) -> None:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
CREATE TRIGGER IF NOT EXISTS files_ai AFTER INSERT ON files BEGIN
|
||||||
|
INSERT INTO files_fts(rowid, path, language, content)
|
||||||
|
VALUES(new.id, new.path, new.language, new.content);
|
||||||
|
END
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
CREATE TRIGGER IF NOT EXISTS files_ad AFTER DELETE ON files BEGIN
|
||||||
|
INSERT INTO files_fts(files_fts, rowid, path, language, content)
|
||||||
|
VALUES('delete', old.id, old.path, old.language, old.content);
|
||||||
|
END
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
CREATE TRIGGER IF NOT EXISTS files_au AFTER UPDATE ON files BEGIN
|
||||||
|
INSERT INTO files_fts(files_fts, rowid, path, language, content)
|
||||||
|
VALUES('delete', old.id, old.path, old.language, old.content);
|
||||||
|
INSERT INTO files_fts(rowid, path, language, content)
|
||||||
|
VALUES(new.id, new.path, new.language, new.content);
|
||||||
|
END
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
def _migrate_fts_to_external(self, conn: sqlite3.Connection) -> None:
|
||||||
|
"""Migrate legacy files_fts (with duplicated content) to external content."""
|
||||||
|
try:
|
||||||
|
conn.execute("BEGIN")
|
||||||
|
conn.execute("DROP TRIGGER IF EXISTS files_ai")
|
||||||
|
conn.execute("DROP TRIGGER IF EXISTS files_ad")
|
||||||
|
conn.execute("DROP TRIGGER IF EXISTS files_au")
|
||||||
|
|
||||||
|
conn.execute("ALTER TABLE files_fts RENAME TO files_fts_legacy")
|
||||||
|
self._create_external_fts(conn)
|
||||||
|
conn.execute("INSERT INTO files_fts(files_fts) VALUES('rebuild')")
|
||||||
|
conn.execute("DROP TABLE files_fts_legacy")
|
||||||
|
conn.commit()
|
||||||
|
except sqlite3.DatabaseError:
|
||||||
|
try:
|
||||||
|
conn.rollback()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
conn.execute("DROP TABLE IF EXISTS files_fts")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
conn.execute("ALTER TABLE files_fts_legacy RENAME TO files_fts")
|
||||||
|
conn.commit()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
raise
|
||||||
|
|
||||||
|
try:
|
||||||
|
conn.execute("VACUUM")
|
||||||
|
except sqlite3.DatabaseError:
|
||||||
|
pass
|
||||||
|
|||||||
1
codex-lens/tests/__init__.py
Normal file
1
codex-lens/tests/__init__.py
Normal file
@@ -0,0 +1 @@
|
|||||||
|
"""CodexLens test suite."""
|
||||||
160
codex-lens/tests/test_storage.py
Normal file
160
codex-lens/tests/test_storage.py
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
"""Tests for CodexLens storage."""
|
||||||
|
|
||||||
|
import sqlite3
|
||||||
|
import pytest
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from codexlens.storage.sqlite_store import SQLiteStore
|
||||||
|
from codexlens.entities import IndexedFile, Symbol
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def temp_db():
|
||||||
|
"""Create a temporary database for testing."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
db_path = Path(tmpdir) / "test.db"
|
||||||
|
store = SQLiteStore(db_path)
|
||||||
|
store.initialize()
|
||||||
|
yield store
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
class TestSQLiteStore:
|
||||||
|
"""Tests for SQLiteStore."""
|
||||||
|
|
||||||
|
def test_initialize(self, temp_db):
|
||||||
|
"""Test database initialization."""
|
||||||
|
stats = temp_db.stats()
|
||||||
|
assert stats["files"] == 0
|
||||||
|
assert stats["symbols"] == 0
|
||||||
|
|
||||||
|
def test_fts_uses_external_content(self, temp_db):
|
||||||
|
"""FTS should be configured as external-content to avoid duplication."""
|
||||||
|
conn = temp_db._get_connection()
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT sql FROM sqlite_master WHERE type='table' AND name='files_fts'"
|
||||||
|
).fetchone()
|
||||||
|
assert row is not None
|
||||||
|
assert "content='files'" in row["sql"] or "content=files" in row["sql"]
|
||||||
|
|
||||||
|
def test_add_file(self, temp_db):
|
||||||
|
"""Test adding a file to the index."""
|
||||||
|
indexed_file = IndexedFile(
|
||||||
|
path="/test/file.py",
|
||||||
|
language="python",
|
||||||
|
symbols=[
|
||||||
|
Symbol(name="hello", kind="function", range=(1, 1)),
|
||||||
|
],
|
||||||
|
chunks=[],
|
||||||
|
)
|
||||||
|
temp_db.add_file(indexed_file, "def hello():\n pass")
|
||||||
|
|
||||||
|
stats = temp_db.stats()
|
||||||
|
assert stats["files"] == 1
|
||||||
|
assert stats["symbols"] == 1
|
||||||
|
|
||||||
|
def test_remove_file(self, temp_db):
|
||||||
|
"""Test removing a file from the index."""
|
||||||
|
indexed_file = IndexedFile(
|
||||||
|
path="/test/file.py",
|
||||||
|
language="python",
|
||||||
|
symbols=[],
|
||||||
|
chunks=[],
|
||||||
|
)
|
||||||
|
temp_db.add_file(indexed_file, "# test")
|
||||||
|
|
||||||
|
assert temp_db.file_exists("/test/file.py")
|
||||||
|
assert temp_db.remove_file("/test/file.py")
|
||||||
|
assert not temp_db.file_exists("/test/file.py")
|
||||||
|
|
||||||
|
def test_search_fts(self, temp_db):
|
||||||
|
"""Test FTS search."""
|
||||||
|
indexed_file = IndexedFile(
|
||||||
|
path="/test/file.py",
|
||||||
|
language="python",
|
||||||
|
symbols=[],
|
||||||
|
chunks=[],
|
||||||
|
)
|
||||||
|
temp_db.add_file(indexed_file, "def hello_world():\n print('hello')")
|
||||||
|
|
||||||
|
results = temp_db.search_fts("hello")
|
||||||
|
assert len(results) == 1
|
||||||
|
assert str(Path("/test/file.py").resolve()) == results[0].path
|
||||||
|
|
||||||
|
def test_search_symbols(self, temp_db):
|
||||||
|
"""Test symbol search."""
|
||||||
|
indexed_file = IndexedFile(
|
||||||
|
path="/test/file.py",
|
||||||
|
language="python",
|
||||||
|
symbols=[
|
||||||
|
Symbol(name="hello_world", kind="function", range=(1, 1)),
|
||||||
|
Symbol(name="goodbye", kind="function", range=(3, 3)),
|
||||||
|
],
|
||||||
|
chunks=[],
|
||||||
|
)
|
||||||
|
temp_db.add_file(indexed_file, "def hello_world():\n pass\ndef goodbye():\n pass")
|
||||||
|
|
||||||
|
results = temp_db.search_symbols("hello")
|
||||||
|
assert len(results) == 1
|
||||||
|
assert results[0].name == "hello_world"
|
||||||
|
|
||||||
|
def test_connection_reuse(self, temp_db):
|
||||||
|
"""Test that connections are reused within the same thread."""
|
||||||
|
conn1 = temp_db._get_connection()
|
||||||
|
conn2 = temp_db._get_connection()
|
||||||
|
assert conn1 is conn2
|
||||||
|
|
||||||
|
def test_migrate_legacy_fts_to_external(self, tmp_path):
|
||||||
|
"""Existing databases should be migrated to external-content FTS."""
|
||||||
|
db_path = tmp_path / "legacy.db"
|
||||||
|
with sqlite3.connect(db_path) as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE files (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
path TEXT UNIQUE NOT NULL,
|
||||||
|
language TEXT NOT NULL,
|
||||||
|
content TEXT NOT NULL,
|
||||||
|
mtime REAL,
|
||||||
|
line_count INTEGER
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
CREATE VIRTUAL TABLE files_fts USING fts5(
|
||||||
|
path UNINDEXED,
|
||||||
|
language UNINDEXED,
|
||||||
|
content
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO files(path, language, content, mtime, line_count)
|
||||||
|
VALUES(?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
(str(Path("/test/file.py").resolve()), "python", "def hello():\n pass", None, 2),
|
||||||
|
)
|
||||||
|
file_id = conn.execute("SELECT id FROM files").fetchone()[0]
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO files_fts(rowid, path, language, content) VALUES(?, ?, ?, ?)",
|
||||||
|
(file_id, str(Path("/test/file.py").resolve()), "python", "def hello():\n pass"),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
store = SQLiteStore(db_path)
|
||||||
|
store.initialize()
|
||||||
|
try:
|
||||||
|
results = store.search_fts("hello")
|
||||||
|
assert len(results) == 1
|
||||||
|
|
||||||
|
conn = store._get_connection()
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT sql FROM sqlite_master WHERE type='table' AND name='files_fts'"
|
||||||
|
).fetchone()
|
||||||
|
assert row is not None
|
||||||
|
assert "content='files'" in row["sql"] or "content=files" in row["sql"]
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
Reference in New Issue
Block a user