Enhance search functionality and indexing pipeline

- Updated `cmd_search` to include line numbers and content in search results.
- Modified `IndexingPipeline` to handle start and end line numbers for chunks.
- Enhanced `FTSEngine` to support storing line metadata in the database.
- Improved `SearchPipeline` to return line numbers and full content in search results.
- Added unit tests for bridge, FTS delete operations, metadata store, and watcher functionality.
- Introduced a `.gitignore` file to exclude specific directories.
This commit is contained in:
catlog22
2026-03-17 14:55:27 +08:00
parent bfe5426b7e
commit 0f02b75be1
25 changed files with 2014 additions and 1482 deletions

View File

@@ -0,0 +1,152 @@
"""Unit tests for bridge.py CLI — argparse parsing, JSON protocol, error handling."""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
from unittest.mock import patch
import pytest
from codexlens_search.bridge import _build_parser, _json_output, _error_exit
# ---------------------------------------------------------------------------
# Parser construction
# ---------------------------------------------------------------------------
class TestParser:
@pytest.fixture(autouse=True)
def _parser(self):
self.parser = _build_parser()
def test_all_subcommands_exist(self):
expected = {
"init", "search", "index-file", "remove-file",
"sync", "watch", "download-models", "status",
}
# parse each subcommand with minimal required args to verify it exists
for cmd in expected:
if cmd == "search":
args = self.parser.parse_args(["search", "--query", "test"])
elif cmd == "index-file":
args = self.parser.parse_args(["index-file", "--file", "x.py"])
elif cmd == "remove-file":
args = self.parser.parse_args(["remove-file", "--file", "x.py"])
elif cmd == "sync":
args = self.parser.parse_args(["sync", "--root", "/tmp"])
elif cmd == "watch":
args = self.parser.parse_args(["watch", "--root", "/tmp"])
else:
args = self.parser.parse_args([cmd])
assert args.command == cmd
def test_global_db_path_default(self):
args = self.parser.parse_args(["status"])
assert args.db_path # has a default
def test_global_db_path_override(self):
args = self.parser.parse_args(["--db-path", "/custom/path", "status"])
assert args.db_path == "/custom/path"
def test_search_args(self):
args = self.parser.parse_args(["search", "-q", "hello", "-k", "5"])
assert args.query == "hello"
assert args.top_k == 5
def test_search_default_top_k(self):
args = self.parser.parse_args(["search", "--query", "test"])
assert args.top_k == 10
def test_sync_glob_default(self):
args = self.parser.parse_args(["sync", "--root", "/tmp"])
assert args.glob == "**/*"
def test_watch_debounce_default(self):
args = self.parser.parse_args(["watch", "--root", "/tmp"])
assert args.debounce_ms == 500
def test_no_command_returns_none(self):
args = self.parser.parse_args([])
assert args.command is None
# ---------------------------------------------------------------------------
# JSON output helpers
# ---------------------------------------------------------------------------
class TestJsonHelpers:
def test_json_output(self, capsys):
_json_output({"key": "value"})
out = capsys.readouterr().out.strip()
parsed = json.loads(out)
assert parsed == {"key": "value"}
def test_json_output_list(self, capsys):
_json_output([1, 2, 3])
out = capsys.readouterr().out.strip()
assert json.loads(out) == [1, 2, 3]
def test_json_output_unicode(self, capsys):
_json_output({"msg": "中文测试"})
out = capsys.readouterr().out.strip()
assert "中文测试" in out
def test_error_exit(self):
with pytest.raises(SystemExit) as exc_info:
_error_exit("something broke")
assert exc_info.value.code == 1
# ---------------------------------------------------------------------------
# cmd_init (lightweight, no model loading)
# ---------------------------------------------------------------------------
class TestCmdInit:
def test_init_creates_databases(self, tmp_path):
"""Init should create metadata.db and fts.db."""
from codexlens_search.bridge import cmd_init
import argparse
db_path = str(tmp_path / "test_idx")
args = argparse.Namespace(db_path=db_path, verbose=False)
cmd_init(args)
assert (Path(db_path) / "metadata.db").exists()
assert (Path(db_path) / "fts.db").exists()
# ---------------------------------------------------------------------------
# cmd_status (lightweight, no model loading)
# ---------------------------------------------------------------------------
class TestCmdStatus:
def test_status_not_initialized(self, tmp_path, capsys):
from codexlens_search.bridge import cmd_status
import argparse
db_path = str(tmp_path / "empty_idx")
Path(db_path).mkdir()
args = argparse.Namespace(db_path=db_path, verbose=False)
cmd_status(args)
out = json.loads(capsys.readouterr().out.strip())
assert out["status"] == "not_initialized"
def test_status_after_init(self, tmp_path, capsys):
from codexlens_search.bridge import cmd_init, cmd_status
import argparse
db_path = str(tmp_path / "idx")
args = argparse.Namespace(db_path=db_path, verbose=False)
cmd_init(args)
# Re-capture after init output
capsys.readouterr()
cmd_status(args)
out = json.loads(capsys.readouterr().out.strip())
assert out["status"] == "ok"
assert out["files_tracked"] == 0
assert out["deleted_chunks"] == 0

View File

@@ -0,0 +1,66 @@
"""Unit tests for FTSEngine delete_by_path and get_chunk_ids_by_path."""
from __future__ import annotations
import pytest
from codexlens_search.search.fts import FTSEngine
@pytest.fixture
def fts(tmp_path):
return FTSEngine(str(tmp_path / "fts.db"))
class TestGetChunkIdsByPath:
def test_empty(self, fts):
assert fts.get_chunk_ids_by_path("a.py") == []
def test_returns_matching_ids(self, fts):
fts.add_documents([
(0, "a.py", "hello world"),
(1, "a.py", "foo bar"),
(2, "b.py", "other content"),
])
ids = fts.get_chunk_ids_by_path("a.py")
assert sorted(ids) == [0, 1]
def test_no_match(self, fts):
fts.add_documents([(0, "a.py", "content")])
assert fts.get_chunk_ids_by_path("b.py") == []
class TestDeleteByPath:
def test_deletes_docs_and_meta(self, fts):
fts.add_documents([
(0, "target.py", "to be deleted"),
(1, "target.py", "also deleted"),
(2, "keep.py", "keep this"),
])
count = fts.delete_by_path("target.py")
assert count == 2
# target.py gone from both tables
assert fts.get_chunk_ids_by_path("target.py") == []
assert fts.get_content(0) == ""
assert fts.get_content(1) == ""
# keep.py still there
assert fts.get_chunk_ids_by_path("keep.py") == [2]
assert fts.get_content(2) == "keep this"
def test_delete_nonexistent_path(self, fts):
count = fts.delete_by_path("nonexistent.py")
assert count == 0
def test_delete_then_search(self, fts):
fts.add_documents([
(0, "a.py", "unique searchable content"),
(1, "b.py", "different content here"),
])
fts.delete_by_path("a.py")
results = fts.exact_search("unique searchable")
assert len(results) == 0
results = fts.exact_search("different")
assert len(results) == 1
assert results[0][0] == 1

View File

@@ -0,0 +1,184 @@
"""Unit tests for MetadataStore — SQLite file-to-chunk mapping + tombstone tracking."""
from __future__ import annotations
import pytest
from codexlens_search.indexing.metadata import MetadataStore
@pytest.fixture
def store(tmp_path):
"""Create a fresh MetadataStore backed by a temp db."""
return MetadataStore(str(tmp_path / "meta.db"))
# ---------------------------------------------------------------------------
# Table creation
# ---------------------------------------------------------------------------
class TestTableCreation:
def test_creates_three_tables(self, store):
"""MetadataStore should create files, chunks, deleted_chunks tables."""
tables = store._conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
).fetchall()
names = {r[0] for r in tables}
assert "files" in names
assert "chunks" in names
assert "deleted_chunks" in names
def test_foreign_keys_enabled(self, store):
"""PRAGMA foreign_keys must be ON."""
row = store._conn.execute("PRAGMA foreign_keys").fetchone()
assert row[0] == 1
def test_wal_mode(self, store):
"""journal_mode should be WAL for concurrency."""
row = store._conn.execute("PRAGMA journal_mode").fetchone()
assert row[0].lower() == "wal"
# ---------------------------------------------------------------------------
# register_file
# ---------------------------------------------------------------------------
class TestRegisterFile:
def test_register_and_retrieve(self, store):
store.register_file("src/main.py", "abc123", 1000.0)
assert store.get_file_hash("src/main.py") == "abc123"
def test_register_updates_existing(self, store):
store.register_file("a.py", "hash1", 1000.0)
store.register_file("a.py", "hash2", 2000.0)
assert store.get_file_hash("a.py") == "hash2"
def test_get_file_hash_returns_none_for_unknown(self, store):
assert store.get_file_hash("nonexistent.py") is None
# ---------------------------------------------------------------------------
# register_chunks
# ---------------------------------------------------------------------------
class TestRegisterChunks:
def test_register_and_retrieve_chunks(self, store):
store.register_file("a.py", "h", 1.0)
store.register_chunks("a.py", [(0, "c0"), (1, "c1"), (2, "c2")])
ids = store.get_chunk_ids_for_file("a.py")
assert sorted(ids) == [0, 1, 2]
def test_empty_chunks_list(self, store):
store.register_file("a.py", "h", 1.0)
store.register_chunks("a.py", [])
assert store.get_chunk_ids_for_file("a.py") == []
def test_chunks_for_unknown_file(self, store):
assert store.get_chunk_ids_for_file("unknown.py") == []
# ---------------------------------------------------------------------------
# mark_file_deleted
# ---------------------------------------------------------------------------
class TestMarkFileDeleted:
def test_tombstones_chunks(self, store):
store.register_file("a.py", "h", 1.0)
store.register_chunks("a.py", [(10, "c10"), (11, "c11")])
count = store.mark_file_deleted("a.py")
assert count == 2
assert store.get_deleted_ids() == {10, 11}
def test_file_removed_after_delete(self, store):
store.register_file("a.py", "h", 1.0)
store.register_chunks("a.py", [(0, "c0")])
store.mark_file_deleted("a.py")
assert store.get_file_hash("a.py") is None
def test_chunks_cascaded_after_delete(self, store):
store.register_file("a.py", "h", 1.0)
store.register_chunks("a.py", [(0, "c0")])
store.mark_file_deleted("a.py")
assert store.get_chunk_ids_for_file("a.py") == []
def test_delete_nonexistent_file(self, store):
count = store.mark_file_deleted("nonexistent.py")
assert count == 0
def test_delete_file_without_chunks(self, store):
store.register_file("empty.py", "h", 1.0)
count = store.mark_file_deleted("empty.py")
assert count == 0
assert store.get_file_hash("empty.py") is None
# ---------------------------------------------------------------------------
# file_needs_update
# ---------------------------------------------------------------------------
class TestFileNeedsUpdate:
def test_new_file_needs_update(self, store):
assert store.file_needs_update("new.py", "any_hash") is True
def test_unchanged_file(self, store):
store.register_file("a.py", "same_hash", 1.0)
assert store.file_needs_update("a.py", "same_hash") is False
def test_changed_file(self, store):
store.register_file("a.py", "old_hash", 1.0)
assert store.file_needs_update("a.py", "new_hash") is True
# ---------------------------------------------------------------------------
# get_deleted_ids / compact_deleted
# ---------------------------------------------------------------------------
class TestDeletedIdsAndCompact:
def test_empty_deleted_ids(self, store):
assert store.get_deleted_ids() == set()
def test_compact_returns_and_clears(self, store):
store.register_file("a.py", "h", 1.0)
store.register_chunks("a.py", [(5, "c5"), (6, "c6")])
store.mark_file_deleted("a.py")
deleted = store.compact_deleted()
assert deleted == {5, 6}
assert store.get_deleted_ids() == set()
def test_compact_noop_when_empty(self, store):
deleted = store.compact_deleted()
assert deleted == set()
# ---------------------------------------------------------------------------
# get_all_files / max_chunk_id
# ---------------------------------------------------------------------------
class TestHelpers:
def test_get_all_files(self, store):
store.register_file("a.py", "h1", 1.0)
store.register_file("b.py", "h2", 2.0)
assert store.get_all_files() == {"a.py": "h1", "b.py": "h2"}
def test_max_chunk_id_empty(self, store):
assert store.max_chunk_id() == -1
def test_max_chunk_id_active(self, store):
store.register_file("a.py", "h", 1.0)
store.register_chunks("a.py", [(0, "c"), (5, "c"), (3, "c")])
assert store.max_chunk_id() == 5
def test_max_chunk_id_includes_deleted(self, store):
store.register_file("a.py", "h", 1.0)
store.register_chunks("a.py", [(10, "c")])
store.mark_file_deleted("a.py")
assert store.max_chunk_id() == 10
def test_max_chunk_id_mixed(self, store):
store.register_file("a.py", "h", 1.0)
store.register_chunks("a.py", [(3, "c")])
store.register_file("b.py", "h2", 1.0)
store.register_chunks("b.py", [(7, "c")])
store.mark_file_deleted("a.py")
# deleted has 3, active has 7
assert store.max_chunk_id() == 7

View File

@@ -0,0 +1,270 @@
"""Unit tests for watcher module — events, FileWatcher debounce/dedup, IncrementalIndexer."""
from __future__ import annotations
import time
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from codexlens_search.watcher.events import ChangeType, FileEvent, WatcherConfig
from codexlens_search.watcher.incremental_indexer import BatchResult, IncrementalIndexer
# ---------------------------------------------------------------------------
# ChangeType enum
# ---------------------------------------------------------------------------
class TestChangeType:
def test_values(self):
assert ChangeType.CREATED.value == "created"
assert ChangeType.MODIFIED.value == "modified"
assert ChangeType.DELETED.value == "deleted"
def test_all_members(self):
assert len(ChangeType) == 3
# ---------------------------------------------------------------------------
# FileEvent
# ---------------------------------------------------------------------------
class TestFileEvent:
def test_creation(self):
e = FileEvent(path=Path("a.py"), change_type=ChangeType.CREATED)
assert e.path == Path("a.py")
assert e.change_type == ChangeType.CREATED
assert isinstance(e.timestamp, float)
def test_custom_timestamp(self):
e = FileEvent(path=Path("b.py"), change_type=ChangeType.DELETED, timestamp=42.0)
assert e.timestamp == 42.0
# ---------------------------------------------------------------------------
# WatcherConfig
# ---------------------------------------------------------------------------
class TestWatcherConfig:
def test_defaults(self):
cfg = WatcherConfig()
assert cfg.debounce_ms == 500
assert ".git" in cfg.ignored_patterns
assert "__pycache__" in cfg.ignored_patterns
assert "node_modules" in cfg.ignored_patterns
def test_custom(self):
cfg = WatcherConfig(debounce_ms=1000, ignored_patterns={".custom"})
assert cfg.debounce_ms == 1000
assert cfg.ignored_patterns == {".custom"}
# ---------------------------------------------------------------------------
# BatchResult
# ---------------------------------------------------------------------------
class TestBatchResult:
def test_defaults(self):
r = BatchResult()
assert r.files_indexed == 0
assert r.files_removed == 0
assert r.chunks_created == 0
assert r.errors == []
def test_total_processed(self):
r = BatchResult(files_indexed=3, files_removed=2)
assert r.total_processed == 5
def test_has_errors(self):
r = BatchResult()
assert r.has_errors is False
r.errors.append("oops")
assert r.has_errors is True
# ---------------------------------------------------------------------------
# IncrementalIndexer — event routing
# ---------------------------------------------------------------------------
class TestIncrementalIndexer:
@pytest.fixture
def mock_pipeline(self):
pipeline = MagicMock()
pipeline.index_file.return_value = MagicMock(
files_processed=1, chunks_created=3
)
return pipeline
def test_routes_created_to_index_file(self, mock_pipeline):
indexer = IncrementalIndexer(mock_pipeline, root=Path("/project"))
events = [
FileEvent(Path("/project/src/new.py"), ChangeType.CREATED),
]
result = indexer.process_events(events)
assert result.files_indexed == 1
mock_pipeline.index_file.assert_called_once()
# CREATED should NOT use force=True
call_kwargs = mock_pipeline.index_file.call_args
assert call_kwargs.kwargs.get("force", call_kwargs[1].get("force")) is False
def test_routes_modified_to_index_file_with_force(self, mock_pipeline):
indexer = IncrementalIndexer(mock_pipeline, root=Path("/project"))
events = [
FileEvent(Path("/project/src/changed.py"), ChangeType.MODIFIED),
]
result = indexer.process_events(events)
assert result.files_indexed == 1
call_kwargs = mock_pipeline.index_file.call_args
assert call_kwargs.kwargs.get("force", call_kwargs[1].get("force")) is True
def test_routes_deleted_to_remove_file(self, mock_pipeline, tmp_path):
root = tmp_path / "project"
root.mkdir()
indexer = IncrementalIndexer(mock_pipeline, root=root)
events = [
FileEvent(root / "src" / "old.py", ChangeType.DELETED),
]
result = indexer.process_events(events)
assert result.files_removed == 1
# On Windows relative_to produces backslashes, normalize
actual_arg = mock_pipeline.remove_file.call_args[0][0]
assert actual_arg.replace("\\", "/") == "src/old.py"
def test_batch_with_mixed_events(self, mock_pipeline):
indexer = IncrementalIndexer(mock_pipeline, root=Path("/project"))
events = [
FileEvent(Path("/project/a.py"), ChangeType.CREATED),
FileEvent(Path("/project/b.py"), ChangeType.MODIFIED),
FileEvent(Path("/project/c.py"), ChangeType.DELETED),
]
result = indexer.process_events(events)
assert result.files_indexed == 2
assert result.files_removed == 1
assert result.total_processed == 3
def test_error_isolation(self, mock_pipeline):
"""One file failure should not stop processing of others."""
call_count = [0]
def side_effect(*args, **kwargs):
call_count[0] += 1
if call_count[0] == 1:
raise RuntimeError("disk error")
return MagicMock(files_processed=1, chunks_created=1)
mock_pipeline.index_file.side_effect = side_effect
indexer = IncrementalIndexer(mock_pipeline, root=Path("/project"))
events = [
FileEvent(Path("/project/fail.py"), ChangeType.CREATED),
FileEvent(Path("/project/ok.py"), ChangeType.CREATED),
]
result = indexer.process_events(events)
assert result.files_indexed == 1 # second succeeded
assert len(result.errors) == 1 # first failed
assert "disk error" in result.errors[0]
def test_empty_events(self, mock_pipeline):
indexer = IncrementalIndexer(mock_pipeline)
result = indexer.process_events([])
assert result.total_processed == 0
mock_pipeline.index_file.assert_not_called()
mock_pipeline.remove_file.assert_not_called()
# ---------------------------------------------------------------------------
# FileWatcher — debounce and dedup logic (unit-level, no actual FS)
# ---------------------------------------------------------------------------
class TestFileWatcherLogic:
"""Test FileWatcher internals without starting a real watchdog Observer."""
@pytest.fixture
def watcher_parts(self):
"""Create a FileWatcher with mocked observer, capture callbacks."""
# Import here since watchdog is optional
from codexlens_search.watcher.file_watcher import FileWatcher, _EVENT_PRIORITY
collected = []
def on_changes(events):
collected.extend(events)
cfg = WatcherConfig(debounce_ms=100)
watcher = FileWatcher(Path("."), cfg, on_changes)
return watcher, collected, _EVENT_PRIORITY
def test_event_priority_ordering(self, watcher_parts):
_, _, priority = watcher_parts
assert priority[ChangeType.DELETED] > priority[ChangeType.MODIFIED]
assert priority[ChangeType.MODIFIED] > priority[ChangeType.CREATED]
def test_dedup_keeps_higher_priority(self, watcher_parts, tmp_path):
watcher, collected, _ = watcher_parts
f = str(tmp_path / "a.py")
watcher._on_raw_event(f, ChangeType.CREATED)
watcher._on_raw_event(f, ChangeType.DELETED)
watcher.flush_now()
assert len(collected) == 1
assert collected[0].change_type == ChangeType.DELETED
def test_dedup_does_not_downgrade(self, watcher_parts, tmp_path):
watcher, collected, _ = watcher_parts
f = str(tmp_path / "b.py")
watcher._on_raw_event(f, ChangeType.DELETED)
watcher._on_raw_event(f, ChangeType.CREATED)
watcher.flush_now()
assert len(collected) == 1
# CREATED (priority 1) < DELETED (priority 3), so DELETED stays
assert collected[0].change_type == ChangeType.DELETED
def test_multiple_files_kept(self, watcher_parts, tmp_path):
watcher, collected, _ = watcher_parts
watcher._on_raw_event(str(tmp_path / "a.py"), ChangeType.CREATED)
watcher._on_raw_event(str(tmp_path / "b.py"), ChangeType.MODIFIED)
watcher._on_raw_event(str(tmp_path / "c.py"), ChangeType.DELETED)
watcher.flush_now()
assert len(collected) == 3
paths = {str(e.path) for e in collected}
assert len(paths) == 3
def test_flush_clears_pending(self, watcher_parts, tmp_path):
watcher, collected, _ = watcher_parts
watcher._on_raw_event(str(tmp_path / "a.py"), ChangeType.CREATED)
watcher.flush_now()
assert len(collected) == 1
collected.clear()
watcher.flush_now()
assert len(collected) == 0
def test_should_watch_filters_ignored(self, watcher_parts):
watcher, _, _ = watcher_parts
assert watcher._should_watch(Path("/project/src/main.py")) is True
assert watcher._should_watch(Path("/project/.git/config")) is False
assert watcher._should_watch(Path("/project/node_modules/foo.js")) is False
assert watcher._should_watch(Path("/project/__pycache__/mod.pyc")) is False
def test_jsonl_serialization(self):
from codexlens_search.watcher.file_watcher import FileWatcher
import json
events = [
FileEvent(Path("/tmp/a.py"), ChangeType.CREATED, 1000.0),
FileEvent(Path("/tmp/b.py"), ChangeType.DELETED, 2000.0),
]
output = FileWatcher.events_to_jsonl(events)
lines = output.strip().split("\n")
assert len(lines) == 2
obj1 = json.loads(lines[0])
assert obj1["change_type"] == "created"
assert obj1["timestamp"] == 1000.0
obj2 = json.loads(lines[1])
assert obj2["change_type"] == "deleted"