mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-03-28 20:01:17 +08:00
feat: Add comprehensive tests for contentPattern and glob pattern matching
- Implemented final verification tests for contentPattern to validate behavior with empty strings, dangerous patterns, and normal patterns. - Created glob pattern matching tests to verify regex conversion and matching functionality. - Developed infinite loop risk tests using Worker threads to isolate potential blocking operations. - Introduced optimized contentPattern tests to validate improvements in the findMatches function. - Added verification tests to assess the effectiveness of contentPattern optimizations. - Conducted safety tests for contentPattern to identify edge cases and potential vulnerabilities. - Implemented unrestricted loop tests to analyze infinite loop risks without match limits. - Developed tests for zero-width pattern detection logic to ensure proper handling of dangerous regex patterns.
This commit is contained in:
catlog22
384
codex-lens/benchmarks/compare_staged_realtime_vs_dense_rerank.py
Normal file
384
codex-lens/benchmarks/compare_staged_realtime_vs_dense_rerank.py
Normal file
@@ -0,0 +1,384 @@
|
||||
#!/usr/bin/env python
|
||||
"""Compare staged realtime LSP pipeline vs direct dense->rerank cascade.
|
||||
|
||||
This benchmark compares two retrieval pipelines:
|
||||
1) staged+realtime: coarse (binary or dense fallback) -> realtime LSP graph expand -> clustering -> rerank
|
||||
2) dense_rerank: dense ANN coarse -> cross-encoder rerank
|
||||
|
||||
Because most repos do not have ground-truth labels, this script reports:
|
||||
- latency statistics
|
||||
- top-k overlap metrics (Jaccard + RBO)
|
||||
- diversity proxies (unique files/dirs)
|
||||
- staged pipeline stage stats (if present)
|
||||
|
||||
Usage:
|
||||
python benchmarks/compare_staged_realtime_vs_dense_rerank.py --source ./src
|
||||
python benchmarks/compare_staged_realtime_vs_dense_rerank.py --queries-file benchmarks/queries.txt
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
# Add src to path (match other benchmark scripts)
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||
|
||||
from codexlens.config import Config
|
||||
from codexlens.search.chain_search import ChainSearchEngine, SearchOptions
|
||||
from codexlens.storage.path_mapper import PathMapper
|
||||
from codexlens.storage.registry import RegistryStore
|
||||
|
||||
|
||||
DEFAULT_QUERIES = [
|
||||
"class Config",
|
||||
"def search",
|
||||
"LspBridge",
|
||||
"graph expansion",
|
||||
"clustering strategy",
|
||||
"error handling",
|
||||
"how to parse json",
|
||||
]
|
||||
|
||||
|
||||
def _now_ms() -> float:
|
||||
return time.perf_counter() * 1000.0
|
||||
|
||||
|
||||
def _safe_relpath(path: str, root: Path) -> str:
|
||||
try:
|
||||
return str(Path(path).resolve().relative_to(root.resolve()))
|
||||
except Exception:
|
||||
return path
|
||||
|
||||
|
||||
def _normalize_path_key(path: str) -> str:
|
||||
"""Normalize file paths for overlap/dedup metrics (Windows-safe)."""
|
||||
try:
|
||||
p = Path(path)
|
||||
# Don't explode on non-files like "<memory>".
|
||||
if str(p) and (p.is_absolute() or re.match(r"^[A-Za-z]:", str(p))):
|
||||
norm = str(p.resolve())
|
||||
else:
|
||||
norm = str(p)
|
||||
except Exception:
|
||||
norm = path
|
||||
norm = norm.replace("/", "\\")
|
||||
if os.name == "nt":
|
||||
norm = norm.lower()
|
||||
return norm
|
||||
|
||||
|
||||
def _extract_stage_stats(errors: List[str]) -> Optional[Dict[str, Any]]:
|
||||
"""Extract STAGE_STATS JSON blob from SearchStats.errors."""
|
||||
for item in errors or []:
|
||||
if not isinstance(item, str):
|
||||
continue
|
||||
if not item.startswith("STAGE_STATS:"):
|
||||
continue
|
||||
payload = item[len("STAGE_STATS:") :]
|
||||
try:
|
||||
return json.loads(payload)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def jaccard_topk(a: List[str], b: List[str]) -> float:
|
||||
sa, sb = set(a), set(b)
|
||||
if not sa and not sb:
|
||||
return 1.0
|
||||
if not sa or not sb:
|
||||
return 0.0
|
||||
return len(sa & sb) / len(sa | sb)
|
||||
|
||||
|
||||
def rbo(a: List[str], b: List[str], p: float = 0.9) -> float:
|
||||
"""Rank-biased overlap for two ranked lists."""
|
||||
if p <= 0.0 or p >= 1.0:
|
||||
raise ValueError("p must be in (0, 1)")
|
||||
if not a and not b:
|
||||
return 1.0
|
||||
|
||||
depth = max(len(a), len(b))
|
||||
seen_a: set[str] = set()
|
||||
seen_b: set[str] = set()
|
||||
|
||||
score = 0.0
|
||||
for d in range(1, depth + 1):
|
||||
if d <= len(a):
|
||||
seen_a.add(a[d - 1])
|
||||
if d <= len(b):
|
||||
seen_b.add(b[d - 1])
|
||||
overlap = len(seen_a & seen_b)
|
||||
score += (overlap / d) * ((1.0 - p) * (p ** (d - 1)))
|
||||
return score
|
||||
|
||||
|
||||
def _unique_parent_dirs(paths: Iterable[str]) -> int:
|
||||
dirs = set()
|
||||
for p in paths:
|
||||
try:
|
||||
dirs.add(str(Path(p).parent))
|
||||
except Exception:
|
||||
continue
|
||||
return len(dirs)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunDetail:
|
||||
strategy: str
|
||||
query: str
|
||||
latency_ms: float
|
||||
num_results: int
|
||||
topk_paths: List[str]
|
||||
stage_stats: Optional[Dict[str, Any]] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompareDetail:
|
||||
query: str
|
||||
staged: RunDetail
|
||||
dense_rerank: RunDetail
|
||||
jaccard_topk: float
|
||||
rbo_topk: float
|
||||
staged_unique_files_topk: int
|
||||
dense_unique_files_topk: int
|
||||
staged_unique_dirs_topk: int
|
||||
dense_unique_dirs_topk: int
|
||||
|
||||
|
||||
def _run_once(
|
||||
engine: ChainSearchEngine,
|
||||
query: str,
|
||||
source_path: Path,
|
||||
*,
|
||||
strategy: str,
|
||||
k: int,
|
||||
coarse_k: int,
|
||||
options: Optional[SearchOptions] = None,
|
||||
) -> RunDetail:
|
||||
gc.collect()
|
||||
start_ms = _now_ms()
|
||||
try:
|
||||
result = engine.cascade_search(
|
||||
query=query,
|
||||
source_path=source_path,
|
||||
k=k,
|
||||
coarse_k=coarse_k,
|
||||
options=options,
|
||||
strategy=strategy,
|
||||
)
|
||||
latency_ms = _now_ms() - start_ms
|
||||
paths_raw = [r.path for r in (result.results or []) if getattr(r, "path", None)]
|
||||
paths = [_normalize_path_key(p) for p in paths_raw]
|
||||
topk: List[str] = []
|
||||
seen: set[str] = set()
|
||||
for p in paths:
|
||||
if p in seen:
|
||||
continue
|
||||
seen.add(p)
|
||||
topk.append(p)
|
||||
if len(topk) >= k:
|
||||
break
|
||||
stage_stats = _extract_stage_stats(getattr(result.stats, "errors", []))
|
||||
return RunDetail(
|
||||
strategy=strategy,
|
||||
query=query,
|
||||
latency_ms=latency_ms,
|
||||
num_results=len(paths),
|
||||
topk_paths=topk,
|
||||
stage_stats=stage_stats,
|
||||
)
|
||||
except Exception as exc:
|
||||
latency_ms = _now_ms() - start_ms
|
||||
return RunDetail(
|
||||
strategy=strategy,
|
||||
query=query,
|
||||
latency_ms=latency_ms,
|
||||
num_results=0,
|
||||
topk_paths=[],
|
||||
stage_stats=None,
|
||||
error=repr(exc),
|
||||
)
|
||||
|
||||
|
||||
def _load_queries(path: Optional[Path], limit: Optional[int]) -> List[str]:
|
||||
if path is None:
|
||||
queries = list(DEFAULT_QUERIES)
|
||||
else:
|
||||
raw = path.read_text(encoding="utf-8", errors="ignore").splitlines()
|
||||
queries = []
|
||||
for line in raw:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
queries.append(line)
|
||||
if limit is not None:
|
||||
return queries[:limit]
|
||||
return queries
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Compare staged realtime LSP pipeline vs direct dense_rerank cascade"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--source",
|
||||
type=Path,
|
||||
default=Path(__file__).parent.parent / "src",
|
||||
help="Source directory to search (default: ./src)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--queries-file",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Optional file with one query per line (# comments supported)",
|
||||
)
|
||||
parser.add_argument("--queries", type=int, default=None, help="Limit number of queries")
|
||||
parser.add_argument("--k", type=int, default=10, help="Final result count (default 10)")
|
||||
parser.add_argument("--coarse-k", type=int, default=100, help="Coarse candidates (default 100)")
|
||||
parser.add_argument("--warmup", type=int, default=1, help="Warmup runs per strategy (default 1)")
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=Path(__file__).parent / "results" / "staged_realtime_vs_dense_rerank.json",
|
||||
help="Output JSON path",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.source.exists():
|
||||
raise SystemExit(f"Source path does not exist: {args.source}")
|
||||
|
||||
queries = _load_queries(args.queries_file, args.queries)
|
||||
if not queries:
|
||||
raise SystemExit("No queries to run")
|
||||
|
||||
# Match CLI behavior: load settings + apply global/workspace .env overrides.
|
||||
# This is important on Windows where ONNX/DirectML can sometimes crash under load;
|
||||
# many users pin EMBEDDING_BACKEND=litellm in ~/.codexlens/.env for stability.
|
||||
config = Config.load()
|
||||
config.cascade_strategy = "staged"
|
||||
config.staged_stage2_mode = "realtime"
|
||||
config.enable_staged_rerank = True
|
||||
# Stability: on some Windows setups, fastembed + DirectML can crash under load.
|
||||
# Dense_rerank uses the embedding backend that matches the index; force CPU here.
|
||||
config.embedding_use_gpu = False
|
||||
registry = RegistryStore()
|
||||
registry.initialize()
|
||||
mapper = PathMapper()
|
||||
engine = ChainSearchEngine(registry=registry, mapper=mapper, config=config)
|
||||
|
||||
try:
|
||||
strategies = ["staged", "dense_rerank"]
|
||||
|
||||
# Warmup
|
||||
if args.warmup > 0:
|
||||
warm_query = queries[0]
|
||||
for s in strategies:
|
||||
for _ in range(args.warmup):
|
||||
try:
|
||||
_run_once(
|
||||
engine,
|
||||
warm_query,
|
||||
args.source,
|
||||
strategy=s,
|
||||
k=min(args.k, 5),
|
||||
coarse_k=min(args.coarse_k, 50),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
comparisons: List[CompareDetail] = []
|
||||
|
||||
for i, query in enumerate(queries, start=1):
|
||||
print(f"[{i}/{len(queries)}] {query}")
|
||||
|
||||
staged = _run_once(
|
||||
engine,
|
||||
query,
|
||||
args.source,
|
||||
strategy="staged",
|
||||
k=args.k,
|
||||
coarse_k=args.coarse_k,
|
||||
)
|
||||
dense = _run_once(
|
||||
engine,
|
||||
query,
|
||||
args.source,
|
||||
strategy="dense_rerank",
|
||||
k=args.k,
|
||||
coarse_k=args.coarse_k,
|
||||
)
|
||||
|
||||
staged_paths = staged.topk_paths
|
||||
dense_paths = dense.topk_paths
|
||||
|
||||
comparisons.append(
|
||||
CompareDetail(
|
||||
query=query,
|
||||
staged=staged,
|
||||
dense_rerank=dense,
|
||||
jaccard_topk=jaccard_topk(staged_paths, dense_paths),
|
||||
rbo_topk=rbo(staged_paths, dense_paths, p=0.9),
|
||||
staged_unique_files_topk=len(set(staged_paths)),
|
||||
dense_unique_files_topk=len(set(dense_paths)),
|
||||
staged_unique_dirs_topk=_unique_parent_dirs(staged_paths),
|
||||
dense_unique_dirs_topk=_unique_parent_dirs(dense_paths),
|
||||
)
|
||||
)
|
||||
|
||||
def _latencies(details: List[RunDetail]) -> List[float]:
|
||||
return [d.latency_ms for d in details if not d.error]
|
||||
|
||||
staged_runs = [c.staged for c in comparisons]
|
||||
dense_runs = [c.dense_rerank for c in comparisons]
|
||||
|
||||
summary = {
|
||||
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"source": str(args.source),
|
||||
"k": args.k,
|
||||
"coarse_k": args.coarse_k,
|
||||
"query_count": len(comparisons),
|
||||
"avg_jaccard_topk": statistics.mean([c.jaccard_topk for c in comparisons]) if comparisons else 0.0,
|
||||
"avg_rbo_topk": statistics.mean([c.rbo_topk for c in comparisons]) if comparisons else 0.0,
|
||||
"staged": {
|
||||
"success": sum(1 for r in staged_runs if not r.error),
|
||||
"avg_latency_ms": statistics.mean(_latencies(staged_runs)) if _latencies(staged_runs) else 0.0,
|
||||
},
|
||||
"dense_rerank": {
|
||||
"success": sum(1 for r in dense_runs if not r.error),
|
||||
"avg_latency_ms": statistics.mean(_latencies(dense_runs)) if _latencies(dense_runs) else 0.0,
|
||||
},
|
||||
}
|
||||
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"summary": summary,
|
||||
"comparisons": [asdict(c) for c in comparisons],
|
||||
}
|
||||
args.output.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
||||
print(f"\nSaved: {args.output}")
|
||||
finally:
|
||||
try:
|
||||
engine.close()
|
||||
except Exception as exc:
|
||||
print(f"WARNING engine.close() failed: {exc!r}", file=sys.stderr)
|
||||
try:
|
||||
registry.close()
|
||||
except Exception as exc:
|
||||
print(f"WARNING registry.close() failed: {exc!r}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
453
codex-lens/benchmarks/results/compare_2026-02-09.json
Normal file
453
codex-lens/benchmarks/results/compare_2026-02-09.json
Normal file
@@ -0,0 +1,453 @@
|
||||
{
|
||||
"summary": {
|
||||
"timestamp": "2026-02-09 11:08:47",
|
||||
"source": "src",
|
||||
"k": 10,
|
||||
"coarse_k": 100,
|
||||
"query_count": 7,
|
||||
"avg_jaccard_topk": 0.41421235160730957,
|
||||
"avg_rbo_topk": 0.22899068093857142,
|
||||
"staged": {
|
||||
"success": 7,
|
||||
"avg_latency_ms": 32009.68328570468
|
||||
},
|
||||
"dense_rerank": {
|
||||
"success": 7,
|
||||
"avg_latency_ms": 2783.3305999977247
|
||||
}
|
||||
},
|
||||
"comparisons": [
|
||||
{
|
||||
"query": "class Config",
|
||||
"staged": {
|
||||
"strategy": "staged",
|
||||
"query": "class Config",
|
||||
"latency_ms": 40875.45489999652,
|
||||
"num_results": 10,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\api\\semantic.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\parsers\\factory.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\graph_expander.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\watcher\\file_watcher.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\lsp\\server.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\embedding_manager.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\api\\references.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\chain_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\__init__.py"
|
||||
],
|
||||
"stage_stats": {
|
||||
"stage_times": {
|
||||
"stage1_binary_ms": 10633.91399383545,
|
||||
"stage2_expand_ms": 12487.980365753174,
|
||||
"stage3_cluster_ms": 10781.587362289429,
|
||||
"stage4_rerank_ms": 6914.837837219238
|
||||
},
|
||||
"stage_counts": {
|
||||
"stage1_candidates": 100,
|
||||
"stage2_expanded": 149,
|
||||
"stage3_clustered": 20,
|
||||
"stage4_reranked": 20
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
"dense_rerank": {
|
||||
"strategy": "dense_rerank",
|
||||
"query": "class Config",
|
||||
"latency_ms": 3111.874899983406,
|
||||
"num_results": 10,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\chunker.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\vector_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\query_parser.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\code_extractor.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\embedding_manager.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\migration_manager.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\registry.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\sqlite_store.py"
|
||||
],
|
||||
"stage_stats": null,
|
||||
"error": null
|
||||
},
|
||||
"jaccard_topk": 0.1111111111111111,
|
||||
"rbo_topk": 0.06741929885142856,
|
||||
"staged_unique_files_topk": 10,
|
||||
"dense_unique_files_topk": 10,
|
||||
"staged_unique_dirs_topk": 8,
|
||||
"dense_unique_dirs_topk": 4
|
||||
},
|
||||
{
|
||||
"query": "def search",
|
||||
"staged": {
|
||||
"strategy": "staged",
|
||||
"query": "def search",
|
||||
"latency_ms": 38541.18510001898,
|
||||
"num_results": 10,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\global_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\ranking.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\ann_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\index_tree.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\graph_expander.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\enrichment.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\vector_store.py"
|
||||
],
|
||||
"stage_stats": {
|
||||
"stage_times": {
|
||||
"stage1_binary_ms": 548.8920211791992,
|
||||
"stage2_expand_ms": 27176.724433898926,
|
||||
"stage3_cluster_ms": 8352.917671203613,
|
||||
"stage4_rerank_ms": 2392.6541805267334
|
||||
},
|
||||
"stage_counts": {
|
||||
"stage1_candidates": 100,
|
||||
"stage2_expanded": 101,
|
||||
"stage3_clustered": 20,
|
||||
"stage4_reranked": 20
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
"dense_rerank": {
|
||||
"strategy": "dense_rerank",
|
||||
"query": "def search",
|
||||
"latency_ms": 2652.75,
|
||||
"num_results": 10,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\query_parser.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\vector_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\hybrid_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\index_tree.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\registry.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\code_extractor.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\chain_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\chunker.py"
|
||||
],
|
||||
"stage_stats": null,
|
||||
"error": null
|
||||
},
|
||||
"jaccard_topk": 0.26666666666666666,
|
||||
"rbo_topk": 0.2983708721671428,
|
||||
"staged_unique_files_topk": 9,
|
||||
"dense_unique_files_topk": 10,
|
||||
"staged_unique_dirs_topk": 4,
|
||||
"dense_unique_dirs_topk": 4
|
||||
},
|
||||
{
|
||||
"query": "LspBridge",
|
||||
"staged": {
|
||||
"strategy": "staged",
|
||||
"query": "LspBridge",
|
||||
"latency_ms": 26319.983999997377,
|
||||
"num_results": 10,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\vector_meta_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\graph_expander.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\registry.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\chunker.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\merkle_tree.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\code_extractor.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\sqlite_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\global_index.py"
|
||||
],
|
||||
"stage_stats": {
|
||||
"stage_times": {
|
||||
"stage1_binary_ms": 514.4834518432617,
|
||||
"stage2_expand_ms": 14329.241514205933,
|
||||
"stage3_cluster_ms": 9249.040842056274,
|
||||
"stage4_rerank_ms": 2159.9059104919434
|
||||
},
|
||||
"stage_counts": {
|
||||
"stage1_candidates": 100,
|
||||
"stage2_expanded": 100,
|
||||
"stage3_clustered": 20,
|
||||
"stage4_reranked": 20
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
"dense_rerank": {
|
||||
"strategy": "dense_rerank",
|
||||
"query": "LspBridge",
|
||||
"latency_ms": 2666.9745999872684,
|
||||
"num_results": 10,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\vector_meta_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\graph_expander.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\registry.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\hybrid_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\index_tree.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\sqlite_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\code_extractor.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\chunker.py"
|
||||
],
|
||||
"stage_stats": null,
|
||||
"error": null
|
||||
},
|
||||
"jaccard_topk": 0.6666666666666666,
|
||||
"rbo_topk": 0.3571430355128571,
|
||||
"staged_unique_files_topk": 10,
|
||||
"dense_unique_files_topk": 10,
|
||||
"staged_unique_dirs_topk": 4,
|
||||
"dense_unique_dirs_topk": 4
|
||||
},
|
||||
{
|
||||
"query": "graph expansion",
|
||||
"staged": {
|
||||
"strategy": "staged",
|
||||
"query": "graph expansion",
|
||||
"latency_ms": 25696.087299972773,
|
||||
"num_results": 10,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\chain_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\gpu_support.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\code_extractor.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\embedding_manager.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\index_tree.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\vector_meta_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\global_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\hybrid_search.py"
|
||||
],
|
||||
"stage_stats": {
|
||||
"stage_times": {
|
||||
"stage1_binary_ms": 560.4684352874756,
|
||||
"stage2_expand_ms": 13951.441526412964,
|
||||
"stage3_cluster_ms": 8879.387140274048,
|
||||
"stage4_rerank_ms": 2229.4514179229736
|
||||
},
|
||||
"stage_counts": {
|
||||
"stage1_candidates": 100,
|
||||
"stage2_expanded": 100,
|
||||
"stage3_clustered": 20,
|
||||
"stage4_reranked": 20
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
"dense_rerank": {
|
||||
"strategy": "dense_rerank",
|
||||
"query": "graph expansion",
|
||||
"latency_ms": 2544.8630999922752,
|
||||
"num_results": 10,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\index_tree.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\migration_manager.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\hybrid_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\chain_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\global_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\registry.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\sqlite_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\vector_store.py"
|
||||
],
|
||||
"stage_stats": null,
|
||||
"error": null
|
||||
},
|
||||
"jaccard_topk": 0.42857142857142855,
|
||||
"rbo_topk": 0.13728894791142857,
|
||||
"staged_unique_files_topk": 10,
|
||||
"dense_unique_files_topk": 10,
|
||||
"staged_unique_dirs_topk": 4,
|
||||
"dense_unique_dirs_topk": 4
|
||||
},
|
||||
{
|
||||
"query": "clustering strategy",
|
||||
"staged": {
|
||||
"strategy": "staged",
|
||||
"query": "clustering strategy",
|
||||
"latency_ms": 27387.41929998994,
|
||||
"num_results": 10,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\ranking.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\registry.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\global_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\embedding_manager.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\chunker.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\sqlite_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\__init__.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\vector_store.py"
|
||||
],
|
||||
"stage_stats": {
|
||||
"stage_times": {
|
||||
"stage1_binary_ms": 625.0262260437012,
|
||||
"stage2_expand_ms": 14211.347103118896,
|
||||
"stage3_cluster_ms": 10269.58680152893,
|
||||
"stage4_rerank_ms": 2208.007335662842
|
||||
},
|
||||
"stage_counts": {
|
||||
"stage1_candidates": 100,
|
||||
"stage2_expanded": 100,
|
||||
"stage3_clustered": 20,
|
||||
"stage4_reranked": 20
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
"dense_rerank": {
|
||||
"strategy": "dense_rerank",
|
||||
"query": "clustering strategy",
|
||||
"latency_ms": 2928.22389999032,
|
||||
"num_results": 10,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\index_tree.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\hybrid_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\code_extractor.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\vector_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\__init__.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\gpu_support.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\chain_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\enrichment.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\chunker.py"
|
||||
],
|
||||
"stage_stats": null,
|
||||
"error": null
|
||||
},
|
||||
"jaccard_topk": 0.17647058823529413,
|
||||
"rbo_topk": 0.07116480920571429,
|
||||
"staged_unique_files_topk": 10,
|
||||
"dense_unique_files_topk": 10,
|
||||
"staged_unique_dirs_topk": 4,
|
||||
"dense_unique_dirs_topk": 4
|
||||
},
|
||||
{
|
||||
"query": "error handling",
|
||||
"staged": {
|
||||
"strategy": "staged",
|
||||
"query": "error handling",
|
||||
"latency_ms": 23732.33979997039,
|
||||
"num_results": 10,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\enrichment.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\vector_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\hybrid_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\chain_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\registry.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\chunker.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\__init__.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\index_tree.py"
|
||||
],
|
||||
"stage_stats": {
|
||||
"stage_times": {
|
||||
"stage1_binary_ms": 504.0884017944336,
|
||||
"stage2_expand_ms": 12899.415016174316,
|
||||
"stage3_cluster_ms": 7881.027936935425,
|
||||
"stage4_rerank_ms": 2372.1535205841064
|
||||
},
|
||||
"stage_counts": {
|
||||
"stage1_candidates": 100,
|
||||
"stage2_expanded": 100,
|
||||
"stage3_clustered": 20,
|
||||
"stage4_reranked": 20
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
"dense_rerank": {
|
||||
"strategy": "dense_rerank",
|
||||
"query": "error handling",
|
||||
"latency_ms": 2946.439900010824,
|
||||
"num_results": 10,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\__init__.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\registry.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\hybrid_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\index_tree.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\chain_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\chunker.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\code_extractor.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\embedding_manager.py"
|
||||
],
|
||||
"stage_stats": null,
|
||||
"error": null
|
||||
},
|
||||
"jaccard_topk": 0.6666666666666666,
|
||||
"rbo_topk": 0.19158624676285715,
|
||||
"staged_unique_files_topk": 10,
|
||||
"dense_unique_files_topk": 10,
|
||||
"staged_unique_dirs_topk": 4,
|
||||
"dense_unique_dirs_topk": 4
|
||||
},
|
||||
{
|
||||
"query": "how to parse json",
|
||||
"staged": {
|
||||
"strategy": "staged",
|
||||
"query": "how to parse json",
|
||||
"latency_ms": 41515.31259998679,
|
||||
"num_results": 9,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\index_tree.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\code_extractor.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\enrichment.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\registry.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\ranking.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\chain_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\hybrid_search.py"
|
||||
],
|
||||
"stage_stats": {
|
||||
"stage_times": {
|
||||
"stage1_binary_ms": 601.7005443572998,
|
||||
"stage2_expand_ms": 30052.319765090942,
|
||||
"stage3_cluster_ms": 8409.791231155396,
|
||||
"stage4_rerank_ms": 2371.1729049682617
|
||||
},
|
||||
"stage_counts": {
|
||||
"stage1_candidates": 100,
|
||||
"stage2_expanded": 100,
|
||||
"stage3_clustered": 20,
|
||||
"stage4_reranked": 20
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
"dense_rerank": {
|
||||
"strategy": "dense_rerank",
|
||||
"query": "how to parse json",
|
||||
"latency_ms": 2632.1878000199795,
|
||||
"num_results": 10,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\chain_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\index_tree.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\code_extractor.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\hybrid_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\ranking.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\chunker.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\sqlite_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\ann_index.py"
|
||||
],
|
||||
"stage_stats": null,
|
||||
"error": null
|
||||
},
|
||||
"jaccard_topk": 0.5833333333333334,
|
||||
"rbo_topk": 0.4799615561585714,
|
||||
"staged_unique_files_topk": 9,
|
||||
"dense_unique_files_topk": 10,
|
||||
"staged_unique_dirs_topk": 4,
|
||||
"dense_unique_dirs_topk": 4
|
||||
}
|
||||
]
|
||||
}
|
||||
73
codex-lens/benchmarks/results/tmp_compare1.json
Normal file
73
codex-lens/benchmarks/results/tmp_compare1.json
Normal file
@@ -0,0 +1,73 @@
|
||||
{
|
||||
"summary": {
|
||||
"timestamp": "2026-02-08 23:48:26",
|
||||
"source": "src",
|
||||
"k": 5,
|
||||
"coarse_k": 50,
|
||||
"query_count": 1,
|
||||
"avg_jaccard_topk": 0.0,
|
||||
"avg_rbo_topk": 0.0,
|
||||
"staged": {
|
||||
"success": 1,
|
||||
"avg_latency_ms": 30093.97499999404
|
||||
},
|
||||
"dense_rerank": {
|
||||
"success": 1,
|
||||
"avg_latency_ms": 331.4424999952316
|
||||
}
|
||||
},
|
||||
"comparisons": [
|
||||
{
|
||||
"query": "class Config",
|
||||
"staged": {
|
||||
"strategy": "staged",
|
||||
"query": "class Config",
|
||||
"latency_ms": 30093.97499999404,
|
||||
"num_results": 5,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\__init__.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\api\\references.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\chain_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\embedding_manager.py"
|
||||
],
|
||||
"stage_stats": {
|
||||
"stage_times": {
|
||||
"stage1_binary_ms": 6421.706914901733,
|
||||
"stage2_expand_ms": 17591.988563537598,
|
||||
"stage3_cluster_ms": 3700.4549503326416,
|
||||
"stage4_rerank_ms": 2340.064525604248
|
||||
},
|
||||
"stage_counts": {
|
||||
"stage1_candidates": 50,
|
||||
"stage2_expanded": 99,
|
||||
"stage3_clustered": 10,
|
||||
"stage4_reranked": 10
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
"dense_rerank": {
|
||||
"strategy": "dense_rerank",
|
||||
"query": "class Config",
|
||||
"latency_ms": 331.4424999952316,
|
||||
"num_results": 5,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\migration_manager.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\registry.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\splade_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\sqlite_store.py"
|
||||
],
|
||||
"stage_stats": null,
|
||||
"error": null
|
||||
},
|
||||
"jaccard_topk": 0.0,
|
||||
"rbo_topk": 0.0,
|
||||
"staged_unique_files_topk": 5,
|
||||
"dense_unique_files_topk": 5,
|
||||
"staged_unique_dirs_topk": 4,
|
||||
"dense_unique_dirs_topk": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
177
codex-lens/benchmarks/results/tmp_compare3_ok_cpu.json
Normal file
177
codex-lens/benchmarks/results/tmp_compare3_ok_cpu.json
Normal file
@@ -0,0 +1,177 @@
|
||||
{
|
||||
"summary": {
|
||||
"timestamp": "2026-02-08 23:58:56",
|
||||
"source": "src",
|
||||
"k": 5,
|
||||
"coarse_k": 50,
|
||||
"query_count": 3,
|
||||
"avg_jaccard_topk": 0.11574074074074074,
|
||||
"avg_rbo_topk": 0.14601366666666662,
|
||||
"staged": {
|
||||
"success": 3,
|
||||
"avg_latency_ms": 27868.044033328693
|
||||
},
|
||||
"dense_rerank": {
|
||||
"success": 3,
|
||||
"avg_latency_ms": 1339.25289999942
|
||||
}
|
||||
},
|
||||
"comparisons": [
|
||||
{
|
||||
"query": "class Config",
|
||||
"staged": {
|
||||
"strategy": "staged",
|
||||
"query": "class Config",
|
||||
"latency_ms": 33643.06179998815,
|
||||
"num_results": 5,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\chain_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\embedding_manager.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\watcher\\incremental_indexer.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\lsp\\server.py"
|
||||
],
|
||||
"stage_stats": {
|
||||
"stage_times": {
|
||||
"stage1_binary_ms": 6201.4524936676025,
|
||||
"stage2_expand_ms": 17306.61702156067,
|
||||
"stage3_cluster_ms": 6829.557418823242,
|
||||
"stage4_rerank_ms": 3267.071485519409
|
||||
},
|
||||
"stage_counts": {
|
||||
"stage1_candidates": 50,
|
||||
"stage2_expanded": 99,
|
||||
"stage3_clustered": 10,
|
||||
"stage4_reranked": 10
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
"dense_rerank": {
|
||||
"strategy": "dense_rerank",
|
||||
"query": "class Config",
|
||||
"latency_ms": 1520.9955999851227,
|
||||
"num_results": 5,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\migration_manager.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\registry.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\splade_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\sqlite_store.py"
|
||||
],
|
||||
"stage_stats": null,
|
||||
"error": null
|
||||
},
|
||||
"jaccard_topk": 0.1111111111111111,
|
||||
"rbo_topk": 0.031347,
|
||||
"staged_unique_files_topk": 5,
|
||||
"dense_unique_files_topk": 5,
|
||||
"staged_unique_dirs_topk": 5,
|
||||
"dense_unique_dirs_topk": 1
|
||||
},
|
||||
{
|
||||
"query": "def search",
|
||||
"staged": {
|
||||
"strategy": "staged",
|
||||
"query": "def search",
|
||||
"latency_ms": 26400.58900000155,
|
||||
"num_results": 5,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\vector_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\index_tree.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\code_extractor.py"
|
||||
],
|
||||
"stage_stats": {
|
||||
"stage_times": {
|
||||
"stage1_binary_ms": 404.60920333862305,
|
||||
"stage2_expand_ms": 20036.258697509766,
|
||||
"stage3_cluster_ms": 4919.439315795898,
|
||||
"stage4_rerank_ms": 1001.8632411956787
|
||||
},
|
||||
"stage_counts": {
|
||||
"stage1_candidates": 50,
|
||||
"stage2_expanded": 51,
|
||||
"stage3_clustered": 10,
|
||||
"stage4_reranked": 10
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
"dense_rerank": {
|
||||
"strategy": "dense_rerank",
|
||||
"query": "def search",
|
||||
"latency_ms": 1264.3862999975681,
|
||||
"num_results": 5,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\chain_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\graph_expander.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\hybrid_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\ranking.py"
|
||||
],
|
||||
"stage_stats": null,
|
||||
"error": null
|
||||
},
|
||||
"jaccard_topk": 0.125,
|
||||
"rbo_topk": 0.20334699999999994,
|
||||
"staged_unique_files_topk": 4,
|
||||
"dense_unique_files_topk": 5,
|
||||
"staged_unique_dirs_topk": 3,
|
||||
"dense_unique_dirs_topk": 2
|
||||
},
|
||||
{
|
||||
"query": "LspBridge",
|
||||
"staged": {
|
||||
"strategy": "staged",
|
||||
"query": "LspBridge",
|
||||
"latency_ms": 23560.481299996376,
|
||||
"num_results": 5,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\vector_meta_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\code_extractor.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\enrichment.py"
|
||||
],
|
||||
"stage_stats": {
|
||||
"stage_times": {
|
||||
"stage1_binary_ms": 385.28990745544434,
|
||||
"stage2_expand_ms": 17787.648677825928,
|
||||
"stage3_cluster_ms": 4374.642372131348,
|
||||
"stage4_rerank_ms": 974.8115539550781
|
||||
},
|
||||
"stage_counts": {
|
||||
"stage1_candidates": 50,
|
||||
"stage2_expanded": 50,
|
||||
"stage3_clustered": 10,
|
||||
"stage4_reranked": 10
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
"dense_rerank": {
|
||||
"strategy": "dense_rerank",
|
||||
"query": "LspBridge",
|
||||
"latency_ms": 1232.3768000155687,
|
||||
"num_results": 5,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\global_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\index_tree.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\path_mapper.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\registry.py"
|
||||
],
|
||||
"stage_stats": null,
|
||||
"error": null
|
||||
},
|
||||
"jaccard_topk": 0.1111111111111111,
|
||||
"rbo_topk": 0.20334699999999994,
|
||||
"staged_unique_files_topk": 5,
|
||||
"dense_unique_files_topk": 5,
|
||||
"staged_unique_dirs_topk": 4,
|
||||
"dense_unique_dirs_topk": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
176
codex-lens/benchmarks/results/tmp_compare3_ok_cpu_dedup.json
Normal file
176
codex-lens/benchmarks/results/tmp_compare3_ok_cpu_dedup.json
Normal file
@@ -0,0 +1,176 @@
|
||||
{
|
||||
"summary": {
|
||||
"timestamp": "2026-02-09 00:08:47",
|
||||
"source": "src",
|
||||
"k": 5,
|
||||
"coarse_k": 50,
|
||||
"query_count": 3,
|
||||
"avg_jaccard_topk": 0.11574074074074074,
|
||||
"avg_rbo_topk": 0.14601366666666662,
|
||||
"staged": {
|
||||
"success": 3,
|
||||
"avg_latency_ms": 31720.555866663653
|
||||
},
|
||||
"dense_rerank": {
|
||||
"success": 3,
|
||||
"avg_latency_ms": 1401.2113333245118
|
||||
}
|
||||
},
|
||||
"comparisons": [
|
||||
{
|
||||
"query": "class Config",
|
||||
"staged": {
|
||||
"strategy": "staged",
|
||||
"query": "class Config",
|
||||
"latency_ms": 40162.88519999385,
|
||||
"num_results": 5,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\embedding_manager.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\watcher\\incremental_indexer.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\chain_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\lsp\\server.py"
|
||||
],
|
||||
"stage_stats": {
|
||||
"stage_times": {
|
||||
"stage1_binary_ms": 6091.366767883301,
|
||||
"stage2_expand_ms": 17540.942907333374,
|
||||
"stage3_cluster_ms": 13169.558048248291,
|
||||
"stage4_rerank_ms": 3317.5392150878906
|
||||
},
|
||||
"stage_counts": {
|
||||
"stage1_candidates": 50,
|
||||
"stage2_expanded": 99,
|
||||
"stage3_clustered": 10,
|
||||
"stage4_reranked": 10
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
"dense_rerank": {
|
||||
"strategy": "dense_rerank",
|
||||
"query": "class Config",
|
||||
"latency_ms": 1571.1398999989033,
|
||||
"num_results": 5,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\migration_manager.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\registry.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\splade_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\sqlite_store.py"
|
||||
],
|
||||
"stage_stats": null,
|
||||
"error": null
|
||||
},
|
||||
"jaccard_topk": 0.1111111111111111,
|
||||
"rbo_topk": 0.031347,
|
||||
"staged_unique_files_topk": 5,
|
||||
"dense_unique_files_topk": 5,
|
||||
"staged_unique_dirs_topk": 5,
|
||||
"dense_unique_dirs_topk": 1
|
||||
},
|
||||
{
|
||||
"query": "def search",
|
||||
"staged": {
|
||||
"strategy": "staged",
|
||||
"query": "def search",
|
||||
"latency_ms": 31623.380899995565,
|
||||
"num_results": 5,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\vector_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\index_tree.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\code_extractor.py"
|
||||
],
|
||||
"stage_stats": {
|
||||
"stage_times": {
|
||||
"stage1_binary_ms": 400.84290504455566,
|
||||
"stage2_expand_ms": 20529.58631515503,
|
||||
"stage3_cluster_ms": 9625.348806381226,
|
||||
"stage4_rerank_ms": 1027.686357498169
|
||||
},
|
||||
"stage_counts": {
|
||||
"stage1_candidates": 50,
|
||||
"stage2_expanded": 51,
|
||||
"stage3_clustered": 10,
|
||||
"stage4_reranked": 10
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
"dense_rerank": {
|
||||
"strategy": "dense_rerank",
|
||||
"query": "def search",
|
||||
"latency_ms": 1376.3304999768734,
|
||||
"num_results": 5,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\chain_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\graph_expander.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\hybrid_search.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\ranking.py"
|
||||
],
|
||||
"stage_stats": null,
|
||||
"error": null
|
||||
},
|
||||
"jaccard_topk": 0.125,
|
||||
"rbo_topk": 0.20334699999999994,
|
||||
"staged_unique_files_topk": 4,
|
||||
"dense_unique_files_topk": 5,
|
||||
"staged_unique_dirs_topk": 3,
|
||||
"dense_unique_dirs_topk": 2
|
||||
},
|
||||
{
|
||||
"query": "LspBridge",
|
||||
"staged": {
|
||||
"strategy": "staged",
|
||||
"query": "LspBridge",
|
||||
"latency_ms": 23375.40150000155,
|
||||
"num_results": 5,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\cli\\commands.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\vector_meta_store.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\semantic\\code_extractor.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\search\\enrichment.py"
|
||||
],
|
||||
"stage_stats": {
|
||||
"stage_times": {
|
||||
"stage1_binary_ms": 392.41671562194824,
|
||||
"stage2_expand_ms": 17760.897397994995,
|
||||
"stage3_cluster_ms": 4194.235563278198,
|
||||
"stage4_rerank_ms": 990.307092666626
|
||||
},
|
||||
"stage_counts": {
|
||||
"stage1_candidates": 50,
|
||||
"stage2_expanded": 50,
|
||||
"stage3_clustered": 10,
|
||||
"stage4_reranked": 10
|
||||
}
|
||||
},
|
||||
"error": null
|
||||
},
|
||||
"dense_rerank": {
|
||||
"strategy": "dense_rerank",
|
||||
"query": "LspBridge",
|
||||
"latency_ms": 1256.1635999977589,
|
||||
"num_results": 5,
|
||||
"topk_paths": [
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\dir_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\global_index.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\index_tree.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\path_mapper.py",
|
||||
"d:\\claude_dms3\\codex-lens\\src\\codexlens\\storage\\registry.py"
|
||||
],
|
||||
"stage_stats": null,
|
||||
"error": null
|
||||
},
|
||||
"jaccard_topk": 0.1111111111111111,
|
||||
"rbo_topk": 0.20334699999999994,
|
||||
"staged_unique_files_topk": 5,
|
||||
"dense_unique_files_topk": 5,
|
||||
"staged_unique_dirs_topk": 4,
|
||||
"dense_unique_dirs_topk": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1101,6 +1101,140 @@ def lsp_status(
|
||||
console.print(f" Initialized: {probe.get('initialized')}")
|
||||
|
||||
|
||||
@app.command(name="reranker-status")
|
||||
def reranker_status(
|
||||
probe: bool = typer.Option(
|
||||
False,
|
||||
"--probe",
|
||||
help="Send a small rerank request to validate connectivity and credentials.",
|
||||
),
|
||||
provider: Optional[str] = typer.Option(
|
||||
None,
|
||||
"--provider",
|
||||
help="Reranker provider: siliconflow | cohere | jina (default: from env, else siliconflow).",
|
||||
),
|
||||
api_base: Optional[str] = typer.Option(
|
||||
None,
|
||||
"--api-base",
|
||||
help="Override API base URL (e.g. https://api.siliconflow.cn or https://api.cohere.ai).",
|
||||
),
|
||||
model: Optional[str] = typer.Option(
|
||||
None,
|
||||
"--model",
|
||||
help="Override reranker model name (provider-specific).",
|
||||
),
|
||||
query: str = typer.Option("ping", "--query", help="Probe query text (used with --probe)."),
|
||||
document: str = typer.Option("pong", "--document", help="Probe document text (used with --probe)."),
|
||||
json_mode: bool = typer.Option(False, "--json", help="Output JSON response."),
|
||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
|
||||
) -> None:
|
||||
"""Show reranker configuration and optionally probe the API backend.
|
||||
|
||||
This is the fastest way to confirm that "重排" can actually execute end-to-end.
|
||||
"""
|
||||
_configure_logging(verbose, json_mode)
|
||||
|
||||
import time
|
||||
|
||||
from codexlens.env_config import load_global_env
|
||||
from codexlens.semantic.reranker.api_reranker import (
|
||||
APIReranker,
|
||||
_normalize_api_base_for_endpoint,
|
||||
)
|
||||
|
||||
env = load_global_env()
|
||||
|
||||
def _env_get(key: str) -> Optional[str]:
|
||||
return (
|
||||
os.environ.get(key)
|
||||
or os.environ.get(f"CODEXLENS_{key}")
|
||||
or env.get(key)
|
||||
or env.get(f"CODEXLENS_{key}")
|
||||
)
|
||||
|
||||
effective_provider = (provider or _env_get("RERANKER_PROVIDER") or "siliconflow").strip()
|
||||
effective_api_base = (api_base or _env_get("RERANKER_API_BASE") or "").strip() or None
|
||||
effective_model = (model or _env_get("RERANKER_MODEL") or "").strip() or None
|
||||
|
||||
# Do not leak secrets; only report whether a key is configured.
|
||||
key_present = bool((_env_get("RERANKER_API_KEY") or "").strip())
|
||||
|
||||
provider_key = effective_provider.strip().lower()
|
||||
defaults = getattr(APIReranker, "_PROVIDER_DEFAULTS", {}).get(provider_key, {})
|
||||
endpoint = defaults.get("endpoint", "/v1/rerank")
|
||||
configured_base = effective_api_base or defaults.get("api_base") or ""
|
||||
normalized_base = _normalize_api_base_for_endpoint(api_base=configured_base, endpoint=endpoint)
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"provider": effective_provider,
|
||||
"api_base": effective_api_base,
|
||||
"endpoint": endpoint,
|
||||
"normalized_api_base": normalized_base or None,
|
||||
"request_url": f"{normalized_base}{endpoint}" if normalized_base else None,
|
||||
"model": effective_model,
|
||||
"api_key_configured": key_present,
|
||||
"probe": None,
|
||||
}
|
||||
|
||||
if probe:
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
reranker = APIReranker(
|
||||
provider=effective_provider,
|
||||
api_base=effective_api_base,
|
||||
model_name=effective_model,
|
||||
)
|
||||
try:
|
||||
scores = reranker.score_pairs([(query, document)])
|
||||
finally:
|
||||
reranker.close()
|
||||
resolved_base = getattr(reranker, "api_base", None)
|
||||
resolved_endpoint = getattr(reranker, "endpoint", None)
|
||||
request_url = (
|
||||
f"{resolved_base}{resolved_endpoint}"
|
||||
if resolved_base and resolved_endpoint
|
||||
else None
|
||||
)
|
||||
payload["probe"] = {
|
||||
"ok": True,
|
||||
"latency_ms": (time.perf_counter() - t0) * 1000.0,
|
||||
"score": float(scores[0]) if scores else None,
|
||||
"normalized_api_base": resolved_base,
|
||||
"request_url": request_url,
|
||||
}
|
||||
except Exception as exc:
|
||||
payload["probe"] = {
|
||||
"ok": False,
|
||||
"latency_ms": (time.perf_counter() - t0) * 1000.0,
|
||||
"error": f"{type(exc).__name__}: {exc}",
|
||||
}
|
||||
|
||||
if json_mode:
|
||||
print_json(success=True, result=payload)
|
||||
return
|
||||
|
||||
console.print("[bold]CodexLens Reranker Status[/bold]")
|
||||
console.print(f" Provider: {payload['provider']}")
|
||||
console.print(f" API Base: {payload['api_base'] or '(default)'}")
|
||||
if payload.get("normalized_api_base"):
|
||||
console.print(f" API Base (normalized): {payload['normalized_api_base']}")
|
||||
console.print(f" Endpoint: {payload.get('endpoint')}")
|
||||
if payload.get("request_url"):
|
||||
console.print(f" Request URL: {payload['request_url']}")
|
||||
console.print(f" Model: {payload['model'] or '(default)'}")
|
||||
console.print(f" API Key: {'set' if key_present else 'missing'}")
|
||||
|
||||
if payload["probe"] is not None:
|
||||
probe_payload = payload["probe"]
|
||||
console.print("\n[bold]Probe:[/bold]")
|
||||
if probe_payload.get("ok"):
|
||||
console.print(f" ✓ OK ({probe_payload.get('latency_ms'):.1f}ms)")
|
||||
console.print(f" Score: {probe_payload.get('score')}")
|
||||
else:
|
||||
console.print(f" ✗ Failed ({probe_payload.get('latency_ms'):.1f}ms)")
|
||||
console.print(f" {probe_payload.get('error')}")
|
||||
|
||||
|
||||
@app.command()
|
||||
def projects(
|
||||
action: str = typer.Argument("list", help="Action: list, show, remove"),
|
||||
|
||||
@@ -79,11 +79,33 @@ class HDBSCANStrategy(BaseClusteringStrategy):
|
||||
# Return each result as its own singleton cluster
|
||||
return [[i] for i in range(n_results)]
|
||||
|
||||
metric = self.config.metric
|
||||
data = embeddings
|
||||
|
||||
# Some hdbscan builds do not recognize metric="cosine" even though it's a
|
||||
# common need for embedding clustering. In that case, compute a precomputed
|
||||
# cosine distance matrix and run HDBSCAN with metric="precomputed".
|
||||
if metric == "cosine":
|
||||
try:
|
||||
from sklearn.metrics import pairwise_distances
|
||||
|
||||
data = pairwise_distances(embeddings, metric="cosine")
|
||||
# Some hdbscan builds are strict about dtype for precomputed distances.
|
||||
# Ensure float64 to avoid Buffer dtype mismatch errors.
|
||||
try:
|
||||
data = data.astype("float64", copy=False)
|
||||
except Exception:
|
||||
pass
|
||||
metric = "precomputed"
|
||||
except Exception:
|
||||
# If we cannot compute distances, fall back to euclidean over raw vectors.
|
||||
metric = "euclidean"
|
||||
|
||||
# Configure HDBSCAN clusterer
|
||||
clusterer = hdbscan.HDBSCAN(
|
||||
min_cluster_size=self.config.min_cluster_size,
|
||||
min_samples=self.config.min_samples,
|
||||
metric=self.config.metric,
|
||||
metric=metric,
|
||||
cluster_selection_epsilon=self.config.cluster_selection_epsilon,
|
||||
allow_single_cluster=self.config.allow_single_cluster,
|
||||
prediction_data=self.config.prediction_data,
|
||||
@@ -91,7 +113,7 @@ class HDBSCANStrategy(BaseClusteringStrategy):
|
||||
|
||||
# Fit and get cluster labels
|
||||
# Labels: -1 = noise, 0+ = cluster index
|
||||
labels = clusterer.fit_predict(embeddings)
|
||||
labels = clusterer.fit_predict(data)
|
||||
|
||||
# Group indices by cluster label
|
||||
cluster_map: dict[int, list[int]] = {}
|
||||
|
||||
@@ -22,16 +22,52 @@ logger = logging.getLogger(__name__)
|
||||
_DEFAULT_ENV_API_KEY = "RERANKER_API_KEY"
|
||||
|
||||
|
||||
def _normalize_api_base_for_endpoint(*, api_base: str, endpoint: str) -> str:
|
||||
"""Normalize api_base to avoid duplicated version paths (e.g. /v1/v1/...).
|
||||
|
||||
httpx joins base_url paths with request paths even when the request path
|
||||
starts with a leading slash. This means:
|
||||
|
||||
base_url="https://host/v1" + endpoint="/v1/rerank"
|
||||
-> "https://host/v1/v1/rerank"
|
||||
|
||||
Many users configure OpenAI-style bases with a trailing "/v1", so we
|
||||
defensively strip that suffix when the endpoint already includes "/v1/".
|
||||
"""
|
||||
cleaned = (api_base or "").strip().rstrip("/")
|
||||
if not cleaned:
|
||||
return cleaned
|
||||
|
||||
endpoint_clean = endpoint or ""
|
||||
|
||||
# If api_base already includes the endpoint suffix (e.g. api_base ends with "/v1/rerank"),
|
||||
# strip it so we don't end up with ".../v1/rerank/v1/rerank".
|
||||
if endpoint_clean.startswith("/") and cleaned.lower().endswith(endpoint_clean.lower()):
|
||||
return cleaned[: -len(endpoint_clean)]
|
||||
|
||||
# Strip a trailing "/v1" if endpoint already includes "/v1/...".
|
||||
if endpoint_clean.startswith("/v1/") and cleaned.lower().endswith("/v1"):
|
||||
return cleaned[:-3]
|
||||
|
||||
return cleaned
|
||||
|
||||
|
||||
def _get_env_with_fallback(key: str, workspace_root: Path | None = None) -> str | None:
|
||||
"""Get environment variable with .env file fallback."""
|
||||
# Check os.environ first
|
||||
if key in os.environ:
|
||||
return os.environ[key]
|
||||
prefixed_key = f"CODEXLENS_{key}"
|
||||
if prefixed_key in os.environ:
|
||||
return os.environ[prefixed_key]
|
||||
|
||||
# Try loading from .env files
|
||||
try:
|
||||
from codexlens.env_config import get_env
|
||||
return get_env(key, workspace_root=workspace_root)
|
||||
value = get_env(key, workspace_root=workspace_root)
|
||||
if value is not None:
|
||||
return value
|
||||
return get_env(prefixed_key, workspace_root=workspace_root)
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
@@ -99,8 +135,11 @@ class APIReranker(BaseReranker):
|
||||
|
||||
# Load api_base from env with .env fallback
|
||||
env_api_base = _get_env_with_fallback("RERANKER_API_BASE", self._workspace_root)
|
||||
self.api_base = (api_base or env_api_base or defaults["api_base"]).strip().rstrip("/")
|
||||
self.endpoint = defaults["endpoint"]
|
||||
self.api_base = _normalize_api_base_for_endpoint(
|
||||
api_base=(api_base or env_api_base or defaults["api_base"]),
|
||||
endpoint=self.endpoint,
|
||||
)
|
||||
|
||||
# Load model from env with .env fallback
|
||||
env_model = _get_env_with_fallback("RERANKER_MODEL", self._workspace_root)
|
||||
|
||||
@@ -72,7 +72,10 @@ def httpx_clients(monkeypatch: pytest.MonkeyPatch) -> list[DummyClient]:
|
||||
def test_api_reranker_requires_api_key(
|
||||
monkeypatch: pytest.MonkeyPatch, httpx_clients: list[DummyClient]
|
||||
) -> None:
|
||||
monkeypatch.delenv("RERANKER_API_KEY", raising=False)
|
||||
# Force empty key in-process so the reranker does not fall back to any
|
||||
# workspace/global .env configuration that may exist on the machine.
|
||||
monkeypatch.setenv("RERANKER_API_KEY", "")
|
||||
monkeypatch.setenv("CODEXLENS_RERANKER_API_KEY", "")
|
||||
|
||||
with pytest.raises(ValueError, match="Missing API key"):
|
||||
APIReranker()
|
||||
@@ -92,10 +95,37 @@ def test_api_reranker_reads_api_key_from_env(
|
||||
assert httpx_clients[0].closed is True
|
||||
|
||||
|
||||
def test_api_reranker_strips_v1_from_api_base_to_avoid_double_v1(
|
||||
monkeypatch: pytest.MonkeyPatch, httpx_clients: list[DummyClient]
|
||||
) -> None:
|
||||
monkeypatch.setenv("RERANKER_API_KEY", "test-key")
|
||||
|
||||
reranker = APIReranker(api_base="https://api.siliconflow.cn/v1", provider="siliconflow")
|
||||
assert len(httpx_clients) == 1
|
||||
# Endpoint already includes /v1, so api_base should not.
|
||||
assert httpx_clients[0].base_url == "https://api.siliconflow.cn"
|
||||
reranker.close()
|
||||
|
||||
|
||||
def test_api_reranker_strips_endpoint_from_api_base_to_avoid_double_endpoint(
|
||||
monkeypatch: pytest.MonkeyPatch, httpx_clients: list[DummyClient]
|
||||
) -> None:
|
||||
monkeypatch.setenv("RERANKER_API_KEY", "test-key")
|
||||
|
||||
reranker = APIReranker(api_base="https://api.siliconflow.cn/v1/rerank", provider="siliconflow")
|
||||
assert len(httpx_clients) == 1
|
||||
# If api_base already includes the endpoint suffix, strip it.
|
||||
assert httpx_clients[0].base_url == "https://api.siliconflow.cn"
|
||||
reranker.close()
|
||||
|
||||
|
||||
def test_api_reranker_scores_pairs_siliconflow(
|
||||
monkeypatch: pytest.MonkeyPatch, httpx_clients: list[DummyClient]
|
||||
) -> None:
|
||||
monkeypatch.delenv("RERANKER_API_KEY", raising=False)
|
||||
# Avoid picking up any machine-local default model from global .env.
|
||||
monkeypatch.setenv("RERANKER_MODEL", "")
|
||||
monkeypatch.setenv("CODEXLENS_RERANKER_MODEL", "")
|
||||
|
||||
reranker = APIReranker(api_key="k", provider="siliconflow")
|
||||
client = httpx_clients[0]
|
||||
@@ -168,4 +198,3 @@ def test_factory_api_backend_constructs_reranker(
|
||||
reranker = get_reranker(backend="api")
|
||||
assert isinstance(reranker, APIReranker)
|
||||
assert len(httpx_clients) == 1
|
||||
|
||||
|
||||
@@ -231,6 +231,26 @@ class TestHDBSCANStrategy:
|
||||
|
||||
assert all_indices == set(range(len(sample_results)))
|
||||
|
||||
def test_cluster_supports_cosine_metric(
|
||||
self, sample_results: List[SearchResult], mock_embeddings
|
||||
):
|
||||
"""Test HDBSCANStrategy can run with metric='cosine' (via precomputed distances)."""
|
||||
try:
|
||||
from codexlens.search.clustering import HDBSCANStrategy
|
||||
except ImportError:
|
||||
pytest.skip("hdbscan not installed")
|
||||
|
||||
config = ClusteringConfig(min_cluster_size=2, min_samples=1, metric="cosine")
|
||||
strategy = HDBSCANStrategy(config)
|
||||
|
||||
clusters = strategy.cluster(mock_embeddings, sample_results)
|
||||
|
||||
all_indices = set()
|
||||
for cluster in clusters:
|
||||
all_indices.update(cluster)
|
||||
|
||||
assert all_indices == set(range(len(sample_results)))
|
||||
|
||||
def test_cluster_empty_results(self, hdbscan_strategy):
|
||||
"""Test cluster() with empty results."""
|
||||
import numpy as np
|
||||
|
||||
Reference in New Issue
Block a user