catlog22/Claude-Code-Workflow
1
0
Fork 0
mirror of https://github.com/catlog22/Claude-Code-Workflow.git synced 2026-02-14 02:42:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: Implement adaptive RRF weights and query intent detection

Browse Source 
- Added integration tests for adaptive RRF weights in hybrid search.
- Enhanced query intent detection with new classifications: keyword, semantic, and mixed.
- Introduced symbol boosting in search results based on explicit symbol matches.
- Implemented embedding-based reranking with configurable options.
- Added global symbol index for efficient symbol lookups across projects.
- Improved file deletion handling on Windows to avoid permission errors.
- Updated chunk configuration to increase overlap for better context.
- Modified package.json test script to target specific test files.
- Created comprehensive writing style guidelines for documentation.
- Added TypeScript tests for query intent detection and adaptive weights.
- Established performance benchmarks for global symbol indexing.
This commit is contained in:
catlog22
2025-12-26 15:08:47 +08:00
parent ecd5085e51
commit 4061ae48c4
29 changed files with 2685 additions and 828 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
codex-lens/src/codexlens/search/hybrid_search.py
View File

@@ -7,12 +7,38 @@ results via Reciprocal Rank Fusion (RRF) algorithm.
from __future__ import annotations
import logging
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from contextlib import contextmanager
from pathlib import Path
from typing import Dict, List, Optional
from typing import Any, Dict, List, Optional
@contextmanager
def timer(name: str, logger: logging.Logger, level: int = logging.DEBUG):
"""Context manager for timing code blocks.
Args:
name: Name of the operation being timed
logger: Logger instance to use
level: Logging level (default DEBUG)
"""
start = time.perf_counter()
try:
yield
finally:
elapsed_ms = (time.perf_counter() - start) * 1000
logger.log(level, "[TIMING] %s: %.2fms", name, elapsed_ms)
from codexlens.config import Config
from codexlens.entities import SearchResult
from codexlens.search.ranking import reciprocal_rank_fusion, tag_search_source
from codexlens.search.ranking import (
apply_symbol_boost,
get_rrf_weights,
reciprocal_rank_fusion,
rerank_results,
tag_search_source,
)
from codexlens.storage.dir_index import DirIndexStore
@@ -34,14 +60,23 @@ class HybridSearchEngine:
"vector": 0.6,
}
def __init__(self, weights: Optional[Dict[str, float]] = None):
def __init__(
self,
weights: Optional[Dict[str, float]] = None,
config: Optional[Config] = None,
embedder: Any = None,
):
"""Initialize hybrid search engine.
Args:
weights: Optional custom RRF weights (default: DEFAULT_WEIGHTS)
config: Optional runtime config (enables optional reranking features)
embedder: Optional embedder instance for embedding-based reranking
"""
self.logger = logging.getLogger(__name__)
self.weights = weights or self.DEFAULT_WEIGHTS.copy()
self._config = config
self.embedder = embedder
def search(
self,
@@ -101,7 +136,8 @@ class HybridSearchEngine:
backends["vector"] = True
# Execute parallel searches
results_map = self._search_parallel(index_path, query, backends, limit)
with timer("parallel_search_total", self.logger):
results_map = self._search_parallel(index_path, query, backends, limit)
# Provide helpful message if pure-vector mode returns no results
if pure_vector and enable_vector and len(results_map.get("vector", [])) == 0:
@@ -120,11 +156,72 @@ class HybridSearchEngine:
if source in results_map
}
fused_results = reciprocal_rank_fusion(results_map, active_weights)
with timer("rrf_fusion", self.logger):
adaptive_weights = get_rrf_weights(query, active_weights)
fused_results = reciprocal_rank_fusion(results_map, adaptive_weights)
# Optional: boost results that include explicit symbol matches
boost_factor = (
self._config.symbol_boost_factor
if self._config is not None
else 1.5
)
with timer("symbol_boost", self.logger):
fused_results = apply_symbol_boost(
fused_results, boost_factor=boost_factor
)
# Optional: embedding-based reranking on top results
if self._config is not None and self._config.enable_reranking:
with timer("reranking", self.logger):
if self.embedder is None:
self.embedder = self._get_reranking_embedder()
fused_results = rerank_results(
query,
fused_results[:100],
self.embedder,
top_k=self._config.reranking_top_k,
)
# Apply final limit
return fused_results[:limit]
def _get_reranking_embedder(self) -> Any:
"""Create an embedder for reranking based on Config embedding settings."""
if self._config is None:
return None
try:
from codexlens.semantic.factory import get_embedder
except Exception as exc:
self.logger.debug("Reranking embedder unavailable: %s", exc)
return None
try:
if self._config.embedding_backend == "fastembed":
return get_embedder(
backend="fastembed",
profile=self._config.embedding_model,
use_gpu=self._config.embedding_use_gpu,
)
if self._config.embedding_backend == "litellm":
return get_embedder(
backend="litellm",
model=self._config.embedding_model,
endpoints=self._config.embedding_endpoints,
strategy=self._config.embedding_strategy,
cooldown=self._config.embedding_cooldown,
)
except Exception as exc:
self.logger.debug("Failed to initialize reranking embedder: %s", exc)
return None
self.logger.debug(
"Unknown embedding backend for reranking: %s",
self._config.embedding_backend,
)
return None
def _search_parallel(
self,
index_path: Path,
@@ -144,25 +241,30 @@ class HybridSearchEngine:
Dictionary mapping source name to results list
"""
results_map: Dict[str, List[SearchResult]] = {}
timing_data: Dict[str, float] = {}
# Use ThreadPoolExecutor for parallel I/O-bound searches
with ThreadPoolExecutor(max_workers=len(backends)) as executor:
# Submit search tasks
# Submit search tasks with timing
future_to_source = {}
submit_times = {}
if backends.get("exact"):
submit_times["exact"] = time.perf_counter()
future = executor.submit(
self._search_exact, index_path, query, limit
)
future_to_source[future] = "exact"
if backends.get("fuzzy"):
submit_times["fuzzy"] = time.perf_counter()
future = executor.submit(
self._search_fuzzy, index_path, query, limit
)
future_to_source[future] = "fuzzy"
if backends.get("vector"):
submit_times["vector"] = time.perf_counter()
future = executor.submit(
self._search_vector, index_path, query, limit
)
@@ -171,18 +273,26 @@ class HybridSearchEngine:
# Collect results as they complete
for future in as_completed(future_to_source):
source = future_to_source[future]
elapsed_ms = (time.perf_counter() - submit_times[source]) * 1000
timing_data[source] = elapsed_ms
try:
results = future.result()
# Tag results with source for debugging
tagged_results = tag_search_source(results, source)
results_map[source] = tagged_results
self.logger.debug(
"Got %d results from %s search", len(results), source
"[TIMING] %s_search: %.2fms (%d results)",
source, elapsed_ms, len(results)
)
except Exception as exc:
self.logger.error("Search failed for %s: %s", source, exc)
results_map[source] = []
# Log timing summary
if timing_data:
timing_str = ", ".join(f"{k}={v:.1f}ms" for k, v in timing_data.items())
self.logger.debug("[TIMING] search_backends: {%s}", timing_str)
return results_map
def _search_exact(
@@ -245,6 +355,8 @@ class HybridSearchEngine:
try:
# Check if semantic chunks table exists
import sqlite3
start_check = time.perf_counter()
try:
with sqlite3.connect(index_path) as conn:
cursor = conn.execute(
@@ -254,6 +366,10 @@ class HybridSearchEngine:
except sqlite3.Error as e:
self.logger.error("Database check failed in vector search: %s", e)
return []
self.logger.debug(
"[TIMING] vector_table_check: %.2fms",
(time.perf_counter() - start_check) * 1000
)
if not has_semantic_table:
self.logger.info(
@@ -267,7 +383,12 @@ class HybridSearchEngine:
from codexlens.semantic.factory import get_embedder
from codexlens.semantic.vector_store import VectorStore
start_init = time.perf_counter()
vector_store = VectorStore(index_path)
self.logger.debug(
"[TIMING] vector_store_init: %.2fms",
(time.perf_counter() - start_init) * 1000
)
# Check if vector store has data
if vector_store.count_chunks() == 0:
@@ -279,6 +400,7 @@ class HybridSearchEngine:
return []
# Get stored model configuration (preferred) or auto-detect from dimension
start_embedder = time.perf_counter()
model_config = vector_store.get_model_config()
if model_config:
backend = model_config.get("backend", "fastembed")
@@ -288,7 +410,7 @@ class HybridSearchEngine:
"Using stored model config: %s backend, %s (%s, %dd)",
backend, model_profile, model_name, model_config["embedding_dim"]
)
# Get embedder based on backend
if backend == "litellm":
embedder = get_embedder(backend="litellm", model=model_name)
@@ -324,21 +446,32 @@ class HybridSearchEngine:
detected_dim
)
embedder = get_embedder(backend="fastembed", profile="code")
self.logger.debug(
"[TIMING] embedder_init: %.2fms",
(time.perf_counter() - start_embedder) * 1000
)
# Generate query embedding
start_embed = time.perf_counter()
query_embedding = embedder.embed_single(query)
self.logger.debug(
"[TIMING] query_embedding: %.2fms",
(time.perf_counter() - start_embed) * 1000
)
# Search for similar chunks
start_search = time.perf_counter()
results = vector_store.search_similar(
query_embedding=query_embedding,
top_k=limit,
min_score=0.0, # Return all results, let RRF handle filtering
return_full_content=True,
)
self.logger.debug(
"[TIMING] vector_similarity_search: %.2fms (%d results)",
(time.perf_counter() - start_search) * 1000, len(results)
)
self.logger.debug("Vector search found %d results", len(results))
return results
except ImportError as exc: