feat(codexlens): Improve search defaults and add explicit SPLADE mode

Config changes:
- Disable SPLADE by default (slow ~360ms), use FTS instead
- Enable use_fts_fallback by default for faster sparse search

CLI improvements:
- Fix duplicate index_app typer definition
- Add cascade_search dispatch for cascade method
- Rename 'mode' to 'method' in search output
- Mark embeddings-status, splade-status as deprecated
- Add enable_splade and enable_cascade to search options

Hybrid search:
- Add enable_splade parameter for explicit SPLADE mode
- Add fallback handling when SPLADE is requested but unavailable
This commit is contained in:
catlog22
2026-01-03 11:49:58 +08:00
parent 2391c77910
commit 713894090d
3 changed files with 584 additions and 27 deletions

View File

@@ -37,7 +37,7 @@ from .output import (
app = typer.Typer(help="CodexLens CLI — local code indexing and search.")
# Index subcommand group for reorganized commands
index_app = typer.Typer(help="Index management commands (embeddings, SPLADE, migrations).")
index_app = typer.Typer(help="Index management commands (init, embeddings, splade, binary, status, migrate, all)")
app.add_typer(index_app, name="index")
@@ -53,10 +53,6 @@ def _deprecated_command_warning(old_name: str, new_name: str) -> None:
f"Use '{new_name}' instead."
)
# Index management subcommand group
index_app = typer.Typer(help="Index management commands (init, embeddings, splade, binary, status, migrate, all)")
app.add_typer(index_app, name="index")
def _configure_logging(verbose: bool, json_mode: bool = False) -> None:
"""Configure logging level.
@@ -659,6 +655,8 @@ def search(
enable_fuzzy=enable_fuzzy,
enable_vector=enable_vector,
pure_vector=pure_vector,
enable_splade=enable_splade,
enable_cascade=enable_cascade,
hybrid_weights=hybrid_weights,
)
@@ -671,7 +669,11 @@ def search(
for fp in file_paths:
console.print(fp)
else:
result = engine.search(query, search_path, options)
# Dispatch to cascade_search for cascade method
if actual_method == "cascade":
result = engine.cascade_search(query, search_path, k=limit, options=options)
else:
result = engine.search(query, search_path, options)
results_list = [
{
"path": r.path,
@@ -686,7 +688,7 @@ def search(
payload = {
"query": query,
"mode": actual_mode,
"method": actual_method,
"count": len(results_list),
"results": results_list,
"stats": {
@@ -699,7 +701,7 @@ def search(
print_json(success=True, result=payload)
else:
render_search_results(result.results, verbose=verbose)
console.print(f"[dim]Mode: {actual_mode} | Searched {result.stats.dirs_searched} directories in {result.stats.time_ms:.1f}ms[/dim]")
console.print(f"[dim]Method: {actual_method} | Searched {result.stats.dirs_searched} directories in {result.stats.time_ms:.1f}ms[/dim]")
except SearchError as exc:
if json_mode:
@@ -1882,7 +1884,7 @@ def model_info(
# ==================== Embedding Management Commands ====================
@app.command(name="embeddings-status")
@app.command(name="embeddings-status", hidden=True, deprecated=True)
def embeddings_status(
path: Optional[Path] = typer.Argument(
None,
@@ -1891,7 +1893,9 @@ def embeddings_status(
),
json_mode: bool = typer.Option(False, "--json", help="Output JSON response."),
) -> None:
"""Check embedding status for one or all indexes.
"""[Deprecated] Use 'codexlens index status' instead.
Check embedding status for one or all indexes.
Shows embedding statistics including:
- Number of chunks generated
@@ -1903,6 +1907,7 @@ def embeddings_status(
codexlens embeddings-status ~/.codexlens/indexes/project/_index.db # Check specific index
codexlens embeddings-status ~/projects/my-app # Check project (auto-finds index)
"""
_deprecated_command_warning("embeddings-status", "index status")
from codexlens.cli.embedding_manager import check_index_embeddings, get_embedding_stats_summary
# Determine what to check
@@ -2655,17 +2660,20 @@ def index_splade(
console.print(f" Database: [dim]{splade_db}[/dim]")
@app.command("splade-status")
@app.command("splade-status", hidden=True, deprecated=True)
def splade_status_command(
path: Path = typer.Argument(..., help="Project path"),
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output."),
) -> None:
"""Show SPLADE index status and statistics.
"""[Deprecated] Use 'codexlens index status' instead.
Show SPLADE index status and statistics.
Examples:
codexlens splade-status ~/projects/my-app
codexlens splade-status .
"""
_deprecated_command_warning("splade-status", "index status")
_configure_logging(verbose)
from codexlens.storage.splade_index import SpladeIndex
@@ -2845,8 +2853,8 @@ def get_binary_index_path(db_path: Path) -> Path:
return db_path.parent / f"{db_path.stem}_binary.bin"
@app.command("cascade-index")
def cascade_index(
@index_app.command("binary")
def index_binary(
path: Annotated[Path, typer.Argument(help="Directory to index")],
force: Annotated[bool, typer.Option("--force", "-f", help="Force regenerate")] = False,
batch_size: Annotated[int, typer.Option("--batch-size", "-b", help="Batch size for embedding")] = 32,
@@ -2866,9 +2874,9 @@ def cascade_index(
- Creates a BinaryANNIndex file for fast coarse retrieval
Examples:
codexlens cascade-index ~/projects/my-app
codexlens cascade-index . --force
codexlens cascade-index . --batch-size 64 --verbose
codexlens index binary ~/projects/my-app
codexlens index binary . --force
codexlens index binary . --batch-size 64 --verbose
"""
_configure_logging(verbose, json_mode)
@@ -3131,6 +3139,384 @@ def cascade_index(
console.print(f" [dim]... and {len(errors_list) - 3} more[/dim]")
# ==================== Index Status Command ====================
@index_app.command("status")
def index_status(
path: Optional[Path] = typer.Argument(
None,
help="Path to project directory or _index.db file. If not specified, uses default index root.",
),
json_mode: bool = typer.Option(False, "--json", help="Output JSON response."),
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output."),
) -> None:
"""Show comprehensive index status (embeddings + SPLADE).
Shows combined status for all index types:
- Dense vector embeddings (HNSW)
- SPLADE sparse embeddings
- Binary cascade embeddings
Examples:
codexlens index status # Check all indexes
codexlens index status ~/projects/my-app # Check specific project
codexlens index status --json # JSON output
"""
_configure_logging(verbose, json_mode)
from codexlens.cli.embedding_manager import check_index_embeddings, get_embedding_stats_summary
from codexlens.storage.splade_index import SpladeIndex
from codexlens.semantic.splade_encoder import check_splade_available
from codexlens.config import SPLADE_DB_NAME
# Determine target path and index root
if path is None:
index_root = _get_index_root()
target_path = None
else:
target_path = path.resolve()
if target_path.is_file() and target_path.name == "_index.db":
index_root = target_path.parent
elif target_path.is_dir():
# Try to find index for this project
registry = RegistryStore()
try:
registry.initialize()
mapper = PathMapper()
index_path = mapper.source_to_index_db(target_path)
if index_path.exists():
index_root = index_path.parent
else:
if json_mode:
print_json(success=False, error=f"No index found for {target_path}")
else:
console.print(f"[red]Error:[/red] No index found for {target_path}")
console.print("Run 'codexlens index init' first to create an index")
raise typer.Exit(code=1)
finally:
registry.close()
else:
if json_mode:
print_json(success=False, error="Path must be _index.db file or directory")
else:
console.print(f"[red]Error:[/red] Path must be _index.db file or directory")
raise typer.Exit(code=1)
# Get embeddings status
embeddings_result = get_embedding_stats_summary(index_root)
# Get SPLADE status
splade_db = index_root / SPLADE_DB_NAME
splade_status = {
"available": False,
"has_index": False,
"stats": None,
"metadata": None,
}
splade_available, splade_err = check_splade_available()
splade_status["available"] = splade_available
if splade_db.exists():
try:
splade_index = SpladeIndex(splade_db)
if splade_index.has_index():
splade_status["has_index"] = True
splade_status["stats"] = splade_index.get_stats()
splade_status["metadata"] = splade_index.get_metadata()
splade_index.close()
except Exception as e:
if verbose:
console.print(f"[yellow]Warning: Failed to read SPLADE index: {e}[/yellow]")
# Build combined result
result = {
"index_root": str(index_root),
"embeddings": embeddings_result.get("result") if embeddings_result.get("success") else None,
"embeddings_error": embeddings_result.get("error") if not embeddings_result.get("success") else None,
"splade": splade_status,
}
if json_mode:
print_json(success=True, result=result)
else:
console.print(f"[bold]Index Status[/bold]")
console.print(f"Index root: [dim]{index_root}[/dim]\n")
# Embeddings section
console.print("[bold]Dense Embeddings (HNSW):[/bold]")
if embeddings_result.get("success"):
data = embeddings_result["result"]
total = data.get("total_indexes", 0)
with_emb = data.get("indexes_with_embeddings", 0)
total_chunks = data.get("total_chunks", 0)
console.print(f" Total indexes: {total}")
console.print(f" Indexes with embeddings: [{'green' if with_emb > 0 else 'yellow'}]{with_emb}[/]/{total}")
console.print(f" Total chunks: {total_chunks:,}")
else:
console.print(f" [yellow]--[/yellow] {embeddings_result.get('error', 'Not available')}")
# SPLADE section
console.print("\n[bold]SPLADE Sparse Index:[/bold]")
if splade_status["has_index"]:
stats = splade_status["stats"] or {}
metadata = splade_status["metadata"] or {}
console.print(f" [green]OK[/green] SPLADE index available")
console.print(f" Chunks: {stats.get('unique_chunks', 0):,}")
console.print(f" Unique tokens: {stats.get('unique_tokens', 0):,}")
console.print(f" Total postings: {stats.get('total_postings', 0):,}")
if metadata.get("model_name"):
console.print(f" Model: {metadata['model_name']}")
elif splade_available:
console.print(f" [yellow]--[/yellow] No SPLADE index found")
console.print(f" [dim]Run 'codexlens index splade <path>' to create one[/dim]")
else:
console.print(f" [yellow]--[/yellow] SPLADE not available: {splade_err}")
# Runtime availability
console.print("\n[bold]Runtime Availability:[/bold]")
console.print(f" SPLADE encoder: {'[green]Yes[/green]' if splade_available else f'[red]No[/red] ({splade_err})'}")
# ==================== Index All Command ====================
@index_app.command("all")
def index_all(
path: Path = typer.Argument(Path("."), exists=True, file_okay=False, dir_okay=True, help="Project root to index."),
language: Optional[List[str]] = typer.Option(
None,
"--language",
"-l",
help="Limit indexing to specific languages (repeat or comma-separated).",
),
workers: Optional[int] = typer.Option(None, "--workers", "-w", min=1, help="Parallel worker processes."),
force: bool = typer.Option(False, "--force", "-f", help="Force full reindex."),
backend: str = typer.Option("fastembed", "--backend", "-b", help="Embedding backend: fastembed or litellm."),
model: str = typer.Option("code", "--model", "-m", help="Embedding model profile or name."),
max_workers: int = typer.Option(1, "--max-workers", min=1, help="Max concurrent API calls."),
skip_splade: bool = typer.Option(False, "--skip-splade", help="Skip SPLADE index generation."),
json_mode: bool = typer.Option(False, "--json", help="Output JSON response."),
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
) -> None:
"""Run all indexing operations in sequence (init, embeddings, splade).
This is a convenience command that runs the complete indexing pipeline:
1. FTS index initialization (index init)
2. Dense vector embeddings (index embeddings)
3. SPLADE sparse index (index splade) - unless --skip-splade
Examples:
codexlens index all ~/projects/my-app
codexlens index all . --force
codexlens index all . --backend litellm --model text-embedding-3-small
codexlens index all . --skip-splade
"""
_configure_logging(verbose, json_mode)
base_path = path.expanduser().resolve()
results = {
"path": str(base_path),
"steps": {},
}
# Step 1: Run init
if not json_mode:
console.print(f"[bold]Step 1/3: Initializing FTS index...[/bold]")
try:
# Import and call the init function directly
from codexlens.config import Config
from codexlens.storage.index_tree import IndexTreeBuilder
config = Config()
languages = _parse_languages(language)
registry = RegistryStore()
registry.initialize()
mapper = PathMapper()
builder = IndexTreeBuilder(registry, mapper, config, incremental=not force)
build_result = builder.build(
source_root=base_path,
languages=languages,
workers=workers,
force_full=force,
)
results["steps"]["init"] = {
"success": True,
"files_indexed": build_result.total_files,
"dirs_indexed": build_result.total_dirs,
"index_root": str(build_result.index_root),
}
if not json_mode:
console.print(f" [green]OK[/green] Indexed {build_result.total_files} files in {build_result.total_dirs} directories")
index_root = Path(build_result.index_root)
registry.close()
except Exception as e:
results["steps"]["init"] = {"success": False, "error": str(e)}
if json_mode:
print_json(success=False, result=results, error=f"Init failed: {e}")
else:
console.print(f" [red]Error:[/red] {e}")
raise typer.Exit(code=1)
# Step 2: Generate embeddings
if not json_mode:
console.print(f"\n[bold]Step 2/3: Generating dense embeddings...[/bold]")
try:
from codexlens.cli.embedding_manager import generate_dense_embeddings_centralized
def progress_update(msg: str):
if not json_mode and verbose:
console.print(f" {msg}")
embed_result = generate_dense_embeddings_centralized(
index_root,
embedding_backend=backend,
model_profile=model,
force=force,
chunk_size=2000,
progress_callback=progress_update,
max_workers=max_workers,
)
if embed_result["success"]:
data = embed_result["result"]
results["steps"]["embeddings"] = {
"success": True,
"chunks_created": data.get("chunks_created", 0),
"files_processed": data.get("files_processed", 0),
}
if not json_mode:
console.print(f" [green]OK[/green] Generated {data.get('chunks_created', 0)} chunks for {data.get('files_processed', 0)} files")
else:
results["steps"]["embeddings"] = {
"success": False,
"error": embed_result.get("error"),
}
if not json_mode:
console.print(f" [yellow]Warning:[/yellow] {embed_result.get('error', 'Unknown error')}")
except Exception as e:
results["steps"]["embeddings"] = {"success": False, "error": str(e)}
if not json_mode:
console.print(f" [yellow]Warning:[/yellow] {e}")
# Step 3: Generate SPLADE index (unless skipped)
if not skip_splade:
if not json_mode:
console.print(f"\n[bold]Step 3/3: Generating SPLADE index...[/bold]")
try:
from codexlens.semantic.splade_encoder import get_splade_encoder, check_splade_available
from codexlens.storage.splade_index import SpladeIndex
from codexlens.semantic.vector_store import VectorStore
from codexlens.config import SPLADE_DB_NAME
ok, err = check_splade_available()
if not ok:
results["steps"]["splade"] = {"success": False, "error": f"SPLADE not available: {err}"}
if not json_mode:
console.print(f" [yellow]Skipped:[/yellow] SPLADE not available ({err})")
else:
# Discover all _index.db files
all_index_dbs = sorted(index_root.rglob("_index.db"))
if not all_index_dbs:
results["steps"]["splade"] = {"success": False, "error": "No index databases found"}
if not json_mode:
console.print(f" [yellow]Skipped:[/yellow] No index databases found")
else:
# Collect chunks
all_chunks = []
global_id = 0
for index_db in all_index_dbs:
try:
vector_store = VectorStore(index_db)
chunks = vector_store.get_all_chunks()
for chunk in chunks:
global_id += 1
all_chunks.append((global_id, chunk, index_db))
vector_store.close()
except Exception:
pass
if all_chunks:
splade_db = index_root / SPLADE_DB_NAME
if splade_db.exists() and force:
splade_db.unlink()
encoder = get_splade_encoder()
splade_index = SpladeIndex(splade_db)
splade_index.create_tables()
chunk_metadata_batch = []
import json as json_module
for gid, chunk, source_db_path in all_chunks:
sparse_vec = encoder.encode_text(chunk.content)
splade_index.add_posting(gid, sparse_vec)
metadata_str = None
if hasattr(chunk, 'metadata') and chunk.metadata:
try:
metadata_str = json_module.dumps(chunk.metadata) if isinstance(chunk.metadata, dict) else chunk.metadata
except Exception:
pass
chunk_metadata_batch.append((
gid,
chunk.file_path or "",
chunk.content,
metadata_str,
str(source_db_path)
))
if chunk_metadata_batch:
splade_index.add_chunks_metadata_batch(chunk_metadata_batch)
splade_index.set_metadata(
model_name=encoder.model_name,
vocab_size=encoder.vocab_size
)
stats = splade_index.get_stats()
results["steps"]["splade"] = {
"success": True,
"chunks": stats['unique_chunks'],
"postings": stats['total_postings'],
}
if not json_mode:
console.print(f" [green]OK[/green] SPLADE index built: {stats['unique_chunks']} chunks, {stats['total_postings']} postings")
else:
results["steps"]["splade"] = {"success": False, "error": "No chunks found"}
if not json_mode:
console.print(f" [yellow]Skipped:[/yellow] No chunks found in indexes")
except Exception as e:
results["steps"]["splade"] = {"success": False, "error": str(e)}
if not json_mode:
console.print(f" [yellow]Warning:[/yellow] {e}")
else:
results["steps"]["splade"] = {"success": True, "skipped": True}
if not json_mode:
console.print(f"\n[bold]Step 3/3: SPLADE index...[/bold]")
console.print(f" [dim]Skipped (--skip-splade)[/dim]")
# Summary
if json_mode:
print_json(success=True, result=results)
else:
console.print(f"\n[bold]Indexing Complete[/bold]")
init_ok = results["steps"].get("init", {}).get("success", False)
emb_ok = results["steps"].get("embeddings", {}).get("success", False)
splade_ok = results["steps"].get("splade", {}).get("success", False)
console.print(f" FTS Index: {'[green]OK[/green]' if init_ok else '[red]Failed[/red]'}")
console.print(f" Embeddings: {'[green]OK[/green]' if emb_ok else '[yellow]Partial/Skipped[/yellow]'}")
console.print(f" SPLADE: {'[green]OK[/green]' if splade_ok else '[yellow]Partial/Skipped[/yellow]'}")
# ==================== Index Migration Commands ====================
# Index version for migration tracking (file-based version marker)
@@ -3284,8 +3670,8 @@ def _check_centralized_storage(index_root: Path) -> Dict[str, Any]:
return result
@app.command(name="index-migrate")
def index_migrate(
@index_app.command("migrate")
def index_migrate_cmd(
path: Annotated[Optional[str], typer.Argument(help="Project path to migrate")] = None,
dry_run: Annotated[bool, typer.Option("--dry-run", help="Show what would be migrated without making changes")] = False,
force: Annotated[bool, typer.Option("--force", help="Force migration even if already migrated")] = False,
@@ -3309,13 +3695,13 @@ def index_migrate(
Use --force to re-run migration even if version marker exists.
Note: For full data migration (SPLADE/vectors consolidation), run:
codexlens splade-index <path> --rebuild
codexlens embeddings-generate <path> --recursive --force
codexlens index splade <path> --rebuild
codexlens index embeddings <path> --force
Examples:
codexlens index-migrate ~/projects/my-app --dry-run
codexlens index-migrate . --force
codexlens index-migrate --json
codexlens index migrate ~/projects/my-app --dry-run
codexlens index migrate . --force
codexlens index migrate --json
"""
_configure_logging(verbose, json_mode)
@@ -3513,3 +3899,152 @@ def index_migrate(
console.print("\n[bold]Recommendations:[/bold]")
for rec in migration_report["recommendations"]:
console.print(f" [cyan]>[/cyan] {rec}")
# ==================== Deprecated Command Aliases ====================
# These commands maintain backward compatibility with the old CLI structure.
# They display deprecation warnings and delegate to the new `index` subcommands.
@app.command("embeddings-generate", hidden=True, deprecated=True)
def embeddings_generate_deprecated(
path: Path = typer.Argument(
...,
exists=True,
help="Path to _index.db file or project directory.",
),
backend: str = typer.Option(
"fastembed",
"--backend",
"-b",
help="Embedding backend: fastembed (local) or litellm (remote API).",
),
model: str = typer.Option(
"code",
"--model",
"-m",
help="Model: profile name for fastembed or model name for litellm.",
),
force: bool = typer.Option(
False,
"--force",
"-f",
help="Force regeneration even if embeddings exist.",
),
chunk_size: int = typer.Option(
2000,
"--chunk-size",
help="Maximum chunk size in characters.",
),
max_workers: int = typer.Option(
1,
"--max-workers",
"-w",
min=1,
help="Max concurrent API calls.",
),
json_mode: bool = typer.Option(False, "--json", help="Output JSON response."),
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output."),
centralized: bool = typer.Option(
True,
"--centralized/--distributed",
"-c/-d",
help="Use centralized vector storage (default) or distributed.",
),
) -> None:
"""[Deprecated] Use 'codexlens index embeddings' instead."""
_deprecated_command_warning("embeddings-generate", "index embeddings")
index_embeddings(
path=path,
backend=backend,
model=model,
force=force,
chunk_size=chunk_size,
max_workers=max_workers,
json_mode=json_mode,
verbose=verbose,
centralized=centralized,
)
@app.command("init", hidden=True, deprecated=True)
def init_deprecated(
path: Path = typer.Argument(Path("."), exists=True, file_okay=False, dir_okay=True, help="Project root to index."),
language: Optional[List[str]] = typer.Option(None, "--language", "-l", help="Limit indexing to specific languages."),
workers: Optional[int] = typer.Option(None, "--workers", "-w", min=1, help="Parallel worker processes."),
force: bool = typer.Option(False, "--force", "-f", help="Force full reindex."),
no_embeddings: bool = typer.Option(False, "--no-embeddings", help="Skip automatic embedding generation."),
backend: str = typer.Option("fastembed", "--backend", "-b", help="Embedding backend."),
model: str = typer.Option("code", "--model", "-m", help="Embedding model."),
max_workers: int = typer.Option(1, "--max-workers", min=1, help="Max concurrent API calls."),
json_mode: bool = typer.Option(False, "--json", help="Output JSON response."),
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
) -> None:
"""[Deprecated] Use 'codexlens index init' instead."""
_deprecated_command_warning("init", "index init")
index_init(
path=path,
language=language,
workers=workers,
force=force,
no_embeddings=no_embeddings,
backend=backend,
model=model,
max_workers=max_workers,
json_mode=json_mode,
verbose=verbose,
)
@app.command("splade-index", hidden=True, deprecated=True)
def splade_index_deprecated(
path: Path = typer.Argument(..., help="Project path to index"),
rebuild: bool = typer.Option(False, "--rebuild", "-r", help="Force rebuild SPLADE index"),
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output."),
) -> None:
"""[Deprecated] Use 'codexlens index splade' instead."""
_deprecated_command_warning("splade-index", "index splade")
index_splade(
path=path,
rebuild=rebuild,
verbose=verbose,
)
@app.command("cascade-index", hidden=True, deprecated=True)
def cascade_index_deprecated(
path: Annotated[Path, typer.Argument(help="Directory to index")],
force: Annotated[bool, typer.Option("--force", "-f", help="Force regenerate")] = False,
batch_size: Annotated[int, typer.Option("--batch-size", "-b", help="Batch size for embedding")] = 32,
json_mode: Annotated[bool, typer.Option("--json", help="Output JSON response")] = False,
verbose: Annotated[bool, typer.Option("--verbose", "-v", help="Enable verbose logging")] = False,
) -> None:
"""[Deprecated] Use 'codexlens index binary' instead."""
_deprecated_command_warning("cascade-index", "index binary")
index_binary(
path=path,
force=force,
batch_size=batch_size,
json_mode=json_mode,
verbose=verbose,
)
@app.command("index-migrate", hidden=True, deprecated=True)
def index_migrate_deprecated(
path: Annotated[Optional[str], typer.Argument(help="Project path to migrate")] = None,
dry_run: Annotated[bool, typer.Option("--dry-run", help="Show what would be migrated")] = False,
force: Annotated[bool, typer.Option("--force", help="Force migration")] = False,
json_mode: Annotated[bool, typer.Option("--json", help="Output JSON response")] = False,
verbose: Annotated[bool, typer.Option("--verbose", "-v", help="Enable verbose output")] = False,
) -> None:
"""[Deprecated] Use 'codexlens index migrate' instead."""
_deprecated_command_warning("index-migrate", "index migrate")
index_migrate_cmd(
path=path,
dry_run=dry_run,
force=force,
json_mode=json_mode,
verbose=verbose,
)

View File

@@ -118,13 +118,13 @@ class Config:
embedding_use_gpu: bool = True # For fastembed: whether to use GPU acceleration
# SPLADE sparse retrieval configuration
enable_splade: bool = True # Enable SPLADE as default sparse backend
enable_splade: bool = False # Disable SPLADE by default (slow ~360ms, use FTS instead)
splade_model: str = "naver/splade-cocondenser-ensembledistil"
splade_threshold: float = 0.01 # Min weight to store in index
splade_onnx_path: Optional[str] = None # Custom ONNX model path
# FTS fallback (disabled by default, available via --use-fts)
use_fts_fallback: bool = False # Use FTS instead of SPLADE
use_fts_fallback: bool = True # Use FTS for sparse search (fast, SPLADE disabled)
# Indexing/search optimizations
global_symbol_index_enabled: bool = True # Enable project-wide symbol index fast path

View File

@@ -112,6 +112,7 @@ class HybridSearchEngine:
enable_fuzzy: bool = True,
enable_vector: bool = False,
pure_vector: bool = False,
enable_splade: bool = False,
) -> List[SearchResult]:
"""Execute hybrid search with parallel retrieval and RRF fusion.
@@ -122,6 +123,7 @@ class HybridSearchEngine:
enable_fuzzy: Enable fuzzy FTS search (default True)
enable_vector: Enable vector search (default False)
pure_vector: If True, only use vector search without FTS fallback (default False)
enable_splade: If True, force SPLADE sparse neural search (default False)
Returns:
List of SearchResult objects sorted by fusion score
@@ -135,6 +137,9 @@ class HybridSearchEngine:
>>> results = engine.search(Path("project/_index.db"),
... "how to authenticate users",
... enable_vector=True, pure_vector=True)
>>> # SPLADE sparse neural search
>>> results = engine.search(Path("project/_index.db"), "auth flow",
... enable_splade=True, enable_vector=True)
>>> for r in results[:5]:
... print(f"{r.path}: {r.score:.3f}")
"""
@@ -158,7 +163,7 @@ class HybridSearchEngine:
# Determine which backends to use
backends = {}
# Check if SPLADE is available
splade_available = False
# Respect config.enable_splade flag and use_fts_fallback flag
@@ -191,6 +196,23 @@ class HybridSearchEngine:
"To use pure vector search, enable vector search mode."
)
backends["exact"] = True
elif enable_splade:
# Explicit SPLADE mode requested via CLI --method splade
if splade_available:
backends["splade"] = True
if enable_vector:
backends["vector"] = True
else:
# SPLADE requested but not available - warn and fallback
self.logger.warning(
"SPLADE search requested but not available. "
"Falling back to FTS. Run 'codexlens index splade' to enable."
)
backends["exact"] = True
if enable_fuzzy:
backends["fuzzy"] = True
if enable_vector:
backends["vector"] = True
else:
# Hybrid mode: default to SPLADE if available, otherwise use FTS
if splade_available: