fix: CodexLens model detection, hybrid search stability, and JSON logging

- Fix model installation detection using fastembed ONNX cache names
- Add embeddings_config table for model metadata tracking
- Fix hybrid search segfault by using single-threaded GPU mode
- Suppress INFO logs in JSON mode to prevent error display
- Add model dropdown filtering to show only installed models

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
catlog22
2025-12-22 21:49:10 +08:00
parent cf58dc0dd3
commit 8203d690cb
11 changed files with 302 additions and 57 deletions

View File

@@ -98,15 +98,17 @@ async function loadCodexLensStatus() {
}
window.cliToolsStatus.codexlens = {
installed: data.ready || false,
version: data.version || null
version: data.version || null,
installedModels: [] // Will be populated by loadSemanticStatus
};
// Update CodexLens badge
updateCodexLensBadge();
// If CodexLens is ready, also check semantic status
// If CodexLens is ready, also check semantic status and models
if (data.ready) {
await loadSemanticStatus();
await loadInstalledModels();
}
return data;
@@ -132,6 +134,37 @@ async function loadSemanticStatus() {
}
}
/**
* Load installed embedding models
*/
async function loadInstalledModels() {
try {
const response = await fetch('/api/codexlens/models');
if (!response.ok) throw new Error('Failed to load models');
const data = await response.json();
if (data.success && data.result && data.result.models) {
// Filter to only installed models
const installedModels = data.result.models
.filter(m => m.installed)
.map(m => m.profile);
// Update window.cliToolsStatus
if (window.cliToolsStatus && window.cliToolsStatus.codexlens) {
window.cliToolsStatus.codexlens.installedModels = installedModels;
window.cliToolsStatus.codexlens.allModels = data.result.models;
}
console.log('[CLI Status] Installed models:', installedModels);
return installedModels;
}
return [];
} catch (err) {
console.error('Failed to load installed models:', err);
return [];
}
}
// ========== Badge Update ==========
function updateCliBadge() {
const badge = document.getElementById('badgeCliTools');

View File

@@ -349,6 +349,50 @@ function getSelectedModel() {
return select ? select.value : 'code';
}
/**
* Build model select options HTML, showing only installed models
* @returns {string} HTML string for select options
*/
function buildModelSelectOptions() {
var installedModels = window.cliToolsStatus?.codexlens?.installedModels || [];
var allModels = window.cliToolsStatus?.codexlens?.allModels || [];
// Model display configuration
var modelConfig = {
'code': { label: t('index.modelCode') || 'Code (768d)', star: true },
'base': { label: t('index.modelBase') || 'Base (768d)', star: false },
'fast': { label: t('index.modelFast') || 'Fast (384d)', star: false },
'minilm': { label: t('index.modelMinilm') || 'MiniLM (384d)', star: false },
'multilingual': { label: t('index.modelMultilingual') || 'Multilingual (1024d)', warn: true },
'balanced': { label: t('index.modelBalanced') || 'Balanced (1024d)', warn: true }
};
// If no models installed, show placeholder
if (installedModels.length === 0) {
return '<option value="" disabled selected>' + (t('index.noModelsInstalled') || 'No models installed') + '</option>';
}
// Build options for installed models only
var options = '';
var firstInstalled = null;
// Preferred order: code, fast, minilm, base, multilingual, balanced
var preferredOrder = ['code', 'fast', 'minilm', 'base', 'multilingual', 'balanced'];
preferredOrder.forEach(function(profile) {
if (installedModels.includes(profile) && modelConfig[profile]) {
var config = modelConfig[profile];
var style = config.warn ? ' style="color: var(--muted-foreground)"' : '';
var suffix = config.star ? ' ⭐' : (config.warn ? ' ⚠️' : '');
var selected = !firstInstalled ? ' selected' : '';
if (!firstInstalled) firstInstalled = profile;
options += '<option value="' + profile + '"' + style + selected + '>' + config.label + suffix + '</option>';
}
});
return options;
}
// ========== Tools Section (Left Column) ==========
function renderToolsSection() {
var container = document.getElementById('tools-section');
@@ -404,12 +448,7 @@ function renderToolsSection() {
(codexLensStatus.ready
? '<span class="tool-status-text success"><i data-lucide="check-circle" class="w-3.5 h-3.5"></i> v' + (codexLensStatus.version || 'installed') + '</span>' +
'<select id="codexlensModelSelect" class="btn-sm bg-muted border border-border rounded text-xs" onclick="event.stopPropagation()" title="' + (t('index.selectModel') || 'Select embedding model') + '">' +
'<option value="code">' + (t('index.modelCode') || 'Code (768d)') + ' ⭐</option>' +
'<option value="base">' + (t('index.modelBase') || 'Base (768d)') + '</option>' +
'<option value="fast">' + (t('index.modelFast') || 'Fast (384d)') + '</option>' +
'<option value="minilm">' + (t('index.modelMinilm') || 'MiniLM (384d)') + '</option>' +
'<option value="multilingual" style="color: var(--muted-foreground)">' + (t('index.modelMultilingual') || 'Multilingual (1024d)') + ' ⚠️</option>' +
'<option value="balanced" style="color: var(--muted-foreground)">' + (t('index.modelBalanced') || 'Balanced (1024d)') + ' ⚠️</option>' +
buildModelSelectOptions() +
'</select>' +
'<button class="btn-sm btn-primary" onclick="event.stopPropagation(); initCodexLensIndex(\'full\', getSelectedModel())" title="' + (t('index.fullDesc') || 'FTS + Semantic search (recommended)') + '"><i data-lucide="layers" class="w-3 h-3"></i> ' + (t('index.fullIndex') || '全部索引') + '</button>' +
'<button class="btn-sm btn-outline" onclick="event.stopPropagation(); initCodexLensIndex(\'vector\', getSelectedModel())" title="' + (t('index.vectorDesc') || 'Semantic search with embeddings') + '"><i data-lucide="sparkles" class="w-3 h-3"></i> ' + (t('index.vectorIndex') || '向量索引') + '</button>' +

View File

@@ -1,6 +1,6 @@
Metadata-Version: 2.4
Name: codex-lens
Version: 0.2.0
Version: 0.1.0
Summary: CodexLens multi-modal code analysis platform
Author: CodexLens contributors
License: MIT
@@ -17,18 +17,18 @@ Requires-Dist: tree-sitter-typescript>=0.23
Requires-Dist: pathspec>=0.11
Provides-Extra: semantic
Requires-Dist: numpy>=1.24; extra == "semantic"
Requires-Dist: fastembed>=0.5; extra == "semantic"
Requires-Dist: fastembed>=0.2; extra == "semantic"
Requires-Dist: hnswlib>=0.8.0; extra == "semantic"
Provides-Extra: semantic-gpu
Requires-Dist: numpy>=1.24; extra == "semantic-gpu"
Requires-Dist: fastembed>=0.5; extra == "semantic-gpu"
Requires-Dist: fastembed>=0.2; extra == "semantic-gpu"
Requires-Dist: hnswlib>=0.8.0; extra == "semantic-gpu"
Requires-Dist: onnxruntime-gpu>=1.18.0; extra == "semantic-gpu"
Requires-Dist: onnxruntime-gpu>=1.15.0; extra == "semantic-gpu"
Provides-Extra: semantic-directml
Requires-Dist: numpy>=1.24; extra == "semantic-directml"
Requires-Dist: fastembed>=0.5; extra == "semantic-directml"
Requires-Dist: fastembed>=0.2; extra == "semantic-directml"
Requires-Dist: hnswlib>=0.8.0; extra == "semantic-directml"
Requires-Dist: onnxruntime-directml>=1.18.0; extra == "semantic-directml"
Requires-Dist: onnxruntime-directml>=1.15.0; extra == "semantic-directml"
Provides-Extra: encoding
Requires-Dist: chardet>=5.0; extra == "encoding"
Provides-Extra: full

View File

@@ -15,17 +15,17 @@ tiktoken>=0.5.0
[semantic]
numpy>=1.24
fastembed>=0.5
fastembed>=0.2
hnswlib>=0.8.0
[semantic-directml]
numpy>=1.24
fastembed>=0.5
fastembed>=0.2
hnswlib>=0.8.0
onnxruntime-directml>=1.18.0
onnxruntime-directml>=1.15.0
[semantic-gpu]
numpy>=1.24
fastembed>=0.5
fastembed>=0.2
hnswlib>=0.8.0
onnxruntime-gpu>=1.18.0
onnxruntime-gpu>=1.15.0

View File

@@ -35,8 +35,17 @@ from .output import (
app = typer.Typer(help="CodexLens CLI — local code indexing and search.")
def _configure_logging(verbose: bool) -> None:
level = logging.DEBUG if verbose else logging.INFO
def _configure_logging(verbose: bool, json_mode: bool = False) -> None:
"""Configure logging level.
In JSON mode, suppress INFO logs to keep stderr clean for error parsing.
Only WARNING and above are shown to avoid mixing logs with JSON output.
"""
if json_mode and not verbose:
# In JSON mode, suppress INFO logs to keep stderr clean
level = logging.WARNING
else:
level = logging.DEBUG if verbose else logging.INFO
logging.basicConfig(level=level, format="%(levelname)s %(message)s")
@@ -95,7 +104,7 @@ def init(
If semantic search dependencies are installed, automatically generates embeddings
after indexing completes. Use --no-embeddings to skip this step.
"""
_configure_logging(verbose)
_configure_logging(verbose, json_mode)
config = Config()
languages = _parse_languages(language)
base_path = path.expanduser().resolve()
@@ -314,7 +323,7 @@ def search(
# Force hybrid mode
codexlens search "authentication" --mode hybrid
"""
_configure_logging(verbose)
_configure_logging(verbose, json_mode)
search_path = path.expanduser().resolve()
# Validate mode
@@ -487,7 +496,7 @@ def symbol(
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
) -> None:
"""Look up symbols by name and optional kind."""
_configure_logging(verbose)
_configure_logging(verbose, json_mode)
search_path = path.expanduser().resolve()
registry: RegistryStore | None = None
@@ -538,7 +547,7 @@ def inspect(
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
) -> None:
"""Analyze a single file and display symbols."""
_configure_logging(verbose)
_configure_logging(verbose, json_mode)
config = Config()
factory = ParserFactory(config)
@@ -588,7 +597,7 @@ def status(
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
) -> None:
"""Show index status and configuration."""
_configure_logging(verbose)
_configure_logging(verbose, json_mode)
registry: RegistryStore | None = None
try:
@@ -648,7 +657,7 @@ def status(
# Embedding manager not available
pass
except Exception as e:
logger.debug(f"Failed to get embeddings status: {e}")
logging.debug(f"Failed to get embeddings status: {e}")
stats = {
"index_root": str(index_root),
@@ -737,7 +746,7 @@ def projects(
- show <path>: Show details for a specific project
- remove <path>: Remove a project from the registry
"""
_configure_logging(verbose)
_configure_logging(verbose, json_mode)
registry: RegistryStore | None = None
try:
@@ -892,7 +901,7 @@ def config(
Config keys:
- index_dir: Directory to store indexes (default: ~/.codexlens/indexes)
"""
_configure_logging(verbose)
_configure_logging(verbose, json_mode)
config_file = Path.home() / ".codexlens" / "config.json"
@@ -1057,7 +1066,7 @@ def migrate(
This is a safe operation that preserves all existing data.
Progress is shown during migration.
"""
_configure_logging(verbose)
_configure_logging(verbose, json_mode)
base_path = path.expanduser().resolve()
registry: RegistryStore | None = None
@@ -1183,7 +1192,7 @@ def clean(
With path, removes that project's indexes.
With --all, removes all indexes (use with caution).
"""
_configure_logging(verbose)
_configure_logging(verbose, json_mode)
try:
mapper = PathMapper()
@@ -1329,7 +1338,7 @@ def semantic_list(
Shows files that have LLM-generated summaries and keywords.
Results are aggregated from all index databases in the project.
"""
_configure_logging(verbose)
_configure_logging(verbose, json_mode)
base_path = path.expanduser().resolve()
registry: Optional[RegistryStore] = None
@@ -1798,7 +1807,7 @@ def embeddings_generate(
codexlens embeddings-generate ~/.codexlens/indexes/project/_index.db # Specific index
codexlens embeddings-generate ~/projects/my-app --model fast --force # Regenerate with fast model
"""
_configure_logging(verbose)
_configure_logging(verbose, json_mode)
from codexlens.cli.embedding_manager import generate_embeddings, generate_embeddings_recursive

View File

@@ -279,6 +279,21 @@ def generate_embeddings(
try:
with VectorStore(index_path) as vector_store:
# Check model compatibility with existing embeddings
if not force:
is_compatible, warning = vector_store.check_model_compatibility(
model_profile, embedder.model_name, embedder.embedding_dim
)
if not is_compatible:
return {
"success": False,
"error": warning,
}
# Set/update model configuration for this index
vector_store.set_model_config(
model_profile, embedder.model_name, embedder.embedding_dim
)
# Use bulk insert mode for efficient batch ANN index building
# This defers ANN updates until end_bulk_insert() is called
with vector_store.bulk_insert():

View File

@@ -16,9 +16,11 @@ except ImportError:
# Model profiles with metadata
# Note: 768d is max recommended dimension for optimal performance/quality balance
# 1024d models are available but not recommended due to higher resource usage
# cache_name: The actual Hugging Face repo name used by fastembed for ONNX caching
MODEL_PROFILES = {
"fast": {
"model_name": "BAAI/bge-small-en-v1.5",
"cache_name": "qdrant/bge-small-en-v1.5-onnx-q", # fastembed uses ONNX version
"dimensions": 384,
"size_mb": 80,
"description": "Fast, lightweight, English-optimized",
@@ -27,6 +29,7 @@ MODEL_PROFILES = {
},
"base": {
"model_name": "BAAI/bge-base-en-v1.5",
"cache_name": "qdrant/bge-base-en-v1.5-onnx-q", # fastembed uses ONNX version
"dimensions": 768,
"size_mb": 220,
"description": "General purpose, good balance of speed and quality",
@@ -35,6 +38,7 @@ MODEL_PROFILES = {
},
"code": {
"model_name": "jinaai/jina-embeddings-v2-base-code",
"cache_name": "jinaai/jina-embeddings-v2-base-code", # Uses original name
"dimensions": 768,
"size_mb": 150,
"description": "Code-optimized, best for programming languages",
@@ -43,6 +47,7 @@ MODEL_PROFILES = {
},
"minilm": {
"model_name": "sentence-transformers/all-MiniLM-L6-v2",
"cache_name": "qdrant/all-MiniLM-L6-v2-onnx", # fastembed uses ONNX version
"dimensions": 384,
"size_mb": 90,
"description": "Popular lightweight model, good quality",
@@ -51,6 +56,7 @@ MODEL_PROFILES = {
},
"multilingual": {
"model_name": "intfloat/multilingual-e5-large",
"cache_name": "qdrant/multilingual-e5-large-onnx", # fastembed uses ONNX version
"dimensions": 1024,
"size_mb": 1000,
"description": "Multilingual + code support (high resource usage)",
@@ -59,6 +65,7 @@ MODEL_PROFILES = {
},
"balanced": {
"model_name": "mixedbread-ai/mxbai-embed-large-v1",
"cache_name": "mixedbread-ai/mxbai-embed-large-v1", # Uses original name
"dimensions": 1024,
"size_mb": 600,
"description": "High accuracy, general purpose (high resource usage)",
@@ -87,6 +94,23 @@ def get_cache_dir() -> Path:
return cache_dir
def _get_model_cache_path(cache_dir: Path, info: Dict) -> Path:
"""Get the actual cache path for a model.
fastembed uses ONNX versions of models with different names than the original.
This function returns the correct path based on the cache_name field.
Args:
cache_dir: The fastembed cache directory
info: Model profile info dictionary
Returns:
Path to the model cache directory
"""
cache_name = info.get("cache_name", info["model_name"])
return cache_dir / f"models--{cache_name.replace('/', '--')}"
def list_models() -> Dict[str, any]:
"""List available model profiles and their installation status.
@@ -106,13 +130,13 @@ def list_models() -> Dict[str, any]:
for profile, info in MODEL_PROFILES.items():
model_name = info["model_name"]
# Check if model is cached
# Check if model is cached using the actual cache name
installed = False
cache_size_mb = 0
if cache_exists:
# Check for model directory in cache
model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}"
# Check for model directory in cache using correct cache_name
model_cache_path = _get_model_cache_path(cache_dir, info)
if model_cache_path.exists():
installed = True
# Calculate cache size
@@ -166,7 +190,8 @@ def download_model(profile: str, progress_callback: Optional[callable] = None) -
"error": f"Unknown profile: {profile}. Available: {', '.join(MODEL_PROFILES.keys())}",
}
model_name = MODEL_PROFILES[profile]["model_name"]
info = MODEL_PROFILES[profile]
model_name = info["model_name"]
try:
# Download model by instantiating TextEmbedding
@@ -179,9 +204,9 @@ def download_model(profile: str, progress_callback: Optional[callable] = None) -
if progress_callback:
progress_callback(f"Model {model_name} downloaded successfully")
# Get cache info
# Get cache info using correct cache_name
cache_dir = get_cache_dir()
model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}"
model_cache_path = _get_model_cache_path(cache_dir, info)
cache_size = 0
if model_cache_path.exists():
@@ -224,9 +249,10 @@ def delete_model(profile: str) -> Dict[str, any]:
"error": f"Unknown profile: {profile}. Available: {', '.join(MODEL_PROFILES.keys())}",
}
model_name = MODEL_PROFILES[profile]["model_name"]
info = MODEL_PROFILES[profile]
model_name = info["model_name"]
cache_dir = get_cache_dir()
model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}"
model_cache_path = _get_model_cache_path(cache_dir, info)
if not model_cache_path.exists():
return {
@@ -281,9 +307,9 @@ def get_model_info(profile: str) -> Dict[str, any]:
info = MODEL_PROFILES[profile]
model_name = info["model_name"]
# Check installation status
# Check installation status using correct cache_name
cache_dir = get_cache_dir()
model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}"
model_cache_path = _get_model_cache_path(cache_dir, info)
installed = model_cache_path.exists()
cache_size_mb = None

View File

@@ -396,7 +396,20 @@ class ChainSearchEngine:
all_results = []
stats = SearchStats()
executor = self._get_executor(options.max_workers)
# Force single-threaded execution for vector/hybrid search to avoid GPU crashes
# DirectML/ONNX have threading issues when multiple threads access GPU resources
effective_workers = options.max_workers
if options.enable_vector or options.hybrid_mode:
effective_workers = 1
self.logger.debug("Using single-threaded mode for vector search (GPU safety)")
# Pre-load embedder to avoid initialization overhead per-search
try:
from codexlens.semantic.embedder import get_embedder
get_embedder(profile="code", use_gpu=True)
except Exception:
pass # Ignore pre-load failures
executor = self._get_executor(effective_workers)
# Submit all search tasks
future_to_path = {
executor.submit(

View File

@@ -274,19 +274,32 @@ class HybridSearchEngine:
)
return []
# Auto-detect embedding dimension and select appropriate profile
detected_dim = vector_store.dimension
if detected_dim is None:
self.logger.info("Vector store dimension unknown, using default profile")
profile = "code" # Default fallback
elif detected_dim == 384:
profile = "fast"
elif detected_dim == 768:
profile = "code"
elif detected_dim == 1024:
profile = "multilingual" # or balanced, both are 1024
# Get stored model configuration (preferred) or auto-detect from dimension
model_config = vector_store.get_model_config()
if model_config:
profile = model_config["model_profile"]
self.logger.debug(
"Using stored model config: %s (%s, %dd)",
profile, model_config["model_name"], model_config["embedding_dim"]
)
else:
profile = "code" # Default fallback
# Fallback: auto-detect from embedding dimension
detected_dim = vector_store.dimension
if detected_dim is None:
self.logger.info("Vector store dimension unknown, using default profile")
profile = "code" # Default fallback
elif detected_dim == 384:
profile = "fast"
elif detected_dim == 768:
profile = "code"
elif detected_dim == 1024:
profile = "multilingual" # or balanced, both are 1024
else:
profile = "code" # Default fallback
self.logger.debug(
"No stored model config, auto-detected profile '%s' from dimension %s",
profile, detected_dim
)
# Use cached embedder (singleton) for performance
embedder = get_embedder(profile=profile)

View File

@@ -116,6 +116,17 @@ class VectorStore:
CREATE INDEX IF NOT EXISTS idx_chunks_file
ON semantic_chunks(file_path)
""")
# Model configuration table - tracks which model generated the embeddings
conn.execute("""
CREATE TABLE IF NOT EXISTS embeddings_config (
id INTEGER PRIMARY KEY CHECK (id = 1),
model_profile TEXT NOT NULL,
model_name TEXT NOT NULL,
embedding_dim INTEGER NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
conn.commit()
def _init_ann_index(self) -> None:
@@ -932,6 +943,92 @@ class VectorStore:
return self._ann_index.count()
return 0
def get_model_config(self) -> Optional[Dict[str, Any]]:
"""Get the model configuration used for embeddings in this store.
Returns:
Dictionary with model_profile, model_name, embedding_dim, or None if not set.
"""
with sqlite3.connect(self.db_path) as conn:
row = conn.execute(
"SELECT model_profile, model_name, embedding_dim, created_at, updated_at "
"FROM embeddings_config WHERE id = 1"
).fetchone()
if row:
return {
"model_profile": row[0],
"model_name": row[1],
"embedding_dim": row[2],
"created_at": row[3],
"updated_at": row[4],
}
return None
def set_model_config(
self, model_profile: str, model_name: str, embedding_dim: int
) -> None:
"""Set the model configuration for embeddings in this store.
This should be called when generating new embeddings. If a different
model was previously used, this will update the configuration.
Args:
model_profile: Model profile name (fast, code, minilm, etc.)
model_name: Full model name (e.g., jinaai/jina-embeddings-v2-base-code)
embedding_dim: Embedding dimension (e.g., 768)
"""
with sqlite3.connect(self.db_path) as conn:
conn.execute(
"""
INSERT INTO embeddings_config (id, model_profile, model_name, embedding_dim)
VALUES (1, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
model_profile = excluded.model_profile,
model_name = excluded.model_name,
embedding_dim = excluded.embedding_dim,
updated_at = CURRENT_TIMESTAMP
""",
(model_profile, model_name, embedding_dim)
)
conn.commit()
def check_model_compatibility(
self, model_profile: str, model_name: str, embedding_dim: int
) -> Tuple[bool, Optional[str]]:
"""Check if the given model is compatible with existing embeddings.
Args:
model_profile: Model profile to check
model_name: Model name to check
embedding_dim: Embedding dimension to check
Returns:
Tuple of (is_compatible, warning_message).
is_compatible is True if no existing config or configs match.
warning_message is a user-friendly message if incompatible.
"""
existing = self.get_model_config()
if existing is None:
return True, None
# Check dimension first (most critical)
if existing["embedding_dim"] != embedding_dim:
return False, (
f"Dimension mismatch: existing embeddings use {existing['embedding_dim']}d "
f"({existing['model_profile']}), but requested model uses {embedding_dim}d "
f"({model_profile}). Use --force to regenerate all embeddings."
)
# Check model (different models with same dimension may have different semantic spaces)
if existing["model_profile"] != model_profile:
return False, (
f"Model mismatch: existing embeddings use '{existing['model_profile']}' "
f"({existing['model_name']}), but requested '{model_profile}' "
f"({model_name}). Use --force to regenerate all embeddings."
)
return True, None
def close(self) -> None:
"""Close the vector store and release resources.

View File

@@ -1,6 +1,6 @@
{
"name": "claude-code-workflow",
"version": "6.2.6",
"version": "6.2.7",
"description": "JSON-driven multi-agent development framework with intelligent CLI orchestration (Gemini/Qwen/Codex), context-first architecture, and automated workflow execution",
"type": "module",
"main": "ccw/src/index.js",