fix: CodexLens model detection, hybrid search stability, and JSON logging

- Fix model installation detection using fastembed ONNX cache names
- Add embeddings_config table for model metadata tracking
- Fix hybrid search segfault by using single-threaded GPU mode
- Suppress INFO logs in JSON mode to prevent error display
- Add model dropdown filtering to show only installed models

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
catlog22
2025-12-22 21:49:10 +08:00
parent cf58dc0dd3
commit 8203d690cb
11 changed files with 302 additions and 57 deletions

View File

@@ -98,15 +98,17 @@ async function loadCodexLensStatus() {
} }
window.cliToolsStatus.codexlens = { window.cliToolsStatus.codexlens = {
installed: data.ready || false, installed: data.ready || false,
version: data.version || null version: data.version || null,
installedModels: [] // Will be populated by loadSemanticStatus
}; };
// Update CodexLens badge // Update CodexLens badge
updateCodexLensBadge(); updateCodexLensBadge();
// If CodexLens is ready, also check semantic status // If CodexLens is ready, also check semantic status and models
if (data.ready) { if (data.ready) {
await loadSemanticStatus(); await loadSemanticStatus();
await loadInstalledModels();
} }
return data; return data;
@@ -132,6 +134,37 @@ async function loadSemanticStatus() {
} }
} }
/**
* Load installed embedding models
*/
async function loadInstalledModels() {
try {
const response = await fetch('/api/codexlens/models');
if (!response.ok) throw new Error('Failed to load models');
const data = await response.json();
if (data.success && data.result && data.result.models) {
// Filter to only installed models
const installedModels = data.result.models
.filter(m => m.installed)
.map(m => m.profile);
// Update window.cliToolsStatus
if (window.cliToolsStatus && window.cliToolsStatus.codexlens) {
window.cliToolsStatus.codexlens.installedModels = installedModels;
window.cliToolsStatus.codexlens.allModels = data.result.models;
}
console.log('[CLI Status] Installed models:', installedModels);
return installedModels;
}
return [];
} catch (err) {
console.error('Failed to load installed models:', err);
return [];
}
}
// ========== Badge Update ========== // ========== Badge Update ==========
function updateCliBadge() { function updateCliBadge() {
const badge = document.getElementById('badgeCliTools'); const badge = document.getElementById('badgeCliTools');

View File

@@ -349,6 +349,50 @@ function getSelectedModel() {
return select ? select.value : 'code'; return select ? select.value : 'code';
} }
/**
* Build model select options HTML, showing only installed models
* @returns {string} HTML string for select options
*/
function buildModelSelectOptions() {
var installedModels = window.cliToolsStatus?.codexlens?.installedModels || [];
var allModels = window.cliToolsStatus?.codexlens?.allModels || [];
// Model display configuration
var modelConfig = {
'code': { label: t('index.modelCode') || 'Code (768d)', star: true },
'base': { label: t('index.modelBase') || 'Base (768d)', star: false },
'fast': { label: t('index.modelFast') || 'Fast (384d)', star: false },
'minilm': { label: t('index.modelMinilm') || 'MiniLM (384d)', star: false },
'multilingual': { label: t('index.modelMultilingual') || 'Multilingual (1024d)', warn: true },
'balanced': { label: t('index.modelBalanced') || 'Balanced (1024d)', warn: true }
};
// If no models installed, show placeholder
if (installedModels.length === 0) {
return '<option value="" disabled selected>' + (t('index.noModelsInstalled') || 'No models installed') + '</option>';
}
// Build options for installed models only
var options = '';
var firstInstalled = null;
// Preferred order: code, fast, minilm, base, multilingual, balanced
var preferredOrder = ['code', 'fast', 'minilm', 'base', 'multilingual', 'balanced'];
preferredOrder.forEach(function(profile) {
if (installedModels.includes(profile) && modelConfig[profile]) {
var config = modelConfig[profile];
var style = config.warn ? ' style="color: var(--muted-foreground)"' : '';
var suffix = config.star ? ' ⭐' : (config.warn ? ' ⚠️' : '');
var selected = !firstInstalled ? ' selected' : '';
if (!firstInstalled) firstInstalled = profile;
options += '<option value="' + profile + '"' + style + selected + '>' + config.label + suffix + '</option>';
}
});
return options;
}
// ========== Tools Section (Left Column) ========== // ========== Tools Section (Left Column) ==========
function renderToolsSection() { function renderToolsSection() {
var container = document.getElementById('tools-section'); var container = document.getElementById('tools-section');
@@ -404,12 +448,7 @@ function renderToolsSection() {
(codexLensStatus.ready (codexLensStatus.ready
? '<span class="tool-status-text success"><i data-lucide="check-circle" class="w-3.5 h-3.5"></i> v' + (codexLensStatus.version || 'installed') + '</span>' + ? '<span class="tool-status-text success"><i data-lucide="check-circle" class="w-3.5 h-3.5"></i> v' + (codexLensStatus.version || 'installed') + '</span>' +
'<select id="codexlensModelSelect" class="btn-sm bg-muted border border-border rounded text-xs" onclick="event.stopPropagation()" title="' + (t('index.selectModel') || 'Select embedding model') + '">' + '<select id="codexlensModelSelect" class="btn-sm bg-muted border border-border rounded text-xs" onclick="event.stopPropagation()" title="' + (t('index.selectModel') || 'Select embedding model') + '">' +
'<option value="code">' + (t('index.modelCode') || 'Code (768d)') + ' ⭐</option>' + buildModelSelectOptions() +
'<option value="base">' + (t('index.modelBase') || 'Base (768d)') + '</option>' +
'<option value="fast">' + (t('index.modelFast') || 'Fast (384d)') + '</option>' +
'<option value="minilm">' + (t('index.modelMinilm') || 'MiniLM (384d)') + '</option>' +
'<option value="multilingual" style="color: var(--muted-foreground)">' + (t('index.modelMultilingual') || 'Multilingual (1024d)') + ' ⚠️</option>' +
'<option value="balanced" style="color: var(--muted-foreground)">' + (t('index.modelBalanced') || 'Balanced (1024d)') + ' ⚠️</option>' +
'</select>' + '</select>' +
'<button class="btn-sm btn-primary" onclick="event.stopPropagation(); initCodexLensIndex(\'full\', getSelectedModel())" title="' + (t('index.fullDesc') || 'FTS + Semantic search (recommended)') + '"><i data-lucide="layers" class="w-3 h-3"></i> ' + (t('index.fullIndex') || '全部索引') + '</button>' + '<button class="btn-sm btn-primary" onclick="event.stopPropagation(); initCodexLensIndex(\'full\', getSelectedModel())" title="' + (t('index.fullDesc') || 'FTS + Semantic search (recommended)') + '"><i data-lucide="layers" class="w-3 h-3"></i> ' + (t('index.fullIndex') || '全部索引') + '</button>' +
'<button class="btn-sm btn-outline" onclick="event.stopPropagation(); initCodexLensIndex(\'vector\', getSelectedModel())" title="' + (t('index.vectorDesc') || 'Semantic search with embeddings') + '"><i data-lucide="sparkles" class="w-3 h-3"></i> ' + (t('index.vectorIndex') || '向量索引') + '</button>' + '<button class="btn-sm btn-outline" onclick="event.stopPropagation(); initCodexLensIndex(\'vector\', getSelectedModel())" title="' + (t('index.vectorDesc') || 'Semantic search with embeddings') + '"><i data-lucide="sparkles" class="w-3 h-3"></i> ' + (t('index.vectorIndex') || '向量索引') + '</button>' +

View File

@@ -1,6 +1,6 @@
Metadata-Version: 2.4 Metadata-Version: 2.4
Name: codex-lens Name: codex-lens
Version: 0.2.0 Version: 0.1.0
Summary: CodexLens multi-modal code analysis platform Summary: CodexLens multi-modal code analysis platform
Author: CodexLens contributors Author: CodexLens contributors
License: MIT License: MIT
@@ -17,18 +17,18 @@ Requires-Dist: tree-sitter-typescript>=0.23
Requires-Dist: pathspec>=0.11 Requires-Dist: pathspec>=0.11
Provides-Extra: semantic Provides-Extra: semantic
Requires-Dist: numpy>=1.24; extra == "semantic" Requires-Dist: numpy>=1.24; extra == "semantic"
Requires-Dist: fastembed>=0.5; extra == "semantic" Requires-Dist: fastembed>=0.2; extra == "semantic"
Requires-Dist: hnswlib>=0.8.0; extra == "semantic" Requires-Dist: hnswlib>=0.8.0; extra == "semantic"
Provides-Extra: semantic-gpu Provides-Extra: semantic-gpu
Requires-Dist: numpy>=1.24; extra == "semantic-gpu" Requires-Dist: numpy>=1.24; extra == "semantic-gpu"
Requires-Dist: fastembed>=0.5; extra == "semantic-gpu" Requires-Dist: fastembed>=0.2; extra == "semantic-gpu"
Requires-Dist: hnswlib>=0.8.0; extra == "semantic-gpu" Requires-Dist: hnswlib>=0.8.0; extra == "semantic-gpu"
Requires-Dist: onnxruntime-gpu>=1.18.0; extra == "semantic-gpu" Requires-Dist: onnxruntime-gpu>=1.15.0; extra == "semantic-gpu"
Provides-Extra: semantic-directml Provides-Extra: semantic-directml
Requires-Dist: numpy>=1.24; extra == "semantic-directml" Requires-Dist: numpy>=1.24; extra == "semantic-directml"
Requires-Dist: fastembed>=0.5; extra == "semantic-directml" Requires-Dist: fastembed>=0.2; extra == "semantic-directml"
Requires-Dist: hnswlib>=0.8.0; extra == "semantic-directml" Requires-Dist: hnswlib>=0.8.0; extra == "semantic-directml"
Requires-Dist: onnxruntime-directml>=1.18.0; extra == "semantic-directml" Requires-Dist: onnxruntime-directml>=1.15.0; extra == "semantic-directml"
Provides-Extra: encoding Provides-Extra: encoding
Requires-Dist: chardet>=5.0; extra == "encoding" Requires-Dist: chardet>=5.0; extra == "encoding"
Provides-Extra: full Provides-Extra: full

View File

@@ -15,17 +15,17 @@ tiktoken>=0.5.0
[semantic] [semantic]
numpy>=1.24 numpy>=1.24
fastembed>=0.5 fastembed>=0.2
hnswlib>=0.8.0 hnswlib>=0.8.0
[semantic-directml] [semantic-directml]
numpy>=1.24 numpy>=1.24
fastembed>=0.5 fastembed>=0.2
hnswlib>=0.8.0 hnswlib>=0.8.0
onnxruntime-directml>=1.18.0 onnxruntime-directml>=1.15.0
[semantic-gpu] [semantic-gpu]
numpy>=1.24 numpy>=1.24
fastembed>=0.5 fastembed>=0.2
hnswlib>=0.8.0 hnswlib>=0.8.0
onnxruntime-gpu>=1.18.0 onnxruntime-gpu>=1.15.0

View File

@@ -35,8 +35,17 @@ from .output import (
app = typer.Typer(help="CodexLens CLI — local code indexing and search.") app = typer.Typer(help="CodexLens CLI — local code indexing and search.")
def _configure_logging(verbose: bool) -> None: def _configure_logging(verbose: bool, json_mode: bool = False) -> None:
level = logging.DEBUG if verbose else logging.INFO """Configure logging level.
In JSON mode, suppress INFO logs to keep stderr clean for error parsing.
Only WARNING and above are shown to avoid mixing logs with JSON output.
"""
if json_mode and not verbose:
# In JSON mode, suppress INFO logs to keep stderr clean
level = logging.WARNING
else:
level = logging.DEBUG if verbose else logging.INFO
logging.basicConfig(level=level, format="%(levelname)s %(message)s") logging.basicConfig(level=level, format="%(levelname)s %(message)s")
@@ -95,7 +104,7 @@ def init(
If semantic search dependencies are installed, automatically generates embeddings If semantic search dependencies are installed, automatically generates embeddings
after indexing completes. Use --no-embeddings to skip this step. after indexing completes. Use --no-embeddings to skip this step.
""" """
_configure_logging(verbose) _configure_logging(verbose, json_mode)
config = Config() config = Config()
languages = _parse_languages(language) languages = _parse_languages(language)
base_path = path.expanduser().resolve() base_path = path.expanduser().resolve()
@@ -314,7 +323,7 @@ def search(
# Force hybrid mode # Force hybrid mode
codexlens search "authentication" --mode hybrid codexlens search "authentication" --mode hybrid
""" """
_configure_logging(verbose) _configure_logging(verbose, json_mode)
search_path = path.expanduser().resolve() search_path = path.expanduser().resolve()
# Validate mode # Validate mode
@@ -487,7 +496,7 @@ def symbol(
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."), verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
) -> None: ) -> None:
"""Look up symbols by name and optional kind.""" """Look up symbols by name and optional kind."""
_configure_logging(verbose) _configure_logging(verbose, json_mode)
search_path = path.expanduser().resolve() search_path = path.expanduser().resolve()
registry: RegistryStore | None = None registry: RegistryStore | None = None
@@ -538,7 +547,7 @@ def inspect(
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."), verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
) -> None: ) -> None:
"""Analyze a single file and display symbols.""" """Analyze a single file and display symbols."""
_configure_logging(verbose) _configure_logging(verbose, json_mode)
config = Config() config = Config()
factory = ParserFactory(config) factory = ParserFactory(config)
@@ -588,7 +597,7 @@ def status(
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."), verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
) -> None: ) -> None:
"""Show index status and configuration.""" """Show index status and configuration."""
_configure_logging(verbose) _configure_logging(verbose, json_mode)
registry: RegistryStore | None = None registry: RegistryStore | None = None
try: try:
@@ -648,7 +657,7 @@ def status(
# Embedding manager not available # Embedding manager not available
pass pass
except Exception as e: except Exception as e:
logger.debug(f"Failed to get embeddings status: {e}") logging.debug(f"Failed to get embeddings status: {e}")
stats = { stats = {
"index_root": str(index_root), "index_root": str(index_root),
@@ -737,7 +746,7 @@ def projects(
- show <path>: Show details for a specific project - show <path>: Show details for a specific project
- remove <path>: Remove a project from the registry - remove <path>: Remove a project from the registry
""" """
_configure_logging(verbose) _configure_logging(verbose, json_mode)
registry: RegistryStore | None = None registry: RegistryStore | None = None
try: try:
@@ -892,7 +901,7 @@ def config(
Config keys: Config keys:
- index_dir: Directory to store indexes (default: ~/.codexlens/indexes) - index_dir: Directory to store indexes (default: ~/.codexlens/indexes)
""" """
_configure_logging(verbose) _configure_logging(verbose, json_mode)
config_file = Path.home() / ".codexlens" / "config.json" config_file = Path.home() / ".codexlens" / "config.json"
@@ -1057,7 +1066,7 @@ def migrate(
This is a safe operation that preserves all existing data. This is a safe operation that preserves all existing data.
Progress is shown during migration. Progress is shown during migration.
""" """
_configure_logging(verbose) _configure_logging(verbose, json_mode)
base_path = path.expanduser().resolve() base_path = path.expanduser().resolve()
registry: RegistryStore | None = None registry: RegistryStore | None = None
@@ -1183,7 +1192,7 @@ def clean(
With path, removes that project's indexes. With path, removes that project's indexes.
With --all, removes all indexes (use with caution). With --all, removes all indexes (use with caution).
""" """
_configure_logging(verbose) _configure_logging(verbose, json_mode)
try: try:
mapper = PathMapper() mapper = PathMapper()
@@ -1329,7 +1338,7 @@ def semantic_list(
Shows files that have LLM-generated summaries and keywords. Shows files that have LLM-generated summaries and keywords.
Results are aggregated from all index databases in the project. Results are aggregated from all index databases in the project.
""" """
_configure_logging(verbose) _configure_logging(verbose, json_mode)
base_path = path.expanduser().resolve() base_path = path.expanduser().resolve()
registry: Optional[RegistryStore] = None registry: Optional[RegistryStore] = None
@@ -1798,7 +1807,7 @@ def embeddings_generate(
codexlens embeddings-generate ~/.codexlens/indexes/project/_index.db # Specific index codexlens embeddings-generate ~/.codexlens/indexes/project/_index.db # Specific index
codexlens embeddings-generate ~/projects/my-app --model fast --force # Regenerate with fast model codexlens embeddings-generate ~/projects/my-app --model fast --force # Regenerate with fast model
""" """
_configure_logging(verbose) _configure_logging(verbose, json_mode)
from codexlens.cli.embedding_manager import generate_embeddings, generate_embeddings_recursive from codexlens.cli.embedding_manager import generate_embeddings, generate_embeddings_recursive

View File

@@ -279,6 +279,21 @@ def generate_embeddings(
try: try:
with VectorStore(index_path) as vector_store: with VectorStore(index_path) as vector_store:
# Check model compatibility with existing embeddings
if not force:
is_compatible, warning = vector_store.check_model_compatibility(
model_profile, embedder.model_name, embedder.embedding_dim
)
if not is_compatible:
return {
"success": False,
"error": warning,
}
# Set/update model configuration for this index
vector_store.set_model_config(
model_profile, embedder.model_name, embedder.embedding_dim
)
# Use bulk insert mode for efficient batch ANN index building # Use bulk insert mode for efficient batch ANN index building
# This defers ANN updates until end_bulk_insert() is called # This defers ANN updates until end_bulk_insert() is called
with vector_store.bulk_insert(): with vector_store.bulk_insert():

View File

@@ -16,9 +16,11 @@ except ImportError:
# Model profiles with metadata # Model profiles with metadata
# Note: 768d is max recommended dimension for optimal performance/quality balance # Note: 768d is max recommended dimension for optimal performance/quality balance
# 1024d models are available but not recommended due to higher resource usage # 1024d models are available but not recommended due to higher resource usage
# cache_name: The actual Hugging Face repo name used by fastembed for ONNX caching
MODEL_PROFILES = { MODEL_PROFILES = {
"fast": { "fast": {
"model_name": "BAAI/bge-small-en-v1.5", "model_name": "BAAI/bge-small-en-v1.5",
"cache_name": "qdrant/bge-small-en-v1.5-onnx-q", # fastembed uses ONNX version
"dimensions": 384, "dimensions": 384,
"size_mb": 80, "size_mb": 80,
"description": "Fast, lightweight, English-optimized", "description": "Fast, lightweight, English-optimized",
@@ -27,6 +29,7 @@ MODEL_PROFILES = {
}, },
"base": { "base": {
"model_name": "BAAI/bge-base-en-v1.5", "model_name": "BAAI/bge-base-en-v1.5",
"cache_name": "qdrant/bge-base-en-v1.5-onnx-q", # fastembed uses ONNX version
"dimensions": 768, "dimensions": 768,
"size_mb": 220, "size_mb": 220,
"description": "General purpose, good balance of speed and quality", "description": "General purpose, good balance of speed and quality",
@@ -35,6 +38,7 @@ MODEL_PROFILES = {
}, },
"code": { "code": {
"model_name": "jinaai/jina-embeddings-v2-base-code", "model_name": "jinaai/jina-embeddings-v2-base-code",
"cache_name": "jinaai/jina-embeddings-v2-base-code", # Uses original name
"dimensions": 768, "dimensions": 768,
"size_mb": 150, "size_mb": 150,
"description": "Code-optimized, best for programming languages", "description": "Code-optimized, best for programming languages",
@@ -43,6 +47,7 @@ MODEL_PROFILES = {
}, },
"minilm": { "minilm": {
"model_name": "sentence-transformers/all-MiniLM-L6-v2", "model_name": "sentence-transformers/all-MiniLM-L6-v2",
"cache_name": "qdrant/all-MiniLM-L6-v2-onnx", # fastembed uses ONNX version
"dimensions": 384, "dimensions": 384,
"size_mb": 90, "size_mb": 90,
"description": "Popular lightweight model, good quality", "description": "Popular lightweight model, good quality",
@@ -51,6 +56,7 @@ MODEL_PROFILES = {
}, },
"multilingual": { "multilingual": {
"model_name": "intfloat/multilingual-e5-large", "model_name": "intfloat/multilingual-e5-large",
"cache_name": "qdrant/multilingual-e5-large-onnx", # fastembed uses ONNX version
"dimensions": 1024, "dimensions": 1024,
"size_mb": 1000, "size_mb": 1000,
"description": "Multilingual + code support (high resource usage)", "description": "Multilingual + code support (high resource usage)",
@@ -59,6 +65,7 @@ MODEL_PROFILES = {
}, },
"balanced": { "balanced": {
"model_name": "mixedbread-ai/mxbai-embed-large-v1", "model_name": "mixedbread-ai/mxbai-embed-large-v1",
"cache_name": "mixedbread-ai/mxbai-embed-large-v1", # Uses original name
"dimensions": 1024, "dimensions": 1024,
"size_mb": 600, "size_mb": 600,
"description": "High accuracy, general purpose (high resource usage)", "description": "High accuracy, general purpose (high resource usage)",
@@ -87,6 +94,23 @@ def get_cache_dir() -> Path:
return cache_dir return cache_dir
def _get_model_cache_path(cache_dir: Path, info: Dict) -> Path:
"""Get the actual cache path for a model.
fastembed uses ONNX versions of models with different names than the original.
This function returns the correct path based on the cache_name field.
Args:
cache_dir: The fastembed cache directory
info: Model profile info dictionary
Returns:
Path to the model cache directory
"""
cache_name = info.get("cache_name", info["model_name"])
return cache_dir / f"models--{cache_name.replace('/', '--')}"
def list_models() -> Dict[str, any]: def list_models() -> Dict[str, any]:
"""List available model profiles and their installation status. """List available model profiles and their installation status.
@@ -106,13 +130,13 @@ def list_models() -> Dict[str, any]:
for profile, info in MODEL_PROFILES.items(): for profile, info in MODEL_PROFILES.items():
model_name = info["model_name"] model_name = info["model_name"]
# Check if model is cached # Check if model is cached using the actual cache name
installed = False installed = False
cache_size_mb = 0 cache_size_mb = 0
if cache_exists: if cache_exists:
# Check for model directory in cache # Check for model directory in cache using correct cache_name
model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}" model_cache_path = _get_model_cache_path(cache_dir, info)
if model_cache_path.exists(): if model_cache_path.exists():
installed = True installed = True
# Calculate cache size # Calculate cache size
@@ -166,7 +190,8 @@ def download_model(profile: str, progress_callback: Optional[callable] = None) -
"error": f"Unknown profile: {profile}. Available: {', '.join(MODEL_PROFILES.keys())}", "error": f"Unknown profile: {profile}. Available: {', '.join(MODEL_PROFILES.keys())}",
} }
model_name = MODEL_PROFILES[profile]["model_name"] info = MODEL_PROFILES[profile]
model_name = info["model_name"]
try: try:
# Download model by instantiating TextEmbedding # Download model by instantiating TextEmbedding
@@ -179,9 +204,9 @@ def download_model(profile: str, progress_callback: Optional[callable] = None) -
if progress_callback: if progress_callback:
progress_callback(f"Model {model_name} downloaded successfully") progress_callback(f"Model {model_name} downloaded successfully")
# Get cache info # Get cache info using correct cache_name
cache_dir = get_cache_dir() cache_dir = get_cache_dir()
model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}" model_cache_path = _get_model_cache_path(cache_dir, info)
cache_size = 0 cache_size = 0
if model_cache_path.exists(): if model_cache_path.exists():
@@ -224,9 +249,10 @@ def delete_model(profile: str) -> Dict[str, any]:
"error": f"Unknown profile: {profile}. Available: {', '.join(MODEL_PROFILES.keys())}", "error": f"Unknown profile: {profile}. Available: {', '.join(MODEL_PROFILES.keys())}",
} }
model_name = MODEL_PROFILES[profile]["model_name"] info = MODEL_PROFILES[profile]
model_name = info["model_name"]
cache_dir = get_cache_dir() cache_dir = get_cache_dir()
model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}" model_cache_path = _get_model_cache_path(cache_dir, info)
if not model_cache_path.exists(): if not model_cache_path.exists():
return { return {
@@ -281,9 +307,9 @@ def get_model_info(profile: str) -> Dict[str, any]:
info = MODEL_PROFILES[profile] info = MODEL_PROFILES[profile]
model_name = info["model_name"] model_name = info["model_name"]
# Check installation status # Check installation status using correct cache_name
cache_dir = get_cache_dir() cache_dir = get_cache_dir()
model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}" model_cache_path = _get_model_cache_path(cache_dir, info)
installed = model_cache_path.exists() installed = model_cache_path.exists()
cache_size_mb = None cache_size_mb = None

View File

@@ -396,7 +396,20 @@ class ChainSearchEngine:
all_results = [] all_results = []
stats = SearchStats() stats = SearchStats()
executor = self._get_executor(options.max_workers) # Force single-threaded execution for vector/hybrid search to avoid GPU crashes
# DirectML/ONNX have threading issues when multiple threads access GPU resources
effective_workers = options.max_workers
if options.enable_vector or options.hybrid_mode:
effective_workers = 1
self.logger.debug("Using single-threaded mode for vector search (GPU safety)")
# Pre-load embedder to avoid initialization overhead per-search
try:
from codexlens.semantic.embedder import get_embedder
get_embedder(profile="code", use_gpu=True)
except Exception:
pass # Ignore pre-load failures
executor = self._get_executor(effective_workers)
# Submit all search tasks # Submit all search tasks
future_to_path = { future_to_path = {
executor.submit( executor.submit(

View File

@@ -274,19 +274,32 @@ class HybridSearchEngine:
) )
return [] return []
# Auto-detect embedding dimension and select appropriate profile # Get stored model configuration (preferred) or auto-detect from dimension
detected_dim = vector_store.dimension model_config = vector_store.get_model_config()
if detected_dim is None: if model_config:
self.logger.info("Vector store dimension unknown, using default profile") profile = model_config["model_profile"]
profile = "code" # Default fallback self.logger.debug(
elif detected_dim == 384: "Using stored model config: %s (%s, %dd)",
profile = "fast" profile, model_config["model_name"], model_config["embedding_dim"]
elif detected_dim == 768: )
profile = "code"
elif detected_dim == 1024:
profile = "multilingual" # or balanced, both are 1024
else: else:
profile = "code" # Default fallback # Fallback: auto-detect from embedding dimension
detected_dim = vector_store.dimension
if detected_dim is None:
self.logger.info("Vector store dimension unknown, using default profile")
profile = "code" # Default fallback
elif detected_dim == 384:
profile = "fast"
elif detected_dim == 768:
profile = "code"
elif detected_dim == 1024:
profile = "multilingual" # or balanced, both are 1024
else:
profile = "code" # Default fallback
self.logger.debug(
"No stored model config, auto-detected profile '%s' from dimension %s",
profile, detected_dim
)
# Use cached embedder (singleton) for performance # Use cached embedder (singleton) for performance
embedder = get_embedder(profile=profile) embedder = get_embedder(profile=profile)

View File

@@ -116,6 +116,17 @@ class VectorStore:
CREATE INDEX IF NOT EXISTS idx_chunks_file CREATE INDEX IF NOT EXISTS idx_chunks_file
ON semantic_chunks(file_path) ON semantic_chunks(file_path)
""") """)
# Model configuration table - tracks which model generated the embeddings
conn.execute("""
CREATE TABLE IF NOT EXISTS embeddings_config (
id INTEGER PRIMARY KEY CHECK (id = 1),
model_profile TEXT NOT NULL,
model_name TEXT NOT NULL,
embedding_dim INTEGER NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
conn.commit() conn.commit()
def _init_ann_index(self) -> None: def _init_ann_index(self) -> None:
@@ -932,6 +943,92 @@ class VectorStore:
return self._ann_index.count() return self._ann_index.count()
return 0 return 0
def get_model_config(self) -> Optional[Dict[str, Any]]:
"""Get the model configuration used for embeddings in this store.
Returns:
Dictionary with model_profile, model_name, embedding_dim, or None if not set.
"""
with sqlite3.connect(self.db_path) as conn:
row = conn.execute(
"SELECT model_profile, model_name, embedding_dim, created_at, updated_at "
"FROM embeddings_config WHERE id = 1"
).fetchone()
if row:
return {
"model_profile": row[0],
"model_name": row[1],
"embedding_dim": row[2],
"created_at": row[3],
"updated_at": row[4],
}
return None
def set_model_config(
self, model_profile: str, model_name: str, embedding_dim: int
) -> None:
"""Set the model configuration for embeddings in this store.
This should be called when generating new embeddings. If a different
model was previously used, this will update the configuration.
Args:
model_profile: Model profile name (fast, code, minilm, etc.)
model_name: Full model name (e.g., jinaai/jina-embeddings-v2-base-code)
embedding_dim: Embedding dimension (e.g., 768)
"""
with sqlite3.connect(self.db_path) as conn:
conn.execute(
"""
INSERT INTO embeddings_config (id, model_profile, model_name, embedding_dim)
VALUES (1, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
model_profile = excluded.model_profile,
model_name = excluded.model_name,
embedding_dim = excluded.embedding_dim,
updated_at = CURRENT_TIMESTAMP
""",
(model_profile, model_name, embedding_dim)
)
conn.commit()
def check_model_compatibility(
self, model_profile: str, model_name: str, embedding_dim: int
) -> Tuple[bool, Optional[str]]:
"""Check if the given model is compatible with existing embeddings.
Args:
model_profile: Model profile to check
model_name: Model name to check
embedding_dim: Embedding dimension to check
Returns:
Tuple of (is_compatible, warning_message).
is_compatible is True if no existing config or configs match.
warning_message is a user-friendly message if incompatible.
"""
existing = self.get_model_config()
if existing is None:
return True, None
# Check dimension first (most critical)
if existing["embedding_dim"] != embedding_dim:
return False, (
f"Dimension mismatch: existing embeddings use {existing['embedding_dim']}d "
f"({existing['model_profile']}), but requested model uses {embedding_dim}d "
f"({model_profile}). Use --force to regenerate all embeddings."
)
# Check model (different models with same dimension may have different semantic spaces)
if existing["model_profile"] != model_profile:
return False, (
f"Model mismatch: existing embeddings use '{existing['model_profile']}' "
f"({existing['model_name']}), but requested '{model_profile}' "
f"({model_name}). Use --force to regenerate all embeddings."
)
return True, None
def close(self) -> None: def close(self) -> None:
"""Close the vector store and release resources. """Close the vector store and release resources.

View File

@@ -1,6 +1,6 @@
{ {
"name": "claude-code-workflow", "name": "claude-code-workflow",
"version": "6.2.6", "version": "6.2.7",
"description": "JSON-driven multi-agent development framework with intelligent CLI orchestration (Gemini/Qwen/Codex), context-first architecture, and automated workflow execution", "description": "JSON-driven multi-agent development framework with intelligent CLI orchestration (Gemini/Qwen/Codex), context-first architecture, and automated workflow execution",
"type": "module", "type": "module",
"main": "ccw/src/index.js", "main": "ccw/src/index.js",