mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-02-05 01:50:27 +08:00
fix: CodexLens model detection, hybrid search stability, and JSON logging
- Fix model installation detection using fastembed ONNX cache names - Add embeddings_config table for model metadata tracking - Fix hybrid search segfault by using single-threaded GPU mode - Suppress INFO logs in JSON mode to prevent error display - Add model dropdown filtering to show only installed models 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -98,15 +98,17 @@ async function loadCodexLensStatus() {
|
|||||||
}
|
}
|
||||||
window.cliToolsStatus.codexlens = {
|
window.cliToolsStatus.codexlens = {
|
||||||
installed: data.ready || false,
|
installed: data.ready || false,
|
||||||
version: data.version || null
|
version: data.version || null,
|
||||||
|
installedModels: [] // Will be populated by loadSemanticStatus
|
||||||
};
|
};
|
||||||
|
|
||||||
// Update CodexLens badge
|
// Update CodexLens badge
|
||||||
updateCodexLensBadge();
|
updateCodexLensBadge();
|
||||||
|
|
||||||
// If CodexLens is ready, also check semantic status
|
// If CodexLens is ready, also check semantic status and models
|
||||||
if (data.ready) {
|
if (data.ready) {
|
||||||
await loadSemanticStatus();
|
await loadSemanticStatus();
|
||||||
|
await loadInstalledModels();
|
||||||
}
|
}
|
||||||
|
|
||||||
return data;
|
return data;
|
||||||
@@ -132,6 +134,37 @@ async function loadSemanticStatus() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load installed embedding models
|
||||||
|
*/
|
||||||
|
async function loadInstalledModels() {
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/codexlens/models');
|
||||||
|
if (!response.ok) throw new Error('Failed to load models');
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
if (data.success && data.result && data.result.models) {
|
||||||
|
// Filter to only installed models
|
||||||
|
const installedModels = data.result.models
|
||||||
|
.filter(m => m.installed)
|
||||||
|
.map(m => m.profile);
|
||||||
|
|
||||||
|
// Update window.cliToolsStatus
|
||||||
|
if (window.cliToolsStatus && window.cliToolsStatus.codexlens) {
|
||||||
|
window.cliToolsStatus.codexlens.installedModels = installedModels;
|
||||||
|
window.cliToolsStatus.codexlens.allModels = data.result.models;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('[CLI Status] Installed models:', installedModels);
|
||||||
|
return installedModels;
|
||||||
|
}
|
||||||
|
return [];
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Failed to load installed models:', err);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ========== Badge Update ==========
|
// ========== Badge Update ==========
|
||||||
function updateCliBadge() {
|
function updateCliBadge() {
|
||||||
const badge = document.getElementById('badgeCliTools');
|
const badge = document.getElementById('badgeCliTools');
|
||||||
|
|||||||
@@ -349,6 +349,50 @@ function getSelectedModel() {
|
|||||||
return select ? select.value : 'code';
|
return select ? select.value : 'code';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build model select options HTML, showing only installed models
|
||||||
|
* @returns {string} HTML string for select options
|
||||||
|
*/
|
||||||
|
function buildModelSelectOptions() {
|
||||||
|
var installedModels = window.cliToolsStatus?.codexlens?.installedModels || [];
|
||||||
|
var allModels = window.cliToolsStatus?.codexlens?.allModels || [];
|
||||||
|
|
||||||
|
// Model display configuration
|
||||||
|
var modelConfig = {
|
||||||
|
'code': { label: t('index.modelCode') || 'Code (768d)', star: true },
|
||||||
|
'base': { label: t('index.modelBase') || 'Base (768d)', star: false },
|
||||||
|
'fast': { label: t('index.modelFast') || 'Fast (384d)', star: false },
|
||||||
|
'minilm': { label: t('index.modelMinilm') || 'MiniLM (384d)', star: false },
|
||||||
|
'multilingual': { label: t('index.modelMultilingual') || 'Multilingual (1024d)', warn: true },
|
||||||
|
'balanced': { label: t('index.modelBalanced') || 'Balanced (1024d)', warn: true }
|
||||||
|
};
|
||||||
|
|
||||||
|
// If no models installed, show placeholder
|
||||||
|
if (installedModels.length === 0) {
|
||||||
|
return '<option value="" disabled selected>' + (t('index.noModelsInstalled') || 'No models installed') + '</option>';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build options for installed models only
|
||||||
|
var options = '';
|
||||||
|
var firstInstalled = null;
|
||||||
|
|
||||||
|
// Preferred order: code, fast, minilm, base, multilingual, balanced
|
||||||
|
var preferredOrder = ['code', 'fast', 'minilm', 'base', 'multilingual', 'balanced'];
|
||||||
|
|
||||||
|
preferredOrder.forEach(function(profile) {
|
||||||
|
if (installedModels.includes(profile) && modelConfig[profile]) {
|
||||||
|
var config = modelConfig[profile];
|
||||||
|
var style = config.warn ? ' style="color: var(--muted-foreground)"' : '';
|
||||||
|
var suffix = config.star ? ' ⭐' : (config.warn ? ' ⚠️' : '');
|
||||||
|
var selected = !firstInstalled ? ' selected' : '';
|
||||||
|
if (!firstInstalled) firstInstalled = profile;
|
||||||
|
options += '<option value="' + profile + '"' + style + selected + '>' + config.label + suffix + '</option>';
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return options;
|
||||||
|
}
|
||||||
|
|
||||||
// ========== Tools Section (Left Column) ==========
|
// ========== Tools Section (Left Column) ==========
|
||||||
function renderToolsSection() {
|
function renderToolsSection() {
|
||||||
var container = document.getElementById('tools-section');
|
var container = document.getElementById('tools-section');
|
||||||
@@ -404,12 +448,7 @@ function renderToolsSection() {
|
|||||||
(codexLensStatus.ready
|
(codexLensStatus.ready
|
||||||
? '<span class="tool-status-text success"><i data-lucide="check-circle" class="w-3.5 h-3.5"></i> v' + (codexLensStatus.version || 'installed') + '</span>' +
|
? '<span class="tool-status-text success"><i data-lucide="check-circle" class="w-3.5 h-3.5"></i> v' + (codexLensStatus.version || 'installed') + '</span>' +
|
||||||
'<select id="codexlensModelSelect" class="btn-sm bg-muted border border-border rounded text-xs" onclick="event.stopPropagation()" title="' + (t('index.selectModel') || 'Select embedding model') + '">' +
|
'<select id="codexlensModelSelect" class="btn-sm bg-muted border border-border rounded text-xs" onclick="event.stopPropagation()" title="' + (t('index.selectModel') || 'Select embedding model') + '">' +
|
||||||
'<option value="code">' + (t('index.modelCode') || 'Code (768d)') + ' ⭐</option>' +
|
buildModelSelectOptions() +
|
||||||
'<option value="base">' + (t('index.modelBase') || 'Base (768d)') + '</option>' +
|
|
||||||
'<option value="fast">' + (t('index.modelFast') || 'Fast (384d)') + '</option>' +
|
|
||||||
'<option value="minilm">' + (t('index.modelMinilm') || 'MiniLM (384d)') + '</option>' +
|
|
||||||
'<option value="multilingual" style="color: var(--muted-foreground)">' + (t('index.modelMultilingual') || 'Multilingual (1024d)') + ' ⚠️</option>' +
|
|
||||||
'<option value="balanced" style="color: var(--muted-foreground)">' + (t('index.modelBalanced') || 'Balanced (1024d)') + ' ⚠️</option>' +
|
|
||||||
'</select>' +
|
'</select>' +
|
||||||
'<button class="btn-sm btn-primary" onclick="event.stopPropagation(); initCodexLensIndex(\'full\', getSelectedModel())" title="' + (t('index.fullDesc') || 'FTS + Semantic search (recommended)') + '"><i data-lucide="layers" class="w-3 h-3"></i> ' + (t('index.fullIndex') || '全部索引') + '</button>' +
|
'<button class="btn-sm btn-primary" onclick="event.stopPropagation(); initCodexLensIndex(\'full\', getSelectedModel())" title="' + (t('index.fullDesc') || 'FTS + Semantic search (recommended)') + '"><i data-lucide="layers" class="w-3 h-3"></i> ' + (t('index.fullIndex') || '全部索引') + '</button>' +
|
||||||
'<button class="btn-sm btn-outline" onclick="event.stopPropagation(); initCodexLensIndex(\'vector\', getSelectedModel())" title="' + (t('index.vectorDesc') || 'Semantic search with embeddings') + '"><i data-lucide="sparkles" class="w-3 h-3"></i> ' + (t('index.vectorIndex') || '向量索引') + '</button>' +
|
'<button class="btn-sm btn-outline" onclick="event.stopPropagation(); initCodexLensIndex(\'vector\', getSelectedModel())" title="' + (t('index.vectorDesc') || 'Semantic search with embeddings') + '"><i data-lucide="sparkles" class="w-3 h-3"></i> ' + (t('index.vectorIndex') || '向量索引') + '</button>' +
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
Metadata-Version: 2.4
|
Metadata-Version: 2.4
|
||||||
Name: codex-lens
|
Name: codex-lens
|
||||||
Version: 0.2.0
|
Version: 0.1.0
|
||||||
Summary: CodexLens multi-modal code analysis platform
|
Summary: CodexLens multi-modal code analysis platform
|
||||||
Author: CodexLens contributors
|
Author: CodexLens contributors
|
||||||
License: MIT
|
License: MIT
|
||||||
@@ -17,18 +17,18 @@ Requires-Dist: tree-sitter-typescript>=0.23
|
|||||||
Requires-Dist: pathspec>=0.11
|
Requires-Dist: pathspec>=0.11
|
||||||
Provides-Extra: semantic
|
Provides-Extra: semantic
|
||||||
Requires-Dist: numpy>=1.24; extra == "semantic"
|
Requires-Dist: numpy>=1.24; extra == "semantic"
|
||||||
Requires-Dist: fastembed>=0.5; extra == "semantic"
|
Requires-Dist: fastembed>=0.2; extra == "semantic"
|
||||||
Requires-Dist: hnswlib>=0.8.0; extra == "semantic"
|
Requires-Dist: hnswlib>=0.8.0; extra == "semantic"
|
||||||
Provides-Extra: semantic-gpu
|
Provides-Extra: semantic-gpu
|
||||||
Requires-Dist: numpy>=1.24; extra == "semantic-gpu"
|
Requires-Dist: numpy>=1.24; extra == "semantic-gpu"
|
||||||
Requires-Dist: fastembed>=0.5; extra == "semantic-gpu"
|
Requires-Dist: fastembed>=0.2; extra == "semantic-gpu"
|
||||||
Requires-Dist: hnswlib>=0.8.0; extra == "semantic-gpu"
|
Requires-Dist: hnswlib>=0.8.0; extra == "semantic-gpu"
|
||||||
Requires-Dist: onnxruntime-gpu>=1.18.0; extra == "semantic-gpu"
|
Requires-Dist: onnxruntime-gpu>=1.15.0; extra == "semantic-gpu"
|
||||||
Provides-Extra: semantic-directml
|
Provides-Extra: semantic-directml
|
||||||
Requires-Dist: numpy>=1.24; extra == "semantic-directml"
|
Requires-Dist: numpy>=1.24; extra == "semantic-directml"
|
||||||
Requires-Dist: fastembed>=0.5; extra == "semantic-directml"
|
Requires-Dist: fastembed>=0.2; extra == "semantic-directml"
|
||||||
Requires-Dist: hnswlib>=0.8.0; extra == "semantic-directml"
|
Requires-Dist: hnswlib>=0.8.0; extra == "semantic-directml"
|
||||||
Requires-Dist: onnxruntime-directml>=1.18.0; extra == "semantic-directml"
|
Requires-Dist: onnxruntime-directml>=1.15.0; extra == "semantic-directml"
|
||||||
Provides-Extra: encoding
|
Provides-Extra: encoding
|
||||||
Requires-Dist: chardet>=5.0; extra == "encoding"
|
Requires-Dist: chardet>=5.0; extra == "encoding"
|
||||||
Provides-Extra: full
|
Provides-Extra: full
|
||||||
|
|||||||
@@ -15,17 +15,17 @@ tiktoken>=0.5.0
|
|||||||
|
|
||||||
[semantic]
|
[semantic]
|
||||||
numpy>=1.24
|
numpy>=1.24
|
||||||
fastembed>=0.5
|
fastembed>=0.2
|
||||||
hnswlib>=0.8.0
|
hnswlib>=0.8.0
|
||||||
|
|
||||||
[semantic-directml]
|
[semantic-directml]
|
||||||
numpy>=1.24
|
numpy>=1.24
|
||||||
fastembed>=0.5
|
fastembed>=0.2
|
||||||
hnswlib>=0.8.0
|
hnswlib>=0.8.0
|
||||||
onnxruntime-directml>=1.18.0
|
onnxruntime-directml>=1.15.0
|
||||||
|
|
||||||
[semantic-gpu]
|
[semantic-gpu]
|
||||||
numpy>=1.24
|
numpy>=1.24
|
||||||
fastembed>=0.5
|
fastembed>=0.2
|
||||||
hnswlib>=0.8.0
|
hnswlib>=0.8.0
|
||||||
onnxruntime-gpu>=1.18.0
|
onnxruntime-gpu>=1.15.0
|
||||||
|
|||||||
@@ -35,8 +35,17 @@ from .output import (
|
|||||||
app = typer.Typer(help="CodexLens CLI — local code indexing and search.")
|
app = typer.Typer(help="CodexLens CLI — local code indexing and search.")
|
||||||
|
|
||||||
|
|
||||||
def _configure_logging(verbose: bool) -> None:
|
def _configure_logging(verbose: bool, json_mode: bool = False) -> None:
|
||||||
level = logging.DEBUG if verbose else logging.INFO
|
"""Configure logging level.
|
||||||
|
|
||||||
|
In JSON mode, suppress INFO logs to keep stderr clean for error parsing.
|
||||||
|
Only WARNING and above are shown to avoid mixing logs with JSON output.
|
||||||
|
"""
|
||||||
|
if json_mode and not verbose:
|
||||||
|
# In JSON mode, suppress INFO logs to keep stderr clean
|
||||||
|
level = logging.WARNING
|
||||||
|
else:
|
||||||
|
level = logging.DEBUG if verbose else logging.INFO
|
||||||
logging.basicConfig(level=level, format="%(levelname)s %(message)s")
|
logging.basicConfig(level=level, format="%(levelname)s %(message)s")
|
||||||
|
|
||||||
|
|
||||||
@@ -95,7 +104,7 @@ def init(
|
|||||||
If semantic search dependencies are installed, automatically generates embeddings
|
If semantic search dependencies are installed, automatically generates embeddings
|
||||||
after indexing completes. Use --no-embeddings to skip this step.
|
after indexing completes. Use --no-embeddings to skip this step.
|
||||||
"""
|
"""
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose, json_mode)
|
||||||
config = Config()
|
config = Config()
|
||||||
languages = _parse_languages(language)
|
languages = _parse_languages(language)
|
||||||
base_path = path.expanduser().resolve()
|
base_path = path.expanduser().resolve()
|
||||||
@@ -314,7 +323,7 @@ def search(
|
|||||||
# Force hybrid mode
|
# Force hybrid mode
|
||||||
codexlens search "authentication" --mode hybrid
|
codexlens search "authentication" --mode hybrid
|
||||||
"""
|
"""
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose, json_mode)
|
||||||
search_path = path.expanduser().resolve()
|
search_path = path.expanduser().resolve()
|
||||||
|
|
||||||
# Validate mode
|
# Validate mode
|
||||||
@@ -487,7 +496,7 @@ def symbol(
|
|||||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
|
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Look up symbols by name and optional kind."""
|
"""Look up symbols by name and optional kind."""
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose, json_mode)
|
||||||
search_path = path.expanduser().resolve()
|
search_path = path.expanduser().resolve()
|
||||||
|
|
||||||
registry: RegistryStore | None = None
|
registry: RegistryStore | None = None
|
||||||
@@ -538,7 +547,7 @@ def inspect(
|
|||||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
|
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Analyze a single file and display symbols."""
|
"""Analyze a single file and display symbols."""
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose, json_mode)
|
||||||
config = Config()
|
config = Config()
|
||||||
factory = ParserFactory(config)
|
factory = ParserFactory(config)
|
||||||
|
|
||||||
@@ -588,7 +597,7 @@ def status(
|
|||||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
|
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Show index status and configuration."""
|
"""Show index status and configuration."""
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose, json_mode)
|
||||||
|
|
||||||
registry: RegistryStore | None = None
|
registry: RegistryStore | None = None
|
||||||
try:
|
try:
|
||||||
@@ -648,7 +657,7 @@ def status(
|
|||||||
# Embedding manager not available
|
# Embedding manager not available
|
||||||
pass
|
pass
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(f"Failed to get embeddings status: {e}")
|
logging.debug(f"Failed to get embeddings status: {e}")
|
||||||
|
|
||||||
stats = {
|
stats = {
|
||||||
"index_root": str(index_root),
|
"index_root": str(index_root),
|
||||||
@@ -737,7 +746,7 @@ def projects(
|
|||||||
- show <path>: Show details for a specific project
|
- show <path>: Show details for a specific project
|
||||||
- remove <path>: Remove a project from the registry
|
- remove <path>: Remove a project from the registry
|
||||||
"""
|
"""
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose, json_mode)
|
||||||
|
|
||||||
registry: RegistryStore | None = None
|
registry: RegistryStore | None = None
|
||||||
try:
|
try:
|
||||||
@@ -892,7 +901,7 @@ def config(
|
|||||||
Config keys:
|
Config keys:
|
||||||
- index_dir: Directory to store indexes (default: ~/.codexlens/indexes)
|
- index_dir: Directory to store indexes (default: ~/.codexlens/indexes)
|
||||||
"""
|
"""
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose, json_mode)
|
||||||
|
|
||||||
config_file = Path.home() / ".codexlens" / "config.json"
|
config_file = Path.home() / ".codexlens" / "config.json"
|
||||||
|
|
||||||
@@ -1057,7 +1066,7 @@ def migrate(
|
|||||||
This is a safe operation that preserves all existing data.
|
This is a safe operation that preserves all existing data.
|
||||||
Progress is shown during migration.
|
Progress is shown during migration.
|
||||||
"""
|
"""
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose, json_mode)
|
||||||
base_path = path.expanduser().resolve()
|
base_path = path.expanduser().resolve()
|
||||||
|
|
||||||
registry: RegistryStore | None = None
|
registry: RegistryStore | None = None
|
||||||
@@ -1183,7 +1192,7 @@ def clean(
|
|||||||
With path, removes that project's indexes.
|
With path, removes that project's indexes.
|
||||||
With --all, removes all indexes (use with caution).
|
With --all, removes all indexes (use with caution).
|
||||||
"""
|
"""
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose, json_mode)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
mapper = PathMapper()
|
mapper = PathMapper()
|
||||||
@@ -1329,7 +1338,7 @@ def semantic_list(
|
|||||||
Shows files that have LLM-generated summaries and keywords.
|
Shows files that have LLM-generated summaries and keywords.
|
||||||
Results are aggregated from all index databases in the project.
|
Results are aggregated from all index databases in the project.
|
||||||
"""
|
"""
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose, json_mode)
|
||||||
base_path = path.expanduser().resolve()
|
base_path = path.expanduser().resolve()
|
||||||
|
|
||||||
registry: Optional[RegistryStore] = None
|
registry: Optional[RegistryStore] = None
|
||||||
@@ -1798,7 +1807,7 @@ def embeddings_generate(
|
|||||||
codexlens embeddings-generate ~/.codexlens/indexes/project/_index.db # Specific index
|
codexlens embeddings-generate ~/.codexlens/indexes/project/_index.db # Specific index
|
||||||
codexlens embeddings-generate ~/projects/my-app --model fast --force # Regenerate with fast model
|
codexlens embeddings-generate ~/projects/my-app --model fast --force # Regenerate with fast model
|
||||||
"""
|
"""
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose, json_mode)
|
||||||
|
|
||||||
from codexlens.cli.embedding_manager import generate_embeddings, generate_embeddings_recursive
|
from codexlens.cli.embedding_manager import generate_embeddings, generate_embeddings_recursive
|
||||||
|
|
||||||
|
|||||||
@@ -279,6 +279,21 @@ def generate_embeddings(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
with VectorStore(index_path) as vector_store:
|
with VectorStore(index_path) as vector_store:
|
||||||
|
# Check model compatibility with existing embeddings
|
||||||
|
if not force:
|
||||||
|
is_compatible, warning = vector_store.check_model_compatibility(
|
||||||
|
model_profile, embedder.model_name, embedder.embedding_dim
|
||||||
|
)
|
||||||
|
if not is_compatible:
|
||||||
|
return {
|
||||||
|
"success": False,
|
||||||
|
"error": warning,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Set/update model configuration for this index
|
||||||
|
vector_store.set_model_config(
|
||||||
|
model_profile, embedder.model_name, embedder.embedding_dim
|
||||||
|
)
|
||||||
# Use bulk insert mode for efficient batch ANN index building
|
# Use bulk insert mode for efficient batch ANN index building
|
||||||
# This defers ANN updates until end_bulk_insert() is called
|
# This defers ANN updates until end_bulk_insert() is called
|
||||||
with vector_store.bulk_insert():
|
with vector_store.bulk_insert():
|
||||||
|
|||||||
@@ -16,9 +16,11 @@ except ImportError:
|
|||||||
# Model profiles with metadata
|
# Model profiles with metadata
|
||||||
# Note: 768d is max recommended dimension for optimal performance/quality balance
|
# Note: 768d is max recommended dimension for optimal performance/quality balance
|
||||||
# 1024d models are available but not recommended due to higher resource usage
|
# 1024d models are available but not recommended due to higher resource usage
|
||||||
|
# cache_name: The actual Hugging Face repo name used by fastembed for ONNX caching
|
||||||
MODEL_PROFILES = {
|
MODEL_PROFILES = {
|
||||||
"fast": {
|
"fast": {
|
||||||
"model_name": "BAAI/bge-small-en-v1.5",
|
"model_name": "BAAI/bge-small-en-v1.5",
|
||||||
|
"cache_name": "qdrant/bge-small-en-v1.5-onnx-q", # fastembed uses ONNX version
|
||||||
"dimensions": 384,
|
"dimensions": 384,
|
||||||
"size_mb": 80,
|
"size_mb": 80,
|
||||||
"description": "Fast, lightweight, English-optimized",
|
"description": "Fast, lightweight, English-optimized",
|
||||||
@@ -27,6 +29,7 @@ MODEL_PROFILES = {
|
|||||||
},
|
},
|
||||||
"base": {
|
"base": {
|
||||||
"model_name": "BAAI/bge-base-en-v1.5",
|
"model_name": "BAAI/bge-base-en-v1.5",
|
||||||
|
"cache_name": "qdrant/bge-base-en-v1.5-onnx-q", # fastembed uses ONNX version
|
||||||
"dimensions": 768,
|
"dimensions": 768,
|
||||||
"size_mb": 220,
|
"size_mb": 220,
|
||||||
"description": "General purpose, good balance of speed and quality",
|
"description": "General purpose, good balance of speed and quality",
|
||||||
@@ -35,6 +38,7 @@ MODEL_PROFILES = {
|
|||||||
},
|
},
|
||||||
"code": {
|
"code": {
|
||||||
"model_name": "jinaai/jina-embeddings-v2-base-code",
|
"model_name": "jinaai/jina-embeddings-v2-base-code",
|
||||||
|
"cache_name": "jinaai/jina-embeddings-v2-base-code", # Uses original name
|
||||||
"dimensions": 768,
|
"dimensions": 768,
|
||||||
"size_mb": 150,
|
"size_mb": 150,
|
||||||
"description": "Code-optimized, best for programming languages",
|
"description": "Code-optimized, best for programming languages",
|
||||||
@@ -43,6 +47,7 @@ MODEL_PROFILES = {
|
|||||||
},
|
},
|
||||||
"minilm": {
|
"minilm": {
|
||||||
"model_name": "sentence-transformers/all-MiniLM-L6-v2",
|
"model_name": "sentence-transformers/all-MiniLM-L6-v2",
|
||||||
|
"cache_name": "qdrant/all-MiniLM-L6-v2-onnx", # fastembed uses ONNX version
|
||||||
"dimensions": 384,
|
"dimensions": 384,
|
||||||
"size_mb": 90,
|
"size_mb": 90,
|
||||||
"description": "Popular lightweight model, good quality",
|
"description": "Popular lightweight model, good quality",
|
||||||
@@ -51,6 +56,7 @@ MODEL_PROFILES = {
|
|||||||
},
|
},
|
||||||
"multilingual": {
|
"multilingual": {
|
||||||
"model_name": "intfloat/multilingual-e5-large",
|
"model_name": "intfloat/multilingual-e5-large",
|
||||||
|
"cache_name": "qdrant/multilingual-e5-large-onnx", # fastembed uses ONNX version
|
||||||
"dimensions": 1024,
|
"dimensions": 1024,
|
||||||
"size_mb": 1000,
|
"size_mb": 1000,
|
||||||
"description": "Multilingual + code support (high resource usage)",
|
"description": "Multilingual + code support (high resource usage)",
|
||||||
@@ -59,6 +65,7 @@ MODEL_PROFILES = {
|
|||||||
},
|
},
|
||||||
"balanced": {
|
"balanced": {
|
||||||
"model_name": "mixedbread-ai/mxbai-embed-large-v1",
|
"model_name": "mixedbread-ai/mxbai-embed-large-v1",
|
||||||
|
"cache_name": "mixedbread-ai/mxbai-embed-large-v1", # Uses original name
|
||||||
"dimensions": 1024,
|
"dimensions": 1024,
|
||||||
"size_mb": 600,
|
"size_mb": 600,
|
||||||
"description": "High accuracy, general purpose (high resource usage)",
|
"description": "High accuracy, general purpose (high resource usage)",
|
||||||
@@ -87,6 +94,23 @@ def get_cache_dir() -> Path:
|
|||||||
return cache_dir
|
return cache_dir
|
||||||
|
|
||||||
|
|
||||||
|
def _get_model_cache_path(cache_dir: Path, info: Dict) -> Path:
|
||||||
|
"""Get the actual cache path for a model.
|
||||||
|
|
||||||
|
fastembed uses ONNX versions of models with different names than the original.
|
||||||
|
This function returns the correct path based on the cache_name field.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
cache_dir: The fastembed cache directory
|
||||||
|
info: Model profile info dictionary
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Path to the model cache directory
|
||||||
|
"""
|
||||||
|
cache_name = info.get("cache_name", info["model_name"])
|
||||||
|
return cache_dir / f"models--{cache_name.replace('/', '--')}"
|
||||||
|
|
||||||
|
|
||||||
def list_models() -> Dict[str, any]:
|
def list_models() -> Dict[str, any]:
|
||||||
"""List available model profiles and their installation status.
|
"""List available model profiles and their installation status.
|
||||||
|
|
||||||
@@ -106,13 +130,13 @@ def list_models() -> Dict[str, any]:
|
|||||||
for profile, info in MODEL_PROFILES.items():
|
for profile, info in MODEL_PROFILES.items():
|
||||||
model_name = info["model_name"]
|
model_name = info["model_name"]
|
||||||
|
|
||||||
# Check if model is cached
|
# Check if model is cached using the actual cache name
|
||||||
installed = False
|
installed = False
|
||||||
cache_size_mb = 0
|
cache_size_mb = 0
|
||||||
|
|
||||||
if cache_exists:
|
if cache_exists:
|
||||||
# Check for model directory in cache
|
# Check for model directory in cache using correct cache_name
|
||||||
model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}"
|
model_cache_path = _get_model_cache_path(cache_dir, info)
|
||||||
if model_cache_path.exists():
|
if model_cache_path.exists():
|
||||||
installed = True
|
installed = True
|
||||||
# Calculate cache size
|
# Calculate cache size
|
||||||
@@ -166,7 +190,8 @@ def download_model(profile: str, progress_callback: Optional[callable] = None) -
|
|||||||
"error": f"Unknown profile: {profile}. Available: {', '.join(MODEL_PROFILES.keys())}",
|
"error": f"Unknown profile: {profile}. Available: {', '.join(MODEL_PROFILES.keys())}",
|
||||||
}
|
}
|
||||||
|
|
||||||
model_name = MODEL_PROFILES[profile]["model_name"]
|
info = MODEL_PROFILES[profile]
|
||||||
|
model_name = info["model_name"]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Download model by instantiating TextEmbedding
|
# Download model by instantiating TextEmbedding
|
||||||
@@ -179,9 +204,9 @@ def download_model(profile: str, progress_callback: Optional[callable] = None) -
|
|||||||
if progress_callback:
|
if progress_callback:
|
||||||
progress_callback(f"Model {model_name} downloaded successfully")
|
progress_callback(f"Model {model_name} downloaded successfully")
|
||||||
|
|
||||||
# Get cache info
|
# Get cache info using correct cache_name
|
||||||
cache_dir = get_cache_dir()
|
cache_dir = get_cache_dir()
|
||||||
model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}"
|
model_cache_path = _get_model_cache_path(cache_dir, info)
|
||||||
|
|
||||||
cache_size = 0
|
cache_size = 0
|
||||||
if model_cache_path.exists():
|
if model_cache_path.exists():
|
||||||
@@ -224,9 +249,10 @@ def delete_model(profile: str) -> Dict[str, any]:
|
|||||||
"error": f"Unknown profile: {profile}. Available: {', '.join(MODEL_PROFILES.keys())}",
|
"error": f"Unknown profile: {profile}. Available: {', '.join(MODEL_PROFILES.keys())}",
|
||||||
}
|
}
|
||||||
|
|
||||||
model_name = MODEL_PROFILES[profile]["model_name"]
|
info = MODEL_PROFILES[profile]
|
||||||
|
model_name = info["model_name"]
|
||||||
cache_dir = get_cache_dir()
|
cache_dir = get_cache_dir()
|
||||||
model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}"
|
model_cache_path = _get_model_cache_path(cache_dir, info)
|
||||||
|
|
||||||
if not model_cache_path.exists():
|
if not model_cache_path.exists():
|
||||||
return {
|
return {
|
||||||
@@ -281,9 +307,9 @@ def get_model_info(profile: str) -> Dict[str, any]:
|
|||||||
info = MODEL_PROFILES[profile]
|
info = MODEL_PROFILES[profile]
|
||||||
model_name = info["model_name"]
|
model_name = info["model_name"]
|
||||||
|
|
||||||
# Check installation status
|
# Check installation status using correct cache_name
|
||||||
cache_dir = get_cache_dir()
|
cache_dir = get_cache_dir()
|
||||||
model_cache_path = cache_dir / f"models--{model_name.replace('/', '--')}"
|
model_cache_path = _get_model_cache_path(cache_dir, info)
|
||||||
installed = model_cache_path.exists()
|
installed = model_cache_path.exists()
|
||||||
|
|
||||||
cache_size_mb = None
|
cache_size_mb = None
|
||||||
|
|||||||
@@ -396,7 +396,20 @@ class ChainSearchEngine:
|
|||||||
all_results = []
|
all_results = []
|
||||||
stats = SearchStats()
|
stats = SearchStats()
|
||||||
|
|
||||||
executor = self._get_executor(options.max_workers)
|
# Force single-threaded execution for vector/hybrid search to avoid GPU crashes
|
||||||
|
# DirectML/ONNX have threading issues when multiple threads access GPU resources
|
||||||
|
effective_workers = options.max_workers
|
||||||
|
if options.enable_vector or options.hybrid_mode:
|
||||||
|
effective_workers = 1
|
||||||
|
self.logger.debug("Using single-threaded mode for vector search (GPU safety)")
|
||||||
|
# Pre-load embedder to avoid initialization overhead per-search
|
||||||
|
try:
|
||||||
|
from codexlens.semantic.embedder import get_embedder
|
||||||
|
get_embedder(profile="code", use_gpu=True)
|
||||||
|
except Exception:
|
||||||
|
pass # Ignore pre-load failures
|
||||||
|
|
||||||
|
executor = self._get_executor(effective_workers)
|
||||||
# Submit all search tasks
|
# Submit all search tasks
|
||||||
future_to_path = {
|
future_to_path = {
|
||||||
executor.submit(
|
executor.submit(
|
||||||
|
|||||||
@@ -274,19 +274,32 @@ class HybridSearchEngine:
|
|||||||
)
|
)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
# Auto-detect embedding dimension and select appropriate profile
|
# Get stored model configuration (preferred) or auto-detect from dimension
|
||||||
detected_dim = vector_store.dimension
|
model_config = vector_store.get_model_config()
|
||||||
if detected_dim is None:
|
if model_config:
|
||||||
self.logger.info("Vector store dimension unknown, using default profile")
|
profile = model_config["model_profile"]
|
||||||
profile = "code" # Default fallback
|
self.logger.debug(
|
||||||
elif detected_dim == 384:
|
"Using stored model config: %s (%s, %dd)",
|
||||||
profile = "fast"
|
profile, model_config["model_name"], model_config["embedding_dim"]
|
||||||
elif detected_dim == 768:
|
)
|
||||||
profile = "code"
|
|
||||||
elif detected_dim == 1024:
|
|
||||||
profile = "multilingual" # or balanced, both are 1024
|
|
||||||
else:
|
else:
|
||||||
profile = "code" # Default fallback
|
# Fallback: auto-detect from embedding dimension
|
||||||
|
detected_dim = vector_store.dimension
|
||||||
|
if detected_dim is None:
|
||||||
|
self.logger.info("Vector store dimension unknown, using default profile")
|
||||||
|
profile = "code" # Default fallback
|
||||||
|
elif detected_dim == 384:
|
||||||
|
profile = "fast"
|
||||||
|
elif detected_dim == 768:
|
||||||
|
profile = "code"
|
||||||
|
elif detected_dim == 1024:
|
||||||
|
profile = "multilingual" # or balanced, both are 1024
|
||||||
|
else:
|
||||||
|
profile = "code" # Default fallback
|
||||||
|
self.logger.debug(
|
||||||
|
"No stored model config, auto-detected profile '%s' from dimension %s",
|
||||||
|
profile, detected_dim
|
||||||
|
)
|
||||||
|
|
||||||
# Use cached embedder (singleton) for performance
|
# Use cached embedder (singleton) for performance
|
||||||
embedder = get_embedder(profile=profile)
|
embedder = get_embedder(profile=profile)
|
||||||
|
|||||||
@@ -116,6 +116,17 @@ class VectorStore:
|
|||||||
CREATE INDEX IF NOT EXISTS idx_chunks_file
|
CREATE INDEX IF NOT EXISTS idx_chunks_file
|
||||||
ON semantic_chunks(file_path)
|
ON semantic_chunks(file_path)
|
||||||
""")
|
""")
|
||||||
|
# Model configuration table - tracks which model generated the embeddings
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS embeddings_config (
|
||||||
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||||
|
model_profile TEXT NOT NULL,
|
||||||
|
model_name TEXT NOT NULL,
|
||||||
|
embedding_dim INTEGER NOT NULL,
|
||||||
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
|
)
|
||||||
|
""")
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
def _init_ann_index(self) -> None:
|
def _init_ann_index(self) -> None:
|
||||||
@@ -932,6 +943,92 @@ class VectorStore:
|
|||||||
return self._ann_index.count()
|
return self._ann_index.count()
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
def get_model_config(self) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Get the model configuration used for embeddings in this store.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dictionary with model_profile, model_name, embedding_dim, or None if not set.
|
||||||
|
"""
|
||||||
|
with sqlite3.connect(self.db_path) as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT model_profile, model_name, embedding_dim, created_at, updated_at "
|
||||||
|
"FROM embeddings_config WHERE id = 1"
|
||||||
|
).fetchone()
|
||||||
|
if row:
|
||||||
|
return {
|
||||||
|
"model_profile": row[0],
|
||||||
|
"model_name": row[1],
|
||||||
|
"embedding_dim": row[2],
|
||||||
|
"created_at": row[3],
|
||||||
|
"updated_at": row[4],
|
||||||
|
}
|
||||||
|
return None
|
||||||
|
|
||||||
|
def set_model_config(
|
||||||
|
self, model_profile: str, model_name: str, embedding_dim: int
|
||||||
|
) -> None:
|
||||||
|
"""Set the model configuration for embeddings in this store.
|
||||||
|
|
||||||
|
This should be called when generating new embeddings. If a different
|
||||||
|
model was previously used, this will update the configuration.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
model_profile: Model profile name (fast, code, minilm, etc.)
|
||||||
|
model_name: Full model name (e.g., jinaai/jina-embeddings-v2-base-code)
|
||||||
|
embedding_dim: Embedding dimension (e.g., 768)
|
||||||
|
"""
|
||||||
|
with sqlite3.connect(self.db_path) as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO embeddings_config (id, model_profile, model_name, embedding_dim)
|
||||||
|
VALUES (1, ?, ?, ?)
|
||||||
|
ON CONFLICT(id) DO UPDATE SET
|
||||||
|
model_profile = excluded.model_profile,
|
||||||
|
model_name = excluded.model_name,
|
||||||
|
embedding_dim = excluded.embedding_dim,
|
||||||
|
updated_at = CURRENT_TIMESTAMP
|
||||||
|
""",
|
||||||
|
(model_profile, model_name, embedding_dim)
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
def check_model_compatibility(
|
||||||
|
self, model_profile: str, model_name: str, embedding_dim: int
|
||||||
|
) -> Tuple[bool, Optional[str]]:
|
||||||
|
"""Check if the given model is compatible with existing embeddings.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
model_profile: Model profile to check
|
||||||
|
model_name: Model name to check
|
||||||
|
embedding_dim: Embedding dimension to check
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (is_compatible, warning_message).
|
||||||
|
is_compatible is True if no existing config or configs match.
|
||||||
|
warning_message is a user-friendly message if incompatible.
|
||||||
|
"""
|
||||||
|
existing = self.get_model_config()
|
||||||
|
if existing is None:
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
# Check dimension first (most critical)
|
||||||
|
if existing["embedding_dim"] != embedding_dim:
|
||||||
|
return False, (
|
||||||
|
f"Dimension mismatch: existing embeddings use {existing['embedding_dim']}d "
|
||||||
|
f"({existing['model_profile']}), but requested model uses {embedding_dim}d "
|
||||||
|
f"({model_profile}). Use --force to regenerate all embeddings."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Check model (different models with same dimension may have different semantic spaces)
|
||||||
|
if existing["model_profile"] != model_profile:
|
||||||
|
return False, (
|
||||||
|
f"Model mismatch: existing embeddings use '{existing['model_profile']}' "
|
||||||
|
f"({existing['model_name']}), but requested '{model_profile}' "
|
||||||
|
f"({model_name}). Use --force to regenerate all embeddings."
|
||||||
|
)
|
||||||
|
|
||||||
|
return True, None
|
||||||
|
|
||||||
def close(self) -> None:
|
def close(self) -> None:
|
||||||
"""Close the vector store and release resources.
|
"""Close the vector store and release resources.
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "claude-code-workflow",
|
"name": "claude-code-workflow",
|
||||||
"version": "6.2.6",
|
"version": "6.2.7",
|
||||||
"description": "JSON-driven multi-agent development framework with intelligent CLI orchestration (Gemini/Qwen/Codex), context-first architecture, and automated workflow execution",
|
"description": "JSON-driven multi-agent development framework with intelligent CLI orchestration (Gemini/Qwen/Codex), context-first architecture, and automated workflow execution",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "ccw/src/index.js",
|
"main": "ccw/src/index.js",
|
||||||
|
|||||||
Reference in New Issue
Block a user