From 504ccfebbc1bc57109c6cfe345e4292b65b45316 Mon Sep 17 00:00:00 2001 From: catlog22 Date: Sat, 3 Jan 2026 22:20:06 +0800 Subject: [PATCH] feat: add reranker models to ProviderCredential and improve FastEmbedReranker scoring - Added `rerankerModels` property to the `ProviderCredential` interface in `litellm-api-config.ts` to support additional reranker configurations. - Introduced a numerically stable sigmoid function in `FastEmbedReranker` for score normalization. - Updated the scoring logic in `FastEmbedReranker` to use raw float scores from the encoder and normalize them using the new sigmoid function. - Adjusted the result mapping to maintain original document order while applying normalization. --- ccw/src/core/routes/litellm-api-routes.ts | 40 + ccw/src/templates/dashboard-js/i18n.js | 8 + .../dashboard-js/views/api-settings.js | 89 +- .../dashboard-js/views/codexlens-manager.js | 1537 ++++++++++++----- ccw/src/types/litellm-api-config.ts | 3 + .../semantic/reranker/fastembed_reranker.py | 51 +- 6 files changed, 1277 insertions(+), 451 deletions(-) diff --git a/ccw/src/core/routes/litellm-api-routes.ts b/ccw/src/core/routes/litellm-api-routes.ts index 98042767..791f82be 100644 --- a/ccw/src/core/routes/litellm-api-routes.ts +++ b/ccw/src/core/routes/litellm-api-routes.ts @@ -789,6 +789,46 @@ export async function handleLiteLLMApiRoutes(ctx: RouteContext): Promise = []; + const modelMap = new Map(); + + for (const provider of config.providers) { + if (!provider.enabled || !provider.rerankerModels) continue; + + for (const model of provider.rerankerModels) { + if (!model.enabled) continue; + + const key = model.id; + if (modelMap.has(key)) { + modelMap.get(key)!.providers.push(provider.name); + } else { + modelMap.set(key, { + modelId: model.id, + modelName: model.name, + providers: [provider.name], + }); + } + } + } + + availableModels.push(...Array.from(modelMap.values())); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + availableModels, + })); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + return true; + } + // GET /api/litellm-api/embedding-pool/discover/:model - Preview auto-discovery results const discoverMatch = pathname.match(/^\/api\/litellm-api\/embedding-pool\/discover\/([^/]+)$/); if (discoverMatch && req.method === 'GET') { diff --git a/ccw/src/templates/dashboard-js/i18n.js b/ccw/src/templates/dashboard-js/i18n.js index 71a7ff9d..163753cc 100644 --- a/ccw/src/templates/dashboard-js/i18n.js +++ b/ccw/src/templates/dashboard-js/i18n.js @@ -1465,6 +1465,10 @@ const i18n = { 'apiSettings.noProvidersFound': 'No providers found', 'apiSettings.llmModels': 'LLM Models', 'apiSettings.embeddingModels': 'Embedding Models', + 'apiSettings.rerankerModels': 'Reranker Models', + 'apiSettings.addRerankerModel': 'Add Reranker Model', + 'apiSettings.rerankerTopK': 'Default Top K', + 'apiSettings.rerankerTopKHint': 'Number of top results to return (default: 10)', 'apiSettings.manageModels': 'Manage', 'apiSettings.addModel': 'Add Model', 'apiSettings.multiKeySettings': 'Multi-Key Settings', @@ -3416,6 +3420,10 @@ const i18n = { 'apiSettings.noProvidersFound': '未找到供应商', 'apiSettings.llmModels': '大语言模型', 'apiSettings.embeddingModels': '向量模型', + 'apiSettings.rerankerModels': '重排模型', + 'apiSettings.addRerankerModel': '添加重排模型', + 'apiSettings.rerankerTopK': '默认 Top K', + 'apiSettings.rerankerTopKHint': '返回的最高排名结果数量（默认：10）', 'apiSettings.manageModels': '管理', 'apiSettings.addModel': '添加模型', 'apiSettings.multiKeySettings': '多密钥设置', diff --git a/ccw/src/templates/dashboard-js/views/api-settings.js b/ccw/src/templates/dashboard-js/views/api-settings.js index 2869567c..5e209e05 100644 --- a/ccw/src/templates/dashboard-js/views/api-settings.js +++ b/ccw/src/templates/dashboard-js/views/api-settings.js @@ -1221,6 +1221,9 @@ function renderProviderDetail(providerId) { '' + + '' + '' + '
' + '
' + : isReranker ? + '
' + + '' + + '' + + '' + t('apiSettings.rerankerTopKHint') + '' + + '
' : '
' + '' + @@ -1691,6 +1702,37 @@ function getEmbeddingPresetsForType(providerType) { return presets[providerType] || presets.custom; } +/** + * Get reranker model presets based on provider type + */ +function getRerankerPresetsForType(providerType) { + const presets = { + openai: [ + { id: 'BAAI/bge-reranker-v2-m3', name: 'BGE Reranker v2 M3', series: 'BGE Reranker', topK: 10 }, + { id: 'BAAI/bge-reranker-large', name: 'BGE Reranker Large', series: 'BGE Reranker', topK: 10 }, + { id: 'BAAI/bge-reranker-base', name: 'BGE Reranker Base', series: 'BGE Reranker', topK: 10 } + ], + cohere: [ + { id: 'rerank-english-v3.0', name: 'Rerank English v3.0', series: 'Cohere Rerank', topK: 10 }, + { id: 'rerank-multilingual-v3.0', name: 'Rerank Multilingual v3.0', series: 'Cohere Rerank', topK: 10 }, + { id: 'rerank-english-v2.0', name: 'Rerank English v2.0', series: 'Cohere Rerank', topK: 10 } + ], + voyage: [ + { id: 'rerank-2', name: 'Rerank 2', series: 'Voyage Rerank', topK: 10 }, + { id: 'rerank-2-lite', name: 'Rerank 2 Lite', series: 'Voyage Rerank', topK: 10 }, + { id: 'rerank-1', name: 'Rerank 1', series: 'Voyage Rerank', topK: 10 } + ], + jina: [ + { id: 'jina-reranker-v2-base-multilingual', name: 'Jina Reranker v2 Multilingual', series: 'Jina Reranker', topK: 10 }, + { id: 'jina-reranker-v1-base-en', name: 'Jina Reranker v1 English', series: 'Jina Reranker', topK: 10 } + ], + custom: [ + { id: 'custom-reranker', name: 'Custom Reranker', series: 'Custom', topK: 10 } + ] + }; + return presets[providerType] || presets.custom; +} + /** * Group presets by series */ @@ -1721,7 +1763,8 @@ function fillModelFromPreset(presetId, modelType) { if (!provider) return; const isLlm = modelType === 'llm'; - const presets = isLlm ? getLlmPresetsForType(provider.type) : getEmbeddingPresetsForType(provider.type); + const isReranker = modelType === 'reranker'; + const presets = isLlm ? getLlmPresetsForType(provider.type) : isReranker ? getRerankerPresetsForType(provider.type) : getEmbeddingPresetsForType(provider.type); const preset = presets.find(function(p) { return p.id === presetId; }); if (preset) { @@ -1732,7 +1775,11 @@ function fillModelFromPreset(presetId, modelType) { if (isLlm && preset.contextWindow) { document.getElementById('model-context-window').value = preset.contextWindow; } - if (!isLlm && preset.dimensions) { + if (isReranker && preset.topK) { + var topKEl = document.getElementById('model-top-k'); + if (topKEl) topKEl.value = preset.topK; + } + if (!isLlm && !isReranker && preset.dimensions) { document.getElementById('model-dimensions').value = preset.dimensions; if (preset.maxTokens) { document.getElementById('model-max-tokens').value = preset.maxTokens; @@ -1748,6 +1795,7 @@ function saveNewModel(event, providerId, modelType) { event.preventDefault(); const isLlm = modelType === 'llm'; + const isReranker = modelType === 'reranker'; const now = new Date().toISOString(); const newModel = { @@ -1769,6 +1817,11 @@ function saveNewModel(event, providerId, modelType) { functionCalling: document.getElementById('cap-function-calling').checked, vision: document.getElementById('cap-vision').checked }; + } else if (isReranker) { + var topKEl = document.getElementById('model-top-k'); + newModel.capabilities = { + topK: topKEl ? parseInt(topKEl.value) || 10 : 10 + }; } else { newModel.capabilities = { embeddingDimension: parseInt(document.getElementById('model-dimensions').value) || 1536, @@ -1780,7 +1833,7 @@ function saveNewModel(event, providerId, modelType) { fetch('/api/litellm-api/providers/' + providerId) .then(function(res) { return res.json(); }) .then(function(provider) { - const modelsKey = isLlm ? 'llmModels' : 'embeddingModels'; + const modelsKey = isLlm ? 'llmModels' : isReranker ? 'rerankerModels' : 'embeddingModels'; const models = provider[modelsKey] || []; // Check for duplicate ID @@ -1824,7 +1877,8 @@ function showModelSettingsModal(providerId, modelId, modelType) { if (!provider) return; var isLlm = modelType === 'llm'; - var models = isLlm ? (provider.llmModels || []) : (provider.embeddingModels || []); + var isReranker = modelType === 'reranker'; + var models = isLlm ? (provider.llmModels || []) : isReranker ? (provider.rerankerModels || []) : (provider.embeddingModels || []); var model = models.find(function(m) { return m.id === modelId; }); if (!model) return; @@ -1834,7 +1888,7 @@ function showModelSettingsModal(providerId, modelId, modelType) { // Calculate endpoint preview URL var providerBase = provider.apiBase || getDefaultApiBase(provider.type); var modelBaseUrl = endpointSettings.baseUrl || providerBase; - var endpointPath = isLlm ? '/chat/completions' : '/embeddings'; + var endpointPath = isLlm ? '/chat/completions' : isReranker ? '/rerank' : '/embeddings'; var endpointPreview = modelBaseUrl + endpointPath; var modalHtml = '' + '
' + - // Model Management + // Model Management - Simplified with Embedding and Reranker sections '
' + '

' + - ' Models' + + ' ' + t('codexlens.models') + '

' + - '
' + - '
' + t('codexlens.loadingModels') + '
' + + // Embedding Models + '
' + + '
' + + ' Embedding Models' + + '
' + + '
' + + '
' + t('codexlens.loadingModels') + '
' + + '
' + + '
' + + // Reranker Models + '
' + + '
' + + ' Reranker Models' + + '
' + + '
' + + '
' + t('common.loading') + '
' + + '
' + '
' + '
' + @@ -502,8 +517,9 @@ function initCodexLensConfigEvents(currentConfig) { // SPLADE status hidden - not currently used // loadSpladeStatus(); - // Load model list + // Load model lists (embedding and reranker) loadModelList(); + loadRerankerModelList(); } // ============================================================ @@ -639,17 +655,57 @@ window.getModelLockState = getModelLockState; // ENVIRONMENT VARIABLES MANAGEMENT // ============================================================ -// Known env variable groups -var ENV_VARIABLES = { - 'RERANKER_API_KEY': { label: 'Reranker API Key', placeholder: 'sk-...', type: 'password' }, - 'RERANKER_API_BASE': { label: 'Reranker API Base', placeholder: 'https://api.openai.com/v1' }, - 'RERANKER_MODEL': { label: 'Reranker Model', placeholder: 'text-embedding-3-small' }, - 'EMBEDDING_API_KEY': { label: 'Embedding API Key', placeholder: 'sk-...', type: 'password' }, - 'EMBEDDING_API_BASE': { label: 'Embedding API Base', placeholder: 'https://api.openai.com/v1' }, - 'EMBEDDING_MODEL': { label: 'Embedding Model', placeholder: 'text-embedding-3-small' }, - 'LITELLM_API_KEY': { label: 'LiteLLM API Key', placeholder: 'sk-...', type: 'password' }, - 'LITELLM_API_BASE': { label: 'LiteLLM API Base', placeholder: 'http://localhost:4000' }, - 'LITELLM_MODEL': { label: 'LiteLLM Model', placeholder: 'gpt-3.5-turbo' } +// Environment variable groups for organized display +var ENV_VAR_GROUPS = { + backend: { + label: 'Backend Selection', + icon: 'toggle-left', + vars: { + 'CODEXLENS_EMBEDDING_BACKEND': { label: 'Embedding Backend', type: 'select', options: ['fastembed', 'litellm'], default: 'fastembed' }, + 'CODEXLENS_RERANKER_BACKEND': { label: 'Reranker Backend', type: 'select', options: ['fastembed', 'litellm'], default: 'fastembed' } + } + }, + local: { + label: 'Local Model Settings', + icon: 'hard-drive', + showWhen: function(env) { return env['CODEXLENS_EMBEDDING_BACKEND'] !== 'litellm' || env['CODEXLENS_RERANKER_BACKEND'] !== 'litellm'; }, + vars: { + 'CODEXLENS_EMBEDDING_MODEL': { label: 'Embedding Model', placeholder: 'fast (code, base, minilm, multilingual, balanced)' }, + 'CODEXLENS_RERANKER_MODEL': { label: 'Reranker Model', placeholder: 'Xenova/ms-marco-MiniLM-L-6-v2' } + } + }, + api: { + label: 'API Settings (LiteLLM)', + icon: 'cloud', + showWhen: function(env) { return env['CODEXLENS_EMBEDDING_BACKEND'] === 'litellm' || env['CODEXLENS_RERANKER_BACKEND'] === 'litellm'; }, + vars: { + 'LITELLM_API_KEY': { label: 'API Key', placeholder: 'sk-...', type: 'password' }, + 'LITELLM_API_BASE': { label: 'API Base URL', placeholder: 'https://api.openai.com/v1' }, + 'LITELLM_EMBEDDING_MODEL': { + label: 'Embedding Model', + type: 'model-select', + placeholder: 'Select or enter model...', + models: [ + { group: 'OpenAI', items: ['text-embedding-3-small', 'text-embedding-3-large', 'text-embedding-ada-002'] }, + { group: 'Cohere', items: ['embed-english-v3.0', 'embed-multilingual-v3.0', 'embed-english-light-v3.0'] }, + { group: 'Voyage', items: ['voyage-3', 'voyage-3-lite', 'voyage-code-3', 'voyage-multilingual-2'] }, + { group: 'SiliconFlow', items: ['BAAI/bge-m3', 'BAAI/bge-large-zh-v1.5', 'BAAI/bge-large-en-v1.5'] }, + { group: 'Jina', items: ['jina-embeddings-v3', 'jina-embeddings-v2-base-en', 'jina-embeddings-v2-base-zh'] } + ] + }, + 'LITELLM_RERANKER_MODEL': { + label: 'Reranker Model', + type: 'model-select', + placeholder: 'Select or enter model...', + models: [ + { group: 'Cohere', items: ['rerank-english-v3.0', 'rerank-multilingual-v3.0', 'rerank-english-v2.0'] }, + { group: 'Voyage', items: ['rerank-2', 'rerank-2-lite', 'rerank-1'] }, + { group: 'SiliconFlow', items: ['BAAI/bge-reranker-v2-m3', 'BAAI/bge-reranker-large', 'BAAI/bge-reranker-base'] }, + { group: 'Jina', items: ['jina-reranker-v2-base-multilingual', 'jina-reranker-v1-base-en'] } + ] + } + } + } }; /** @@ -662,34 +718,170 @@ async function loadEnvVariables() { container.innerHTML = '
Loading...
'; try { - var response = await fetch('/api/codexlens/env'); - var result = await response.json(); + // Fetch env vars and configured models in parallel + var [envResponse, embeddingPoolResponse, rerankerPoolResponse] = await Promise.all([ + fetch('/api/codexlens/env'), + fetch('/api/litellm-api/embedding-pool').catch(function() { return null; }), + fetch('/api/litellm-api/reranker-pool').catch(function() { return null; }) + ]); + + var result = await envResponse.json(); if (!result.success) { - container.innerHTML = '
' + (result.error || 'Failed to load') + '
'; + container.innerHTML = '
' + escapeHtml(result.error || 'Failed to load') + '
'; return; } + // Get configured embedding models from API settings + var configuredEmbeddingModels = []; + if (embeddingPoolResponse && embeddingPoolResponse.ok) { + var poolData = await embeddingPoolResponse.json(); + configuredEmbeddingModels = poolData.availableModels || []; + } + + // Get configured reranker models from API settings + var configuredRerankerModels = []; + if (rerankerPoolResponse && rerankerPoolResponse.ok) { + var rerankerData = await rerankerPoolResponse.json(); + configuredRerankerModels = rerankerData.availableModels || []; + } + var env = result.env || {}; - var html = '
'; + var html = '
'; - // Render known variables with their values - for (var key in ENV_VARIABLES) { - var config = ENV_VARIABLES[key]; - var value = env[key] || ''; - var inputType = config.type || 'text'; + // Get available LiteLLM providers + var litellmProviders = window.litellmApiConfig?.providers || []; - html += '
' + - '' + - '' + - '
'; + // Render each group + for (var groupKey in ENV_VAR_GROUPS) { + var group = ENV_VAR_GROUPS[groupKey]; + + // Check if this group should be shown + if (group.showWhen && !group.showWhen(env)) { + continue; + } + + html += '
' + + '
' + + '' + + group.label + + '
' + + '
'; + + // Add provider selector for API group + if (groupKey === 'api' && litellmProviders.length > 0) { + html += '
' + + '' + + '
'; + } + + for (var key in group.vars) { + var config = group.vars[key]; + var value = env[key] || config.default || ''; + + if (config.type === 'select') { + html += '
' + + '' + + '
'; + } else if (config.type === 'model-select') { + // Model selector with grouped options and custom input support + var datalistId = 'models-' + key.replace(/_/g, '-').toLowerCase(); + var isEmbeddingModel = key === 'LITELLM_EMBEDDING_MODEL'; + var isRerankerModel = key === 'LITELLM_RERANKER_MODEL'; + + html += '
' + + '' + + '
' + + '' + + '' + + '
' + + ''; + + // For embedding models: use configured models from API settings first + if (isEmbeddingModel && configuredEmbeddingModels.length > 0) { + html += ''; + configuredEmbeddingModels.forEach(function(model) { + var providers = model.providers ? model.providers.join(', ') : ''; + html += ''; + }); + // Add separator and fallback options + if (config.models) { + html += ''; + config.models.forEach(function(group) { + group.items.forEach(function(model) { + // Skip if already in configured list + var exists = configuredEmbeddingModels.some(function(m) { return m.modelId === model; }); + if (!exists) { + html += ''; + } + }); + }); + } + } else if (isRerankerModel && configuredRerankerModels.length > 0) { + // For reranker models: use configured models from API settings first + html += ''; + configuredRerankerModels.forEach(function(model) { + var providers = model.providers ? model.providers.join(', ') : ''; + html += ''; + }); + // Add separator and fallback options + if (config.models) { + html += ''; + config.models.forEach(function(group) { + group.items.forEach(function(model) { + // Skip if already in configured list + var exists = configuredRerankerModels.some(function(m) { return m.modelId === model; }); + if (!exists) { + html += ''; + } + }); + }); + } + } else if (config.models) { + // Fallback: use static model list + config.models.forEach(function(group) { + group.items.forEach(function(model) { + html += ''; + }); + }); + } + + html += '
'; + } else { + var inputType = config.type || 'text'; + html += '
' + + '' + + '' + + '
'; + } + } + + html += '
'; } html += '
' + '
' + '' + '' + - '' + - '
'; - - return html; -} - -/** - * Toggle manual download guide visibility - */ -function toggleManualDownloadGuide() { - var content = document.getElementById('manualDownloadContent'); - var chevron = document.getElementById('manualDownloadChevron'); - - if (content && chevron) { - content.classList.toggle('hidden'); - chevron.style.transform = content.classList.contains('hidden') ? '' : 'rotate(90deg)'; - } -} - /** * Copy text to clipboard */ @@ -1353,91 +1485,106 @@ function copyToClipboard(text) { } /** - * Load model list + * Load model list (simplified version) */ async function loadModelList() { var container = document.getElementById('modelListContainer'); if (!container) return; try { + // Get config for backend info + var configResponse = await fetch('/api/codexlens/config'); + var config = await configResponse.json(); + var embeddingBackend = config.embedding_backend || 'fastembed'; + var response = await fetch('/api/codexlens/models'); var result = await response.json(); + var html = '
'; + + // Show current backend status + var backendLabel = embeddingBackend === 'litellm' ? 'API (LiteLLM)' : 'Local (FastEmbed)'; + var backendIcon = embeddingBackend === 'litellm' ? 'cloud' : 'hard-drive'; + html += + '
' + + '
' + + '' + + '' + backendLabel + '' + + '
' + + 'via Environment Variables' + + '
'; + if (!result.success) { - // Check if the error is specifically about fastembed not being installed var errorMsg = result.error || ''; if (errorMsg.includes('fastembed not installed') || errorMsg.includes('Semantic')) { - container.innerHTML = - '
' + t('codexlens.semanticNotInstalled') + '
'; + html += '
' + t('codexlens.semanticNotInstalled') + '
'; } else { - // Show actual error message for other failures - container.innerHTML = - '
' + t('codexlens.modelListError') + ': ' + (errorMsg || t('common.unknownError')) + '
'; + html += '
' + escapeHtml(errorMsg || t('common.unknownError')) + '
'; } + html += '
'; + container.innerHTML = html; + if (window.lucide) lucide.createIcons(); return; } if (!result.result || !result.result.models) { - container.innerHTML = - '
' + t('codexlens.noModelsAvailable') + '
'; + html += '
' + t('codexlens.noModelsAvailable') + '
'; + html += '
'; + container.innerHTML = html; + if (window.lucide) lucide.createIcons(); return; } - var models = result.result.models; - var html = '
'; + // Show models for local backend + if (embeddingBackend !== 'litellm') { + var models = result.result.models; + models.forEach(function(model) { + var statusIcon = model.installed + ? '' + : ''; - models.forEach(function(model) { - var statusIcon = model.installed - ? '' - : ''; + var sizeText = model.installed + ? model.actual_size_mb.toFixed(0) + ' MB' + : '~' + model.estimated_size_mb + ' MB'; - var sizeText = model.installed - ? model.actual_size_mb.toFixed(1) + ' MB' - : '~' + model.estimated_size_mb + ' MB'; + var actionBtn = model.installed + ? '' + : ''; - var actionBtn = model.installed - ? '' - : ''; - - html += - '
' + - '
' + - '
' + - '
' + - statusIcon + - '' + model.profile + '' + - '(' + model.dimensions + ' dims)' + - '
' + - '
' + model.model_name + '
' + - '
' + model.use_case + '
' + + html += + '
' + + '
' + + statusIcon + + '' + model.profile + '' + + '' + model.dimensions + 'd' + '
' + - '
' + - '
' + sizeText + '
' + + '
' + + '' + sizeText + '' + actionBtn + '
' + - '
' + + '
'; + }); + } else { + // LiteLLM backend - show API info + html += + '
' + + '' + + '
Using API embeddings
' + + '
Model configured via CODEXLENS_EMBEDDING_MODEL
' + '
'; - }); + } html += '
'; - - // Add manual download guide section - html += buildManualDownloadGuide(); - container.innerHTML = html; if (window.lucide) lucide.createIcons(); } catch (err) { container.innerHTML = - '
' + t('common.error') + ': ' + err.message + '
'; + '
' + escapeHtml(err.message) + '
'; } } /** - * Download model with progress simulation and manual download info + * Download model (simplified version) */ async function downloadModel(profile) { var modelCard = document.getElementById('model-' + profile); @@ -1445,84 +1592,16 @@ async function downloadModel(profile) { var originalHTML = modelCard.innerHTML; - // Get model info for size estimation - var modelSizes = { - 'fast': { size: 80, time: '1-2' }, - 'code': { size: 150, time: '2-5' } - }; - - var modelInfo = modelSizes[profile] || { size: 100, time: '2-5' }; - - // Show detailed download UI with progress simulation + // Show loading state modelCard.innerHTML = - '
' + + '
' + '
' + - '
' + - '' + (t('codexlens.downloadingModel') || 'Downloading') + ' ' + profile + '' + + '
' + + 'Downloading ' + profile + '...' + '
' + - '
' + - '
' + - '
' + - '
' + - '
' + - '' + (t('codexlens.connectingToHuggingFace') || 'Connecting to Hugging Face...') + '' + - '~' + modelInfo.size + ' MB' + - '
' + - '
' + - '
' + - '
' + - '' + - '' + (t('codexlens.downloadTimeEstimate') || 'Estimated time') + ': ' + modelInfo.time + ' ' + (t('common.minutes') || 'minutes') + '' + - '
' + - '
' + - '' + - '' + (t('codexlens.manualDownloadHint') || 'Manual download') + ': codexlens model-download ' + profile + '' + - '
' + - '
' + - '' + + '' + '
'; - if (window.lucide) lucide.createIcons(); - - // Start progress simulation - var progressBar = document.getElementById('model-progress-' + profile); - var statusText = document.getElementById('model-status-' + profile); - var simulatedProgress = 0; - var progressInterval = null; - var downloadAborted = false; - - // Store abort controller for cancellation - window['modelDownloadAbort_' + profile] = function() { - downloadAborted = true; - if (progressInterval) clearInterval(progressInterval); - }; - - // Simulate progress based on model size - var progressStages = [ - { percent: 10, msg: t('codexlens.downloadingModelFiles') || 'Downloading model files...' }, - { percent: 30, msg: t('codexlens.downloadingWeights') || 'Downloading model weights...' }, - { percent: 60, msg: t('codexlens.downloadingTokenizer') || 'Downloading tokenizer...' }, - { percent: 80, msg: t('codexlens.verifyingModel') || 'Verifying model...' }, - { percent: 95, msg: t('codexlens.finalizingDownload') || 'Finalizing...' } - ]; - - var stageIndex = 0; - var baseInterval = Math.max(2000, modelInfo.size * 30); // Slower for larger models - - progressInterval = setInterval(function() { - if (downloadAborted) return; - - if (stageIndex < progressStages.length) { - var stage = progressStages[stageIndex]; - simulatedProgress = stage.percent; - if (progressBar) progressBar.style.width = simulatedProgress + '%'; - if (statusText) statusText.textContent = stage.msg; - stageIndex++; - } - }, baseInterval); - try { var response = await fetch('/api/codexlens/models/download', { method: 'POST', @@ -1530,105 +1609,28 @@ async function downloadModel(profile) { body: JSON.stringify({ profile: profile }) }); - // Clear simulation - if (progressInterval) clearInterval(progressInterval); - - if (downloadAborted) { - modelCard.innerHTML = originalHTML; - if (window.lucide) lucide.createIcons(); - return; - } - var result = await response.json(); if (result.success) { - // Show completion - if (progressBar) progressBar.style.width = '100%'; - if (statusText) statusText.textContent = t('codexlens.downloadComplete') || 'Download complete!'; - - showRefreshToast(t('codexlens.modelDownloaded') + ': ' + profile, 'success'); - - // Refresh model list after short delay - setTimeout(function() { - loadModelList(); - }, 500); + showRefreshToast('Model downloaded: ' + profile, 'success'); + loadModelList(); } else { - showRefreshToast(t('codexlens.modelDownloadFailed') + ': ' + result.error, 'error'); - showModelDownloadError(modelCard, profile, result.error, originalHTML); + showRefreshToast('Download failed: ' + result.error, 'error'); + modelCard.innerHTML = originalHTML; + if (window.lucide) lucide.createIcons(); } } catch (err) { - if (progressInterval) clearInterval(progressInterval); - showRefreshToast(t('common.error') + ': ' + err.message, 'error'); - showModelDownloadError(modelCard, profile, err.message, originalHTML); - } - - // Cleanup abort function - delete window['modelDownloadAbort_' + profile]; -} - -/** - * Show model download error with manual download instructions - */ -function showModelDownloadError(modelCard, profile, error, originalHTML) { - var modelNames = { - 'fast': 'BAAI/bge-small-en-v1.5', - 'code': 'jinaai/jina-embeddings-v2-base-code' - }; - - var modelName = modelNames[profile] || profile; - var hfUrl = 'https://huggingface.co/' + modelName; - - modelCard.innerHTML = - '
' + - '
' + - '' + - '
' + - '
' + (t('codexlens.downloadFailed') || 'Download failed') + '
' + - '
' + error + '
' + - '
' + - '
' + - '
' + - '
' + (t('codexlens.manualDownloadOptions') || 'Manual download options') + ':
' + - '
' + - '
' + - '1.' + - '' + (t('codexlens.cliDownload') || 'CLI') + ': codexlens model-download ' + profile + '' + - '
' + - '
' + - '2.' + - '' + (t('codexlens.huggingfaceDownload') || 'Hugging Face') + ': ' + modelName + '' + - '
' + - '
' + - '
' + - '
' + - '' + - '' + - '
' + - '
'; - - if (window.lucide) lucide.createIcons(); -} - -/** - * Cancel model download - */ -function cancelModelDownload(profile) { - if (window['modelDownloadAbort_' + profile]) { - window['modelDownloadAbort_' + profile](); - showRefreshToast(t('codexlens.downloadCanceled') || 'Download canceled', 'info'); - loadModelList(); + showRefreshToast('Error: ' + err.message, 'error'); + modelCard.innerHTML = originalHTML; + if (window.lucide) lucide.createIcons(); } } /** - * Delete model + * Delete model (simplified) */ async function deleteModel(profile) { - if (!confirm(t('codexlens.deleteModelConfirm') + ' ' + profile + '?')) { + if (!confirm('Delete model ' + profile + '?')) { return; } @@ -1637,8 +1639,11 @@ async function deleteModel(profile) { var originalHTML = modelCard.innerHTML; modelCard.innerHTML = - '
' + - '' + t('codexlens.deleting') + '' + + '
' + + '
' + + '
' + + 'Deleting...' + + '
' + '
'; try { @@ -1651,20 +1656,378 @@ async function deleteModel(profile) { var result = await response.json(); if (result.success) { - showRefreshToast(t('codexlens.modelDeleted') + ': ' + profile, 'success'); - await loadModelList(); + showRefreshToast('Model deleted: ' + profile, 'success'); + loadModelList(); } else { - showRefreshToast(t('codexlens.modelDeleteFailed') + ': ' + result.error, 'error'); + showRefreshToast('Delete failed: ' + result.error, 'error'); modelCard.innerHTML = originalHTML; if (window.lucide) lucide.createIcons(); } } catch (err) { - showRefreshToast(t('common.error') + ': ' + err.message, 'error'); + showRefreshToast('Error: ' + err.message, 'error'); modelCard.innerHTML = originalHTML; if (window.lucide) lucide.createIcons(); } } +// ============================================================ +// RERANKER MODEL MANAGEMENT +// ============================================================ + +// Available reranker models (fastembed TextCrossEncoder) +var RERANKER_MODELS = [ + { id: 'ms-marco-mini', name: 'Xenova/ms-marco-MiniLM-L-6-v2', size: 80, desc: 'Fast, lightweight' }, + { id: 'ms-marco-12', name: 'Xenova/ms-marco-MiniLM-L-12-v2', size: 120, desc: 'Better accuracy' }, + { id: 'bge-base', name: 'BAAI/bge-reranker-base', size: 1040, desc: 'High quality' }, + { id: 'jina-tiny', name: 'jinaai/jina-reranker-v1-tiny-en', size: 130, desc: 'Tiny, fast' }, + { id: 'jina-turbo', name: 'jinaai/jina-reranker-v1-turbo-en', size: 150, desc: 'Balanced' } +]; + +/** + * Load reranker model list + */ +async function loadRerankerModelList() { + var container = document.getElementById('rerankerModelListContainer'); + if (!container) return; + + try { + // Get current reranker config + var response = await fetch('/api/codexlens/reranker/config'); + var config = await response.json(); + var currentModel = config.model_name || 'Xenova/ms-marco-MiniLM-L-6-v2'; + var currentBackend = config.backend || 'fastembed'; + + var html = '
'; + + // Show current backend status + var backendLabel = currentBackend === 'litellm' ? 'API (LiteLLM)' : 'Local (FastEmbed)'; + var backendIcon = currentBackend === 'litellm' ? 'cloud' : 'hard-drive'; + html += + '
' + + '
' + + '' + + '' + backendLabel + '' + + '
' + + 'via Environment Variables' + + '
'; + + // Show models for local backend only + if (currentBackend === 'fastembed' || currentBackend === 'onnx') { + RERANKER_MODELS.forEach(function(model) { + var isActive = currentModel === model.name; + var statusIcon = isActive + ? '' + : ''; + + var actionBtn = isActive + ? 'Active' + : ''; + + html += + '
' + + '
' + + statusIcon + + '' + model.id + '' + + '' + model.desc + '' + + '
' + + '
' + + '~' + model.size + ' MB' + + actionBtn + + '
' + + '
'; + }); + } else { + // LiteLLM backend - show API info + html += + '
' + + '' + + '
Using API reranker
' + + '
Model configured via CODEXLENS_RERANKER_MODEL
' + + '
'; + } + + html += '
'; + container.innerHTML = html; + if (window.lucide) lucide.createIcons(); + } catch (err) { + container.innerHTML = + '
' + escapeHtml(err.message) + '
'; + } +} + +/** + * Update reranker backend + */ +async function updateRerankerBackend(backend) { + try { + var response = await fetch('/api/codexlens/reranker/config', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ backend: backend }) + }); + var result = await response.json(); + + if (result.success) { + showRefreshToast('Reranker backend updated: ' + backend, 'success'); + loadRerankerModelList(); + } else { + showRefreshToast('Failed to update: ' + (result.error || 'Unknown error'), 'error'); + } + } catch (err) { + showRefreshToast('Error: ' + err.message, 'error'); + } +} + +/** + * Select reranker model + */ +async function selectRerankerModel(modelName) { + try { + var response = await fetch('/api/codexlens/reranker/config', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ model_name: modelName }) + }); + var result = await response.json(); + + if (result.success) { + showRefreshToast('Reranker model selected: ' + modelName.split('/').pop(), 'success'); + loadRerankerModelList(); + } else { + showRefreshToast('Failed to select: ' + (result.error || 'Unknown error'), 'error'); + } + } catch (err) { + showRefreshToast('Error: ' + err.message, 'error'); + } +} + +// ============================================================ +// MODEL TAB & MODE MANAGEMENT +// ============================================================ + +/** + * Switch between Embedding and Reranker tabs + */ +function switchModelTab(tabName) { + console.log('[CodexLens] Switching to tab:', tabName); + + // Update tab buttons using direct style manipulation for reliability + var tabs = document.querySelectorAll('.model-tab'); + tabs.forEach(function(tab) { + var isActive = tab.getAttribute('data-tab') === tabName; + if (isActive) { + tab.className = 'model-tab flex-1 px-4 py-2.5 text-sm font-medium border-b-2 border-primary text-primary'; + tab.style.backgroundColor = 'rgba(var(--primary), 0.05)'; + } else { + tab.className = 'model-tab flex-1 px-4 py-2.5 text-sm font-medium border-b-2 border-transparent text-muted-foreground'; + tab.style.backgroundColor = ''; + } + }); + + // Update tab content + var embeddingContent = document.getElementById('embeddingTabContent'); + var rerankerContent = document.getElementById('rerankerTabContent'); + + if (embeddingContent && rerankerContent) { + if (tabName === 'embedding') { + embeddingContent.style.display = 'block'; + rerankerContent.style.display = 'none'; + } else { + embeddingContent.style.display = 'none'; + rerankerContent.style.display = 'block'; + } + } +} + +/** + * Update model mode (Local vs API) + */ +function updateModelMode(mode) { + var gpuContainer = document.getElementById('gpuSelectContainer'); + var modeSelect = document.getElementById('modelModeSelect'); + + // Show/hide GPU selector based on mode + if (gpuContainer) { + if (mode === 'local') { + gpuContainer.classList.remove('hidden'); + loadGpuDevicesForModeSelector(); + } else { + gpuContainer.classList.add('hidden'); + } + } + + // Store mode preference (will be saved when locked) + if (modeSelect) { + modeSelect.setAttribute('data-current-mode', mode); + } +} + +/** + * Load GPU devices for mode selector + */ +async function loadGpuDevicesForModeSelector() { + var gpuSelect = document.getElementById('gpuDeviceSelect'); + if (!gpuSelect) return; + + try { + var response = await fetch('/api/codexlens/gpu/devices'); + var result = await response.json(); + + var html = ''; + if (result.devices && result.devices.length > 0) { + result.devices.forEach(function(device, index) { + html += ''; + }); + } + gpuSelect.innerHTML = html; + } catch (err) { + console.error('Failed to load GPU devices:', err); + } +} + +/** + * Toggle model mode lock (save configuration) + */ +async function toggleModelModeLock() { + var lockBtn = document.getElementById('modelModeLockBtn'); + var modeSelect = document.getElementById('modelModeSelect'); + var gpuSelect = document.getElementById('gpuDeviceSelect'); + + if (!lockBtn || !modeSelect) return; + + var isLocked = lockBtn.getAttribute('data-locked') === 'true'; + + if (isLocked) { + // Unlock - enable editing + lockBtn.setAttribute('data-locked', 'false'); + lockBtn.innerHTML = 'Lock'; + lockBtn.classList.remove('btn-primary'); + lockBtn.classList.add('btn-outline'); + modeSelect.disabled = false; + if (gpuSelect) gpuSelect.disabled = false; + if (window.lucide) lucide.createIcons(); + } else { + // Lock - save configuration + var mode = modeSelect.value; + var gpuDevice = gpuSelect ? gpuSelect.value : 'auto'; + + try { + // Save embedding backend preference + var embeddingBackend = mode === 'local' ? 'fastembed' : 'litellm'; + await fetch('/api/codexlens/config', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + embedding_backend: embeddingBackend, + gpu_device: gpuDevice + }) + }); + + // Save reranker backend preference + var rerankerBackend = mode === 'local' ? 'fastembed' : 'litellm'; + await fetch('/api/codexlens/reranker/config', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ backend: rerankerBackend }) + }); + + // Update UI to locked state + lockBtn.setAttribute('data-locked', 'true'); + lockBtn.innerHTML = 'Locked'; + lockBtn.classList.remove('btn-outline'); + lockBtn.classList.add('btn-primary'); + modeSelect.disabled = true; + if (gpuSelect) gpuSelect.disabled = true; + if (window.lucide) lucide.createIcons(); + + showRefreshToast('Configuration saved: ' + (mode === 'local' ? 'Local (FastEmbed)' : 'API (LiteLLM)'), 'success'); + + // Refresh model lists to reflect new backend + loadModelList(); + loadRerankerModelList(); + } catch (err) { + showRefreshToast('Failed to save configuration: ' + err.message, 'error'); + } + } +} + +/** + * Initialize model mode from saved config + */ +async function initModelModeFromConfig() { + var modeSelect = document.getElementById('modelModeSelect'); + var gpuContainer = document.getElementById('gpuSelectContainer'); + + if (!modeSelect) return; + + try { + var response = await fetch('/api/codexlens/config'); + var config = await response.json(); + + var embeddingBackend = config.embedding_backend || 'fastembed'; + var mode = embeddingBackend === 'litellm' ? 'api' : 'local'; + + modeSelect.value = mode; + modeSelect.setAttribute('data-current-mode', mode); + + // Show GPU selector for local mode + if (gpuContainer) { + if (mode === 'local') { + gpuContainer.classList.remove('hidden'); + loadGpuDevicesForModeSelector(); + } else { + gpuContainer.classList.add('hidden'); + } + } + } catch (err) { + console.error('Failed to load model mode config:', err); + } +} + +/** + * Update compact semantic status badge in header + */ +async function updateSemanticStatusBadge() { + var badge = document.getElementById('semanticStatusBadge'); + if (!badge) return; + + try { + var response = await fetch('/api/codexlens/semantic/status'); + var result = await response.json(); + + if (result.available) { + var accelerator = result.accelerator || 'CPU'; + var badgeClass = 'bg-success/20 text-success'; + var icon = 'check-circle'; + + if (accelerator === 'CUDA') { + badgeClass = 'bg-green-500/20 text-green-600'; + icon = 'zap'; + } else if (accelerator === 'DirectML') { + badgeClass = 'bg-blue-500/20 text-blue-600'; + icon = 'cpu'; + } + + badge.innerHTML = + '' + + '' + + accelerator + + ''; + } else { + badge.innerHTML = + '' + + '' + + 'Not Ready' + + ''; + } + + if (window.lucide) lucide.createIcons(); + } catch (err) { + badge.innerHTML = + 'Error'; + } +} + // ============================================================ // CODEXLENS ACTIONS // ============================================================ @@ -2451,13 +2814,23 @@ async function renderCodexLensManager() { loadSemanticDepsStatus(); loadModelList(); + loadRerankerModelList(); + + // Initialize model mode and semantic status badge + updateSemanticStatusBadge(); + loadGpuDevicesForModeSelector(); + + // Initialize file watcher status + initWatcherStatus(); // Load index stats for the Index Manager section if (isInstalled) { loadIndexStatsForPage(); + // Check index health based on git history + checkIndexHealth(); } } catch (err) { - container.innerHTML = '

' + t('common.error') + ': ' + err.message + '

'; + container.innerHTML = '

' + t('common.error') + ': ' + escapeHtml(err.message) + '

'; if (window.lucide) lucide.createIcons(); } } @@ -2520,22 +2893,6 @@ function buildCodexLensManagerPage(config) { '' + 'Incremental Update' + '' + - // Watchdog Section - '
' + - '
' + - '
' + - '' + - 'File Watcher' + - '
' + - '
' + - 'Stopped' + - '' + - '
' + - '
' + - '

Auto-update index when files change

' + - '
' + '

' + t('codexlens.indexTypeHint') + ' Configure embedding model in Environment Variables below.

' + '
' + '
' + @@ -2579,21 +2936,100 @@ function buildCodexLensManagerPage(config) { '
' + // Right Column '
' + - // Semantic Dependencies - '
' + - '

' + t('codexlens.semanticDeps') + '

' + - '
' + - '
' + - '
' + t('codexlens.checkingDeps') + + // Combined: Semantic Status + Model Management with Tabs + '
' + + // Compact Header with Semantic Status + '
' + + '
' + + '

' + t('codexlens.modelManagement') + '

' + + '
' + + 'Checking...' + + '
' + + '
' + + '
' + + // GPU Config Section (for local mode) + '
' + + '
' + + 'GPU:' + + '' + + '
' + + '

Backend configured in Environment Variables below

' + + '
' + + // Tabs for Embedding / Reranker + '
' + + '
' + + '' + + '' + + '
' + + '
' + + // Tab Content + '
' + + // Embedding Tab Content + '
' + + '
' + + '
' + + '
' + t('codexlens.loadingModels') + + '
' + + '
' + + '
' + + // Reranker Tab Content + '' + '
' + '
' + - // Model Management - '
' + - '

' + t('codexlens.modelManagement') + '

' + - '
' + - '
' + - '
' + t('codexlens.loadingModels') + + // File Watcher Card + '
' + + '
' + + '
' + + '
' + + '' + + '

File Watcher

' + + '
' + + '
' + + 'Stopped' + + '' + + '
' + + '
' + + '
' + + '
' + + '

Monitor file changes and auto-update index

' + + // Stats row + '
' + + '
' + + '
-
' + + '
Files
' + + '
' + + '
' + + '
0
' + + '
Changes
' + + '
' + + '
' + + '
-
' + + '
Uptime
' + + '
' + + '
' + + // Recent activity log + '
' + + '
' + + 'Recent Activity' + + '' + + '
' + + '
' + + '
No activity yet. Start watcher to monitor files.
' + + '
' + '
' + '
' + '
' + @@ -2606,6 +3042,7 @@ function buildCodexLensManagerPage(config) { '' + '' + t('index.manager') + '' + '-' + + '...' + '
' + '
' + '' + '
' + '
' + + // Index Health Details + '' + '
' + '
' + '' + @@ -2932,7 +3379,7 @@ window.toggleWatcher = async function toggleWatcher() { /** * Update watcher UI state */ -function updateWatcherUI(running) { +function updateWatcherUI(running, stats) { var statusBadge = document.getElementById('watcherStatusBadge'); if (statusBadge) { var badgeClass = running ? 'bg-success/20 text-success' : 'bg-muted text-muted-foreground'; @@ -2947,12 +3394,191 @@ function updateWatcherUI(running) { if (window.lucide) lucide.createIcons(); } + + // Update stats if provided + if (stats) { + var filesCount = document.getElementById('watcherFilesCount'); + var changesCount = document.getElementById('watcherChangesCount'); + var uptimeDisplay = document.getElementById('watcherUptimeDisplay'); + + if (filesCount) filesCount.textContent = stats.files_watched || '-'; + if (changesCount) changesCount.textContent = stats.changes_detected || '0'; + if (uptimeDisplay) uptimeDisplay.textContent = formatUptime(stats.uptime_seconds); + } + + // Start or stop polling based on running state + if (running) { + startWatcherPolling(); + } else { + stopWatcherPolling(); + } +} + +// Watcher polling interval +var watcherPollInterval = null; +var watcherStartTime = null; +var watcherChangesCount = 0; + +/** + * Format uptime in human readable format + */ +function formatUptime(seconds) { + if (!seconds || seconds < 0) return '-'; + if (seconds < 60) return Math.floor(seconds) + 's'; + if (seconds < 3600) return Math.floor(seconds / 60) + 'm'; + var hours = Math.floor(seconds / 3600); + var mins = Math.floor((seconds % 3600) / 60); + return hours + 'h ' + mins + 'm'; +} + +/** + * Start polling watcher status + */ +function startWatcherPolling() { + if (watcherPollInterval) return; // Already polling + + watcherStartTime = Date.now(); + watcherPollInterval = setInterval(async function() { + try { + var response = await fetch('/api/codexlens/watch/status'); + var result = await response.json(); + + if (result.success && result.running) { + // Update uptime + var uptimeDisplay = document.getElementById('watcherUptimeDisplay'); + if (uptimeDisplay) { + var uptime = (Date.now() - watcherStartTime) / 1000; + uptimeDisplay.textContent = formatUptime(uptime); + } + + // Update files count if available + if (result.files_watched !== undefined) { + var filesCount = document.getElementById('watcherFilesCount'); + if (filesCount) filesCount.textContent = result.files_watched; + } + + // Check for new events + if (result.recent_events && result.recent_events.length > 0) { + result.recent_events.forEach(function(event) { + addWatcherLogEntry(event.type, event.path); + }); + } + } else if (!result.running) { + // Watcher stopped externally + updateWatcherUI(false); + stopWatcherPolling(); + } + } catch (err) { + console.warn('[Watcher] Poll error:', err); + } + }, 3000); // Poll every 3 seconds +} + +/** + * Stop polling watcher status + */ +function stopWatcherPolling() { + if (watcherPollInterval) { + clearInterval(watcherPollInterval); + watcherPollInterval = null; + } + watcherStartTime = null; +} + +/** + * Add entry to watcher activity log + */ +function addWatcherLogEntry(type, path) { + var logContainer = document.getElementById('watcherActivityLog'); + if (!logContainer) return; + + // Clear "no activity" message if present + var noActivity = logContainer.querySelector('.text-muted-foreground:only-child'); + if (noActivity && noActivity.textContent.includes('No activity')) { + logContainer.innerHTML = ''; + } + + // Increment changes count + watcherChangesCount++; + var changesCount = document.getElementById('watcherChangesCount'); + if (changesCount) changesCount.textContent = watcherChangesCount; + + // Create log entry + var timestamp = new Date().toLocaleTimeString(); + var typeColors = { + 'created': 'text-success', + 'modified': 'text-warning', + 'deleted': 'text-destructive', + 'renamed': 'text-primary' + }; + var typeIcons = { + 'created': '+', + 'modified': '~', + 'deleted': '-', + 'renamed': '→' + }; + + var colorClass = typeColors[type] || 'text-muted-foreground'; + var icon = typeIcons[type] || '•'; + + // Get just the filename + var filename = path.split(/[/\\]/).pop(); + + var entry = document.createElement('div'); + entry.className = 'flex items-center gap-2 py-0.5'; + entry.innerHTML = + '' + timestamp + '' + + '' + icon + '' + + '' + escapeHtml(filename) + ''; + + // Add to top of log + logContainer.insertBefore(entry, logContainer.firstChild); + + // Keep only last 50 entries + while (logContainer.children.length > 50) { + logContainer.removeChild(logContainer.lastChild); + } +} + +/** + * Clear watcher activity log + */ +function clearWatcherLog() { + var logContainer = document.getElementById('watcherActivityLog'); + if (logContainer) { + logContainer.innerHTML = '
Log cleared. Waiting for file changes...
'; + } + watcherChangesCount = 0; + var changesCount = document.getElementById('watcherChangesCount'); + if (changesCount) changesCount.textContent = '0'; +} + +/** + * Initialize watcher status on page load + */ +async function initWatcherStatus() { + try { + var response = await fetch('/api/codexlens/watch/status'); + var result = await response.json(); + if (result.success) { + updateWatcherUI(result.running, { + files_watched: result.files_watched, + changes_detected: 0, + uptime_seconds: result.uptime_seconds + }); + } + } catch (err) { + console.warn('[Watcher] Failed to get initial status:', err); + } } // Make functions globally accessible window.runIncrementalUpdate = runIncrementalUpdate; window.toggleWatcher = toggleWatcher; window.updateWatcherUI = updateWatcherUI; +window.addWatcherLogEntry = addWatcherLogEntry; +window.clearWatcherLog = clearWatcherLog; +window.initWatcherStatus = initWatcherStatus; /** * Initialize CodexLens Manager page event handlers @@ -3042,7 +3668,7 @@ async function loadIndexStatsForPage() { console.error('[CodexLens] Failed to load index stats:', err); var tbody = document.getElementById('indexTableBody'); if (tbody) { - tbody.innerHTML = '' + err.message + ''; + tbody.innerHTML = '' + escapeHtml(err.message) + ''; } } } @@ -3130,6 +3756,93 @@ function formatTimeAgoSimple(isoString) { return date.toLocaleDateString(); } +/** + * Check and display index health for current workspace + */ +async function checkIndexHealth() { + var healthBadge = document.getElementById('indexHealthBadge'); + var healthDetails = document.getElementById('indexHealthDetails'); + var lastUpdateEl = document.getElementById('indexLastUpdate'); + var commitsSinceEl = document.getElementById('indexCommitsSince'); + + if (!healthBadge) return; + + try { + // Get current workspace index info + var indexResponse = await fetch('/api/codexlens/indexes'); + var indexData = await indexResponse.json(); + var indexes = indexData.indexes || []; + + // Find current workspace index (newest one or matching current path) + var currentIndex = indexes.length > 0 ? indexes[0] : null; + + if (!currentIndex) { + healthBadge.className = 'text-xs px-2 py-0.5 rounded-full bg-muted text-muted-foreground'; + healthBadge.textContent = 'No Index'; + if (healthDetails) healthDetails.classList.add('hidden'); + return; + } + + var lastIndexTime = currentIndex.lastModified ? new Date(currentIndex.lastModified) : null; + + // Get git commits since last index + var commitsSince = 0; + try { + var gitResponse = await fetch('/api/git/commits-since?since=' + encodeURIComponent(currentIndex.lastModified || '')); + var gitData = await gitResponse.json(); + commitsSince = gitData.count || 0; + } catch (gitErr) { + console.warn('[CodexLens] Could not get git history:', gitErr); + // Fallback: estimate based on time + if (lastIndexTime) { + var hoursSince = (Date.now() - lastIndexTime.getTime()) / (1000 * 60 * 60); + commitsSince = Math.floor(hoursSince / 2); // Rough estimate + } + } + + // Determine health status + var healthStatus = 'good'; + var healthText = 'Up to date'; + var healthClass = 'bg-success/20 text-success'; + + if (commitsSince > 50 || (lastIndexTime && (Date.now() - lastIndexTime.getTime()) > 7 * 24 * 60 * 60 * 1000)) { + // More than 50 commits or 7 days old + healthStatus = 'outdated'; + healthText = 'Outdated'; + healthClass = 'bg-destructive/20 text-destructive'; + } else if (commitsSince > 10 || (lastIndexTime && (Date.now() - lastIndexTime.getTime()) > 24 * 60 * 60 * 1000)) { + // More than 10 commits or 1 day old + healthStatus = 'stale'; + healthText = 'Stale'; + healthClass = 'bg-warning/20 text-warning'; + } + + // Update badge + healthBadge.className = 'text-xs px-2 py-0.5 rounded-full ' + healthClass; + healthBadge.textContent = healthText; + + // Update details section + if (healthDetails && healthStatus !== 'good') { + healthDetails.classList.remove('hidden'); + if (lastUpdateEl) lastUpdateEl.textContent = lastIndexTime ? formatTimeAgoSimple(currentIndex.lastModified) : 'Unknown'; + if (commitsSinceEl) { + commitsSinceEl.textContent = commitsSince; + commitsSinceEl.className = 'font-medium ' + (commitsSince > 20 ? 'text-destructive' : commitsSince > 5 ? 'text-warning' : 'text-foreground'); + } + } else if (healthDetails) { + healthDetails.classList.add('hidden'); + } + + } catch (err) { + console.error('[CodexLens] Failed to check index health:', err); + healthBadge.className = 'text-xs px-2 py-0.5 rounded-full bg-muted text-muted-foreground'; + healthBadge.textContent = 'Unknown'; + } +} + +// Make function globally accessible +window.checkIndexHealth = checkIndexHealth; + /** * Clean a specific project's index from the page */ diff --git a/ccw/src/types/litellm-api-config.ts b/ccw/src/types/litellm-api-config.ts index b21bfa71..4fcf1ebb 100644 --- a/ccw/src/types/litellm-api-config.ts +++ b/ccw/src/types/litellm-api-config.ts @@ -226,6 +226,9 @@ export interface ProviderCredential { /** Embedding models configured for this provider */ embeddingModels?: ModelDefinition[]; + /** Reranker models configured for this provider */ + rerankerModels?: ModelDefinition[]; + /** Creation timestamp (ISO 8601) */ createdAt: string; diff --git a/codex-lens/src/codexlens/semantic/reranker/fastembed_reranker.py b/codex-lens/src/codexlens/semantic/reranker/fastembed_reranker.py index d4d54c77..c38d4aa0 100644 --- a/codex-lens/src/codexlens/semantic/reranker/fastembed_reranker.py +++ b/codex-lens/src/codexlens/semantic/reranker/fastembed_reranker.py @@ -125,6 +125,16 @@ class FastEmbedReranker(BaseReranker): logger.debug("FastEmbed reranker model loaded successfully") + @staticmethod + def _sigmoid(x: float) -> float: + """Numerically stable sigmoid function.""" + if x < -709: + return 0.0 + if x > 709: + return 1.0 + import math + return 1.0 / (1.0 + math.exp(-x)) + def score_pairs( self, pairs: Sequence[tuple[str, str]], @@ -165,8 +175,8 @@ class FastEmbedReranker(BaseReranker): indices = [idx for idx, _ in indexed_docs] try: - # TextCrossEncoder.rerank returns list of RerankResult with score attribute - results = list( + # TextCrossEncoder.rerank returns raw float scores in same order as input + raw_scores = list( self._encoder.rerank( query=query, documents=docs, @@ -174,22 +184,12 @@ class FastEmbedReranker(BaseReranker): ) ) - # Map scores back to original positions - # Results are returned in descending score order, but we need original order - for result in results: - # Each result has 'index' (position in input docs) and 'score' - doc_idx = result.index if hasattr(result, "index") else 0 - score = result.score if hasattr(result, "score") else 0.0 - - if doc_idx < len(indices): - original_idx = indices[doc_idx] - # Normalize score to [0, 1] using sigmoid if needed - # FastEmbed typically returns scores in [0, 1] already - if score < 0 or score > 1: - import math - - score = 1.0 / (1.0 + math.exp(-score)) - scores[original_idx] = float(score) + # Map scores back to original positions and normalize with sigmoid + for i, raw_score in enumerate(raw_scores): + if i < len(indices): + original_idx = indices[i] + # Normalize score to [0, 1] using stable sigmoid + scores[original_idx] = self._sigmoid(float(raw_score)) except Exception as e: logger.warning("FastEmbed rerank failed for query: %s", str(e)[:100]) @@ -227,7 +227,8 @@ class FastEmbedReranker(BaseReranker): return [] try: - results = list( + # TextCrossEncoder.rerank returns raw float scores in same order as input + raw_scores = list( self._encoder.rerank( query=query, documents=list(documents), @@ -235,13 +236,13 @@ class FastEmbedReranker(BaseReranker): ) ) - # Convert to our format: (score, document, original_index) + # Convert to our format: (normalized_score, document, original_index) ranked = [] - for result in results: - idx = result.index if hasattr(result, "index") else 0 - score = result.score if hasattr(result, "score") else 0.0 - doc = documents[idx] if idx < len(documents) else "" - ranked.append((float(score), doc, idx)) + for idx, raw_score in enumerate(raw_scores): + if idx < len(documents): + # Normalize score to [0, 1] using stable sigmoid + normalized = self._sigmoid(float(raw_score)) + ranked.append((normalized, documents[idx], idx)) # Sort by score descending ranked.sort(key=lambda x: x[0], reverse=True)