mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-02-13 02:41:50 +08:00
feat: 添加动态批量大小计算，优化嵌入管理和配置系统
This commit is contained in:
catlog22
@@ -93,6 +93,96 @@ export async function handleCodexLensConfigRoutes(ctx: RouteContext): Promise<bo
|
||||
return true;
|
||||
}
|
||||
|
||||
// API: CodexLens Workspace Status - Get FTS and Vector index status for current workspace
|
||||
if (pathname === '/api/codexlens/workspace-status') {
|
||||
try {
|
||||
const venvStatus = await checkVenvStatus();
|
||||
|
||||
// Default response when not installed
|
||||
if (!venvStatus.ready) {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
success: true,
|
||||
hasIndex: false,
|
||||
fts: { percent: 0, indexedFiles: 0, totalFiles: 0 },
|
||||
vector: { percent: 0, filesWithEmbeddings: 0, totalFiles: 0, totalChunks: 0 }
|
||||
}));
|
||||
return true;
|
||||
}
|
||||
|
||||
// Get project info for current workspace
|
||||
const projectResult = await executeCodexLens(['projects', 'get', initialPath, '--json']);
|
||||
|
||||
if (!projectResult.success) {
|
||||
// No index for this workspace
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
success: true,
|
||||
hasIndex: false,
|
||||
fts: { percent: 0, indexedFiles: 0, totalFiles: 0 },
|
||||
vector: { percent: 0, filesWithEmbeddings: 0, totalFiles: 0, totalChunks: 0 }
|
||||
}));
|
||||
return true;
|
||||
}
|
||||
|
||||
// Parse project data
|
||||
let projectData: any = null;
|
||||
try {
|
||||
const parsed = extractJSON(projectResult.output ?? '');
|
||||
if (parsed.success && parsed.result) {
|
||||
projectData = parsed.result;
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
console.error('[CodexLens] Failed to parse project data:', e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
|
||||
if (!projectData) {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
success: true,
|
||||
hasIndex: false,
|
||||
fts: { percent: 0, indexedFiles: 0, totalFiles: 0 },
|
||||
vector: { percent: 0, filesWithEmbeddings: 0, totalFiles: 0, totalChunks: 0 }
|
||||
}));
|
||||
return true;
|
||||
}
|
||||
|
||||
// Calculate FTS and Vector percentages
|
||||
const totalFiles = projectData.total_files || 0;
|
||||
const indexedFiles = projectData.indexed_files || projectData.total_files || 0;
|
||||
const filesWithEmbeddings = projectData.files_with_embeddings || projectData.embedded_files || 0;
|
||||
const totalChunks = projectData.total_chunks || projectData.embedded_chunks || 0;
|
||||
|
||||
// FTS percentage (all indexed files have FTS)
|
||||
const ftsPercent = totalFiles > 0 ? Math.round((indexedFiles / totalFiles) * 100) : 0;
|
||||
|
||||
// Vector percentage (files with embeddings)
|
||||
const vectorPercent = totalFiles > 0 ? Math.round((filesWithEmbeddings / totalFiles) * 1000) / 10 : 0;
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
success: true,
|
||||
hasIndex: true,
|
||||
path: initialPath,
|
||||
fts: {
|
||||
percent: ftsPercent,
|
||||
indexedFiles,
|
||||
totalFiles
|
||||
},
|
||||
vector: {
|
||||
percent: vectorPercent,
|
||||
filesWithEmbeddings,
|
||||
totalFiles,
|
||||
totalChunks
|
||||
}
|
||||
}));
|
||||
} catch (err: unknown) {
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ success: false, error: err instanceof Error ? err.message : String(err) }));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// API: CodexLens Bootstrap (Install)
|
||||
if (pathname === '/api/codexlens/bootstrap' && req.method === 'POST') {
|
||||
handlePostRequest(req, res, async () => {
|
||||
@@ -164,9 +254,10 @@ export async function handleCodexLensConfigRoutes(ctx: RouteContext): Promise<bo
|
||||
return true;
|
||||
}
|
||||
|
||||
const [configResult, statusResult] = await Promise.all([
|
||||
// Use projects list for accurate index_count (same source as /api/codexlens/indexes)
|
||||
const [configResult, projectsResult] = await Promise.all([
|
||||
executeCodexLens(['config', '--json']),
|
||||
executeCodexLens(['status', '--json'])
|
||||
executeCodexLens(['projects', 'list', '--json'])
|
||||
]);
|
||||
|
||||
// Parse config (extract JSON from output that may contain log messages)
|
||||
@@ -190,16 +281,27 @@ export async function handleCodexLensConfigRoutes(ctx: RouteContext): Promise<bo
|
||||
}
|
||||
}
|
||||
|
||||
// Parse status to get index_count (projects_count)
|
||||
if (statusResult.success) {
|
||||
// Parse projects list to get index_count (consistent with /api/codexlens/indexes)
|
||||
if (projectsResult.success) {
|
||||
try {
|
||||
const status = extractJSON(statusResult.output ?? '');
|
||||
if (status.success && status.result) {
|
||||
responseData.index_count = status.result.projects_count || 0;
|
||||
const projectsData = extractJSON(projectsResult.output ?? '');
|
||||
if (projectsData.success && Array.isArray(projectsData.result)) {
|
||||
// Filter out test/temp projects (same logic as /api/codexlens/indexes)
|
||||
const validProjects = projectsData.result.filter((project: any) => {
|
||||
if (project.source_root && (
|
||||
project.source_root.includes('\\Temp\\') ||
|
||||
project.source_root.includes('/tmp/') ||
|
||||
project.total_files === 0
|
||||
)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
responseData.index_count = validProjects.length;
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
console.error('[CodexLens] Failed to parse status:', e instanceof Error ? e.message : String(e));
|
||||
console.error('[CodexLens] Status output:', (statusResult.output ?? '').substring(0, 200));
|
||||
console.error('[CodexLens] Failed to parse projects list:', e instanceof Error ? e.message : String(e));
|
||||
console.error('[CodexLens] Projects output:', (projectsResult.output ?? '').substring(0, 200));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
import {
|
||||
cancelIndexing,
|
||||
checkVenvStatus,
|
||||
checkSemanticStatus,
|
||||
ensureLiteLLMEmbedderReady,
|
||||
executeCodexLens,
|
||||
isIndexingInProgress,
|
||||
@@ -230,11 +231,29 @@ export async function handleCodexLensIndexRoutes(ctx: RouteContext): Promise<boo
|
||||
const resolvedEmbeddingBackend = typeof embeddingBackend === 'string' && embeddingBackend.trim().length > 0 ? embeddingBackend : 'fastembed';
|
||||
const resolvedMaxWorkers = typeof maxWorkers === 'number' ? maxWorkers : Number(maxWorkers);
|
||||
|
||||
// Ensure LiteLLM backend dependencies are installed before running the CLI
|
||||
if (resolvedIndexType !== 'normal' && resolvedEmbeddingBackend === 'litellm') {
|
||||
const installResult = await ensureLiteLLMEmbedderReady();
|
||||
if (!installResult.success) {
|
||||
return { success: false, error: installResult.error || 'Failed to prepare LiteLLM embedder', status: 500 };
|
||||
// Pre-check: Verify embedding backend availability before proceeding with vector indexing
|
||||
// This prevents silent degradation where vector indexing is skipped without error
|
||||
if (resolvedIndexType !== 'normal') {
|
||||
if (resolvedEmbeddingBackend === 'litellm') {
|
||||
// For litellm backend, ensure ccw-litellm is installed
|
||||
const installResult = await ensureLiteLLMEmbedderReady();
|
||||
if (!installResult.success) {
|
||||
return {
|
||||
success: false,
|
||||
error: installResult.error || 'LiteLLM embedding backend is not available. Please install ccw-litellm first.',
|
||||
status: 500
|
||||
};
|
||||
}
|
||||
} else {
|
||||
// For fastembed backend (default), check semantic dependencies
|
||||
const semanticStatus = await checkSemanticStatus();
|
||||
if (!semanticStatus.available) {
|
||||
return {
|
||||
success: false,
|
||||
error: semanticStatus.error || 'FastEmbed semantic backend is not available. Please install semantic dependencies first (CodeLens Settings → Install Semantic).',
|
||||
status: 500
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
installSemantic,
|
||||
} from '../../../tools/codex-lens.js';
|
||||
import type { GpuMode } from '../../../tools/codex-lens.js';
|
||||
import { loadLiteLLMApiConfig } from '../../../config/litellm-api-config-manager.js';
|
||||
import { loadLiteLLMApiConfig, getAvailableModelsForType, getProvider, getAllProviders } from '../../../config/litellm-api-config-manager.js';
|
||||
import {
|
||||
isUvAvailable,
|
||||
createCodexLensUvManager,
|
||||
@@ -317,16 +317,21 @@ export async function handleCodexLensSemanticRoutes(ctx: RouteContext): Promise<
|
||||
config_source: 'default'
|
||||
};
|
||||
|
||||
// Load LiteLLM endpoints for dropdown
|
||||
// Load LiteLLM reranker models for dropdown (from litellm-api-config providers)
|
||||
try {
|
||||
const litellmConfig = loadLiteLLMApiConfig(initialPath);
|
||||
if (litellmConfig.endpoints && Array.isArray(litellmConfig.endpoints)) {
|
||||
rerankerConfig.litellm_endpoints = litellmConfig.endpoints.map(
|
||||
(ep: any) => ep.alias || ep.name || ep.baseUrl
|
||||
).filter(Boolean);
|
||||
const availableRerankerModels = getAvailableModelsForType(initialPath, 'reranker');
|
||||
if (availableRerankerModels && Array.isArray(availableRerankerModels)) {
|
||||
// Return full model info for frontend to use
|
||||
(rerankerConfig as any).litellm_models = availableRerankerModels.map((m: any) => ({
|
||||
modelId: m.modelId,
|
||||
modelName: m.modelName,
|
||||
providers: m.providers
|
||||
}));
|
||||
// Keep litellm_endpoints for backward compatibility (just model IDs)
|
||||
rerankerConfig.litellm_endpoints = availableRerankerModels.map((m: any) => m.modelId);
|
||||
}
|
||||
} catch {
|
||||
// LiteLLM config not available, continue with empty endpoints
|
||||
// LiteLLM config not available, continue with empty models
|
||||
}
|
||||
|
||||
// If CodexLens is installed, try to get actual config
|
||||
@@ -407,6 +412,97 @@ export async function handleCodexLensSemanticRoutes(ctx: RouteContext): Promise<
|
||||
try {
|
||||
const updates: string[] = [];
|
||||
|
||||
// Special handling for litellm backend - auto-configure from litellm-api-config
|
||||
if (resolvedBackend === 'litellm' && (resolvedModelName || resolvedLiteLLMEndpoint)) {
|
||||
const selectedModel = resolvedModelName || resolvedLiteLLMEndpoint;
|
||||
|
||||
// Find the provider that has this model
|
||||
const providers = getAllProviders(initialPath);
|
||||
let providerWithModel: any = null;
|
||||
let foundModel: any = null;
|
||||
|
||||
for (const provider of providers) {
|
||||
if (!provider.enabled || !provider.rerankerModels) continue;
|
||||
const model = provider.rerankerModels.find((m: any) => m.id === selectedModel && m.enabled);
|
||||
if (model) {
|
||||
providerWithModel = provider;
|
||||
foundModel = model;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (providerWithModel) {
|
||||
// Set backend to litellm
|
||||
const backendResult = await executeCodexLens(['config', 'set', 'reranker_backend', 'litellm', '--json']);
|
||||
if (backendResult.success) updates.push('backend');
|
||||
|
||||
// Set model
|
||||
const modelResult = await executeCodexLens(['config', 'set', 'reranker_model', selectedModel, '--json']);
|
||||
if (modelResult.success) updates.push('model_name');
|
||||
|
||||
// Auto-configure API credentials from provider
|
||||
// Write to CodexLens .env file for persistence
|
||||
const { writeFileSync, existsSync, readFileSync } = await import('fs');
|
||||
const { join } = await import('path');
|
||||
const { homedir } = await import('os');
|
||||
|
||||
const codexlensDir = join(homedir(), '.codexlens');
|
||||
const envFile = join(codexlensDir, '.env');
|
||||
|
||||
// Read existing .env content
|
||||
let envContent = '';
|
||||
if (existsSync(envFile)) {
|
||||
envContent = readFileSync(envFile, 'utf-8');
|
||||
}
|
||||
|
||||
// Update or add RERANKER_API_KEY and RERANKER_API_BASE
|
||||
const apiKey = providerWithModel.apiKey;
|
||||
const apiBase = providerWithModel.apiBase;
|
||||
|
||||
// Helper to update env var in content
|
||||
const updateEnvVar = (content: string, key: string, value: string): string => {
|
||||
const regex = new RegExp(`^${key}=.*$`, 'm');
|
||||
const newLine = `${key}="${value}"`;
|
||||
if (regex.test(content)) {
|
||||
return content.replace(regex, newLine);
|
||||
} else {
|
||||
return content.trim() + '\n' + newLine;
|
||||
}
|
||||
};
|
||||
|
||||
if (apiKey) {
|
||||
envContent = updateEnvVar(envContent, 'RERANKER_API_KEY', apiKey);
|
||||
envContent = updateEnvVar(envContent, 'CODEXLENS_RERANKER_API_KEY', apiKey);
|
||||
process.env.RERANKER_API_KEY = apiKey;
|
||||
updates.push('api_key (auto-configured)');
|
||||
}
|
||||
if (apiBase) {
|
||||
envContent = updateEnvVar(envContent, 'RERANKER_API_BASE', apiBase);
|
||||
envContent = updateEnvVar(envContent, 'CODEXLENS_RERANKER_API_BASE', apiBase);
|
||||
process.env.RERANKER_API_BASE = apiBase;
|
||||
updates.push('api_base (auto-configured)');
|
||||
}
|
||||
|
||||
// Write updated .env
|
||||
writeFileSync(envFile, envContent.trim() + '\n', 'utf-8');
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: `LiteLLM backend configured with model: ${selectedModel}`,
|
||||
updated_fields: updates,
|
||||
provider: providerWithModel.name,
|
||||
auto_configured: true
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
success: false,
|
||||
error: `Model "${selectedModel}" not found in any enabled LiteLLM provider. Please configure it in API Settings first.`,
|
||||
status: 400
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Standard handling for non-litellm backends
|
||||
// Set backend
|
||||
if (resolvedBackend) {
|
||||
const result = await executeCodexLens(['config', 'set', 'reranker_backend', resolvedBackend, '--json']);
|
||||
@@ -425,8 +521,8 @@ export async function handleCodexLensSemanticRoutes(ctx: RouteContext): Promise<
|
||||
if (result.success) updates.push('api_provider');
|
||||
}
|
||||
|
||||
// Set LiteLLM endpoint
|
||||
if (resolvedLiteLLMEndpoint) {
|
||||
// Set LiteLLM endpoint (for backward compatibility)
|
||||
if (resolvedLiteLLMEndpoint && resolvedBackend !== 'litellm') {
|
||||
const result = await executeCodexLens([
|
||||
'config',
|
||||
'set',
|
||||
|
||||
Reference in New Issue
Block a user