mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-03-22 19:18:47 +08:00
feat: add MCP server for semantic code search with FastMCP integration
This commit is contained in:
catlog22
@@ -1,37 +1,13 @@
|
||||
/**
|
||||
* Memory Embedder Bridge - TypeScript interface to Python memory embedder
|
||||
* Memory Embedder Bridge - STUB (v1 Python bridge removed)
|
||||
*
|
||||
* This module provides a TypeScript bridge to the Python memory_embedder.py script,
|
||||
* which generates and searches embeddings for memory chunks using CodexLens's embedder.
|
||||
*
|
||||
* Features:
|
||||
* - Reuses CodexLens venv at ~/.codexlens/venv
|
||||
* - JSON protocol communication
|
||||
* - Three commands: embed, search, status
|
||||
* - Automatic availability checking
|
||||
* - Stage1 output embedding for V2 pipeline
|
||||
* The Python memory_embedder.py bridge has been removed. This module provides
|
||||
* no-op stubs so that existing consumers compile without errors.
|
||||
*/
|
||||
|
||||
import { spawn } from 'child_process';
|
||||
import { join, dirname } from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { getCodexLensHiddenPython } from '../utils/codexlens-path.js';
|
||||
import { getCoreMemoryStore } from './core-memory-store.js';
|
||||
import type { Stage1Output } from './core-memory-store.js';
|
||||
import { StoragePaths } from '../config/storage-paths.js';
|
||||
const V1_REMOVED = 'Memory embedder Python bridge has been removed (v1 cleanup).';
|
||||
|
||||
// Get directory of this module
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// Venv paths (reuse CodexLens venv)
|
||||
const VENV_PYTHON = getCodexLensHiddenPython();
|
||||
|
||||
// Script path
|
||||
const EMBEDDER_SCRIPT = join(__dirname, '..', '..', 'scripts', 'memory_embedder.py');
|
||||
|
||||
// Types
|
||||
// Types (kept for backward compatibility)
|
||||
export interface EmbedResult {
|
||||
success: boolean;
|
||||
chunks_processed: number;
|
||||
@@ -78,197 +54,6 @@ export interface SearchOptions {
|
||||
sourceType?: 'core_memory' | 'workflow' | 'cli_history';
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if embedder is available (venv and script exist)
|
||||
* @returns True if embedder is available
|
||||
*/
|
||||
export function isEmbedderAvailable(): boolean {
|
||||
// Check venv python exists
|
||||
if (!existsSync(VENV_PYTHON)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check script exists
|
||||
if (!existsSync(EMBEDDER_SCRIPT)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run Python script with arguments
|
||||
* @param args - Command line arguments
|
||||
* @param timeout - Timeout in milliseconds
|
||||
* @returns JSON output from script
|
||||
*/
|
||||
function runPython(args: string[], timeout: number = 300000): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Check availability
|
||||
if (!isEmbedderAvailable()) {
|
||||
reject(
|
||||
new Error(
|
||||
'Memory embedder not available. Ensure CodexLens venv exists at ~/.codexlens/venv'
|
||||
)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Spawn Python process
|
||||
const child = spawn(VENV_PYTHON, [EMBEDDER_SCRIPT, ...args], {
|
||||
shell: false,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
timeout,
|
||||
windowsHide: true,
|
||||
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
|
||||
});
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
|
||||
child.stdout.on('data', (data) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
|
||||
child.stderr.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
child.on('close', (code) => {
|
||||
if (code === 0) {
|
||||
resolve(stdout.trim());
|
||||
} else {
|
||||
reject(new Error(`Python script failed (exit code ${code}): ${stderr || stdout}`));
|
||||
}
|
||||
});
|
||||
|
||||
child.on('error', (err) => {
|
||||
if ((err as NodeJS.ErrnoException).code === 'ETIMEDOUT') {
|
||||
reject(new Error('Python script timed out'));
|
||||
} else {
|
||||
reject(new Error(`Failed to spawn Python: ${err.message}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embeddings for memory chunks
|
||||
* @param dbPath - Path to SQLite database
|
||||
* @param options - Embedding options
|
||||
* @returns Embedding result
|
||||
*/
|
||||
export async function generateEmbeddings(
|
||||
dbPath: string,
|
||||
options: EmbedOptions = {}
|
||||
): Promise<EmbedResult> {
|
||||
const { sourceId, batchSize = 8, force = false } = options;
|
||||
|
||||
// Build arguments
|
||||
const args = ['embed', dbPath];
|
||||
|
||||
if (sourceId) {
|
||||
args.push('--source-id', sourceId);
|
||||
}
|
||||
|
||||
if (batchSize !== 8) {
|
||||
args.push('--batch-size', batchSize.toString());
|
||||
}
|
||||
|
||||
if (force) {
|
||||
args.push('--force');
|
||||
}
|
||||
|
||||
try {
|
||||
// Default timeout: 5 minutes
|
||||
const output = await runPython(args, 300000);
|
||||
const result = JSON.parse(output) as EmbedResult;
|
||||
return result;
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
chunks_processed: 0,
|
||||
chunks_failed: 0,
|
||||
elapsed_time: 0,
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search memory chunks using semantic search
|
||||
* @param dbPath - Path to SQLite database
|
||||
* @param query - Search query text
|
||||
* @param options - Search options
|
||||
* @returns Search results
|
||||
*/
|
||||
export async function searchMemories(
|
||||
dbPath: string,
|
||||
query: string,
|
||||
options: SearchOptions = {}
|
||||
): Promise<SearchResult> {
|
||||
const { topK = 10, minScore = 0.3, sourceType } = options;
|
||||
|
||||
// Build arguments
|
||||
const args = ['search', dbPath, query];
|
||||
|
||||
if (topK !== 10) {
|
||||
args.push('--top-k', topK.toString());
|
||||
}
|
||||
|
||||
if (minScore !== 0.3) {
|
||||
args.push('--min-score', minScore.toString());
|
||||
}
|
||||
|
||||
if (sourceType) {
|
||||
args.push('--type', sourceType);
|
||||
}
|
||||
|
||||
try {
|
||||
// Default timeout: 30 seconds
|
||||
const output = await runPython(args, 30000);
|
||||
const result = JSON.parse(output) as SearchResult;
|
||||
return result;
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
matches: [],
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get embedding status statistics
|
||||
* @param dbPath - Path to SQLite database
|
||||
* @returns Embedding status
|
||||
*/
|
||||
export async function getEmbeddingStatus(dbPath: string): Promise<EmbeddingStatus> {
|
||||
// Build arguments
|
||||
const args = ['status', dbPath];
|
||||
|
||||
try {
|
||||
// Default timeout: 30 seconds
|
||||
const output = await runPython(args, 30000);
|
||||
const result = JSON.parse(output) as EmbeddingStatus;
|
||||
return { ...result, success: true };
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
total_chunks: 0,
|
||||
embedded_chunks: 0,
|
||||
pending_chunks: 0,
|
||||
by_type: {},
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Memory V2: Stage1 Output Embedding
|
||||
// ============================================================================
|
||||
|
||||
/** Result of stage1 embedding operation */
|
||||
export interface Stage1EmbedResult {
|
||||
success: boolean;
|
||||
chunksCreated: number;
|
||||
@@ -276,98 +61,54 @@ export interface Stage1EmbedResult {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunk and embed stage1_outputs (raw_memory + rollout_summary) for semantic search.
|
||||
*
|
||||
* Reads all stage1_outputs from the DB, chunks their raw_memory and rollout_summary
|
||||
* content, inserts chunks into memory_chunks with source_type='cli_history' and
|
||||
* metadata indicating the V2 origin, then triggers embedding generation.
|
||||
*
|
||||
* Uses source_id format: "s1:{thread_id}" to differentiate from regular cli_history chunks.
|
||||
*
|
||||
* @param projectPath - Project root path
|
||||
* @param force - Force re-chunking even if chunks exist
|
||||
* @returns Embedding result
|
||||
*/
|
||||
export async function embedStage1Outputs(
|
||||
projectPath: string,
|
||||
force: boolean = false
|
||||
): Promise<Stage1EmbedResult> {
|
||||
try {
|
||||
const store = getCoreMemoryStore(projectPath);
|
||||
const stage1Outputs = store.listStage1Outputs();
|
||||
|
||||
if (stage1Outputs.length === 0) {
|
||||
return { success: true, chunksCreated: 0, chunksEmbedded: 0 };
|
||||
}
|
||||
|
||||
let totalChunksCreated = 0;
|
||||
|
||||
for (const output of stage1Outputs) {
|
||||
const sourceId = `s1:${output.thread_id}`;
|
||||
|
||||
// Check if already chunked
|
||||
const existingChunks = store.getChunks(sourceId);
|
||||
if (existingChunks.length > 0 && !force) continue;
|
||||
|
||||
// Delete old chunks if force
|
||||
if (force && existingChunks.length > 0) {
|
||||
store.deleteChunks(sourceId);
|
||||
}
|
||||
|
||||
// Combine raw_memory and rollout_summary for richer semantic content
|
||||
const combinedContent = [
|
||||
output.rollout_summary ? `## Summary\n${output.rollout_summary}` : '',
|
||||
output.raw_memory ? `## Raw Memory\n${output.raw_memory}` : '',
|
||||
].filter(Boolean).join('\n\n');
|
||||
|
||||
if (!combinedContent.trim()) continue;
|
||||
|
||||
// Chunk using the store's built-in chunking
|
||||
const chunks = store.chunkContent(combinedContent, sourceId, 'cli_history');
|
||||
|
||||
// Insert chunks with V2 metadata
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
store.insertChunk({
|
||||
source_id: sourceId,
|
||||
source_type: 'cli_history',
|
||||
chunk_index: i,
|
||||
content: chunks[i],
|
||||
metadata: JSON.stringify({
|
||||
v2_source: 'stage1_output',
|
||||
thread_id: output.thread_id,
|
||||
generated_at: output.generated_at,
|
||||
}),
|
||||
created_at: new Date().toISOString(),
|
||||
});
|
||||
totalChunksCreated++;
|
||||
}
|
||||
}
|
||||
|
||||
// If we created chunks, generate embeddings
|
||||
let chunksEmbedded = 0;
|
||||
if (totalChunksCreated > 0) {
|
||||
const paths = StoragePaths.project(projectPath);
|
||||
const dbPath = join(paths.root, 'core-memory', 'core_memory.db');
|
||||
|
||||
const embedResult = await generateEmbeddings(dbPath, { force: false });
|
||||
if (embedResult.success) {
|
||||
chunksEmbedded = embedResult.chunks_processed;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
chunksCreated: totalChunksCreated,
|
||||
chunksEmbedded,
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
chunksCreated: 0,
|
||||
chunksEmbedded: 0,
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
export function isEmbedderAvailable(): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function generateEmbeddings(
|
||||
_dbPath: string,
|
||||
_options: EmbedOptions = {}
|
||||
): Promise<EmbedResult> {
|
||||
return {
|
||||
success: false,
|
||||
chunks_processed: 0,
|
||||
chunks_failed: 0,
|
||||
elapsed_time: 0,
|
||||
error: V1_REMOVED,
|
||||
};
|
||||
}
|
||||
|
||||
export async function searchMemories(
|
||||
_dbPath: string,
|
||||
_query: string,
|
||||
_options: SearchOptions = {}
|
||||
): Promise<SearchResult> {
|
||||
return {
|
||||
success: false,
|
||||
matches: [],
|
||||
error: V1_REMOVED,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getEmbeddingStatus(_dbPath: string): Promise<EmbeddingStatus> {
|
||||
return {
|
||||
success: false,
|
||||
total_chunks: 0,
|
||||
embedded_chunks: 0,
|
||||
pending_chunks: 0,
|
||||
by_type: {},
|
||||
error: V1_REMOVED,
|
||||
};
|
||||
}
|
||||
|
||||
export async function embedStage1Outputs(
|
||||
_projectPath: string,
|
||||
_force: boolean = false
|
||||
): Promise<Stage1EmbedResult> {
|
||||
return {
|
||||
success: false,
|
||||
chunksCreated: 0,
|
||||
chunksEmbedded: 0,
|
||||
error: V1_REMOVED,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
/**
|
||||
* CodexLens Routes Module
|
||||
* Handles all CodexLens-related API endpoints.
|
||||
*/
|
||||
|
||||
import type { RouteContext } from './types.js';
|
||||
import { handleCodexLensConfigRoutes } from './codexlens/config-handlers.js';
|
||||
import { handleCodexLensIndexRoutes } from './codexlens/index-handlers.js';
|
||||
import { handleCodexLensSemanticRoutes } from './codexlens/semantic-handlers.js';
|
||||
import { handleCodexLensWatcherRoutes } from './codexlens/watcher-handlers.js';
|
||||
|
||||
/**
|
||||
* Handle CodexLens routes
|
||||
* @returns true if route was handled, false otherwise
|
||||
*/
|
||||
export async function handleCodexLensRoutes(ctx: RouteContext): Promise<boolean> {
|
||||
if (await handleCodexLensIndexRoutes(ctx)) return true;
|
||||
if (await handleCodexLensConfigRoutes(ctx)) return true;
|
||||
if (await handleCodexLensSemanticRoutes(ctx)) return true;
|
||||
if (await handleCodexLensWatcherRoutes(ctx)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
# CodexLens Routes
|
||||
|
||||
CodexLens-related HTTP endpoints are handled by `ccw/src/core/routes/codexlens-routes.ts`, which delegates to handler modules in this directory. Each handler returns `true` when it handles the current request.
|
||||
|
||||
## File Map
|
||||
|
||||
- `ccw/src/core/routes/codexlens/utils.ts` – shared helpers (ANSI stripping + robust JSON extraction from CLI output).
|
||||
- `ccw/src/core/routes/codexlens/index-handlers.ts` – index/project management endpoints:
|
||||
- `GET /api/codexlens/indexes`
|
||||
- `POST /api/codexlens/clean`
|
||||
- `POST /api/codexlens/init`
|
||||
- `POST /api/codexlens/cancel`
|
||||
- `GET /api/codexlens/indexing-status`
|
||||
- `ccw/src/core/routes/codexlens/config-handlers.ts` – install/config/environment endpoints:
|
||||
- `GET /api/codexlens/status`
|
||||
- `GET /api/codexlens/dashboard-init`
|
||||
- `POST /api/codexlens/bootstrap`
|
||||
- `POST /api/codexlens/uninstall`
|
||||
- `GET /api/codexlens/config`
|
||||
- `POST /api/codexlens/config`
|
||||
- GPU: `GET /api/codexlens/gpu/detect`, `GET /api/codexlens/gpu/list`, `POST /api/codexlens/gpu/select`, `POST /api/codexlens/gpu/reset`
|
||||
- Models: `GET /api/codexlens/models`, `POST /api/codexlens/models/download`, `POST /api/codexlens/models/delete`, `GET /api/codexlens/models/info`
|
||||
- Env: `GET /api/codexlens/env`, `POST /api/codexlens/env`
|
||||
- `ccw/src/core/routes/codexlens/semantic-handlers.ts` – semantic search + reranker + SPLADE endpoints:
|
||||
- Semantic: `GET /api/codexlens/semantic/status`, `GET /api/codexlens/semantic/metadata`, `POST /api/codexlens/semantic/install`
|
||||
- Search: `GET /api/codexlens/search`, `GET /api/codexlens/search_files`, `GET /api/codexlens/symbol`, `POST /api/codexlens/enhance`
|
||||
- Reranker: `GET /api/codexlens/reranker/config`, `POST /api/codexlens/reranker/config`, `GET /api/codexlens/reranker/models`, `POST /api/codexlens/reranker/models/download`, `POST /api/codexlens/reranker/models/delete`, `GET /api/codexlens/reranker/models/info`
|
||||
- SPLADE: `GET /api/codexlens/splade/status`, `POST /api/codexlens/splade/install`, `GET /api/codexlens/splade/index-status`, `POST /api/codexlens/splade/rebuild`
|
||||
- `ccw/src/core/routes/codexlens/watcher-handlers.ts` – file watcher endpoints:
|
||||
- `GET /api/codexlens/watch/status`
|
||||
- `POST /api/codexlens/watch/start`
|
||||
- `POST /api/codexlens/watch/stop`
|
||||
- Also exports `stopWatcherForUninstall()` used during uninstall flow.
|
||||
|
||||
## Notes
|
||||
|
||||
- CodexLens CLI output may include logging + ANSI escapes even with `--json`; handlers use `extractJSON()` from `utils.ts` to parse reliably.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,459 +0,0 @@
|
||||
/**
|
||||
* CodexLens index management handlers.
|
||||
*/
|
||||
|
||||
import {
|
||||
cancelIndexing,
|
||||
checkVenvStatus,
|
||||
checkSemanticStatus,
|
||||
ensureLiteLLMEmbedderReady,
|
||||
executeCodexLens,
|
||||
isIndexingInProgress,
|
||||
} from '../../../tools/codex-lens.js';
|
||||
import type { ProgressInfo } from '../../../tools/codex-lens.js';
|
||||
import type { RouteContext } from '../types.js';
|
||||
import { extractJSON, formatSize } from './utils.js';
|
||||
|
||||
/**
|
||||
* Handle CodexLens index routes
|
||||
* @returns true if route was handled, false otherwise
|
||||
*/
|
||||
export async function handleCodexLensIndexRoutes(ctx: RouteContext): Promise<boolean> {
|
||||
const { pathname, url, req, res, initialPath, handlePostRequest, broadcastToClients } = ctx;
|
||||
|
||||
// API: CodexLens Index List - Get all indexed projects with details
|
||||
if (pathname === '/api/codexlens/indexes') {
|
||||
try {
|
||||
// Check if CodexLens is installed first (without auto-installing)
|
||||
const venvStatus = await checkVenvStatus();
|
||||
if (!venvStatus.ready) {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ success: true, indexes: [], totalSize: 0, totalSizeFormatted: '0 B' }));
|
||||
return true;
|
||||
}
|
||||
|
||||
// Execute all CLI commands in parallel
|
||||
const [configResult, projectsResult, statusResult] = await Promise.all([
|
||||
executeCodexLens(['config', '--json']),
|
||||
executeCodexLens(['projects', 'list', '--json']),
|
||||
executeCodexLens(['status', '--json'])
|
||||
]);
|
||||
|
||||
let indexDir = '';
|
||||
if (configResult.success) {
|
||||
try {
|
||||
const config = extractJSON(configResult.output ?? '');
|
||||
if (config.success && config.result) {
|
||||
// CLI returns index_dir (not index_root)
|
||||
indexDir = config.result.index_dir || config.result.index_root || '';
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
console.error('[CodexLens] Failed to parse config for index list:', e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
}
|
||||
|
||||
let indexes: any[] = [];
|
||||
let totalSize = 0;
|
||||
let vectorIndexCount = 0;
|
||||
let normalIndexCount = 0;
|
||||
|
||||
if (projectsResult.success) {
|
||||
try {
|
||||
const projectsData = extractJSON(projectsResult.output ?? '');
|
||||
if (projectsData.success && Array.isArray(projectsData.result)) {
|
||||
const { stat, readdir } = await import('fs/promises');
|
||||
const { existsSync } = await import('fs');
|
||||
const { basename, join } = await import('path');
|
||||
|
||||
for (const project of projectsData.result) {
|
||||
// Skip test/temp projects
|
||||
if (project.source_root && (
|
||||
project.source_root.includes('\\Temp\\') ||
|
||||
project.source_root.includes('/tmp/') ||
|
||||
project.total_files === 0
|
||||
)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let projectSize = 0;
|
||||
let hasVectorIndex = false;
|
||||
let hasNormalIndex = true; // All projects have FTS index
|
||||
let lastModified = null;
|
||||
|
||||
// Try to get actual index size from index_root
|
||||
if (project.index_root && existsSync(project.index_root)) {
|
||||
try {
|
||||
const files = await readdir(project.index_root);
|
||||
for (const file of files) {
|
||||
try {
|
||||
const filePath = join(project.index_root, file);
|
||||
const fileStat = await stat(filePath);
|
||||
projectSize += fileStat.size;
|
||||
if (!lastModified || fileStat.mtime > lastModified) {
|
||||
lastModified = fileStat.mtime;
|
||||
}
|
||||
// Check for vector/embedding files
|
||||
if (file.includes('vector') || file.includes('embedding') ||
|
||||
file.endsWith('.faiss') || file.endsWith('.npy') ||
|
||||
file.includes('semantic_chunks')) {
|
||||
hasVectorIndex = true;
|
||||
}
|
||||
} catch {
|
||||
// Skip files we can't stat
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Can't read index directory
|
||||
}
|
||||
}
|
||||
|
||||
if (hasVectorIndex) vectorIndexCount++;
|
||||
if (hasNormalIndex) normalIndexCount++;
|
||||
totalSize += projectSize;
|
||||
|
||||
// Use source_root as the display name
|
||||
const displayName = project.source_root ? basename(project.source_root) : `project_${project.id}`;
|
||||
|
||||
indexes.push({
|
||||
id: displayName,
|
||||
path: project.source_root || '',
|
||||
indexPath: project.index_root || '',
|
||||
size: projectSize,
|
||||
sizeFormatted: formatSize(projectSize),
|
||||
fileCount: project.total_files || 0,
|
||||
dirCount: project.total_dirs || 0,
|
||||
hasVectorIndex,
|
||||
hasNormalIndex,
|
||||
status: project.status || 'active',
|
||||
lastModified: lastModified ? lastModified.toISOString() : null
|
||||
});
|
||||
}
|
||||
|
||||
// Sort by file count (most files first), then by name
|
||||
indexes.sort((a, b) => {
|
||||
if (b.fileCount !== a.fileCount) return b.fileCount - a.fileCount;
|
||||
return a.id.localeCompare(b.id);
|
||||
});
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
console.error('[CodexLens] Failed to parse projects list:', e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
}
|
||||
|
||||
// Parse summary stats from status command (already fetched in parallel)
|
||||
let statusSummary: any = {};
|
||||
|
||||
if (statusResult.success) {
|
||||
try {
|
||||
const status = extractJSON(statusResult.output ?? '');
|
||||
if (status.success && status.result) {
|
||||
statusSummary = {
|
||||
totalProjects: status.result.projects_count || indexes.length,
|
||||
totalFiles: status.result.total_files || 0,
|
||||
totalDirs: status.result.total_dirs || 0,
|
||||
// Keep calculated totalSize for consistency with per-project sizes
|
||||
// status.index_size_bytes includes shared resources (models, cache)
|
||||
indexSizeBytes: totalSize,
|
||||
indexSizeMb: totalSize / (1024 * 1024),
|
||||
embeddings: status.result.embeddings || {},
|
||||
// Store full index dir size separately for reference
|
||||
fullIndexDirSize: status.result.index_size_bytes || 0,
|
||||
fullIndexDirSizeFormatted: formatSize(status.result.index_size_bytes || 0)
|
||||
};
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
console.error('[CodexLens] Failed to parse status:', e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
}
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
success: true,
|
||||
indexDir,
|
||||
indexes,
|
||||
summary: {
|
||||
totalProjects: indexes.length,
|
||||
totalSize,
|
||||
totalSizeFormatted: formatSize(totalSize),
|
||||
vectorIndexCount,
|
||||
normalIndexCount,
|
||||
...statusSummary
|
||||
}
|
||||
}));
|
||||
} catch (err: unknown) {
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ success: false, error: err instanceof Error ? err.message : String(err) }));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// API: CodexLens Clean (Clean indexes)
|
||||
if (pathname === '/api/codexlens/clean' && req.method === 'POST') {
|
||||
handlePostRequest(req, res, async (body) => {
|
||||
const { all = false, path } = body as { all?: unknown; path?: unknown };
|
||||
|
||||
try {
|
||||
const args = ['clean'];
|
||||
if (all === true) {
|
||||
args.push('--all');
|
||||
} else if (typeof path === 'string' && path.trim().length > 0) {
|
||||
// Path is passed as a positional argument, not as a flag
|
||||
args.push(path);
|
||||
}
|
||||
args.push('--json');
|
||||
|
||||
const result = await executeCodexLens(args);
|
||||
if (result.success) {
|
||||
return { success: true, message: 'Indexes cleaned successfully' };
|
||||
} else {
|
||||
return { success: false, error: result.error || 'Failed to clean indexes', status: 500 };
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
return { success: false, error: err instanceof Error ? err.message : String(err), status: 500 };
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
// API: CodexLens Init (Initialize workspace index)
|
||||
if (pathname === '/api/codexlens/init' && req.method === 'POST') {
|
||||
handlePostRequest(req, res, async (body) => {
|
||||
const { path: projectPath, indexType = 'vector', embeddingModel = 'code', embeddingBackend = 'fastembed', maxWorkers = 1 } = body as {
|
||||
path?: unknown;
|
||||
indexType?: unknown;
|
||||
embeddingModel?: unknown;
|
||||
embeddingBackend?: unknown;
|
||||
maxWorkers?: unknown;
|
||||
};
|
||||
const targetPath = typeof projectPath === 'string' && projectPath.trim().length > 0 ? projectPath : initialPath;
|
||||
const resolvedIndexType = indexType === 'normal' ? 'normal' : 'vector';
|
||||
const resolvedEmbeddingModel = typeof embeddingModel === 'string' && embeddingModel.trim().length > 0 ? embeddingModel : 'code';
|
||||
const resolvedEmbeddingBackend = typeof embeddingBackend === 'string' && embeddingBackend.trim().length > 0 ? embeddingBackend : 'fastembed';
|
||||
const resolvedMaxWorkers = typeof maxWorkers === 'number' ? maxWorkers : Number(maxWorkers);
|
||||
|
||||
// Pre-check: Verify embedding backend availability before proceeding with vector indexing
|
||||
// This prevents silent degradation where vector indexing is skipped without error
|
||||
if (resolvedIndexType !== 'normal') {
|
||||
if (resolvedEmbeddingBackend === 'litellm') {
|
||||
// For litellm backend, ensure ccw-litellm is installed
|
||||
const installResult = await ensureLiteLLMEmbedderReady();
|
||||
if (!installResult.success) {
|
||||
return {
|
||||
success: false,
|
||||
error: installResult.error || 'LiteLLM embedding backend is not available. Please install ccw-litellm first.',
|
||||
status: 500
|
||||
};
|
||||
}
|
||||
} else {
|
||||
// For fastembed backend (default), check semantic dependencies
|
||||
const semanticStatus = await checkSemanticStatus();
|
||||
if (!semanticStatus.available) {
|
||||
return {
|
||||
success: false,
|
||||
error: semanticStatus.error || 'FastEmbed semantic backend is not available. Please install semantic dependencies first (CodeLens Settings → Install Semantic).',
|
||||
status: 500
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build CLI arguments based on index type
|
||||
// Use 'index init' subcommand (new CLI structure)
|
||||
// --force flag ensures full reindex (not incremental)
|
||||
const args = ['index', 'init', targetPath, '--force', '--json'];
|
||||
if (resolvedIndexType === 'normal') {
|
||||
args.push('--no-embeddings');
|
||||
} else {
|
||||
// Add embedding model selection for vector index (use --model, not --embedding-model)
|
||||
args.push('--model', resolvedEmbeddingModel);
|
||||
// Add embedding backend if not using default fastembed (use --backend, not --embedding-backend)
|
||||
if (resolvedEmbeddingBackend && resolvedEmbeddingBackend !== 'fastembed') {
|
||||
args.push('--backend', resolvedEmbeddingBackend);
|
||||
}
|
||||
// Add max workers for concurrent API calls (useful for litellm backend)
|
||||
if (!Number.isNaN(resolvedMaxWorkers) && resolvedMaxWorkers > 1) {
|
||||
args.push('--max-workers', String(resolvedMaxWorkers));
|
||||
}
|
||||
}
|
||||
|
||||
// Broadcast start event
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { stage: 'start', message: 'Starting index...', percent: 0, path: targetPath, indexType: resolvedIndexType }
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await executeCodexLens(args, {
|
||||
cwd: targetPath,
|
||||
timeout: 1800000, // 30 minutes for large codebases
|
||||
onProgress: (progress: ProgressInfo) => {
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { ...progress, path: targetPath }
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
if (result.success) {
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { stage: 'complete', message: 'Index complete', percent: 100, path: targetPath }
|
||||
});
|
||||
|
||||
try {
|
||||
const parsed = extractJSON(result.output ?? '');
|
||||
return { success: true, result: parsed };
|
||||
} catch {
|
||||
return { success: true, output: result.output ?? '' };
|
||||
}
|
||||
} else {
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { stage: 'error', message: result.error || 'Unknown error', percent: 0, path: targetPath }
|
||||
});
|
||||
return { success: false, error: result.error, status: 500 };
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { stage: 'error', message, percent: 0, path: targetPath }
|
||||
});
|
||||
return { success: false, error: message, status: 500 };
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
// API: Cancel CodexLens Indexing
|
||||
if (pathname === '/api/codexlens/cancel' && req.method === 'POST') {
|
||||
const result = cancelIndexing();
|
||||
|
||||
// Broadcast cancellation event
|
||||
if (result.success) {
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { stage: 'cancelled', message: 'Indexing cancelled by user', percent: 0 }
|
||||
});
|
||||
}
|
||||
|
||||
res.writeHead(result.success ? 200 : 400, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(result));
|
||||
return true;
|
||||
}
|
||||
|
||||
// API: CodexLens Update (Incremental index update)
|
||||
if (pathname === '/api/codexlens/update' && req.method === 'POST') {
|
||||
handlePostRequest(req, res, async (body) => {
|
||||
const { path: projectPath, indexType = 'vector', embeddingModel = 'code', embeddingBackend = 'fastembed', maxWorkers = 1 } = body as {
|
||||
path?: unknown;
|
||||
indexType?: unknown;
|
||||
embeddingModel?: unknown;
|
||||
embeddingBackend?: unknown;
|
||||
maxWorkers?: unknown;
|
||||
};
|
||||
const targetPath = typeof projectPath === 'string' && projectPath.trim().length > 0 ? projectPath : initialPath;
|
||||
const resolvedIndexType = indexType === 'normal' ? 'normal' : 'vector';
|
||||
const resolvedEmbeddingModel = typeof embeddingModel === 'string' && embeddingModel.trim().length > 0 ? embeddingModel : 'code';
|
||||
const resolvedEmbeddingBackend = typeof embeddingBackend === 'string' && embeddingBackend.trim().length > 0 ? embeddingBackend : 'fastembed';
|
||||
const resolvedMaxWorkers = typeof maxWorkers === 'number' ? maxWorkers : Number(maxWorkers);
|
||||
|
||||
// Pre-check: Verify embedding backend availability before proceeding with vector indexing
|
||||
if (resolvedIndexType !== 'normal') {
|
||||
if (resolvedEmbeddingBackend === 'litellm') {
|
||||
const installResult = await ensureLiteLLMEmbedderReady();
|
||||
if (!installResult.success) {
|
||||
return {
|
||||
success: false,
|
||||
error: installResult.error || 'LiteLLM embedding backend is not available. Please install ccw-litellm first.',
|
||||
status: 500
|
||||
};
|
||||
}
|
||||
} else {
|
||||
const semanticStatus = await checkSemanticStatus();
|
||||
if (!semanticStatus.available) {
|
||||
return {
|
||||
success: false,
|
||||
error: semanticStatus.error || 'FastEmbed semantic backend is not available. Please install semantic dependencies first.',
|
||||
status: 500
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build CLI arguments for incremental update using 'index init' without --force
|
||||
// 'index init' defaults to incremental mode (skip unchanged files)
|
||||
// 'index update' is only for single-file updates in hooks
|
||||
const args = ['index', 'init', targetPath, '--json'];
|
||||
if (resolvedIndexType === 'normal') {
|
||||
args.push('--no-embeddings');
|
||||
} else {
|
||||
args.push('--model', resolvedEmbeddingModel);
|
||||
if (resolvedEmbeddingBackend && resolvedEmbeddingBackend !== 'fastembed') {
|
||||
args.push('--backend', resolvedEmbeddingBackend);
|
||||
}
|
||||
if (!Number.isNaN(resolvedMaxWorkers) && resolvedMaxWorkers > 1) {
|
||||
args.push('--max-workers', String(resolvedMaxWorkers));
|
||||
}
|
||||
}
|
||||
|
||||
// Broadcast start event
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { stage: 'start', message: 'Starting incremental index update...', percent: 0, path: targetPath, indexType: resolvedIndexType }
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await executeCodexLens(args, {
|
||||
cwd: targetPath,
|
||||
timeout: 1800000,
|
||||
onProgress: (progress: ProgressInfo) => {
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { ...progress, path: targetPath }
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
if (result.success) {
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { stage: 'complete', message: 'Incremental update complete', percent: 100, path: targetPath }
|
||||
});
|
||||
|
||||
try {
|
||||
const parsed = extractJSON(result.output ?? '');
|
||||
return { success: true, result: parsed };
|
||||
} catch {
|
||||
return { success: true, output: result.output ?? '' };
|
||||
}
|
||||
} else {
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { stage: 'error', message: result.error || 'Unknown error', percent: 0, path: targetPath }
|
||||
});
|
||||
return { success: false, error: result.error, status: 500 };
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { stage: 'error', message, percent: 0, path: targetPath }
|
||||
});
|
||||
return { success: false, error: message, status: 500 };
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
// API: Check if indexing is in progress
|
||||
if (pathname === '/api/codexlens/indexing-status') {
|
||||
const inProgress = isIndexingInProgress();
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ success: true, inProgress }));
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,96 +0,0 @@
|
||||
/**
|
||||
* CodexLens route utilities.
|
||||
*
|
||||
* CodexLens CLI can emit logging + ANSI escapes even with --json, so helpers
|
||||
* here normalize output for reliable JSON parsing.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Strip ANSI color codes from string.
|
||||
* Rich library adds color codes even with --json flag.
|
||||
*/
|
||||
export function stripAnsiCodes(str: string): string {
|
||||
// ANSI escape code pattern: \x1b[...m or \x1b]...
|
||||
return str.replace(/\x1b\[[0-9;]*m/g, '')
|
||||
.replace(/\x1b\][0-9;]*\x07/g, '')
|
||||
.replace(/\x1b\][^\x07]*\x07/g, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Format file size to human readable string.
|
||||
*/
|
||||
export function formatSize(bytes: number): string {
|
||||
if (bytes === 0) return '0 B';
|
||||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
const k = 1024;
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
const size = parseFloat((bytes / Math.pow(k, i)).toFixed(i < 2 ? 0 : 1));
|
||||
return size + ' ' + units[i];
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract JSON from CLI output that may contain logging messages.
|
||||
* CodexLens CLI outputs logs like "INFO ..." before the JSON.
|
||||
* Also strips ANSI color codes that Rich library adds.
|
||||
* Handles trailing content after JSON (e.g., "INFO: Done" messages).
|
||||
*/
|
||||
export function extractJSON(output: string): any {
|
||||
// Strip ANSI color codes first
|
||||
const cleanOutput = stripAnsiCodes(output);
|
||||
|
||||
// Find the first { or [ character (start of JSON)
|
||||
const jsonStart = cleanOutput.search(/[{\[]/);
|
||||
if (jsonStart === -1) {
|
||||
throw new Error('No JSON found in output');
|
||||
}
|
||||
|
||||
const startChar = cleanOutput[jsonStart];
|
||||
const endChar = startChar === '{' ? '}' : ']';
|
||||
|
||||
// Find matching closing brace/bracket using a simple counter
|
||||
let depth = 0;
|
||||
let inString = false;
|
||||
let escapeNext = false;
|
||||
let jsonEnd = -1;
|
||||
|
||||
for (let i = jsonStart; i < cleanOutput.length; i++) {
|
||||
const char = cleanOutput[i];
|
||||
|
||||
if (escapeNext) {
|
||||
escapeNext = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (char === '\\' && inString) {
|
||||
escapeNext = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (char === '"') {
|
||||
inString = !inString;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!inString) {
|
||||
if (char === startChar) {
|
||||
depth++;
|
||||
} else if (char === endChar) {
|
||||
depth--;
|
||||
if (depth === 0) {
|
||||
jsonEnd = i + 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (jsonEnd === -1) {
|
||||
// Fallback: try to parse from start to end (original behavior)
|
||||
const jsonString = cleanOutput.substring(jsonStart);
|
||||
return JSON.parse(jsonString);
|
||||
}
|
||||
|
||||
const jsonString = cleanOutput.substring(jsonStart, jsonEnd);
|
||||
return JSON.parse(jsonString);
|
||||
}
|
||||
|
||||
@@ -1,322 +0,0 @@
|
||||
/**
|
||||
* CodexLens file watcher handlers.
|
||||
*
|
||||
* Maintains watcher process state across requests to support dashboard controls.
|
||||
*/
|
||||
|
||||
import {
|
||||
checkVenvStatus,
|
||||
executeCodexLens,
|
||||
getVenvPythonPath,
|
||||
useCodexLensV2,
|
||||
} from '../../../tools/codex-lens.js';
|
||||
import type { RouteContext } from '../types.js';
|
||||
import { extractJSON, stripAnsiCodes } from './utils.js';
|
||||
import type { ChildProcess } from 'child_process';
|
||||
|
||||
// File watcher state (persisted across requests)
|
||||
let watcherProcess: any = null;
|
||||
let watcherStats = {
|
||||
running: false,
|
||||
root_path: '',
|
||||
events_processed: 0,
|
||||
start_time: null as Date | null
|
||||
};
|
||||
|
||||
export async function stopWatcherForUninstall(): Promise<void> {
|
||||
if (!watcherStats.running || !watcherProcess) return;
|
||||
|
||||
try {
|
||||
watcherProcess.kill('SIGTERM');
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
if (watcherProcess && !watcherProcess.killed) {
|
||||
watcherProcess.kill('SIGKILL');
|
||||
}
|
||||
} catch {
|
||||
// Ignore errors stopping watcher
|
||||
}
|
||||
|
||||
watcherStats = {
|
||||
running: false,
|
||||
root_path: '',
|
||||
events_processed: 0,
|
||||
start_time: null
|
||||
};
|
||||
watcherProcess = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Spawn v2 bridge watcher subprocess.
|
||||
* Runs 'codexlens-search watch --root X --debounce-ms Y' and reads JSONL stdout.
|
||||
* @param root - Root directory to watch
|
||||
* @param debounceMs - Debounce interval in milliseconds
|
||||
* @returns Spawned child process
|
||||
*/
|
||||
function spawnV2Watcher(root: string, debounceMs: number): ChildProcess {
|
||||
const { spawn } = require('child_process') as typeof import('child_process');
|
||||
return spawn('codexlens-search', [
|
||||
'watch',
|
||||
'--root', root,
|
||||
'--debounce-ms', String(debounceMs),
|
||||
'--db-path', require('path').join(root, '.codexlens'),
|
||||
], {
|
||||
cwd: root,
|
||||
shell: false,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
windowsHide: true,
|
||||
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle CodexLens watcher routes
|
||||
* @returns true if route was handled, false otherwise
|
||||
*/
|
||||
export async function handleCodexLensWatcherRoutes(ctx: RouteContext): Promise<boolean> {
|
||||
const { pathname, req, res, initialPath, handlePostRequest, broadcastToClients } = ctx;
|
||||
|
||||
// API: Get File Watcher Status
|
||||
if (pathname === '/api/codexlens/watch/status') {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
success: true,
|
||||
running: watcherStats.running,
|
||||
root_path: watcherStats.root_path,
|
||||
events_processed: watcherStats.events_processed,
|
||||
start_time: watcherStats.start_time?.toISOString() || null,
|
||||
uptime_seconds: watcherStats.start_time
|
||||
? Math.floor((Date.now() - watcherStats.start_time.getTime()) / 1000)
|
||||
: 0
|
||||
}));
|
||||
return true;
|
||||
}
|
||||
|
||||
// API: Start File Watcher
|
||||
if (pathname === '/api/codexlens/watch/start' && req.method === 'POST') {
|
||||
handlePostRequest(req, res, async (body) => {
|
||||
const { path: watchPath, debounce_ms = 1000 } = body as { path?: unknown; debounce_ms?: unknown };
|
||||
const targetPath = typeof watchPath === 'string' && watchPath.trim().length > 0 ? watchPath : initialPath;
|
||||
const resolvedDebounceMs = typeof debounce_ms === 'number' ? debounce_ms : Number(debounce_ms);
|
||||
const debounceMs = !Number.isNaN(resolvedDebounceMs) && resolvedDebounceMs > 0 ? resolvedDebounceMs : 1000;
|
||||
|
||||
if (watcherStats.running) {
|
||||
return { success: false, error: 'Watcher already running', status: 400 };
|
||||
}
|
||||
|
||||
try {
|
||||
const { spawn } = await import('child_process');
|
||||
const { existsSync, statSync } = await import('fs');
|
||||
|
||||
// Validate path exists and is a directory
|
||||
if (!existsSync(targetPath)) {
|
||||
return { success: false, error: `Path does not exist: ${targetPath}`, status: 400 };
|
||||
}
|
||||
const pathStat = statSync(targetPath);
|
||||
if (!pathStat.isDirectory()) {
|
||||
return { success: false, error: `Path is not a directory: ${targetPath}`, status: 400 };
|
||||
}
|
||||
|
||||
// Route to v2 or v1 watcher based on feature flag
|
||||
if (useCodexLensV2()) {
|
||||
// v2 bridge watcher: codexlens-search watch
|
||||
console.log('[CodexLens] Using v2 bridge watcher');
|
||||
watcherProcess = spawnV2Watcher(targetPath, debounceMs);
|
||||
} else {
|
||||
// v1 watcher: python -m codexlens watch
|
||||
const venvStatus = await checkVenvStatus();
|
||||
if (!venvStatus.ready) {
|
||||
return { success: false, error: 'CodexLens not installed', status: 400 };
|
||||
}
|
||||
|
||||
// Verify directory is indexed before starting watcher
|
||||
try {
|
||||
const statusResult = await executeCodexLens(['projects', 'list', '--json']);
|
||||
if (statusResult.success && statusResult.output) {
|
||||
const parsed = extractJSON(statusResult.output);
|
||||
const projects = parsed.result || parsed || [];
|
||||
const normalizedTarget = targetPath.toLowerCase().replace(/\\/g, '/');
|
||||
const isIndexed = Array.isArray(projects) && projects.some((p: { source_root?: string }) =>
|
||||
p.source_root && p.source_root.toLowerCase().replace(/\\/g, '/') === normalizedTarget
|
||||
);
|
||||
if (!isIndexed) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Directory is not indexed: ${targetPath}. Run 'codexlens init' first.`,
|
||||
status: 400
|
||||
};
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('[CodexLens] Could not verify index status:', err);
|
||||
// Continue anyway - watcher will fail with proper error if not indexed
|
||||
}
|
||||
|
||||
// Spawn watch process using Python (no shell: true for security)
|
||||
const pythonPath = getVenvPythonPath();
|
||||
const args = ['-m', 'codexlens', 'watch', targetPath, '--debounce', String(debounceMs)];
|
||||
watcherProcess = spawn(pythonPath, args, {
|
||||
cwd: targetPath,
|
||||
shell: false,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
windowsHide: true,
|
||||
env: { ...process.env, PYTHONIOENCODING: 'utf-8' }
|
||||
});
|
||||
}
|
||||
|
||||
watcherStats = {
|
||||
running: true,
|
||||
root_path: targetPath,
|
||||
events_processed: 0,
|
||||
start_time: new Date()
|
||||
};
|
||||
|
||||
// Capture stderr for error messages (capped at 4KB to prevent memory leak)
|
||||
const MAX_STDERR_SIZE = 4096;
|
||||
let stderrBuffer = '';
|
||||
if (watcherProcess.stderr) {
|
||||
watcherProcess.stderr.on('data', (data: Buffer) => {
|
||||
stderrBuffer += data.toString();
|
||||
// Cap buffer size to prevent memory leak in long-running watchers
|
||||
if (stderrBuffer.length > MAX_STDERR_SIZE) {
|
||||
stderrBuffer = stderrBuffer.slice(-MAX_STDERR_SIZE);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Handle process output for event counting
|
||||
const isV2Watcher = useCodexLensV2();
|
||||
let stdoutLineBuffer = '';
|
||||
if (watcherProcess.stdout) {
|
||||
watcherProcess.stdout.on('data', (data: Buffer) => {
|
||||
const output = data.toString();
|
||||
|
||||
if (isV2Watcher) {
|
||||
// v2 bridge outputs JSONL - parse line by line
|
||||
stdoutLineBuffer += output;
|
||||
const lines = stdoutLineBuffer.split('\n');
|
||||
// Keep incomplete last line in buffer
|
||||
stdoutLineBuffer = lines.pop() || '';
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
try {
|
||||
const event = JSON.parse(trimmed);
|
||||
// Count file change events (created, modified, deleted, moved)
|
||||
if (event.event && event.event !== 'watching') {
|
||||
watcherStats.events_processed += 1;
|
||||
}
|
||||
} catch {
|
||||
// Not valid JSON, skip
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// v1 watcher: count text-based event messages
|
||||
const matches = output.match(/Processed \d+ events?/g);
|
||||
if (matches) {
|
||||
watcherStats.events_processed += matches.length;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Handle spawn errors (e.g., ENOENT)
|
||||
watcherProcess.on('error', (err: Error) => {
|
||||
console.error(`[CodexLens] Watcher spawn error: ${err.message}`);
|
||||
watcherStats.running = false;
|
||||
watcherProcess = null;
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_WATCHER_STATUS',
|
||||
payload: { running: false, error: `Spawn error: ${err.message}` }
|
||||
});
|
||||
});
|
||||
|
||||
// Handle process exit
|
||||
watcherProcess.on('exit', (code: number) => {
|
||||
watcherStats.running = false;
|
||||
watcherProcess = null;
|
||||
console.log(`[CodexLens] Watcher exited with code ${code}`);
|
||||
|
||||
// Broadcast error if exited with non-zero code
|
||||
if (code !== 0) {
|
||||
const errorMsg = stderrBuffer.trim() || `Exited with code ${code}`;
|
||||
const cleanError = stripAnsiCodes(errorMsg);
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_WATCHER_STATUS',
|
||||
payload: { running: false, error: cleanError }
|
||||
});
|
||||
} else {
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_WATCHER_STATUS',
|
||||
payload: { running: false }
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Broadcast watcher started
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_WATCHER_STATUS',
|
||||
payload: { running: true, path: targetPath }
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: 'Watcher started',
|
||||
path: targetPath,
|
||||
pid: watcherProcess.pid
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
return { success: false, error: err instanceof Error ? err.message : String(err), status: 500 };
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
// API: Stop File Watcher
|
||||
if (pathname === '/api/codexlens/watch/stop' && req.method === 'POST') {
|
||||
handlePostRequest(req, res, async () => {
|
||||
if (!watcherStats.running || !watcherProcess) {
|
||||
return { success: false, error: 'Watcher not running', status: 400 };
|
||||
}
|
||||
|
||||
try {
|
||||
watcherProcess.kill('SIGTERM');
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
if (watcherProcess && !watcherProcess.killed) {
|
||||
watcherProcess.kill('SIGKILL');
|
||||
}
|
||||
|
||||
const finalStats = {
|
||||
events_processed: watcherStats.events_processed,
|
||||
uptime_seconds: watcherStats.start_time
|
||||
? Math.floor((Date.now() - watcherStats.start_time.getTime()) / 1000)
|
||||
: 0
|
||||
};
|
||||
|
||||
watcherStats = {
|
||||
running: false,
|
||||
root_path: '',
|
||||
events_processed: 0,
|
||||
start_time: null
|
||||
};
|
||||
watcherProcess = null;
|
||||
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_WATCHER_STATUS',
|
||||
payload: { running: false }
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: 'Watcher stopped',
|
||||
...finalStats
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
return { success: false, error: err instanceof Error ? err.message : String(err), status: 500 };
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -3,17 +3,6 @@
|
||||
* Handles LiteLLM provider management, endpoint configuration, and cache management
|
||||
*/
|
||||
import { z } from 'zod';
|
||||
import { spawn } from 'child_process';
|
||||
import {
|
||||
getSystemPythonCommand,
|
||||
parsePythonCommandSpec,
|
||||
type PythonCommandSpec,
|
||||
} from '../../utils/python-utils.js';
|
||||
import {
|
||||
isUvAvailable,
|
||||
createCodexLensUvManager
|
||||
} from '../../utils/uv-manager.js';
|
||||
import { ensureLiteLLMEmbedderReady } from '../../tools/codex-lens.js';
|
||||
import type { RouteContext } from './types.js';
|
||||
|
||||
// ========== Input Validation Schemas ==========
|
||||
@@ -81,106 +70,13 @@ import {
|
||||
type EmbeddingPoolConfig,
|
||||
} from '../../config/litellm-api-config-manager.js';
|
||||
import { getContextCacheStore } from '../../tools/context-cache-store.js';
|
||||
import { getLiteLLMClient } from '../../tools/litellm-client.js';
|
||||
import { testApiKeyConnection, getDefaultApiBase } from '../services/api-key-tester.js';
|
||||
|
||||
interface CcwLitellmEnvCheck {
|
||||
python: string;
|
||||
installed: boolean;
|
||||
version?: string;
|
||||
error?: string;
|
||||
}
|
||||
const V1_REMOVED = 'Python bridge has been removed (v1 cleanup).';
|
||||
|
||||
interface CcwLitellmStatusResponse {
|
||||
/**
|
||||
* Whether ccw-litellm is installed in the CodexLens venv.
|
||||
* This is the environment used for the LiteLLM embedding backend.
|
||||
*/
|
||||
installed: boolean;
|
||||
version?: string;
|
||||
error?: string;
|
||||
checks?: {
|
||||
codexLensVenv: CcwLitellmEnvCheck;
|
||||
systemPython?: CcwLitellmEnvCheck;
|
||||
};
|
||||
}
|
||||
|
||||
function checkCcwLitellmImport(
|
||||
pythonCmd: string | PythonCommandSpec,
|
||||
options: { timeout: number }
|
||||
): Promise<CcwLitellmEnvCheck> {
|
||||
const { timeout } = options;
|
||||
const pythonSpec = typeof pythonCmd === 'string' ? parsePythonCommandSpec(pythonCmd) : pythonCmd;
|
||||
|
||||
const sanitizePythonError = (stderrText: string): string | undefined => {
|
||||
const trimmed = stderrText.trim();
|
||||
if (!trimmed) return undefined;
|
||||
const lines = trimmed
|
||||
.split(/\r?\n/g)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean);
|
||||
// Prefer the final exception line (avoids leaking full traceback + file paths)
|
||||
return lines[lines.length - 1] || undefined;
|
||||
};
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(pythonSpec.command, [...pythonSpec.args, '-c', 'import ccw_litellm; print(ccw_litellm.__version__)'], {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
timeout,
|
||||
windowsHide: true,
|
||||
shell: false,
|
||||
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
|
||||
});
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
|
||||
child.stdout?.on('data', (data: Buffer) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
|
||||
child.stderr?.on('data', (data: Buffer) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
child.on('close', (code: number | null) => {
|
||||
const version = stdout.trim();
|
||||
const error = sanitizePythonError(stderr);
|
||||
|
||||
if (code === 0 && version) {
|
||||
resolve({ python: pythonSpec.display, installed: true, version });
|
||||
return;
|
||||
}
|
||||
|
||||
if (code === null) {
|
||||
resolve({ python: pythonSpec.display, installed: false, error: `Timed out after ${timeout}ms` });
|
||||
return;
|
||||
}
|
||||
|
||||
resolve({ python: pythonSpec.display, installed: false, error: error || undefined });
|
||||
});
|
||||
|
||||
child.on('error', (err) => {
|
||||
resolve({ python: pythonSpec.display, installed: false, error: err.message });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Cache for ccw-litellm status check
|
||||
let ccwLitellmStatusCache: {
|
||||
data: CcwLitellmStatusResponse | null;
|
||||
timestamp: number;
|
||||
ttl: number;
|
||||
} = {
|
||||
data: null,
|
||||
timestamp: 0,
|
||||
ttl: 5 * 60 * 1000, // 5 minutes
|
||||
};
|
||||
|
||||
// Clear cache (call after install)
|
||||
// Clear cache (no-op stub, kept for backward compatibility)
|
||||
export function clearCcwLitellmStatusCache() {
|
||||
ccwLitellmStatusCache.data = null;
|
||||
ccwLitellmStatusCache.timestamp = 0;
|
||||
// no-op: Python bridge removed
|
||||
}
|
||||
|
||||
function sanitizeProviderForResponse(provider: any): any {
|
||||
@@ -922,57 +818,10 @@ export async function handleLiteLLMApiRoutes(ctx: RouteContext): Promise<boolean
|
||||
// CCW-LiteLLM Package Management
|
||||
// ===========================
|
||||
|
||||
// GET /api/litellm-api/ccw-litellm/status - Check ccw-litellm installation status
|
||||
// Supports ?refresh=true to bypass cache
|
||||
// GET /api/litellm-api/ccw-litellm/status - Stub (v1 Python bridge removed)
|
||||
if (pathname === '/api/litellm-api/ccw-litellm/status' && req.method === 'GET') {
|
||||
const forceRefresh = url.searchParams.get('refresh') === 'true';
|
||||
|
||||
// Check cache first (unless force refresh)
|
||||
if (!forceRefresh && ccwLitellmStatusCache.data &&
|
||||
Date.now() - ccwLitellmStatusCache.timestamp < ccwLitellmStatusCache.ttl) {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(ccwLitellmStatusCache.data));
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
const uv = createCodexLensUvManager();
|
||||
const venvPython = uv.getVenvPython();
|
||||
const statusTimeout = process.platform === 'win32' ? 15000 : 10000;
|
||||
const codexLensVenv = uv.isVenvValid()
|
||||
? await checkCcwLitellmImport(venvPython, { timeout: statusTimeout })
|
||||
: { python: venvPython, installed: false, error: 'CodexLens venv not valid' };
|
||||
|
||||
// Diagnostics only: if not installed in venv, also check system python so users understand mismatches.
|
||||
// NOTE: `installed` flag remains the CodexLens venv status (we want isolated venv dependencies).
|
||||
const systemPython = !codexLensVenv.installed
|
||||
? await checkCcwLitellmImport(getSystemPythonCommand(), { timeout: statusTimeout })
|
||||
: undefined;
|
||||
|
||||
const result: CcwLitellmStatusResponse = {
|
||||
installed: codexLensVenv.installed,
|
||||
version: codexLensVenv.version,
|
||||
error: codexLensVenv.error,
|
||||
checks: {
|
||||
codexLensVenv,
|
||||
...(systemPython ? { systemPython } : {}),
|
||||
},
|
||||
};
|
||||
|
||||
// Update cache
|
||||
ccwLitellmStatusCache = {
|
||||
data: result,
|
||||
timestamp: Date.now(),
|
||||
ttl: 5 * 60 * 1000,
|
||||
};
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(result));
|
||||
} catch (err) {
|
||||
const errorResult = { installed: false, error: (err as Error).message };
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(errorResult));
|
||||
}
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ installed: false, error: V1_REMOVED }));
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -1367,96 +1216,18 @@ export async function handleLiteLLMApiRoutes(ctx: RouteContext): Promise<boolean
|
||||
return true;
|
||||
}
|
||||
|
||||
// POST /api/litellm-api/ccw-litellm/install - Install ccw-litellm package
|
||||
// POST /api/litellm-api/ccw-litellm/install - Stub (v1 Python bridge removed)
|
||||
if (pathname === '/api/litellm-api/ccw-litellm/install' && req.method === 'POST') {
|
||||
handlePostRequest(req, res, async () => {
|
||||
try {
|
||||
// Delegate entirely to ensureLiteLLMEmbedderReady for consistent installation
|
||||
// This uses unified package discovery and handles UV → pip fallback
|
||||
const result = await ensureLiteLLMEmbedderReady();
|
||||
|
||||
if (result.success) {
|
||||
clearCcwLitellmStatusCache();
|
||||
broadcastToClients({
|
||||
type: 'CCW_LITELLM_INSTALLED',
|
||||
payload: { timestamp: new Date().toISOString(), method: 'unified' }
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (err) {
|
||||
return { success: false, error: (err as Error).message };
|
||||
}
|
||||
return { success: false, error: V1_REMOVED };
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
// POST /api/litellm-api/ccw-litellm/uninstall - Uninstall ccw-litellm package
|
||||
// POST /api/litellm-api/ccw-litellm/uninstall - Stub (v1 Python bridge removed)
|
||||
if (pathname === '/api/litellm-api/ccw-litellm/uninstall' && req.method === 'POST') {
|
||||
handlePostRequest(req, res, async () => {
|
||||
try {
|
||||
// Priority 1: Use UV to uninstall from CodexLens venv
|
||||
if (await isUvAvailable()) {
|
||||
const uv = createCodexLensUvManager();
|
||||
if (uv.isVenvValid()) {
|
||||
console.log('[ccw-litellm uninstall] Using UV to uninstall from CodexLens venv...');
|
||||
const uvResult = await uv.uninstall(['ccw-litellm']);
|
||||
clearCcwLitellmStatusCache();
|
||||
|
||||
if (uvResult.success) {
|
||||
broadcastToClients({
|
||||
type: 'CCW_LITELLM_UNINSTALLED',
|
||||
payload: { timestamp: new Date().toISOString() }
|
||||
});
|
||||
return { success: true, message: 'ccw-litellm uninstalled successfully via UV' };
|
||||
}
|
||||
console.log('[ccw-litellm uninstall] UV uninstall failed, falling back to pip:', uvResult.error);
|
||||
}
|
||||
}
|
||||
|
||||
// Priority 2: Fallback to system pip uninstall
|
||||
console.log('[ccw-litellm uninstall] Using pip fallback...');
|
||||
const pythonCmd = getSystemPythonCommand();
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const proc = spawn(
|
||||
pythonCmd.command,
|
||||
[...pythonCmd.args, '-m', 'pip', 'uninstall', '-y', 'ccw-litellm'],
|
||||
{
|
||||
shell: false,
|
||||
timeout: 120000,
|
||||
windowsHide: true,
|
||||
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
|
||||
},
|
||||
);
|
||||
let output = '';
|
||||
let error = '';
|
||||
proc.stdout?.on('data', (data) => { output += data.toString(); });
|
||||
proc.stderr?.on('data', (data) => { error += data.toString(); });
|
||||
proc.on('close', (code) => {
|
||||
// Clear status cache after uninstallation attempt
|
||||
clearCcwLitellmStatusCache();
|
||||
|
||||
if (code === 0) {
|
||||
broadcastToClients({
|
||||
type: 'CCW_LITELLM_UNINSTALLED',
|
||||
payload: { timestamp: new Date().toISOString() }
|
||||
});
|
||||
resolve({ success: true, message: 'ccw-litellm uninstalled successfully' });
|
||||
} else {
|
||||
// Check if package was not installed
|
||||
if (error.includes('not installed') || output.includes('not installed')) {
|
||||
resolve({ success: true, message: 'ccw-litellm was not installed' });
|
||||
} else {
|
||||
resolve({ success: false, error: error || output || 'Uninstallation failed' });
|
||||
}
|
||||
}
|
||||
});
|
||||
proc.on('error', (err) => resolve({ success: false, error: err.message }));
|
||||
});
|
||||
} catch (err) {
|
||||
return { success: false, error: (err as Error).message };
|
||||
}
|
||||
return { success: false, error: V1_REMOVED };
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import { existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { homedir } from 'os';
|
||||
import { getCliToolsStatus } from '../../tools/cli-executor.js';
|
||||
import { checkVenvStatus, checkSemanticStatus } from '../../tools/codex-lens.js';
|
||||
import type { RouteContext } from './types.js';
|
||||
|
||||
// Performance logging helper
|
||||
@@ -80,36 +79,14 @@ export async function handleStatusRoutes(ctx: RouteContext): Promise<boolean> {
|
||||
const ccwInstallStatus = checkCcwInstallStatus();
|
||||
perfLog('checkCcwInstallStatus', ccwStart);
|
||||
|
||||
// Execute all status checks in parallel with individual timing
|
||||
// Execute async status checks
|
||||
const cliStart = Date.now();
|
||||
const codexStart = Date.now();
|
||||
const semanticStart = Date.now();
|
||||
|
||||
const [cliStatus, codexLensStatus, semanticStatus] = await Promise.all([
|
||||
getCliToolsStatus().then(result => {
|
||||
perfLog('getCliToolsStatus', cliStart, { toolCount: Object.keys(result).length });
|
||||
return result;
|
||||
}),
|
||||
checkVenvStatus().then(result => {
|
||||
perfLog('checkVenvStatus', codexStart, { ready: result.ready });
|
||||
return result;
|
||||
}),
|
||||
// Always check semantic status (will return available: false if CodexLens not ready)
|
||||
checkSemanticStatus()
|
||||
.then(result => {
|
||||
perfLog('checkSemanticStatus', semanticStart, { available: result.available });
|
||||
return result;
|
||||
})
|
||||
.catch(() => {
|
||||
perfLog('checkSemanticStatus (error)', semanticStart);
|
||||
return { available: false, backend: null };
|
||||
})
|
||||
]);
|
||||
const cliStatus = await getCliToolsStatus();
|
||||
perfLog('getCliToolsStatus', cliStart, { toolCount: Object.keys(cliStatus).length });
|
||||
|
||||
const response = {
|
||||
cli: cliStatus,
|
||||
codexLens: codexLensStatus,
|
||||
semantic: semanticStatus,
|
||||
ccwInstall: ccwInstallStatus,
|
||||
timestamp: new Date().toISOString()
|
||||
};
|
||||
|
||||
@@ -16,7 +16,6 @@ import { handleUnifiedMemoryRoutes } from './routes/unified-memory-routes.js';
|
||||
import { handleMcpRoutes } from './routes/mcp-routes.js';
|
||||
import { handleHooksRoutes } from './routes/hooks-routes.js';
|
||||
import { handleUnsplashRoutes, handleBackgroundRoutes } from './routes/unsplash-routes.js';
|
||||
import { handleCodexLensRoutes } from './routes/codexlens-routes.js';
|
||||
import { handleGraphRoutes } from './routes/graph-routes.js';
|
||||
import { handleSystemRoutes } from './routes/system-routes.js';
|
||||
import { handleFilesRoutes } from './routes/files-routes.js';
|
||||
@@ -66,7 +65,6 @@ import { getCliSessionManager } from './services/cli-session-manager.js';
|
||||
import { QueueSchedulerService } from './services/queue-scheduler-service.js';
|
||||
|
||||
// Import status check functions for warmup
|
||||
import { checkSemanticStatus, checkVenvStatus } from '../tools/codex-lens.js';
|
||||
import { getCliToolsStatus } from '../tools/cli-executor.js';
|
||||
|
||||
import type { ServerConfig } from '../types/config.js';
|
||||
@@ -302,28 +300,6 @@ async function warmupCaches(initialPath: string): Promise<void> {
|
||||
|
||||
// Run all warmup tasks in parallel for faster startup
|
||||
const warmupTasks = [
|
||||
// Warmup semantic status cache (Python process startup - can be slow first time)
|
||||
(async () => {
|
||||
const taskStart = Date.now();
|
||||
try {
|
||||
const semanticStatus = await checkSemanticStatus();
|
||||
console.log(`[WARMUP] Semantic status: ${semanticStatus.available ? 'available' : 'not available'} (${Date.now() - taskStart}ms)`);
|
||||
} catch (err) {
|
||||
console.warn(`[WARMUP] Semantic status check failed: ${(err as Error).message}`);
|
||||
}
|
||||
})(),
|
||||
|
||||
// Warmup venv status cache
|
||||
(async () => {
|
||||
const taskStart = Date.now();
|
||||
try {
|
||||
const venvStatus = await checkVenvStatus();
|
||||
console.log(`[WARMUP] Venv status: ${venvStatus.ready ? 'ready' : 'not ready'} (${Date.now() - taskStart}ms)`);
|
||||
} catch (err) {
|
||||
console.warn(`[WARMUP] Venv status check failed: ${(err as Error).message}`);
|
||||
}
|
||||
})(),
|
||||
|
||||
// Warmup CLI tools status cache
|
||||
(async () => {
|
||||
const taskStart = Date.now();
|
||||
@@ -598,11 +574,6 @@ export async function startServer(options: ServerOptions = {}): Promise<http.Ser
|
||||
if (await handleUnsplashRoutes(routeContext)) return;
|
||||
}
|
||||
|
||||
// CodexLens routes (/api/codexlens/*)
|
||||
if (pathname.startsWith('/api/codexlens/')) {
|
||||
if (await handleCodexLensRoutes(routeContext)) return;
|
||||
}
|
||||
|
||||
// LiteLLM routes (/api/litellm/*)
|
||||
if (pathname.startsWith('/api/litellm/')) {
|
||||
if (await handleLiteLLMRoutes(routeContext)) return;
|
||||
|
||||
@@ -1,79 +1,37 @@
|
||||
/**
|
||||
* Unified Vector Index - TypeScript bridge to unified_memory_embedder.py
|
||||
* Unified Vector Index - STUB (v1 Python bridge removed)
|
||||
*
|
||||
* Provides HNSW-backed vector indexing and search for all memory content
|
||||
* (core_memory, cli_history, workflow, entity, pattern) via CodexLens VectorStore.
|
||||
*
|
||||
* Features:
|
||||
* - JSON stdin/stdout protocol to Python embedder
|
||||
* - Content chunking (paragraph -> sentence splitting, CHUNK_SIZE=1500, OVERLAP=200)
|
||||
* - Batch embedding via CodexLens EmbedderFactory
|
||||
* - HNSW approximate nearest neighbor search (sub-10ms for 1000 chunks)
|
||||
* - Category-based filtering
|
||||
* The Python unified_memory_embedder.py bridge has been removed. This module
|
||||
* provides no-op stubs so that existing consumers compile without errors.
|
||||
*/
|
||||
|
||||
import { spawn } from 'child_process';
|
||||
import { join, dirname } from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { getCodexLensHiddenPython } from '../utils/codexlens-path.js';
|
||||
import { StoragePaths, ensureStorageDir } from '../config/storage-paths.js';
|
||||
const V1_REMOVED = 'Unified vector index Python bridge has been removed (v1 cleanup).';
|
||||
|
||||
// Get directory of this module
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types (kept for backward compatibility)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Venv python path (reuse CodexLens venv)
|
||||
const VENV_PYTHON = getCodexLensHiddenPython();
|
||||
|
||||
// Script path
|
||||
const EMBEDDER_SCRIPT = join(__dirname, '..', '..', 'scripts', 'unified_memory_embedder.py');
|
||||
|
||||
// Chunking constants (match existing core-memory-store.ts)
|
||||
const CHUNK_SIZE = 1500;
|
||||
const OVERLAP = 200;
|
||||
|
||||
// =============================================================================
|
||||
// Types
|
||||
// =============================================================================
|
||||
|
||||
/** Valid source types for vector content */
|
||||
export type SourceType = 'core_memory' | 'workflow' | 'cli_history';
|
||||
|
||||
/** Valid category values for vector filtering */
|
||||
export type VectorCategory = 'core_memory' | 'cli_history' | 'workflow' | 'entity' | 'pattern';
|
||||
|
||||
/** Metadata attached to each chunk in the vector store */
|
||||
export interface ChunkMetadata {
|
||||
/** Source identifier (e.g., memory ID, session ID) */
|
||||
source_id: string;
|
||||
/** Source type */
|
||||
source_type: SourceType;
|
||||
/** Category for filtering */
|
||||
category: VectorCategory;
|
||||
/** Chunk index within the source */
|
||||
chunk_index?: number;
|
||||
/** Additional metadata */
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/** A chunk to be embedded and indexed */
|
||||
export interface VectorChunk {
|
||||
/** Text content */
|
||||
content: string;
|
||||
/** Source identifier */
|
||||
source_id: string;
|
||||
/** Source type */
|
||||
source_type: SourceType;
|
||||
/** Category for filtering */
|
||||
category: VectorCategory;
|
||||
/** Chunk index */
|
||||
chunk_index: number;
|
||||
/** Additional metadata */
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** Result of an embed operation */
|
||||
export interface EmbedResult {
|
||||
success: boolean;
|
||||
chunks_processed: number;
|
||||
@@ -82,7 +40,6 @@ export interface EmbedResult {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/** A single search match */
|
||||
export interface VectorSearchMatch {
|
||||
content: string;
|
||||
score: number;
|
||||
@@ -93,7 +50,6 @@ export interface VectorSearchMatch {
|
||||
metadata: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** Result of a search operation */
|
||||
export interface VectorSearchResult {
|
||||
success: boolean;
|
||||
matches: VectorSearchMatch[];
|
||||
@@ -102,14 +58,12 @@ export interface VectorSearchResult {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/** Search options */
|
||||
export interface VectorSearchOptions {
|
||||
topK?: number;
|
||||
minScore?: number;
|
||||
category?: VectorCategory;
|
||||
}
|
||||
|
||||
/** Index status information */
|
||||
export interface VectorIndexStatus {
|
||||
success: boolean;
|
||||
total_chunks: number;
|
||||
@@ -126,7 +80,6 @@ export interface VectorIndexStatus {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/** Reindex result */
|
||||
export interface ReindexResult {
|
||||
success: boolean;
|
||||
hnsw_count?: number;
|
||||
@@ -134,344 +87,73 @@ export interface ReindexResult {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Python Bridge
|
||||
// =============================================================================
|
||||
// ---------------------------------------------------------------------------
|
||||
// No-op implementations
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Check if the unified embedder is available (venv and script exist)
|
||||
*/
|
||||
export function isUnifiedEmbedderAvailable(): boolean {
|
||||
if (!existsSync(VENV_PYTHON)) {
|
||||
return false;
|
||||
}
|
||||
if (!existsSync(EMBEDDER_SCRIPT)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run Python script with JSON stdin/stdout protocol.
|
||||
*
|
||||
* @param request - JSON request object to send via stdin
|
||||
* @param timeout - Timeout in milliseconds (default: 5 minutes)
|
||||
* @returns Parsed JSON response
|
||||
*/
|
||||
function runPython<T>(request: Record<string, unknown>, timeout: number = 300000): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!isUnifiedEmbedderAvailable()) {
|
||||
reject(
|
||||
new Error(
|
||||
'Unified embedder not available. Ensure CodexLens venv exists at ~/.codexlens/venv'
|
||||
)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const child = spawn(VENV_PYTHON, [EMBEDDER_SCRIPT], {
|
||||
shell: false,
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
timeout,
|
||||
windowsHide: true,
|
||||
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
|
||||
});
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
|
||||
child.stdout.on('data', (data) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
|
||||
child.stderr.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
child.on('close', (code) => {
|
||||
if (code === 0 && stdout.trim()) {
|
||||
try {
|
||||
resolve(JSON.parse(stdout.trim()) as T);
|
||||
} catch {
|
||||
reject(new Error(`Failed to parse Python output: ${stdout.substring(0, 500)}`));
|
||||
}
|
||||
} else {
|
||||
reject(new Error(`Python script failed (exit code ${code}): ${stderr || stdout}`));
|
||||
}
|
||||
});
|
||||
|
||||
child.on('error', (err) => {
|
||||
if ((err as NodeJS.ErrnoException).code === 'ETIMEDOUT') {
|
||||
reject(new Error('Python script timed out'));
|
||||
} else {
|
||||
reject(new Error(`Failed to spawn Python: ${err.message}`));
|
||||
}
|
||||
});
|
||||
|
||||
// Write JSON request to stdin and close
|
||||
const jsonInput = JSON.stringify(request);
|
||||
child.stdin.write(jsonInput);
|
||||
child.stdin.end();
|
||||
});
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Content Chunking
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Chunk content into smaller pieces for embedding.
|
||||
* Uses paragraph-first, sentence-fallback strategy with overlap.
|
||||
*
|
||||
* Matches the chunking logic in core-memory-store.ts:
|
||||
* - CHUNK_SIZE = 1500 characters
|
||||
* - OVERLAP = 200 characters
|
||||
* - Split by paragraph boundaries (\n\n) first
|
||||
* - Fall back to sentence boundaries (. ) for oversized paragraphs
|
||||
*
|
||||
* @param content - Text content to chunk
|
||||
* @returns Array of chunk strings
|
||||
*/
|
||||
export function chunkContent(content: string): string[] {
|
||||
const chunks: string[] = [];
|
||||
|
||||
// Split by paragraph boundaries first
|
||||
const paragraphs = content.split(/\n\n+/);
|
||||
let currentChunk = '';
|
||||
|
||||
for (const paragraph of paragraphs) {
|
||||
// If adding this paragraph would exceed chunk size
|
||||
if (currentChunk.length + paragraph.length > CHUNK_SIZE && currentChunk.length > 0) {
|
||||
chunks.push(currentChunk.trim());
|
||||
|
||||
// Start new chunk with overlap
|
||||
const overlapText = currentChunk.slice(-OVERLAP);
|
||||
currentChunk = overlapText + '\n\n' + paragraph;
|
||||
} else {
|
||||
currentChunk += (currentChunk ? '\n\n' : '') + paragraph;
|
||||
}
|
||||
}
|
||||
|
||||
// Add remaining chunk
|
||||
if (currentChunk.trim()) {
|
||||
chunks.push(currentChunk.trim());
|
||||
}
|
||||
|
||||
// If chunks are still too large, split by sentences
|
||||
const finalChunks: string[] = [];
|
||||
for (const chunk of chunks) {
|
||||
if (chunk.length <= CHUNK_SIZE) {
|
||||
finalChunks.push(chunk);
|
||||
} else {
|
||||
// Split by sentence boundaries
|
||||
const sentences = chunk.split(/\. +/);
|
||||
let sentenceChunk = '';
|
||||
|
||||
for (const sentence of sentences) {
|
||||
const sentenceWithPeriod = sentence + '. ';
|
||||
if (
|
||||
sentenceChunk.length + sentenceWithPeriod.length > CHUNK_SIZE &&
|
||||
sentenceChunk.length > 0
|
||||
) {
|
||||
finalChunks.push(sentenceChunk.trim());
|
||||
const overlapText = sentenceChunk.slice(-OVERLAP);
|
||||
sentenceChunk = overlapText + sentenceWithPeriod;
|
||||
} else {
|
||||
sentenceChunk += sentenceWithPeriod;
|
||||
}
|
||||
}
|
||||
|
||||
if (sentenceChunk.trim()) {
|
||||
finalChunks.push(sentenceChunk.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return finalChunks.length > 0 ? finalChunks : [content];
|
||||
// Minimal chunking for backward compat - just return the content as-is
|
||||
if (!content.trim()) return [];
|
||||
return [content];
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// UnifiedVectorIndex Class
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Unified vector index backed by CodexLens VectorStore (HNSW).
|
||||
*
|
||||
* Provides content chunking, embedding, storage, and search for all
|
||||
* memory content types through a single interface.
|
||||
*/
|
||||
export class UnifiedVectorIndex {
|
||||
private storePath: string;
|
||||
constructor(_projectPath: string) {}
|
||||
|
||||
/**
|
||||
* Create a UnifiedVectorIndex for a project.
|
||||
*
|
||||
* @param projectPath - Project root path (used to resolve storage location)
|
||||
*/
|
||||
constructor(projectPath: string) {
|
||||
const paths = StoragePaths.project(projectPath);
|
||||
this.storePath = paths.unifiedVectors.root;
|
||||
ensureStorageDir(this.storePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Index content by chunking, embedding, and storing in VectorStore.
|
||||
*
|
||||
* @param content - Text content to index
|
||||
* @param metadata - Metadata for all chunks (source_id, source_type, category)
|
||||
* @returns Embed result
|
||||
*/
|
||||
async indexContent(
|
||||
content: string,
|
||||
metadata: ChunkMetadata
|
||||
_content: string,
|
||||
_metadata: ChunkMetadata
|
||||
): Promise<EmbedResult> {
|
||||
if (!content.trim()) {
|
||||
return {
|
||||
success: true,
|
||||
chunks_processed: 0,
|
||||
chunks_failed: 0,
|
||||
elapsed_time: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Chunk content
|
||||
const textChunks = chunkContent(content);
|
||||
|
||||
// Build chunk objects for Python
|
||||
const chunks: VectorChunk[] = textChunks.map((text, index) => ({
|
||||
content: text,
|
||||
source_id: metadata.source_id,
|
||||
source_type: metadata.source_type,
|
||||
category: metadata.category,
|
||||
chunk_index: metadata.chunk_index != null ? metadata.chunk_index + index : index,
|
||||
metadata: { ...metadata },
|
||||
}));
|
||||
|
||||
try {
|
||||
const result = await runPython<EmbedResult>({
|
||||
operation: 'embed',
|
||||
store_path: this.storePath,
|
||||
chunks,
|
||||
batch_size: 8,
|
||||
});
|
||||
return result;
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
chunks_processed: 0,
|
||||
chunks_failed: textChunks.length,
|
||||
elapsed_time: 0,
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
return {
|
||||
success: false,
|
||||
chunks_processed: 0,
|
||||
chunks_failed: 0,
|
||||
elapsed_time: 0,
|
||||
error: V1_REMOVED,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Search the vector index using semantic similarity.
|
||||
*
|
||||
* @param query - Natural language search query
|
||||
* @param options - Search options (topK, minScore, category)
|
||||
* @returns Search results sorted by relevance
|
||||
*/
|
||||
async search(
|
||||
query: string,
|
||||
options: VectorSearchOptions = {}
|
||||
_query: string,
|
||||
_options: VectorSearchOptions = {}
|
||||
): Promise<VectorSearchResult> {
|
||||
const { topK = 10, minScore = 0.3, category } = options;
|
||||
|
||||
try {
|
||||
const result = await runPython<VectorSearchResult>({
|
||||
operation: 'search',
|
||||
store_path: this.storePath,
|
||||
query,
|
||||
top_k: topK,
|
||||
min_score: minScore,
|
||||
category: category || null,
|
||||
});
|
||||
return result;
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
matches: [],
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
return {
|
||||
success: false,
|
||||
matches: [],
|
||||
error: V1_REMOVED,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Search the vector index using a pre-computed embedding vector.
|
||||
* Bypasses text embedding, directly querying HNSW with a raw vector.
|
||||
*
|
||||
* @param vector - Pre-computed embedding vector (array of floats)
|
||||
* @param options - Search options (topK, minScore, category)
|
||||
* @returns Search results sorted by relevance
|
||||
*/
|
||||
async searchByVector(
|
||||
vector: number[],
|
||||
options: VectorSearchOptions = {}
|
||||
_vector: number[],
|
||||
_options: VectorSearchOptions = {}
|
||||
): Promise<VectorSearchResult> {
|
||||
const { topK = 10, minScore = 0.3, category } = options;
|
||||
|
||||
try {
|
||||
const result = await runPython<VectorSearchResult>({
|
||||
operation: 'search_by_vector',
|
||||
store_path: this.storePath,
|
||||
vector,
|
||||
top_k: topK,
|
||||
min_score: minScore,
|
||||
category: category || null,
|
||||
});
|
||||
return result;
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
matches: [],
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
return {
|
||||
success: false,
|
||||
matches: [],
|
||||
error: V1_REMOVED,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild the HNSW index from scratch.
|
||||
*
|
||||
* @returns Reindex result
|
||||
*/
|
||||
async reindexAll(): Promise<ReindexResult> {
|
||||
try {
|
||||
const result = await runPython<ReindexResult>({
|
||||
operation: 'reindex',
|
||||
store_path: this.storePath,
|
||||
});
|
||||
return result;
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
return {
|
||||
success: false,
|
||||
error: V1_REMOVED,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current status of the vector index.
|
||||
*
|
||||
* @returns Index status including chunk counts, HNSW availability, dimension
|
||||
*/
|
||||
async getStatus(): Promise<VectorIndexStatus> {
|
||||
try {
|
||||
const result = await runPython<VectorIndexStatus>({
|
||||
operation: 'status',
|
||||
store_path: this.storePath,
|
||||
});
|
||||
return result;
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
total_chunks: 0,
|
||||
hnsw_available: false,
|
||||
hnsw_count: 0,
|
||||
dimension: 0,
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
return {
|
||||
success: false,
|
||||
total_chunks: 0,
|
||||
hnsw_available: false,
|
||||
hnsw_count: 0,
|
||||
dimension: 0,
|
||||
error: V1_REMOVED,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user