/** * CodexLens Tool - Bridge between CCW and CodexLens Python package * Provides code indexing and semantic search via spawned Python process * * Features: * - Automatic venv bootstrap at ~/.codexlens/venv * - JSON protocol communication * - Symbol extraction and semantic search * - FTS5 full-text search */ import { z } from 'zod'; import type { ToolSchema, ToolResult } from '../types/tool.js'; import { spawn, execSync } from 'child_process'; import { existsSync, mkdirSync } from 'fs'; import { join, dirname } from 'path'; import { homedir } from 'os'; import { fileURLToPath } from 'url'; // Get directory of this module const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); // CodexLens configuration const CODEXLENS_DATA_DIR = join(homedir(), '.codexlens'); const CODEXLENS_VENV = join(CODEXLENS_DATA_DIR, 'venv'); const VENV_PYTHON = process.platform === 'win32' ? join(CODEXLENS_VENV, 'Scripts', 'python.exe') : join(CODEXLENS_VENV, 'bin', 'python'); // Bootstrap status cache let bootstrapChecked = false; let bootstrapReady = false; // Define Zod schema for validation const ParamsSchema = z.object({ action: z.enum([ 'init', 'search', 'search_files', 'symbol', 'status', 'config_show', 'config_set', 'config_migrate', 'clean', 'bootstrap', 'check', ]), path: z.string().optional(), query: z.string().optional(), mode: z.enum(['text', 'semantic']).default('text'), file: z.string().optional(), key: z.string().optional(), // For config_set action value: z.string().optional(), // For config_set action newPath: z.string().optional(), // For config_migrate action all: z.boolean().optional(), // For clean action languages: z.array(z.string()).optional(), limit: z.number().default(20), format: z.enum(['json', 'table', 'plain']).default('json'), }); type Params = z.infer; interface ReadyStatus { ready: boolean; error?: string; version?: string; } interface SemanticStatus { available: boolean; backend?: string; error?: string; } interface BootstrapResult { success: boolean; error?: string; message?: string; } interface ExecuteResult { success: boolean; output?: string; error?: string; message?: string; results?: unknown; files?: unknown; symbols?: unknown; status?: unknown; config?: unknown; cleanResult?: unknown; ready?: boolean; version?: string; } interface ExecuteOptions { timeout?: number; cwd?: string; } /** * Detect available Python 3 executable * @returns Python executable command */ function getSystemPython(): string { const commands = process.platform === 'win32' ? ['python', 'py', 'python3'] : ['python3', 'python']; for (const cmd of commands) { try { const version = execSync(`${cmd} --version 2>&1`, { encoding: 'utf8' }); if (version.includes('Python 3')) { return cmd; } } catch { // Try next command } } throw new Error('Python 3 not found. Please install Python 3 and ensure it is in PATH.'); } /** * Check if CodexLens venv exists and has required packages * @returns Ready status */ async function checkVenvStatus(): Promise { // Check venv exists if (!existsSync(CODEXLENS_VENV)) { return { ready: false, error: 'Venv not found' }; } // Check python executable exists if (!existsSync(VENV_PYTHON)) { return { ready: false, error: 'Python executable not found in venv' }; } // Check codexlens is importable return new Promise((resolve) => { const child = spawn(VENV_PYTHON, ['-c', 'import codexlens; print(codexlens.__version__)'], { stdio: ['ignore', 'pipe', 'pipe'], timeout: 10000, }); let stdout = ''; let stderr = ''; child.stdout.on('data', (data) => { stdout += data.toString(); }); child.stderr.on('data', (data) => { stderr += data.toString(); }); child.on('close', (code) => { if (code === 0) { resolve({ ready: true, version: stdout.trim() }); } else { resolve({ ready: false, error: `CodexLens not installed: ${stderr}` }); } }); child.on('error', (err) => { resolve({ ready: false, error: `Failed to check venv: ${err.message}` }); }); }); } /** * Check if semantic search dependencies are installed * @returns Semantic status */ async function checkSemanticStatus(): Promise { // First check if CodexLens is installed const venvStatus = await checkVenvStatus(); if (!venvStatus.ready) { return { available: false, error: 'CodexLens not installed' }; } // Check semantic module availability return new Promise((resolve) => { const checkCode = ` import sys try: from codexlens.semantic import SEMANTIC_AVAILABLE, SEMANTIC_BACKEND if SEMANTIC_AVAILABLE: print(f"available:{SEMANTIC_BACKEND}") else: print("unavailable") except Exception as e: print(f"error:{e}") `; const child = spawn(VENV_PYTHON, ['-c', checkCode], { stdio: ['ignore', 'pipe', 'pipe'], timeout: 15000, }); let stdout = ''; let stderr = ''; child.stdout.on('data', (data) => { stdout += data.toString(); }); child.stderr.on('data', (data) => { stderr += data.toString(); }); child.on('close', (code) => { const output = stdout.trim(); if (output.startsWith('available:')) { const backend = output.split(':')[1]; resolve({ available: true, backend }); } else if (output === 'unavailable') { resolve({ available: false, error: 'Semantic dependencies not installed' }); } else { resolve({ available: false, error: output || stderr || 'Unknown error' }); } }); child.on('error', (err) => { resolve({ available: false, error: `Check failed: ${err.message}` }); }); }); } /** * Install semantic search dependencies (fastembed, ONNX-based, ~200MB) * @returns Bootstrap result */ async function installSemantic(): Promise { // First ensure CodexLens is installed const venvStatus = await checkVenvStatus(); if (!venvStatus.ready) { return { success: false, error: 'CodexLens not installed. Install CodexLens first.' }; } const pipPath = process.platform === 'win32' ? join(CODEXLENS_VENV, 'Scripts', 'pip.exe') : join(CODEXLENS_VENV, 'bin', 'pip'); return new Promise((resolve) => { console.log('[CodexLens] Installing semantic search dependencies (fastembed)...'); console.log('[CodexLens] Using ONNX-based fastembed backend (~200MB)'); const child = spawn(pipPath, ['install', 'numpy>=1.24', 'fastembed>=0.2'], { stdio: ['ignore', 'pipe', 'pipe'], timeout: 600000, // 10 minutes for potential model download }); let stdout = ''; let stderr = ''; child.stdout.on('data', (data) => { stdout += data.toString(); // Log progress const line = data.toString().trim(); if (line.includes('Downloading') || line.includes('Installing') || line.includes('Collecting')) { console.log(`[CodexLens] ${line}`); } }); child.stderr.on('data', (data) => { stderr += data.toString(); }); child.on('close', (code) => { if (code === 0) { console.log('[CodexLens] Semantic dependencies installed successfully'); resolve({ success: true }); } else { resolve({ success: false, error: `Installation failed: ${stderr || stdout}` }); } }); child.on('error', (err) => { resolve({ success: false, error: `Failed to run pip: ${err.message}` }); }); }); } /** * Bootstrap CodexLens venv with required packages * @returns Bootstrap result */ async function bootstrapVenv(): Promise { // Ensure data directory exists if (!existsSync(CODEXLENS_DATA_DIR)) { mkdirSync(CODEXLENS_DATA_DIR, { recursive: true }); } // Create venv if not exists if (!existsSync(CODEXLENS_VENV)) { try { console.log('[CodexLens] Creating virtual environment...'); const pythonCmd = getSystemPython(); execSync(`${pythonCmd} -m venv "${CODEXLENS_VENV}"`, { stdio: 'inherit' }); } catch (err) { return { success: false, error: `Failed to create venv: ${(err as Error).message}` }; } } // Install codexlens with semantic extras try { console.log('[CodexLens] Installing codexlens package...'); const pipPath = process.platform === 'win32' ? join(CODEXLENS_VENV, 'Scripts', 'pip.exe') : join(CODEXLENS_VENV, 'bin', 'pip'); // Try multiple local paths, then fall back to PyPI const possiblePaths = [ join(process.cwd(), 'codex-lens'), join(__dirname, '..', '..', '..', 'codex-lens'), // ccw/src/tools -> project root join(homedir(), 'codex-lens'), ]; let installed = false; for (const localPath of possiblePaths) { if (existsSync(join(localPath, 'pyproject.toml'))) { console.log(`[CodexLens] Installing from local path: ${localPath}`); execSync(`"${pipPath}" install -e "${localPath}"`, { stdio: 'inherit' }); installed = true; break; } } if (!installed) { console.log('[CodexLens] Installing from PyPI...'); execSync(`"${pipPath}" install codexlens`, { stdio: 'inherit' }); } return { success: true }; } catch (err) { return { success: false, error: `Failed to install codexlens: ${(err as Error).message}` }; } } /** * Ensure CodexLens is ready to use * @returns Ready status */ async function ensureReady(): Promise { // Use cached result if already checked if (bootstrapChecked && bootstrapReady) { return { ready: true }; } // Check current status const status = await checkVenvStatus(); if (status.ready) { bootstrapChecked = true; bootstrapReady = true; return { ready: true, version: status.version }; } // Attempt bootstrap const bootstrap = await bootstrapVenv(); if (!bootstrap.success) { return { ready: false, error: bootstrap.error }; } // Verify after bootstrap const recheck = await checkVenvStatus(); bootstrapChecked = true; bootstrapReady = recheck.ready; return recheck; } /** * Execute CodexLens CLI command * @param args - CLI arguments * @param options - Execution options * @returns Execution result */ async function executeCodexLens(args: string[], options: ExecuteOptions = {}): Promise { const { timeout = 60000, cwd = process.cwd() } = options; // Ensure ready const readyStatus = await ensureReady(); if (!readyStatus.ready) { return { success: false, error: readyStatus.error }; } return new Promise((resolve) => { const child = spawn(VENV_PYTHON, ['-m', 'codexlens', ...args], { cwd, stdio: ['ignore', 'pipe', 'pipe'], }); let stdout = ''; let stderr = ''; let timedOut = false; child.stdout.on('data', (data) => { stdout += data.toString(); }); child.stderr.on('data', (data) => { stderr += data.toString(); }); const timeoutId = setTimeout(() => { timedOut = true; child.kill('SIGTERM'); }, timeout); child.on('close', (code) => { clearTimeout(timeoutId); if (timedOut) { resolve({ success: false, error: 'Command timed out' }); } else if (code === 0) { resolve({ success: true, output: stdout.trim() }); } else { resolve({ success: false, error: stderr || `Exit code: ${code}` }); } }); child.on('error', (err) => { clearTimeout(timeoutId); resolve({ success: false, error: `Spawn failed: ${err.message}` }); }); }); } /** * Initialize CodexLens index for a directory * @param params - Parameters * @returns Execution result */ async function initIndex(params: Params): Promise { const { path = '.', languages } = params; const args = ['init', path]; if (languages && languages.length > 0) { args.push('--languages', languages.join(',')); } return executeCodexLens(args, { cwd: path }); } /** * Search code using CodexLens * @param params - Search parameters * @returns Execution result */ async function searchCode(params: Params): Promise { const { query, path = '.', limit = 20 } = params; if (!query) { return { success: false, error: 'Query is required for search action' }; } const args = ['search', query, '--limit', limit.toString(), '--json']; const result = await executeCodexLens(args, { cwd: path }); if (result.success && result.output) { try { result.results = JSON.parse(result.output); delete result.output; } catch { // Keep raw output if JSON parse fails } } return result; } /** * Search code and return only file paths * @param params - Search parameters * @returns Execution result */ async function searchFiles(params: Params): Promise { const { query, path = '.', limit = 20 } = params; if (!query) { return { success: false, error: 'Query is required for search_files action' }; } const args = ['search', query, '--files-only', '--limit', limit.toString(), '--json']; const result = await executeCodexLens(args, { cwd: path }); if (result.success && result.output) { try { result.files = JSON.parse(result.output); delete result.output; } catch { // Keep raw output if JSON parse fails } } return result; } /** * Extract symbols from a file * @param params - Parameters * @returns Execution result */ async function extractSymbols(params: Params): Promise { const { file } = params; if (!file) { return { success: false, error: 'File is required for symbol action' }; } const args = ['symbol', file, '--json']; const result = await executeCodexLens(args); if (result.success && result.output) { try { result.symbols = JSON.parse(result.output); delete result.output; } catch { // Keep raw output if JSON parse fails } } return result; } /** * Get index status * @param params - Parameters * @returns Execution result */ async function getStatus(params: Params): Promise { const { path = '.' } = params; const args = ['status', '--json']; const result = await executeCodexLens(args, { cwd: path }); if (result.success && result.output) { try { result.status = JSON.parse(result.output); delete result.output; } catch { // Keep raw output if JSON parse fails } } return result; } /** * Show configuration * @param params - Parameters * @returns Execution result */ async function configShow(): Promise { const args = ['config', 'show', '--json']; const result = await executeCodexLens(args); if (result.success && result.output) { try { result.config = JSON.parse(result.output); delete result.output; } catch { // Keep raw output if JSON parse fails } } return result; } /** * Set configuration value * @param params - Parameters * @returns Execution result */ async function configSet(params: Params): Promise { const { key, value } = params; if (!key) { return { success: false, error: 'key is required for config_set action' }; } if (!value) { return { success: false, error: 'value is required for config_set action' }; } const args = ['config', 'set', key, value, '--json']; const result = await executeCodexLens(args); if (result.success && result.output) { try { result.config = JSON.parse(result.output); delete result.output; } catch { // Keep raw output if JSON parse fails } } return result; } /** * Migrate indexes to new location * @param params - Parameters * @returns Execution result */ async function configMigrate(params: Params): Promise { const { newPath } = params; if (!newPath) { return { success: false, error: 'newPath is required for config_migrate action' }; } const args = ['config', 'migrate', newPath, '--json']; const result = await executeCodexLens(args, { timeout: 300000 }); // 5 min for migration if (result.success && result.output) { try { result.config = JSON.parse(result.output); delete result.output; } catch { // Keep raw output if JSON parse fails } } return result; } /** * Clean indexes * @param params - Parameters * @returns Execution result */ async function cleanIndexes(params: Params): Promise { const { path, all } = params; const args = ['clean']; if (all) { args.push('--all'); } else if (path) { args.push(path); } args.push('--json'); const result = await executeCodexLens(args); if (result.success && result.output) { try { result.cleanResult = JSON.parse(result.output); delete result.output; } catch { // Keep raw output if JSON parse fails } } return result; } // Tool schema for MCP export const schema: ToolSchema = { name: 'codex_lens', description: `CodexLens - Code indexing and search. Usage: codex_lens(action="init", path=".") # Index directory codex_lens(action="search", query="func", path=".") # Search code codex_lens(action="search_files", query="x") # Search, return paths only codex_lens(action="symbol", file="f.py") # Extract symbols codex_lens(action="status") # Index status codex_lens(action="config_show") # Show configuration codex_lens(action="config_set", key="index_dir", value="/path/to/indexes") # Set config codex_lens(action="config_migrate", newPath="/new/path") # Migrate indexes codex_lens(action="clean") # Show clean status codex_lens(action="clean", path=".") # Clean specific project codex_lens(action="clean", all=true) # Clean all indexes`, inputSchema: { type: 'object', properties: { action: { type: 'string', enum: [ 'init', 'search', 'search_files', 'symbol', 'status', 'config_show', 'config_set', 'config_migrate', 'clean', 'bootstrap', 'check', ], description: 'Action to perform', }, path: { type: 'string', description: 'Target path (for init, search, search_files, status, clean)', }, query: { type: 'string', description: 'Search query (for search and search_files actions)', }, mode: { type: 'string', enum: ['text', 'semantic'], description: 'Search mode (default: text)', default: 'text', }, file: { type: 'string', description: 'File path (for symbol action)', }, key: { type: 'string', description: 'Config key (for config_set action, e.g., "index_dir")', }, value: { type: 'string', description: 'Config value (for config_set action)', }, newPath: { type: 'string', description: 'New index path (for config_migrate action)', }, all: { type: 'boolean', description: 'Clean all indexes (for clean action)', default: false, }, languages: { type: 'array', items: { type: 'string' }, description: 'Languages to index (for init action)', }, limit: { type: 'number', description: 'Maximum results (for search and search_files actions)', default: 20, }, format: { type: 'string', enum: ['json', 'table', 'plain'], description: 'Output format', default: 'json', }, }, required: ['action'], }, }; // Handler function export async function handler(params: Record): Promise> { const parsed = ParamsSchema.safeParse(params); if (!parsed.success) { return { success: false, error: `Invalid params: ${parsed.error.message}` }; } const { action } = parsed.data; try { let result: ExecuteResult; switch (action) { case 'init': result = await initIndex(parsed.data); break; case 'search': result = await searchCode(parsed.data); break; case 'search_files': result = await searchFiles(parsed.data); break; case 'symbol': result = await extractSymbols(parsed.data); break; case 'status': result = await getStatus(parsed.data); break; case 'config_show': result = await configShow(); break; case 'config_set': result = await configSet(parsed.data); break; case 'config_migrate': result = await configMigrate(parsed.data); break; case 'clean': result = await cleanIndexes(parsed.data); break; case 'bootstrap': { // Force re-bootstrap bootstrapChecked = false; bootstrapReady = false; const bootstrapResult = await bootstrapVenv(); result = bootstrapResult.success ? { success: true, message: 'CodexLens bootstrapped successfully' } : { success: false, error: bootstrapResult.error }; break; } case 'check': { const checkResult = await checkVenvStatus(); result = { success: checkResult.ready, ready: checkResult.ready, error: checkResult.error, version: checkResult.version, }; break; } default: throw new Error( `Unknown action: ${action}. Valid actions: init, search, search_files, symbol, status, config_show, config_set, config_migrate, clean, bootstrap, check` ); } return result.success ? { success: true, result } : { success: false, error: result.error }; } catch (error) { return { success: false, error: (error as Error).message }; } } // Export for direct usage export { ensureReady, executeCodexLens, checkVenvStatus, bootstrapVenv, checkSemanticStatus, installSemantic }; // Backward-compatible export for tests export const codexLensTool = { name: schema.name, description: schema.description, parameters: schema.inputSchema, execute: async (params: Record) => { const result = await handler(params); // Return the result directly - tests expect {success: boolean, ...} format return result.success ? result.result : { success: false, error: result.error }; } };