/** * CodexLens Tool - Bridge between CCW and CodexLens Python package * Provides code indexing and semantic search via spawned Python process * * Features: * - Automatic venv bootstrap at ~/.codexlens/venv * - JSON protocol communication * - Symbol extraction and semantic search * - FTS5 full-text search */ import { spawn, execSync } from 'child_process'; import { existsSync, mkdirSync } from 'fs'; import { join, dirname } from 'path'; import { homedir } from 'os'; import { fileURLToPath } from 'url'; // Get directory of this module const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); // CodexLens configuration const CODEXLENS_DATA_DIR = join(homedir(), '.codexlens'); const CODEXLENS_VENV = join(CODEXLENS_DATA_DIR, 'venv'); const VENV_PYTHON = process.platform === 'win32' ? join(CODEXLENS_VENV, 'Scripts', 'python.exe') : join(CODEXLENS_VENV, 'bin', 'python'); // Bootstrap status cache let bootstrapChecked = false; let bootstrapReady = false; /** * Detect available Python 3 executable * @returns {string} - Python executable command */ function getSystemPython() { const commands = process.platform === 'win32' ? ['python', 'py', 'python3'] : ['python3', 'python']; for (const cmd of commands) { try { const version = execSync(`${cmd} --version 2>&1`, { encoding: 'utf8' }); if (version.includes('Python 3')) { return cmd; } } catch { // Try next command } } throw new Error('Python 3 not found. Please install Python 3 and ensure it is in PATH.'); } /** * Check if CodexLens venv exists and has required packages * @returns {Promise<{ready: boolean, error?: string}>} */ async function checkVenvStatus() { // Check venv exists if (!existsSync(CODEXLENS_VENV)) { return { ready: false, error: 'Venv not found' }; } // Check python executable exists if (!existsSync(VENV_PYTHON)) { return { ready: false, error: 'Python executable not found in venv' }; } // Check codexlens is importable return new Promise((resolve) => { const child = spawn(VENV_PYTHON, ['-c', 'import codexlens; print(codexlens.__version__)'], { stdio: ['ignore', 'pipe', 'pipe'], timeout: 10000 }); let stdout = ''; let stderr = ''; child.stdout.on('data', (data) => { stdout += data.toString(); }); child.stderr.on('data', (data) => { stderr += data.toString(); }); child.on('close', (code) => { if (code === 0) { resolve({ ready: true, version: stdout.trim() }); } else { resolve({ ready: false, error: `CodexLens not installed: ${stderr}` }); } }); child.on('error', (err) => { resolve({ ready: false, error: `Failed to check venv: ${err.message}` }); }); }); } /** * Bootstrap CodexLens venv with required packages * @returns {Promise<{success: boolean, error?: string}>} */ async function bootstrapVenv() { // Ensure data directory exists if (!existsSync(CODEXLENS_DATA_DIR)) { mkdirSync(CODEXLENS_DATA_DIR, { recursive: true }); } // Create venv if not exists if (!existsSync(CODEXLENS_VENV)) { try { console.log('[CodexLens] Creating virtual environment...'); const pythonCmd = getSystemPython(); execSync(`${pythonCmd} -m venv "${CODEXLENS_VENV}"`, { stdio: 'inherit' }); } catch (err) { return { success: false, error: `Failed to create venv: ${err.message}` }; } } // Install codexlens with semantic extras try { console.log('[CodexLens] Installing codexlens package...'); const pipPath = process.platform === 'win32' ? join(CODEXLENS_VENV, 'Scripts', 'pip.exe') : join(CODEXLENS_VENV, 'bin', 'pip'); // Try multiple local paths, then fall back to PyPI const possiblePaths = [ join(process.cwd(), 'codex-lens'), join(__dirname, '..', '..', '..', 'codex-lens'), // ccw/src/tools -> project root join(homedir(), 'codex-lens'), ]; let installed = false; for (const localPath of possiblePaths) { if (existsSync(join(localPath, 'pyproject.toml'))) { console.log(`[CodexLens] Installing from local path: ${localPath}`); execSync(`"${pipPath}" install -e "${localPath}"`, { stdio: 'inherit' }); installed = true; break; } } if (!installed) { console.log('[CodexLens] Installing from PyPI...'); execSync(`"${pipPath}" install codexlens`, { stdio: 'inherit' }); } return { success: true }; } catch (err) { return { success: false, error: `Failed to install codexlens: ${err.message}` }; } } /** * Ensure CodexLens is ready to use * @returns {Promise<{ready: boolean, error?: string}>} */ async function ensureReady() { // Use cached result if already checked if (bootstrapChecked && bootstrapReady) { return { ready: true }; } // Check current status const status = await checkVenvStatus(); if (status.ready) { bootstrapChecked = true; bootstrapReady = true; return { ready: true, version: status.version }; } // Attempt bootstrap const bootstrap = await bootstrapVenv(); if (!bootstrap.success) { return { ready: false, error: bootstrap.error }; } // Verify after bootstrap const recheck = await checkVenvStatus(); bootstrapChecked = true; bootstrapReady = recheck.ready; return recheck; } /** * Execute CodexLens CLI command * @param {string[]} args - CLI arguments * @param {Object} options - Execution options * @returns {Promise<{success: boolean, output?: string, error?: string}>} */ async function executeCodexLens(args, options = {}) { const { timeout = 60000, cwd = process.cwd() } = options; // Ensure ready const readyStatus = await ensureReady(); if (!readyStatus.ready) { return { success: false, error: readyStatus.error }; } return new Promise((resolve) => { const child = spawn(VENV_PYTHON, ['-m', 'codexlens', ...args], { cwd, stdio: ['ignore', 'pipe', 'pipe'] }); let stdout = ''; let stderr = ''; let timedOut = false; child.stdout.on('data', (data) => { stdout += data.toString(); }); child.stderr.on('data', (data) => { stderr += data.toString(); }); const timeoutId = setTimeout(() => { timedOut = true; child.kill('SIGTERM'); }, timeout); child.on('close', (code) => { clearTimeout(timeoutId); if (timedOut) { resolve({ success: false, error: 'Command timed out' }); } else if (code === 0) { resolve({ success: true, output: stdout.trim() }); } else { resolve({ success: false, error: stderr || `Exit code: ${code}` }); } }); child.on('error', (err) => { clearTimeout(timeoutId); resolve({ success: false, error: `Spawn failed: ${err.message}` }); }); }); } /** * Initialize CodexLens index for a directory * @param {Object} params - Parameters * @returns {Promise} */ async function initIndex(params) { const { path = '.', languages } = params; const args = ['init', path]; if (languages && languages.length > 0) { args.push('--languages', languages.join(',')); } return executeCodexLens(args, { cwd: path }); } /** * Search code using CodexLens * @param {Object} params - Search parameters * @returns {Promise} */ async function searchCode(params) { const { query, path = '.', mode = 'text', limit = 20 } = params; const args = ['search', query, '--limit', limit.toString(), '--json']; // Note: semantic mode requires semantic extras to be installed // Currently not exposed via CLI flag, uses standard FTS search const result = await executeCodexLens(args, { cwd: path }); if (result.success) { try { result.results = JSON.parse(result.output); delete result.output; } catch { // Keep raw output if JSON parse fails } } return result; } /** * Extract symbols from a file * @param {Object} params - Parameters * @returns {Promise} */ async function extractSymbols(params) { const { file } = params; const args = ['symbol', file, '--json']; const result = await executeCodexLens(args); if (result.success) { try { result.symbols = JSON.parse(result.output); delete result.output; } catch { // Keep raw output if JSON parse fails } } return result; } /** * Get index status * @param {Object} params - Parameters * @returns {Promise} */ async function getStatus(params) { const { path = '.' } = params; const args = ['status', '--json']; const result = await executeCodexLens(args, { cwd: path }); if (result.success) { try { result.status = JSON.parse(result.output); delete result.output; } catch { // Keep raw output if JSON parse fails } } return result; } /** * Update specific files in the index * @param {Object} params - Parameters * @returns {Promise} */ async function updateFiles(params) { const { files, path = '.' } = params; if (!files || !Array.isArray(files) || files.length === 0) { return { success: false, error: 'files parameter is required and must be a non-empty array' }; } const args = ['update', ...files, '--json']; const result = await executeCodexLens(args, { cwd: path }); if (result.success) { try { result.updateResult = JSON.parse(result.output); delete result.output; } catch { // Keep raw output if JSON parse fails } } return result; } /** * Main execute function - routes to appropriate handler * @param {Object} params - Execution parameters * @returns {Promise} */ async function execute(params) { const { action, ...rest } = params; switch (action) { case 'init': return initIndex(rest); case 'search': return searchCode(rest); case 'symbol': return extractSymbols(rest); case 'status': return getStatus(rest); case 'update': return updateFiles(rest); case 'bootstrap': // Force re-bootstrap bootstrapChecked = false; bootstrapReady = false; const bootstrapResult = await bootstrapVenv(); return bootstrapResult.success ? { success: true, message: 'CodexLens bootstrapped successfully' } : { success: false, error: bootstrapResult.error }; case 'check': // Check venv status return checkVenvStatus(); default: throw new Error(`Unknown action: ${action}. Valid actions: init, search, symbol, status, update, bootstrap, check`); } } /** * CodexLens Tool Definition */ export const codexLensTool = { name: 'codex_lens', description: `Code indexing and semantic search via CodexLens Python package. Actions: - init: Initialize index for a directory - search: Search code (text or semantic mode) - symbol: Extract symbols from a file - status: Get index status - update: Incrementally update specific files (add/modify/remove) - bootstrap: Force re-install CodexLens venv - check: Check venv readiness Features: - Automatic venv bootstrap at ~/.codexlens/venv - SQLite FTS5 full-text search - Tree-sitter symbol extraction - Incremental updates for changed files - Optional semantic search with embeddings`, parameters: { type: 'object', properties: { action: { type: 'string', enum: ['init', 'search', 'symbol', 'status', 'update', 'bootstrap', 'check'], description: 'Action to perform' }, path: { type: 'string', description: 'Target path (for init, search, status, update)' }, query: { type: 'string', description: 'Search query (for search action)' }, mode: { type: 'string', enum: ['text', 'semantic'], description: 'Search mode (default: text)', default: 'text' }, file: { type: 'string', description: 'File path (for symbol action)' }, files: { type: 'array', items: { type: 'string' }, description: 'File paths to update (for update action)' }, languages: { type: 'array', items: { type: 'string' }, description: 'Languages to index (for init action)' }, limit: { type: 'number', description: 'Maximum results (for search action)', default: 20 }, format: { type: 'string', enum: ['json', 'table', 'plain'], description: 'Output format', default: 'json' } }, required: ['action'] }, execute }; // Export for direct usage export { ensureReady, executeCodexLens, checkVenvStatus, bootstrapVenv };