feat(ccw): migrate backend to TypeScript

- Convert 40 JS files to TypeScript (CLI, tools, core, MCP server)
- Add Zod for runtime parameter validation
- Add type definitions in src/types/
- Keep src/templates/ as JavaScript (dashboard frontend)
- Update bin entries to use dist/
- Add tsconfig.json with strict mode
- Add backward-compatible exports for tests
- All 39 tests passing

Breaking changes: None (backward compatible)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
catlog22
2025-12-13 10:43:15 +08:00
parent d4e59770d0
commit 25ac862f46
93 changed files with 5531 additions and 9302 deletions

712
ccw/src/tools/codex-lens.ts Normal file
View File

@@ -0,0 +1,712 @@
/**
* CodexLens Tool - Bridge between CCW and CodexLens Python package
* Provides code indexing and semantic search via spawned Python process
*
* Features:
* - Automatic venv bootstrap at ~/.codexlens/venv
* - JSON protocol communication
* - Symbol extraction and semantic search
* - FTS5 full-text search
*/
import { z } from 'zod';
import type { ToolSchema, ToolResult } from '../types/tool.js';
import { spawn, execSync } from 'child_process';
import { existsSync, mkdirSync } from 'fs';
import { join, dirname } from 'path';
import { homedir } from 'os';
import { fileURLToPath } from 'url';
// Get directory of this module
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
// CodexLens configuration
const CODEXLENS_DATA_DIR = join(homedir(), '.codexlens');
const CODEXLENS_VENV = join(CODEXLENS_DATA_DIR, 'venv');
const VENV_PYTHON =
process.platform === 'win32'
? join(CODEXLENS_VENV, 'Scripts', 'python.exe')
: join(CODEXLENS_VENV, 'bin', 'python');
// Bootstrap status cache
let bootstrapChecked = false;
let bootstrapReady = false;
// Define Zod schema for validation
const ParamsSchema = z.object({
action: z.enum(['init', 'search', 'search_files', 'symbol', 'status', 'update', 'bootstrap', 'check']),
path: z.string().optional(),
query: z.string().optional(),
mode: z.enum(['text', 'semantic']).default('text'),
file: z.string().optional(),
files: z.array(z.string()).optional(),
languages: z.array(z.string()).optional(),
limit: z.number().default(20),
format: z.enum(['json', 'table', 'plain']).default('json'),
});
type Params = z.infer<typeof ParamsSchema>;
interface ReadyStatus {
ready: boolean;
error?: string;
version?: string;
}
interface SemanticStatus {
available: boolean;
backend?: string;
error?: string;
}
interface BootstrapResult {
success: boolean;
error?: string;
message?: string;
}
interface ExecuteResult {
success: boolean;
output?: string;
error?: string;
message?: string;
results?: unknown;
files?: unknown;
symbols?: unknown;
status?: unknown;
updateResult?: unknown;
ready?: boolean;
version?: string;
}
interface ExecuteOptions {
timeout?: number;
cwd?: string;
}
/**
* Detect available Python 3 executable
* @returns Python executable command
*/
function getSystemPython(): string {
const commands = process.platform === 'win32' ? ['python', 'py', 'python3'] : ['python3', 'python'];
for (const cmd of commands) {
try {
const version = execSync(`${cmd} --version 2>&1`, { encoding: 'utf8' });
if (version.includes('Python 3')) {
return cmd;
}
} catch {
// Try next command
}
}
throw new Error('Python 3 not found. Please install Python 3 and ensure it is in PATH.');
}
/**
* Check if CodexLens venv exists and has required packages
* @returns Ready status
*/
async function checkVenvStatus(): Promise<ReadyStatus> {
// Check venv exists
if (!existsSync(CODEXLENS_VENV)) {
return { ready: false, error: 'Venv not found' };
}
// Check python executable exists
if (!existsSync(VENV_PYTHON)) {
return { ready: false, error: 'Python executable not found in venv' };
}
// Check codexlens is importable
return new Promise((resolve) => {
const child = spawn(VENV_PYTHON, ['-c', 'import codexlens; print(codexlens.__version__)'], {
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 10000,
});
let stdout = '';
let stderr = '';
child.stdout.on('data', (data) => {
stdout += data.toString();
});
child.stderr.on('data', (data) => {
stderr += data.toString();
});
child.on('close', (code) => {
if (code === 0) {
resolve({ ready: true, version: stdout.trim() });
} else {
resolve({ ready: false, error: `CodexLens not installed: ${stderr}` });
}
});
child.on('error', (err) => {
resolve({ ready: false, error: `Failed to check venv: ${err.message}` });
});
});
}
/**
* Check if semantic search dependencies are installed
* @returns Semantic status
*/
async function checkSemanticStatus(): Promise<SemanticStatus> {
// First check if CodexLens is installed
const venvStatus = await checkVenvStatus();
if (!venvStatus.ready) {
return { available: false, error: 'CodexLens not installed' };
}
// Check semantic module availability
return new Promise((resolve) => {
const checkCode = `
import sys
try:
from codexlens.semantic import SEMANTIC_AVAILABLE, SEMANTIC_BACKEND
if SEMANTIC_AVAILABLE:
print(f"available:{SEMANTIC_BACKEND}")
else:
print("unavailable")
except Exception as e:
print(f"error:{e}")
`;
const child = spawn(VENV_PYTHON, ['-c', checkCode], {
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 15000,
});
let stdout = '';
let stderr = '';
child.stdout.on('data', (data) => {
stdout += data.toString();
});
child.stderr.on('data', (data) => {
stderr += data.toString();
});
child.on('close', (code) => {
const output = stdout.trim();
if (output.startsWith('available:')) {
const backend = output.split(':')[1];
resolve({ available: true, backend });
} else if (output === 'unavailable') {
resolve({ available: false, error: 'Semantic dependencies not installed' });
} else {
resolve({ available: false, error: output || stderr || 'Unknown error' });
}
});
child.on('error', (err) => {
resolve({ available: false, error: `Check failed: ${err.message}` });
});
});
}
/**
* Install semantic search dependencies (fastembed, ONNX-based, ~200MB)
* @returns Bootstrap result
*/
async function installSemantic(): Promise<BootstrapResult> {
// First ensure CodexLens is installed
const venvStatus = await checkVenvStatus();
if (!venvStatus.ready) {
return { success: false, error: 'CodexLens not installed. Install CodexLens first.' };
}
const pipPath =
process.platform === 'win32'
? join(CODEXLENS_VENV, 'Scripts', 'pip.exe')
: join(CODEXLENS_VENV, 'bin', 'pip');
return new Promise((resolve) => {
console.log('[CodexLens] Installing semantic search dependencies (fastembed)...');
console.log('[CodexLens] Using ONNX-based fastembed backend (~200MB)');
const child = spawn(pipPath, ['install', 'numpy>=1.24', 'fastembed>=0.2'], {
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 600000, // 10 minutes for potential model download
});
let stdout = '';
let stderr = '';
child.stdout.on('data', (data) => {
stdout += data.toString();
// Log progress
const line = data.toString().trim();
if (line.includes('Downloading') || line.includes('Installing') || line.includes('Collecting')) {
console.log(`[CodexLens] ${line}`);
}
});
child.stderr.on('data', (data) => {
stderr += data.toString();
});
child.on('close', (code) => {
if (code === 0) {
console.log('[CodexLens] Semantic dependencies installed successfully');
resolve({ success: true });
} else {
resolve({ success: false, error: `Installation failed: ${stderr || stdout}` });
}
});
child.on('error', (err) => {
resolve({ success: false, error: `Failed to run pip: ${err.message}` });
});
});
}
/**
* Bootstrap CodexLens venv with required packages
* @returns Bootstrap result
*/
async function bootstrapVenv(): Promise<BootstrapResult> {
// Ensure data directory exists
if (!existsSync(CODEXLENS_DATA_DIR)) {
mkdirSync(CODEXLENS_DATA_DIR, { recursive: true });
}
// Create venv if not exists
if (!existsSync(CODEXLENS_VENV)) {
try {
console.log('[CodexLens] Creating virtual environment...');
const pythonCmd = getSystemPython();
execSync(`${pythonCmd} -m venv "${CODEXLENS_VENV}"`, { stdio: 'inherit' });
} catch (err) {
return { success: false, error: `Failed to create venv: ${(err as Error).message}` };
}
}
// Install codexlens with semantic extras
try {
console.log('[CodexLens] Installing codexlens package...');
const pipPath =
process.platform === 'win32'
? join(CODEXLENS_VENV, 'Scripts', 'pip.exe')
: join(CODEXLENS_VENV, 'bin', 'pip');
// Try multiple local paths, then fall back to PyPI
const possiblePaths = [
join(process.cwd(), 'codex-lens'),
join(__dirname, '..', '..', '..', 'codex-lens'), // ccw/src/tools -> project root
join(homedir(), 'codex-lens'),
];
let installed = false;
for (const localPath of possiblePaths) {
if (existsSync(join(localPath, 'pyproject.toml'))) {
console.log(`[CodexLens] Installing from local path: ${localPath}`);
execSync(`"${pipPath}" install -e "${localPath}"`, { stdio: 'inherit' });
installed = true;
break;
}
}
if (!installed) {
console.log('[CodexLens] Installing from PyPI...');
execSync(`"${pipPath}" install codexlens`, { stdio: 'inherit' });
}
return { success: true };
} catch (err) {
return { success: false, error: `Failed to install codexlens: ${(err as Error).message}` };
}
}
/**
* Ensure CodexLens is ready to use
* @returns Ready status
*/
async function ensureReady(): Promise<ReadyStatus> {
// Use cached result if already checked
if (bootstrapChecked && bootstrapReady) {
return { ready: true };
}
// Check current status
const status = await checkVenvStatus();
if (status.ready) {
bootstrapChecked = true;
bootstrapReady = true;
return { ready: true, version: status.version };
}
// Attempt bootstrap
const bootstrap = await bootstrapVenv();
if (!bootstrap.success) {
return { ready: false, error: bootstrap.error };
}
// Verify after bootstrap
const recheck = await checkVenvStatus();
bootstrapChecked = true;
bootstrapReady = recheck.ready;
return recheck;
}
/**
* Execute CodexLens CLI command
* @param args - CLI arguments
* @param options - Execution options
* @returns Execution result
*/
async function executeCodexLens(args: string[], options: ExecuteOptions = {}): Promise<ExecuteResult> {
const { timeout = 60000, cwd = process.cwd() } = options;
// Ensure ready
const readyStatus = await ensureReady();
if (!readyStatus.ready) {
return { success: false, error: readyStatus.error };
}
return new Promise((resolve) => {
const child = spawn(VENV_PYTHON, ['-m', 'codexlens', ...args], {
cwd,
stdio: ['ignore', 'pipe', 'pipe'],
});
let stdout = '';
let stderr = '';
let timedOut = false;
child.stdout.on('data', (data) => {
stdout += data.toString();
});
child.stderr.on('data', (data) => {
stderr += data.toString();
});
const timeoutId = setTimeout(() => {
timedOut = true;
child.kill('SIGTERM');
}, timeout);
child.on('close', (code) => {
clearTimeout(timeoutId);
if (timedOut) {
resolve({ success: false, error: 'Command timed out' });
} else if (code === 0) {
resolve({ success: true, output: stdout.trim() });
} else {
resolve({ success: false, error: stderr || `Exit code: ${code}` });
}
});
child.on('error', (err) => {
clearTimeout(timeoutId);
resolve({ success: false, error: `Spawn failed: ${err.message}` });
});
});
}
/**
* Initialize CodexLens index for a directory
* @param params - Parameters
* @returns Execution result
*/
async function initIndex(params: Params): Promise<ExecuteResult> {
const { path = '.', languages } = params;
const args = ['init', path];
if (languages && languages.length > 0) {
args.push('--languages', languages.join(','));
}
return executeCodexLens(args, { cwd: path });
}
/**
* Search code using CodexLens
* @param params - Search parameters
* @returns Execution result
*/
async function searchCode(params: Params): Promise<ExecuteResult> {
const { query, path = '.', limit = 20 } = params;
if (!query) {
return { success: false, error: 'Query is required for search action' };
}
const args = ['search', query, '--limit', limit.toString(), '--json'];
const result = await executeCodexLens(args, { cwd: path });
if (result.success && result.output) {
try {
result.results = JSON.parse(result.output);
delete result.output;
} catch {
// Keep raw output if JSON parse fails
}
}
return result;
}
/**
* Search code and return only file paths
* @param params - Search parameters
* @returns Execution result
*/
async function searchFiles(params: Params): Promise<ExecuteResult> {
const { query, path = '.', limit = 20 } = params;
if (!query) {
return { success: false, error: 'Query is required for search_files action' };
}
const args = ['search', query, '--files-only', '--limit', limit.toString(), '--json'];
const result = await executeCodexLens(args, { cwd: path });
if (result.success && result.output) {
try {
result.files = JSON.parse(result.output);
delete result.output;
} catch {
// Keep raw output if JSON parse fails
}
}
return result;
}
/**
* Extract symbols from a file
* @param params - Parameters
* @returns Execution result
*/
async function extractSymbols(params: Params): Promise<ExecuteResult> {
const { file } = params;
if (!file) {
return { success: false, error: 'File is required for symbol action' };
}
const args = ['symbol', file, '--json'];
const result = await executeCodexLens(args);
if (result.success && result.output) {
try {
result.symbols = JSON.parse(result.output);
delete result.output;
} catch {
// Keep raw output if JSON parse fails
}
}
return result;
}
/**
* Get index status
* @param params - Parameters
* @returns Execution result
*/
async function getStatus(params: Params): Promise<ExecuteResult> {
const { path = '.' } = params;
const args = ['status', '--json'];
const result = await executeCodexLens(args, { cwd: path });
if (result.success && result.output) {
try {
result.status = JSON.parse(result.output);
delete result.output;
} catch {
// Keep raw output if JSON parse fails
}
}
return result;
}
/**
* Update specific files in the index
* @param params - Parameters
* @returns Execution result
*/
async function updateFiles(params: Params): Promise<ExecuteResult> {
const { files, path = '.' } = params;
if (!files || !Array.isArray(files) || files.length === 0) {
return { success: false, error: 'files parameter is required and must be a non-empty array' };
}
const args = ['update', ...files, '--json'];
const result = await executeCodexLens(args, { cwd: path });
if (result.success && result.output) {
try {
result.updateResult = JSON.parse(result.output);
delete result.output;
} catch {
// Keep raw output if JSON parse fails
}
}
return result;
}
// Tool schema for MCP
export const schema: ToolSchema = {
name: 'codex_lens',
description: `CodexLens - Code indexing and search.
Usage:
codex_lens(action="init", path=".") # Index directory
codex_lens(action="search", query="func", path=".") # Search code
codex_lens(action="search_files", query="x") # Search, return paths only
codex_lens(action="symbol", file="f.py") # Extract symbols
codex_lens(action="status") # Index status
codex_lens(action="update", files=["a.js"]) # Update specific files`,
inputSchema: {
type: 'object',
properties: {
action: {
type: 'string',
enum: ['init', 'search', 'search_files', 'symbol', 'status', 'update', 'bootstrap', 'check'],
description: 'Action to perform',
},
path: {
type: 'string',
description: 'Target path (for init, search, search_files, status, update)',
},
query: {
type: 'string',
description: 'Search query (for search and search_files actions)',
},
mode: {
type: 'string',
enum: ['text', 'semantic'],
description: 'Search mode (default: text)',
default: 'text',
},
file: {
type: 'string',
description: 'File path (for symbol action)',
},
files: {
type: 'array',
items: { type: 'string' },
description: 'File paths to update (for update action)',
},
languages: {
type: 'array',
items: { type: 'string' },
description: 'Languages to index (for init action)',
},
limit: {
type: 'number',
description: 'Maximum results (for search and search_files actions)',
default: 20,
},
format: {
type: 'string',
enum: ['json', 'table', 'plain'],
description: 'Output format',
default: 'json',
},
},
required: ['action'],
},
};
// Handler function
export async function handler(params: Record<string, unknown>): Promise<ToolResult<ExecuteResult>> {
const parsed = ParamsSchema.safeParse(params);
if (!parsed.success) {
return { success: false, error: `Invalid params: ${parsed.error.message}` };
}
const { action } = parsed.data;
try {
let result: ExecuteResult;
switch (action) {
case 'init':
result = await initIndex(parsed.data);
break;
case 'search':
result = await searchCode(parsed.data);
break;
case 'search_files':
result = await searchFiles(parsed.data);
break;
case 'symbol':
result = await extractSymbols(parsed.data);
break;
case 'status':
result = await getStatus(parsed.data);
break;
case 'update':
result = await updateFiles(parsed.data);
break;
case 'bootstrap': {
// Force re-bootstrap
bootstrapChecked = false;
bootstrapReady = false;
const bootstrapResult = await bootstrapVenv();
result = bootstrapResult.success
? { success: true, message: 'CodexLens bootstrapped successfully' }
: { success: false, error: bootstrapResult.error };
break;
}
case 'check': {
const checkResult = await checkVenvStatus();
result = {
success: checkResult.ready,
ready: checkResult.ready,
error: checkResult.error,
version: checkResult.version,
};
break;
}
default:
throw new Error(
`Unknown action: ${action}. Valid actions: init, search, search_files, symbol, status, update, bootstrap, check`
);
}
return result.success ? { success: true, result } : { success: false, error: result.error };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
// Export for direct usage
export { ensureReady, executeCodexLens, checkVenvStatus, bootstrapVenv, checkSemanticStatus, installSemantic };
// Backward-compatible export for tests
export const codexLensTool = {
name: schema.name,
description: schema.description,
parameters: schema.inputSchema,
execute: async (params: Record<string, unknown>) => {
const result = await handler(params);
// Return the result directly - tests expect {success: boolean, ...} format
return result.success ? result.result : { success: false, error: result.error };
}
};