Add tests and documentation for CodexLens LSP tool

- Introduced a new test script for the CodexLens LSP tool to validate core functionalities including symbol search, find definition, find references, and get hover.
- Created comprehensive documentation for the MCP endpoint design, detailing the architecture, features, and integration with the CCW MCP Manager.
- Developed a detailed implementation plan for transitioning to a real LSP server, outlining phases, architecture, and acceptance criteria.
This commit is contained in:
catlog22
2026-01-19 23:26:35 +08:00
parent eeaefa7208
commit 3fe630f221
24 changed files with 3044 additions and 509 deletions

View File

@@ -159,6 +159,11 @@ export class CliHistoryStore {
stdout TEXT,
stderr TEXT,
truncated INTEGER DEFAULT 0,
cached INTEGER DEFAULT 0,
stdout_full TEXT,
stderr_full TEXT,
parsed_output TEXT,
final_output TEXT,
FOREIGN KEY (conversation_id) REFERENCES conversations(id) ON DELETE CASCADE,
UNIQUE(conversation_id, turn_number)
);
@@ -325,36 +330,34 @@ export class CliHistoryStore {
// Add cached output columns to turns table for non-streaming mode
const turnsInfo = this.db.prepare('PRAGMA table_info(turns)').all() as Array<{ name: string }>;
const hasCached = turnsInfo.some(col => col.name === 'cached');
const hasStdoutFull = turnsInfo.some(col => col.name === 'stdout_full');
const hasStderrFull = turnsInfo.some(col => col.name === 'stderr_full');
const hasParsedOutput = turnsInfo.some(col => col.name === 'parsed_output');
const hasFinalOutput = turnsInfo.some(col => col.name === 'final_output');
const turnsColumns = new Set(turnsInfo.map(col => col.name));
if (!hasCached) {
console.log('[CLI History] Migrating database: adding cached column to turns table...');
this.db.exec('ALTER TABLE turns ADD COLUMN cached INTEGER DEFAULT 0;');
console.log('[CLI History] Migration complete: cached column added');
// Collect all missing columns
const missingTurnsColumns: string[] = [];
const turnsColumnDefs: Record<string, string> = {
'cached': 'INTEGER DEFAULT 0',
'stdout_full': 'TEXT',
'stderr_full': 'TEXT',
'parsed_output': 'TEXT',
'final_output': 'TEXT'
};
// Silently detect missing columns
for (const [col, def] of Object.entries(turnsColumnDefs)) {
if (!turnsColumns.has(col)) {
missingTurnsColumns.push(col);
}
}
if (!hasStdoutFull) {
console.log('[CLI History] Migrating database: adding stdout_full column to turns table...');
this.db.exec('ALTER TABLE turns ADD COLUMN stdout_full TEXT;');
console.log('[CLI History] Migration complete: stdout_full column added');
}
if (!hasStderrFull) {
console.log('[CLI History] Migrating database: adding stderr_full column to turns table...');
this.db.exec('ALTER TABLE turns ADD COLUMN stderr_full TEXT;');
console.log('[CLI History] Migration complete: stderr_full column added');
}
if (!hasParsedOutput) {
console.log('[CLI History] Migrating database: adding parsed_output column to turns table...');
this.db.exec('ALTER TABLE turns ADD COLUMN parsed_output TEXT;');
console.log('[CLI History] Migration complete: parsed_output column added');
}
if (!hasFinalOutput) {
console.log('[CLI History] Migrating database: adding final_output column to turns table...');
this.db.exec('ALTER TABLE turns ADD COLUMN final_output TEXT;');
console.log('[CLI History] Migration complete: final_output column added');
// Batch migration - only output log if there are columns to migrate
if (missingTurnsColumns.length > 0) {
console.log(`[CLI History] Migrating turns table: adding ${missingTurnsColumns.length} columns (${missingTurnsColumns.join(', ')})...`);
for (const col of missingTurnsColumns) {
this.db.exec(`ALTER TABLE turns ADD COLUMN ${col} ${turnsColumnDefs[col]};`);
}
console.log('[CLI History] Migration complete: turns table updated');
}
} catch (err) {
console.error('[CLI History] Migration error:', (err as Error).message);

View File

@@ -0,0 +1,405 @@
/**
* CodexLens LSP Tool - Provides LSP-like code intelligence via CodexLens Python API
*
* Features:
* - symbol_search: Search symbols across workspace
* - find_definition: Go to symbol definition
* - find_references: Find all symbol references
* - get_hover: Get hover information for symbols
*/
import { z } from 'zod';
import type { ToolSchema, ToolResult } from '../types/tool.js';
import { spawn } from 'child_process';
import { join } from 'path';
import { homedir } from 'os';
import { getProjectRoot } from '../utils/path-validator.js';
// CodexLens venv configuration
const CODEXLENS_VENV =
process.platform === 'win32'
? join(homedir(), '.codexlens', 'venv', 'Scripts', 'python.exe')
: join(homedir(), '.codexlens', 'venv', 'bin', 'python');
// Define Zod schema for validation
const ParamsSchema = z.object({
action: z.enum(['symbol_search', 'find_definition', 'find_references', 'get_hover']),
project_root: z.string().optional().describe('Project root directory (auto-detected if not provided)'),
symbol_name: z.string().describe('Symbol name to search/query'),
symbol_kind: z.string().optional().describe('Symbol kind filter (class, function, method, etc.)'),
file_context: z.string().optional().describe('Current file path for proximity ranking'),
limit: z.number().default(50).describe('Maximum number of results to return'),
kind_filter: z.array(z.string()).optional().describe('List of symbol kinds to filter (for symbol_search)'),
file_pattern: z.string().optional().describe('Glob pattern to filter files (for symbol_search)'),
});
type Params = z.infer<typeof ParamsSchema>;
/**
* Result types
*/
interface SymbolInfo {
name: string;
kind: string;
file_path: string;
range: {
start_line: number;
end_line: number;
};
score?: number;
}
interface DefinitionResult {
name: string;
kind: string;
file_path: string;
range: {
start_line: number;
end_line: number;
};
}
interface ReferenceResult {
file_path: string;
line: number;
column: number;
}
interface HoverInfo {
name: string;
kind: string;
signature: string;
file_path: string;
start_line: number;
}
type LSPResult = {
success: boolean;
results?: SymbolInfo[] | DefinitionResult[] | ReferenceResult[] | HoverInfo;
error?: string;
action: string;
metadata?: Record<string, unknown>;
};
/**
* Execute CodexLens Python API call
*/
async function executeCodexLensAPI(
apiFunction: string,
args: Record<string, unknown>,
timeout: number = 30000
): Promise<LSPResult> {
return new Promise((resolve) => {
// Build Python script to call API function
const pythonScript = `
import json
import sys
from dataclasses import is_dataclass, asdict
from codexlens.api import ${apiFunction}
def to_serializable(obj):
"""Recursively convert dataclasses to dicts for JSON serialization."""
if obj is None:
return None
if is_dataclass(obj) and not isinstance(obj, type):
return asdict(obj)
if isinstance(obj, list):
return [to_serializable(item) for item in obj]
if isinstance(obj, dict):
return {key: to_serializable(value) for key, value in obj.items()}
if isinstance(obj, tuple):
return tuple(to_serializable(item) for item in obj)
return obj
try:
args = ${JSON.stringify(args)}
result = ${apiFunction}(**args)
# Convert result to JSON-serializable format
output = to_serializable(result)
print(json.dumps({"success": True, "result": output}))
except Exception as e:
print(json.dumps({"success": False, "error": str(e)}), file=sys.stderr)
sys.exit(1)
`;
const child = spawn(CODEXLENS_VENV, ['-c', pythonScript], {
stdio: ['ignore', 'pipe', 'pipe'],
timeout,
});
let stdout = '';
let stderr = '';
child.stdout.on('data', (data) => {
stdout += data.toString();
});
child.stderr.on('data', (data) => {
stderr += data.toString();
});
child.on('close', (code) => {
if (code !== 0) {
try {
const errorData = JSON.parse(stderr);
resolve({
success: false,
error: errorData.error || 'Unknown error',
action: apiFunction,
});
} catch {
resolve({
success: false,
error: stderr || `Process exited with code ${code}`,
action: apiFunction,
});
}
return;
}
try {
const data = JSON.parse(stdout);
resolve({
success: data.success,
results: data.result,
action: apiFunction,
});
} catch (err) {
resolve({
success: false,
error: `Failed to parse output: ${(err as Error).message}`,
action: apiFunction,
});
}
});
child.on('error', (err) => {
resolve({
success: false,
error: `Failed to execute: ${err.message}`,
action: apiFunction,
});
});
});
}
/**
* Handler: symbol_search
*/
async function handleSymbolSearch(params: Params): Promise<LSPResult> {
const projectRoot = params.project_root || getProjectRoot();
const args: Record<string, unknown> = {
project_root: projectRoot,
query: params.symbol_name,
limit: params.limit,
};
if (params.kind_filter) {
args.kind_filter = params.kind_filter;
}
if (params.file_pattern) {
args.file_pattern = params.file_pattern;
}
return executeCodexLensAPI('workspace_symbols', args);
}
/**
* Handler: find_definition
*/
async function handleFindDefinition(params: Params): Promise<LSPResult> {
const projectRoot = params.project_root || getProjectRoot();
const args: Record<string, unknown> = {
project_root: projectRoot,
symbol_name: params.symbol_name,
limit: params.limit,
};
if (params.symbol_kind) {
args.symbol_kind = params.symbol_kind;
}
if (params.file_context) {
args.file_context = params.file_context;
}
return executeCodexLensAPI('find_definition', args);
}
/**
* Handler: find_references
*/
async function handleFindReferences(params: Params): Promise<LSPResult> {
const projectRoot = params.project_root || getProjectRoot();
const args: Record<string, unknown> = {
project_root: projectRoot,
symbol_name: params.symbol_name,
limit: params.limit,
};
if (params.symbol_kind) {
args.symbol_kind = params.symbol_kind;
}
return executeCodexLensAPI('find_references', args);
}
/**
* Handler: get_hover
*/
async function handleGetHover(params: Params): Promise<LSPResult> {
const projectRoot = params.project_root || getProjectRoot();
const args: Record<string, unknown> = {
project_root: projectRoot,
symbol_name: params.symbol_name,
};
if (params.file_context) {
args.file_path = params.file_context;
}
return executeCodexLensAPI('get_hover', args);
}
/**
* Main handler function
*/
export async function handler(params: Record<string, unknown>): Promise<ToolResult<LSPResult>> {
try {
// Validate parameters
const validatedParams = ParamsSchema.parse(params);
// Route to appropriate handler based on action
let result: LSPResult;
switch (validatedParams.action) {
case 'symbol_search':
result = await handleSymbolSearch(validatedParams);
break;
case 'find_definition':
result = await handleFindDefinition(validatedParams);
break;
case 'find_references':
result = await handleFindReferences(validatedParams);
break;
case 'get_hover':
result = await handleGetHover(validatedParams);
break;
default:
return {
success: false,
error: `Unknown action: ${(validatedParams as any).action}`,
result: null as any,
};
}
if (!result.success) {
return {
success: false,
error: result.error || 'Unknown error',
result: null as any,
};
}
return {
success: true,
result,
};
} catch (err) {
if (err instanceof z.ZodError) {
return {
success: false,
error: `Parameter validation failed: ${err.issues.map((e: z.ZodIssue) => `${e.path.join('.')}: ${e.message}`).join(', ')}`,
result: null as any,
};
}
return {
success: false,
error: `Execution failed: ${(err as Error).message}`,
result: null as any,
};
}
}
/**
* Tool schema for MCP
*/
export const schema: ToolSchema = {
name: 'codex_lens_lsp',
description: `LSP-like code intelligence tool powered by CodexLens indexing.
**Actions:**
- symbol_search: Search for symbols across the workspace
- find_definition: Find the definition of a symbol
- find_references: Find all references to a symbol
- get_hover: Get hover information for a symbol
**Usage Examples:**
Search symbols:
codex_lens_lsp(action="symbol_search", symbol_name="MyClass")
codex_lens_lsp(action="symbol_search", symbol_name="auth", kind_filter=["function", "method"])
codex_lens_lsp(action="symbol_search", symbol_name="User", file_pattern="*.py")
Find definition:
codex_lens_lsp(action="find_definition", symbol_name="authenticate")
codex_lens_lsp(action="find_definition", symbol_name="User", symbol_kind="class")
Find references:
codex_lens_lsp(action="find_references", symbol_name="login")
Get hover info:
codex_lens_lsp(action="get_hover", symbol_name="processPayment")
**Requirements:**
- CodexLens must be installed and indexed: run smart_search(action="init") first
- Python environment with codex-lens package available`,
inputSchema: {
type: 'object',
properties: {
action: {
type: 'string',
enum: ['symbol_search', 'find_definition', 'find_references', 'get_hover'],
description: 'LSP action to perform',
},
symbol_name: {
type: 'string',
description: 'Symbol name to search/query (required)',
},
project_root: {
type: 'string',
description: 'Project root directory (auto-detected if not provided)',
},
symbol_kind: {
type: 'string',
description: 'Symbol kind filter: class, function, method, variable, etc. (optional)',
},
file_context: {
type: 'string',
description: 'Current file path for proximity ranking (optional)',
},
limit: {
type: 'number',
description: 'Maximum number of results to return (default: 50)',
default: 50,
},
kind_filter: {
type: 'array',
items: { type: 'string' },
description: 'List of symbol kinds to include (for symbol_search)',
},
file_pattern: {
type: 'string',
description: 'Glob pattern to filter files (for symbol_search)',
},
},
required: ['action', 'symbol_name'],
},
};

View File

@@ -20,6 +20,8 @@ import * as cliExecutorMod from './cli-executor.js';
import * as smartSearchMod from './smart-search.js';
import { executeInitWithProgress } from './smart-search.js';
// codex_lens removed - functionality integrated into smart_search
import * as codexLensLspMod from './codex-lens-lsp.js';
import * as vscodeLspMod from './vscode-lsp.js';
import * as readFileMod from './read-file.js';
import * as coreMemoryMod from './core-memory.js';
import * as contextCacheMod from './context-cache.js';
@@ -358,6 +360,8 @@ registerTool(toLegacyTool(sessionManagerMod));
registerTool(toLegacyTool(cliExecutorMod));
registerTool(toLegacyTool(smartSearchMod));
// codex_lens removed - functionality integrated into smart_search
registerTool(toLegacyTool(codexLensLspMod));
registerTool(toLegacyTool(vscodeLspMod));
registerTool(toLegacyTool(readFileMod));
registerTool(toLegacyTool(coreMemoryMod));
registerTool(toLegacyTool(contextCacheMod));

317
ccw/src/tools/vscode-lsp.ts Normal file
View File

@@ -0,0 +1,317 @@
/**
* VSCode LSP Tool - Provides LSP-like code intelligence via VSCode Bridge Extension
*
* Features:
* - get_definition: Find symbol definition
* - get_references: Find all symbol references
* - get_hover: Get hover information for symbols
* - get_document_symbols: List all symbols in file
*
* Requirements:
* - ccw-vscode-bridge extension must be running in VSCode
* - File must be open/accessible in VSCode workspace
*/
import { z } from 'zod';
import type { ToolSchema, ToolResult } from '../types/tool.js';
// VSCode Bridge configuration
const BRIDGE_URL = 'http://127.0.0.1:3457';
const REQUEST_TIMEOUT = 10000; // 10 seconds
// Define Zod schema for validation
const ParamsSchema = z.object({
action: z.enum(['get_definition', 'get_references', 'get_hover', 'get_document_symbols']),
file_path: z.string().describe('Absolute path to the file'),
line: z.number().optional().describe('Line number (1-based)'),
character: z.number().optional().describe('Character position (1-based)'),
});
type Params = z.infer<typeof ParamsSchema>;
/**
* Result types from VSCode LSP
*/
interface Position {
line: number;
character: number;
}
interface Range {
start: Position;
end: Position;
}
interface Location {
uri: string;
range: Range;
}
interface HoverContent {
contents: string[];
range?: Range;
}
interface DocumentSymbol {
name: string;
kind: string;
range: Range;
selectionRange: Range;
detail?: string;
parent?: string;
}
type LSPResult = {
success: boolean;
result?: Location | Location[] | HoverContent[] | DocumentSymbol[];
error?: string;
action: string;
};
/**
* Execute HTTP request to VSCode Bridge
*/
async function callVSCodeBridge(
endpoint: string,
payload: Record<string, unknown>
): Promise<LSPResult> {
const url = `${BRIDGE_URL}${endpoint}`;
try {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), REQUEST_TIMEOUT);
const response = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(payload),
signal: controller.signal,
});
clearTimeout(timeoutId);
if (!response.ok) {
const errorBody = await response.json();
return {
success: false,
error: errorBody.error || `HTTP ${response.status}: ${response.statusText}`,
action: endpoint,
};
}
const data = await response.json();
return {
success: data.success !== false,
result: data.result,
action: endpoint,
};
} catch (err) {
const error = err as Error;
if (error.name === 'AbortError') {
return {
success: false,
error: `Request timed out after ${REQUEST_TIMEOUT}ms`,
action: endpoint,
};
}
if ((error as any).code === 'ECONNREFUSED') {
return {
success: false,
error: `Could not connect to VSCode Bridge at ${BRIDGE_URL}. Is the ccw-vscode-bridge extension running in VSCode?`,
action: endpoint,
};
}
return {
success: false,
error: `Request failed: ${error.message}`,
action: endpoint,
};
}
}
/**
* Handler: get_definition
*/
async function handleGetDefinition(params: Params): Promise<LSPResult> {
if (params.line === undefined || params.character === undefined) {
return {
success: false,
error: 'line and character are required for get_definition',
action: 'get_definition',
};
}
return callVSCodeBridge('/get_definition', {
file_path: params.file_path,
line: params.line,
character: params.character,
});
}
/**
* Handler: get_references
*/
async function handleGetReferences(params: Params): Promise<LSPResult> {
if (params.line === undefined || params.character === undefined) {
return {
success: false,
error: 'line and character are required for get_references',
action: 'get_references',
};
}
return callVSCodeBridge('/get_references', {
file_path: params.file_path,
line: params.line,
character: params.character,
});
}
/**
* Handler: get_hover
*/
async function handleGetHover(params: Params): Promise<LSPResult> {
if (params.line === undefined || params.character === undefined) {
return {
success: false,
error: 'line and character are required for get_hover',
action: 'get_hover',
};
}
return callVSCodeBridge('/get_hover', {
file_path: params.file_path,
line: params.line,
character: params.character,
});
}
/**
* Handler: get_document_symbols
*/
async function handleGetDocumentSymbols(params: Params): Promise<LSPResult> {
return callVSCodeBridge('/get_document_symbols', {
file_path: params.file_path,
});
}
/**
* Main handler function
*/
export async function handler(params: Record<string, unknown>): Promise<ToolResult<LSPResult>> {
try {
// Validate parameters
const validatedParams = ParamsSchema.parse(params);
// Route to appropriate handler based on action
let result: LSPResult;
switch (validatedParams.action) {
case 'get_definition':
result = await handleGetDefinition(validatedParams);
break;
case 'get_references':
result = await handleGetReferences(validatedParams);
break;
case 'get_hover':
result = await handleGetHover(validatedParams);
break;
case 'get_document_symbols':
result = await handleGetDocumentSymbols(validatedParams);
break;
default:
return {
success: false,
error: `Unknown action: ${(validatedParams as any).action}`,
result: null as any,
};
}
if (!result.success) {
return {
success: false,
error: result.error || 'Unknown error',
result: null as any,
};
}
return {
success: true,
result,
};
} catch (err) {
if (err instanceof z.ZodError) {
return {
success: false,
error: `Parameter validation failed: ${err.issues.map((e: z.ZodIssue) => `${e.path.join('.')}: ${e.message}`).join(', ')}`,
result: null as any,
};
}
return {
success: false,
error: `Execution failed: ${(err as Error).message}`,
result: null as any,
};
}
}
/**
* Tool schema for MCP
*/
export const schema: ToolSchema = {
name: 'vscode_lsp',
description: `Access live VSCode LSP features via ccw-vscode-bridge extension.
**Actions:**
- get_definition: Find the definition of a symbol at a given position
- get_references: Find all references to a symbol at a given position
- get_hover: Get hover information for a symbol at a given position
- get_document_symbols: List all symbols in a given file
**Usage Examples:**
Find definition:
vscode_lsp(action="get_definition", file_path="/path/to/file.ts", line=10, character=5)
Find references:
vscode_lsp(action="get_references", file_path="/path/to/file.py", line=42, character=15)
Get hover info:
vscode_lsp(action="get_hover", file_path="/path/to/file.go", line=100, character=20)
List document symbols:
vscode_lsp(action="get_document_symbols", file_path="/path/to/file.rs")
**Requirements:**
- ccw-vscode-bridge extension must be installed and active in VSCode
- File must be part of the VSCode workspace
- VSCode Bridge HTTP server running on http://127.0.0.1:3457`,
inputSchema: {
type: 'object',
properties: {
action: {
type: 'string',
enum: ['get_definition', 'get_references', 'get_hover', 'get_document_symbols'],
description: 'LSP action to perform',
},
file_path: {
type: 'string',
description: 'Absolute path to the file (required)',
},
line: {
type: 'number',
description: 'Line number (1-based, required for definition/references/hover)',
},
character: {
type: 'number',
description: 'Character position (1-based, required for definition/references/hover)',
},
},
required: ['action', 'file_path'],
},
};