feat: enhance search, ranking, reranker and CLI tooling across ccw and codex-lens

Major improvements to smart-search, chain-search cascade, ranking pipeline,
reranker factory, CLI history store, codex-lens integration, and uv-manager.
Simplify command-generator skill by inlining phases. Add comprehensive tests.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
catlog22
2026-03-16 20:35:08 +08:00
parent 1cd96b90e8
commit 5a4b18d9b1
73 changed files with 14684 additions and 2442 deletions

View File

@@ -29,7 +29,7 @@ import {
projectExists,
getStorageLocationInstructions
} from '../tools/storage-manager.js';
import { getHistoryStore, findProjectWithExecution } from '../tools/cli-history-store.js';
import { getHistoryStore, findProjectWithExecution, getRegisteredExecutionHistory } from '../tools/cli-history-store.js';
import { createSpinner } from '../utils/ui.js';
import { loadClaudeCliSettings } from '../tools/claude-cli-tools.js';
@@ -421,11 +421,15 @@ async function outputAction(conversationId: string | undefined, options: OutputV
if (!result) {
const hint = options.project
? `in project: ${options.project}`
: 'in current directory or parent directories';
: 'in registered CCW project history';
console.error(chalk.red(`Error: Execution not found: ${conversationId}`));
console.error(chalk.gray(` Searched ${hint}`));
console.error(chalk.gray(' Tip: use the real CCW execution ID, not an outer task label.'));
console.error(chalk.gray(' Capture [CCW_EXEC_ID=...] from stderr, or start with --id <your-id>.'));
console.error(chalk.gray(' Discover IDs via: ccw cli show or ccw cli history'));
console.error(chalk.gray('Usage: ccw cli output <conversation-id> [--project <path>]'));
process.exit(1);
return;
}
if (options.raw) {
@@ -1394,7 +1398,7 @@ async function showAction(options: { all?: boolean }): Promise<void> {
// 2. Get recent history from SQLite
const historyLimit = options.all ? 100 : 20;
const history = await getExecutionHistoryAsync(process.cwd(), { limit: historyLimit, recursive: true });
const history = getRegisteredExecutionHistory({ limit: historyLimit });
const historyById = new Map(history.executions.map(exec => [exec.id, exec]));
// 3. Build unified list: active first, then history (de-duped)
@@ -1595,7 +1599,7 @@ async function historyAction(options: HistoryOptions): Promise<void> {
console.log(chalk.bold.cyan('\n CLI Execution History\n'));
// Use recursive: true to aggregate history from parent and child projects (matches Dashboard behavior)
const history = await getExecutionHistoryAsync(process.cwd(), { limit: parseInt(limit, 10), tool, status, recursive: true });
const history = getRegisteredExecutionHistory({ limit: parseInt(limit, 10), tool, status });
if (history.executions.length === 0) {
console.log(chalk.gray(' No executions found.\n'));
@@ -1650,7 +1654,14 @@ async function detailAction(conversationId: string | undefined): Promise<void> {
process.exit(1);
}
const conversation = getConversationDetail(process.cwd(), conversationId);
let conversation = getConversationDetail(process.cwd(), conversationId);
if (!conversation) {
const found = findProjectWithExecution(conversationId, process.cwd());
if (found) {
conversation = getConversationDetail(found.projectPath, conversationId);
}
}
if (!conversation) {
console.error(chalk.red(`Error: Conversation not found: ${conversationId}`));

View File

@@ -16,7 +16,7 @@ import { spawn } from 'child_process';
import { join, dirname } from 'path';
import { existsSync } from 'fs';
import { fileURLToPath } from 'url';
import { getCodexLensPython } from '../utils/codexlens-path.js';
import { getCodexLensHiddenPython } from '../utils/codexlens-path.js';
import { getCoreMemoryStore } from './core-memory-store.js';
import type { Stage1Output } from './core-memory-store.js';
import { StoragePaths } from '../config/storage-paths.js';
@@ -26,7 +26,7 @@ const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
// Venv paths (reuse CodexLens venv)
const VENV_PYTHON = getCodexLensPython();
const VENV_PYTHON = getCodexLensHiddenPython();
// Script path
const EMBEDDER_SCRIPT = join(__dirname, '..', '..', 'scripts', 'memory_embedder.py');
@@ -116,8 +116,11 @@ function runPython(args: string[], timeout: number = 300000): Promise<string> {
// Spawn Python process
const child = spawn(VENV_PYTHON, [EMBEDDER_SCRIPT, ...args], {
shell: false,
stdio: ['ignore', 'pipe', 'pipe'],
timeout,
windowsHide: true,
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
});
let stdout = '';

View File

@@ -8,7 +8,7 @@ import {
executeCodexLens,
installSemantic,
} from '../../../tools/codex-lens.js';
import { getCodexLensPython } from '../../../utils/codexlens-path.js';
import { getCodexLensHiddenPython } from '../../../utils/codexlens-path.js';
import { spawn } from 'child_process';
import type { GpuMode } from '../../../tools/codex-lens.js';
import { loadLiteLLMApiConfig, getAvailableModelsForType, getProvider, getAllProviders } from '../../../config/litellm-api-config-manager.js';
@@ -59,10 +59,13 @@ except Exception as e:
sys.exit(1)
`;
const pythonPath = getCodexLensPython();
const pythonPath = getCodexLensHiddenPython();
const child = spawn(pythonPath, ['-c', pythonScript], {
shell: false,
stdio: ['ignore', 'pipe', 'pipe'],
timeout,
windowsHide: true,
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
});
let stdout = '';

View File

@@ -126,8 +126,10 @@ export async function handleCodexLensWatcherRoutes(ctx: RouteContext): Promise<b
const args = ['-m', 'codexlens', 'watch', targetPath, '--debounce', String(debounceMs)];
watcherProcess = spawn(pythonPath, args, {
cwd: targetPath,
shell: false,
stdio: ['ignore', 'pipe', 'pipe'],
env: { ...process.env }
windowsHide: true,
env: { ...process.env, PYTHONIOENCODING: 'utf-8' }
});
watcherStats = {

View File

@@ -4,7 +4,11 @@
*/
import { z } from 'zod';
import { spawn } from 'child_process';
import { getSystemPython } from '../../utils/python-utils.js';
import {
getSystemPythonCommand,
parsePythonCommandSpec,
type PythonCommandSpec,
} from '../../utils/python-utils.js';
import {
isUvAvailable,
createCodexLensUvManager
@@ -102,10 +106,11 @@ interface CcwLitellmStatusResponse {
}
function checkCcwLitellmImport(
pythonCmd: string,
options: { timeout: number; shell?: boolean }
pythonCmd: string | PythonCommandSpec,
options: { timeout: number }
): Promise<CcwLitellmEnvCheck> {
const { timeout, shell = false } = options;
const { timeout } = options;
const pythonSpec = typeof pythonCmd === 'string' ? parsePythonCommandSpec(pythonCmd) : pythonCmd;
const sanitizePythonError = (stderrText: string): string | undefined => {
const trimmed = stderrText.trim();
@@ -119,11 +124,12 @@ function checkCcwLitellmImport(
};
return new Promise((resolve) => {
const child = spawn(pythonCmd, ['-c', 'import ccw_litellm; print(ccw_litellm.__version__)'], {
const child = spawn(pythonSpec.command, [...pythonSpec.args, '-c', 'import ccw_litellm; print(ccw_litellm.__version__)'], {
stdio: ['ignore', 'pipe', 'pipe'],
timeout,
windowsHide: true,
shell,
shell: false,
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
});
let stdout = '';
@@ -142,20 +148,20 @@ function checkCcwLitellmImport(
const error = sanitizePythonError(stderr);
if (code === 0 && version) {
resolve({ python: pythonCmd, installed: true, version });
resolve({ python: pythonSpec.display, installed: true, version });
return;
}
if (code === null) {
resolve({ python: pythonCmd, installed: false, error: `Timed out after ${timeout}ms` });
resolve({ python: pythonSpec.display, installed: false, error: `Timed out after ${timeout}ms` });
return;
}
resolve({ python: pythonCmd, installed: false, error: error || undefined });
resolve({ python: pythonSpec.display, installed: false, error: error || undefined });
});
child.on('error', (err) => {
resolve({ python: pythonCmd, installed: false, error: err.message });
resolve({ python: pythonSpec.display, installed: false, error: err.message });
});
});
}
@@ -940,7 +946,7 @@ export async function handleLiteLLMApiRoutes(ctx: RouteContext): Promise<boolean
// Diagnostics only: if not installed in venv, also check system python so users understand mismatches.
// NOTE: `installed` flag remains the CodexLens venv status (we want isolated venv dependencies).
const systemPython = !codexLensVenv.installed
? await checkCcwLitellmImport(getSystemPython(), { timeout: statusTimeout, shell: true })
? await checkCcwLitellmImport(getSystemPythonCommand(), { timeout: statusTimeout })
: undefined;
const result: CcwLitellmStatusResponse = {
@@ -1410,10 +1416,19 @@ export async function handleLiteLLMApiRoutes(ctx: RouteContext): Promise<boolean
// Priority 2: Fallback to system pip uninstall
console.log('[ccw-litellm uninstall] Using pip fallback...');
const pythonCmd = getSystemPython();
const pythonCmd = getSystemPythonCommand();
return new Promise((resolve) => {
const proc = spawn(pythonCmd, ['-m', 'pip', 'uninstall', '-y', 'ccw-litellm'], { shell: true, timeout: 120000 });
const proc = spawn(
pythonCmd.command,
[...pythonCmd.args, '-m', 'pip', 'uninstall', '-y', 'ccw-litellm'],
{
shell: false,
timeout: 120000,
windowsHide: true,
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
},
);
let output = '';
let error = '';
proc.stdout?.on('data', (data) => { output += data.toString(); });

View File

@@ -16,7 +16,7 @@ import { spawn } from 'child_process';
import { join, dirname } from 'path';
import { existsSync } from 'fs';
import { fileURLToPath } from 'url';
import { getCodexLensPython } from '../utils/codexlens-path.js';
import { getCodexLensHiddenPython } from '../utils/codexlens-path.js';
import { StoragePaths, ensureStorageDir } from '../config/storage-paths.js';
// Get directory of this module
@@ -24,7 +24,7 @@ const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
// Venv python path (reuse CodexLens venv)
const VENV_PYTHON = getCodexLensPython();
const VENV_PYTHON = getCodexLensHiddenPython();
// Script path
const EMBEDDER_SCRIPT = join(__dirname, '..', '..', 'scripts', 'unified_memory_embedder.py');
@@ -170,8 +170,11 @@ function runPython<T>(request: Record<string, unknown>, timeout: number = 300000
}
const child = spawn(VENV_PYTHON, [EMBEDDER_SCRIPT], {
shell: false,
stdio: ['pipe', 'pipe', 'pipe'],
timeout,
windowsHide: true,
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
});
let stdout = '';

View File

@@ -1532,6 +1532,197 @@ export function closeAllStores(): void {
storeCache.clear();
}
function collectHistoryDatabasePaths(): string[] {
const projectsDir = join(getCCWHome(), 'projects');
if (!existsSync(projectsDir)) {
return [];
}
const historyDbPaths: string[] = [];
const visitedDirs = new Set<string>();
const skipDirs = new Set(['cache', 'cli-history', 'config', 'memory']);
function scanDirectory(dir: string): void {
const resolvedDir = resolve(dir);
if (visitedDirs.has(resolvedDir)) {
return;
}
visitedDirs.add(resolvedDir);
const historyDb = join(resolvedDir, 'cli-history', 'history.db');
if (existsSync(historyDb)) {
historyDbPaths.push(historyDb);
}
try {
const entries = readdirSync(resolvedDir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isDirectory() || skipDirs.has(entry.name)) {
continue;
}
scanDirectory(join(resolvedDir, entry.name));
}
} catch {
// Ignore unreadable directories during best-effort global scans.
}
}
scanDirectory(projectsDir);
return historyDbPaths;
}
function getConversationLocationColumns(db: Database.Database): {
projectRootSelect: string;
relativePathSelect: string;
} {
const tableInfo = db.prepare(`PRAGMA table_info(conversations)`).all() as Array<{ name: string }>;
const hasProjectRoot = tableInfo.some(col => col.name === 'project_root');
const hasRelativePath = tableInfo.some(col => col.name === 'relative_path');
return {
projectRootSelect: hasProjectRoot ? 'c.project_root AS project_root' : `'' AS project_root`,
relativePathSelect: hasRelativePath ? 'c.relative_path AS relative_path' : `'' AS relative_path`
};
}
function normalizeHistoryTimestamp(updatedAt: unknown, createdAt: unknown): number {
const parsedUpdatedAt = typeof updatedAt === 'string' ? Date.parse(updatedAt) : NaN;
if (!Number.isNaN(parsedUpdatedAt)) {
return parsedUpdatedAt;
}
const parsedCreatedAt = typeof createdAt === 'string' ? Date.parse(createdAt) : NaN;
return Number.isNaN(parsedCreatedAt) ? 0 : parsedCreatedAt;
}
export function getRegisteredExecutionHistory(options: {
limit?: number;
offset?: number;
tool?: string | null;
status?: string | null;
category?: ExecutionCategory | null;
} = {}): {
total: number;
count: number;
executions: (HistoryIndexEntry & { sourceDir?: string })[];
} {
const {
limit = 50,
offset = 0,
tool = null,
status = null,
category = null
} = options;
const perStoreLimit = Math.max(limit + offset, limit, 1);
const allExecutions: (HistoryIndexEntry & { sourceDir?: string })[] = [];
let totalCount = 0;
for (const historyDb of collectHistoryDatabasePaths()) {
let db: Database.Database | null = null;
try {
db = new Database(historyDb, { readonly: true });
const { projectRootSelect, relativePathSelect } = getConversationLocationColumns(db);
let whereClause = '1=1';
const params: Record<string, string | number> = { limit: perStoreLimit };
if (tool) {
whereClause += ' AND c.tool = @tool';
params.tool = tool;
}
if (status) {
whereClause += ' AND c.latest_status = @status';
params.status = status;
}
if (category) {
whereClause += ' AND c.category = @category';
params.category = category;
}
const countRow = db.prepare(`
SELECT COUNT(*) AS count
FROM conversations c
WHERE ${whereClause}
`).get(params) as { count?: number } | undefined;
totalCount += countRow?.count || 0;
const rows = db.prepare(`
SELECT
c.id,
c.created_at AS timestamp,
c.updated_at,
c.tool,
c.latest_status AS status,
c.category,
c.total_duration_ms AS duration_ms,
c.turn_count,
c.prompt_preview,
${projectRootSelect},
${relativePathSelect}
FROM conversations c
WHERE ${whereClause}
ORDER BY c.updated_at DESC
LIMIT @limit
`).all(params) as Array<{
id: string;
timestamp: string;
updated_at?: string;
tool: string;
status: string;
category?: ExecutionCategory;
duration_ms: number;
turn_count?: number;
prompt_preview: unknown;
project_root?: string;
relative_path?: string;
}>;
for (const row of rows) {
allExecutions.push({
id: row.id,
timestamp: row.timestamp,
updated_at: row.updated_at,
tool: row.tool,
status: row.status,
category: row.category || 'user',
duration_ms: row.duration_ms,
turn_count: row.turn_count,
prompt_preview: typeof row.prompt_preview === 'string'
? row.prompt_preview
: (row.prompt_preview ? JSON.stringify(row.prompt_preview) : ''),
sourceDir: row.project_root || row.relative_path || undefined
});
}
} catch {
// Skip databases that are unavailable or incompatible.
} finally {
db?.close();
}
}
allExecutions.sort((a, b) => normalizeHistoryTimestamp(b.updated_at, b.timestamp) - normalizeHistoryTimestamp(a.updated_at, a.timestamp));
const dedupedExecutions: (HistoryIndexEntry & { sourceDir?: string })[] = [];
const seenIds = new Set<string>();
for (const execution of allExecutions) {
if (seenIds.has(execution.id)) {
continue;
}
seenIds.add(execution.id);
dedupedExecutions.push(execution);
}
const pagedExecutions = dedupedExecutions.slice(offset, offset + limit);
return {
total: dedupedExecutions.length || totalCount,
count: pagedExecutions.length,
executions: pagedExecutions
};
}
/**
* Find project path that contains the given execution
* Searches upward through parent directories and all registered projects
@@ -1579,43 +1770,28 @@ export function findProjectWithExecution(
// Strategy 2: Search in all registered projects (global search)
// This covers cases where execution might be in a completely different project tree
const projectsDir = join(getCCWHome(), 'projects');
if (existsSync(projectsDir)) {
for (const historyDb of collectHistoryDatabasePaths()) {
let db: Database.Database | null = null;
try {
const entries = readdirSync(projectsDir, { withFileTypes: true });
db = new Database(historyDb, { readonly: true });
const { projectRootSelect } = getConversationLocationColumns(db);
const row = db.prepare(`
SELECT ${projectRootSelect}
FROM conversations c
WHERE c.id = ?
LIMIT 1
`).get(conversationId) as { project_root?: string } | undefined;
for (const entry of entries) {
if (!entry.isDirectory()) continue;
const projectId = entry.name;
const historyDb = join(projectsDir, projectId, 'cli-history', 'history.db');
if (!existsSync(historyDb)) continue;
try {
// Open and query this database directly
const db = new Database(historyDb, { readonly: true });
const turn = db.prepare(`
SELECT * FROM turns
WHERE conversation_id = ?
ORDER BY turn_number DESC
LIMIT 1
`).get(conversationId);
db.close();
if (turn) {
// Found in this project - return the projectId
// Note: projectPath is set to projectId since we don't have the original path stored
return { projectPath: projectId, projectId };
}
} catch {
// Skip this database (might be corrupted or locked)
continue;
}
if (row?.project_root) {
return {
projectPath: row.project_root,
projectId: getProjectId(row.project_root)
};
}
} catch {
// Failed to read projects directory
// Skip this database (might be corrupted or locked)
} finally {
db?.close();
}
}

View File

@@ -13,10 +13,10 @@ import type { ToolSchema, ToolResult } from '../types/tool.js';
import { spawn } from 'child_process';
import { join } from 'path';
import { getProjectRoot } from '../utils/path-validator.js';
import { getCodexLensPython } from '../utils/codexlens-path.js';
import { getCodexLensHiddenPython } from '../utils/codexlens-path.js';
// CodexLens venv configuration
const CODEXLENS_VENV = getCodexLensPython();
const CODEXLENS_VENV = getCodexLensHiddenPython();
// Define Zod schema for validation
const ParamsSchema = z.object({
@@ -122,8 +122,11 @@ except Exception as e:
`;
const child = spawn(CODEXLENS_VENV, ['-c', pythonScript], {
shell: false,
stdio: ['ignore', 'pipe', 'pipe'],
timeout,
windowsHide: true,
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
});
let stdout = '';

View File

@@ -11,10 +11,15 @@
import { z } from 'zod';
import type { ToolSchema, ToolResult } from '../types/tool.js';
import { spawn, execSync, exec } from 'child_process';
import { existsSync, mkdirSync, rmSync } from 'fs';
import { join } from 'path';
import { getSystemPython, parsePythonVersion, isPythonVersionCompatible } from '../utils/python-utils.js';
import { spawn, spawnSync, execSync, exec, type SpawnOptions, type SpawnSyncOptionsWithStringEncoding } from 'child_process';
import { existsSync, mkdirSync, rmSync, statSync } from 'fs';
import { join, resolve } from 'path';
import {
getSystemPythonCommand,
parsePythonVersion,
isPythonVersionCompatible,
type PythonCommandSpec,
} from '../utils/python-utils.js';
import { EXEC_TIMEOUTS } from '../utils/exec-constants.js';
import {
UvManager,
@@ -26,6 +31,7 @@ import {
getCodexLensDataDir,
getCodexLensVenvDir,
getCodexLensPython,
getCodexLensHiddenPython,
getCodexLensPip,
} from '../utils/codexlens-path.js';
import {
@@ -58,6 +64,10 @@ interface SemanticStatusCache {
let semanticStatusCache: SemanticStatusCache | null = null;
const SEMANTIC_STATUS_TTL = 5 * 60 * 1000; // 5 minutes TTL
type HiddenCodexLensSpawnSyncOptions = Omit<SpawnSyncOptionsWithStringEncoding, 'encoding'> & {
encoding?: BufferEncoding;
};
// Track running indexing process for cancellation
let currentIndexingProcess: ReturnType<typeof spawn> | null = null;
let currentIndexingAborted = false;
@@ -69,13 +79,34 @@ const VENV_CHECK_TIMEOUT = process.platform === 'win32' ? 15000 : 10000;
* Pre-flight check: verify Python 3.9+ is available before attempting bootstrap.
* Returns an error message if Python is not suitable, or null if OK.
*/
function probePythonVersion(
pythonCommand: PythonCommandSpec,
runner: typeof spawnSync = spawnSync,
): string {
const result = runner(
pythonCommand.command,
[...pythonCommand.args, '--version'],
buildCodexLensSpawnSyncOptions({
timeout: EXEC_TIMEOUTS.PYTHON_VERSION,
}),
);
if (result.error) {
throw result.error;
}
const versionOutput = `${result.stdout ?? ''}${result.stderr ?? ''}`.trim();
if (result.status !== 0) {
throw new Error(versionOutput || `Python version probe exited with code ${String(result.status)}`);
}
return versionOutput;
}
function preFlightCheck(): string | null {
try {
const pythonCmd = getSystemPython();
const version = execSync(`${pythonCmd} --version 2>&1`, {
encoding: 'utf8',
timeout: EXEC_TIMEOUTS.PYTHON_VERSION,
}).trim();
const pythonCommand = getSystemPythonCommand();
const version = probePythonVersion(pythonCommand);
const parsed = parsePythonVersion(version);
if (!parsed) {
return `Cannot parse Python version from: "${version}". Ensure Python 3.9+ is installed.`;
@@ -244,7 +275,7 @@ async function checkVenvStatus(force = false): Promise<ReadyStatus> {
return result;
}
const pythonPath = getCodexLensPython();
const pythonPath = getCodexLensHiddenPython();
// Check python executable exists
if (!existsSync(pythonPath)) {
@@ -259,18 +290,21 @@ async function checkVenvStatus(force = false): Promise<ReadyStatus> {
console.log('[PERF][CodexLens] checkVenvStatus spawning Python...');
return new Promise((resolve) => {
const child = spawn(pythonPath, ['-c', 'import sys; import codexlens; import watchdog; print(f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"); print(codexlens.__version__)'], {
stdio: ['ignore', 'pipe', 'pipe'],
timeout: VENV_CHECK_TIMEOUT,
});
const child = spawn(
pythonPath,
['-c', 'import sys; import codexlens; import watchdog; print(f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"); print(codexlens.__version__)'],
buildCodexLensSpawnOptions(venvPath, VENV_CHECK_TIMEOUT, {
stdio: ['ignore', 'pipe', 'pipe'],
}),
);
let stdout = '';
let stderr = '';
child.stdout.on('data', (data) => {
child.stdout?.on('data', (data) => {
stdout += data.toString();
});
child.stderr.on('data', (data) => {
child.stderr?.on('data', (data) => {
stderr += data.toString();
});
@@ -380,18 +414,21 @@ try:
except Exception as e:
print(json.dumps({"available": False, "error": str(e)}))
`;
const child = spawn(getCodexLensPython(), ['-c', checkCode], {
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 15000,
});
const child = spawn(
getCodexLensHiddenPython(),
['-c', checkCode],
buildCodexLensSpawnOptions(getCodexLensVenvDir(), 15000, {
stdio: ['ignore', 'pipe', 'pipe'],
}),
);
let stdout = '';
let stderr = '';
child.stdout.on('data', (data) => {
child.stdout?.on('data', (data) => {
stdout += data.toString();
});
child.stderr.on('data', (data) => {
child.stderr?.on('data', (data) => {
stderr += data.toString();
});
@@ -441,13 +478,16 @@ async function ensureLiteLLMEmbedderReady(): Promise<BootstrapResult> {
// Check if ccw_litellm can be imported
const importStatus = await new Promise<{ ok: boolean; error?: string }>((resolve) => {
const child = spawn(getCodexLensPython(), ['-c', 'import ccw_litellm; print("OK")'], {
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 15000,
});
const child = spawn(
getCodexLensHiddenPython(),
['-c', 'import ccw_litellm; print("OK")'],
buildCodexLensSpawnOptions(getCodexLensVenvDir(), 15000, {
stdio: ['ignore', 'pipe', 'pipe'],
}),
);
let stderr = '';
child.stderr.on('data', (data) => {
child.stderr?.on('data', (data) => {
stderr += data.toString();
});
@@ -522,10 +562,19 @@ async function ensureLiteLLMEmbedderReady(): Promise<BootstrapResult> {
const venvPython = getCodexLensPython();
console.warn(`[CodexLens] pip not found at: ${pipPath}. Attempting to bootstrap pip with ensurepip...`);
try {
execSync(`\"${venvPython}\" -m ensurepip --upgrade`, {
stdio: 'inherit',
timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL,
});
const ensurePipResult = spawnSync(
venvPython,
['-m', 'ensurepip', '--upgrade'],
buildCodexLensSpawnSyncOptions({
timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL,
}),
);
if (ensurePipResult.error) {
throw ensurePipResult.error;
}
if (ensurePipResult.status !== 0) {
throw new Error(`ensurepip exited with code ${String(ensurePipResult.status)}`);
}
} catch (err) {
console.warn(`[CodexLens] ensurepip failed: ${(err as Error).message}`);
}
@@ -549,13 +598,36 @@ async function ensureLiteLLMEmbedderReady(): Promise<BootstrapResult> {
try {
if (localPath) {
const pipFlag = editable ? '-e' : '';
const pipInstallSpec = editable ? `"${localPath}"` : `"${localPath}"`;
const pipArgs = editable ? ['install', '-e', localPath] : ['install', localPath];
console.log(`[CodexLens] Installing ccw-litellm from local path with pip: ${localPath} (editable: ${editable})`);
execSync(`"${pipPath}" install ${pipFlag} ${pipInstallSpec}`.replace(/ +/g, ' '), { stdio: 'inherit', timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL });
const installResult = spawnSync(
pipPath,
pipArgs,
buildCodexLensSpawnSyncOptions({
timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL,
}),
);
if (installResult.error) {
throw installResult.error;
}
if (installResult.status !== 0) {
throw new Error(`pip install exited with code ${String(installResult.status)}`);
}
} else {
console.log('[CodexLens] Installing ccw-litellm from PyPI with pip...');
execSync(`"${pipPath}" install ccw-litellm`, { stdio: 'inherit', timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL });
const installResult = spawnSync(
pipPath,
['install', 'ccw-litellm'],
buildCodexLensSpawnSyncOptions({
timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL,
}),
);
if (installResult.error) {
throw installResult.error;
}
if (installResult.status !== 0) {
throw new Error(`pip install exited with code ${String(installResult.status)}`);
}
}
return {
@@ -609,7 +681,7 @@ interface PythonEnvInfo {
* DirectML requires: 64-bit Python, version 3.8-3.12
*/
async function checkPythonEnvForDirectML(): Promise<PythonEnvInfo> {
const pythonPath = getCodexLensPython();
const pythonPath = getCodexLensHiddenPython();
if (!existsSync(pythonPath)) {
return { version: '', majorMinor: '', architecture: 0, compatible: false, error: 'Python not found in venv' };
@@ -619,8 +691,19 @@ async function checkPythonEnvForDirectML(): Promise<PythonEnvInfo> {
// Get Python version and architecture in one call
// Use % formatting instead of f-string to avoid Windows shell escaping issues with curly braces
const checkScript = `import sys, struct; print('%d.%d.%d|%d' % (sys.version_info.major, sys.version_info.minor, sys.version_info.micro, struct.calcsize('P') * 8))`;
const result = execSync(`"${pythonPath}" -c "${checkScript}"`, { encoding: 'utf-8', timeout: 10000 }).trim();
const [version, archStr] = result.split('|');
const result = spawnSync(
pythonPath,
['-c', checkScript],
buildCodexLensSpawnSyncOptions({ timeout: 10000 }),
);
if (result.error) {
throw result.error;
}
const output = `${result.stdout ?? ''}${result.stderr ?? ''}`.trim();
if (result.status !== 0) {
throw new Error(output || `Python probe exited with code ${String(result.status)}`);
}
const [version, archStr] = output.split('|');
const architecture = parseInt(archStr, 10);
const [major, minor] = version.split('.').map(Number);
const majorMinor = `${major}.${minor}`;
@@ -898,15 +981,18 @@ async function installSemantic(gpuMode: GpuMode = 'cpu'): Promise<BootstrapResul
console.log(`[CodexLens] Packages: ${packages.join(', ')}`);
// Install ONNX Runtime first with force-reinstall to ensure clean state
const installOnnx = spawn(pipPath, ['install', '--force-reinstall', onnxPackage], {
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 600000, // 10 minutes for GPU packages
});
const installOnnx = spawn(
pipPath,
['install', '--force-reinstall', onnxPackage],
buildCodexLensSpawnOptions(getCodexLensVenvDir(), 600000, {
stdio: ['ignore', 'pipe', 'pipe'],
}),
);
let onnxStdout = '';
let onnxStderr = '';
installOnnx.stdout.on('data', (data) => {
installOnnx.stdout?.on('data', (data) => {
onnxStdout += data.toString();
const line = data.toString().trim();
if (line.includes('Downloading') || line.includes('Installing')) {
@@ -914,7 +1000,7 @@ async function installSemantic(gpuMode: GpuMode = 'cpu'): Promise<BootstrapResul
}
});
installOnnx.stderr.on('data', (data) => {
installOnnx.stderr?.on('data', (data) => {
onnxStderr += data.toString();
});
@@ -927,15 +1013,18 @@ async function installSemantic(gpuMode: GpuMode = 'cpu'): Promise<BootstrapResul
console.log(`[CodexLens] ${onnxPackage} installed successfully`);
// Now install remaining packages
const child = spawn(pipPath, ['install', ...packages], {
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 600000,
});
const child = spawn(
pipPath,
['install', ...packages],
buildCodexLensSpawnOptions(getCodexLensVenvDir(), 600000, {
stdio: ['ignore', 'pipe', 'pipe'],
}),
);
let stdout = '';
let stderr = '';
child.stdout.on('data', (data) => {
child.stdout?.on('data', (data) => {
stdout += data.toString();
const line = data.toString().trim();
if (line.includes('Downloading') || line.includes('Installing') || line.includes('Collecting')) {
@@ -943,7 +1032,7 @@ async function installSemantic(gpuMode: GpuMode = 'cpu'): Promise<BootstrapResul
}
});
child.stderr.on('data', (data) => {
child.stderr?.on('data', (data) => {
stderr += data.toString();
});
@@ -1028,8 +1117,20 @@ async function bootstrapVenv(): Promise<BootstrapResult> {
if (!existsSync(venvDir)) {
try {
console.log('[CodexLens] Creating virtual environment...');
const pythonCmd = getSystemPython();
execSync(`${pythonCmd} -m venv "${venvDir}"`, { stdio: 'inherit', timeout: EXEC_TIMEOUTS.PROCESS_SPAWN });
const pythonCmd = getSystemPythonCommand();
const createResult = spawnSync(
pythonCmd.command,
[...pythonCmd.args, '-m', 'venv', venvDir],
buildCodexLensSpawnSyncOptions({
timeout: EXEC_TIMEOUTS.PROCESS_SPAWN,
}),
);
if (createResult.error) {
throw createResult.error;
}
if (createResult.status !== 0) {
throw new Error(`venv creation exited with code ${String(createResult.status)}`);
}
} catch (err) {
return {
success: false,
@@ -1049,10 +1150,19 @@ async function bootstrapVenv(): Promise<BootstrapResult> {
const venvPython = getCodexLensPython();
console.warn(`[CodexLens] pip not found at: ${pipPath}. Attempting to bootstrap pip with ensurepip...`);
try {
execSync(`\"${venvPython}\" -m ensurepip --upgrade`, {
stdio: 'inherit',
timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL,
});
const ensurePipResult = spawnSync(
venvPython,
['-m', 'ensurepip', '--upgrade'],
buildCodexLensSpawnSyncOptions({
timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL,
}),
);
if (ensurePipResult.error) {
throw ensurePipResult.error;
}
if (ensurePipResult.status !== 0) {
throw new Error(`ensurepip exited with code ${String(ensurePipResult.status)}`);
}
} catch (err) {
console.warn(`[CodexLens] ensurepip failed: ${(err as Error).message}`);
}
@@ -1063,8 +1173,20 @@ async function bootstrapVenv(): Promise<BootstrapResult> {
console.warn('[CodexLens] pip still missing after ensurepip; recreating venv with system Python...');
try {
rmSync(venvDir, { recursive: true, force: true });
const pythonCmd = getSystemPython();
execSync(`${pythonCmd} -m venv \"${venvDir}\"`, { stdio: 'inherit', timeout: EXEC_TIMEOUTS.PROCESS_SPAWN });
const pythonCmd = getSystemPythonCommand();
const recreateResult = spawnSync(
pythonCmd.command,
[...pythonCmd.args, '-m', 'venv', venvDir],
buildCodexLensSpawnSyncOptions({
timeout: EXEC_TIMEOUTS.PROCESS_SPAWN,
}),
);
if (recreateResult.error) {
throw recreateResult.error;
}
if (recreateResult.status !== 0) {
throw new Error(`venv recreation exited with code ${String(recreateResult.status)}`);
}
} catch (err) {
return {
success: false,
@@ -1090,9 +1212,21 @@ async function bootstrapVenv(): Promise<BootstrapResult> {
}
const editable = isDevEnvironment() && !discovery.insideNodeModules;
const pipFlag = editable ? ' -e' : '';
const pipArgs = editable ? ['install', '-e', discovery.path] : ['install', discovery.path];
console.log(`[CodexLens] Installing from local path: ${discovery.path} (editable: ${editable})`);
execSync(`"${pipPath}" install${pipFlag} "${discovery.path}"`, { stdio: 'inherit', timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL });
const installResult = spawnSync(
pipPath,
pipArgs,
buildCodexLensSpawnSyncOptions({
timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL,
}),
);
if (installResult.error) {
throw installResult.error;
}
if (installResult.status !== 0) {
throw new Error(`pip install exited with code ${String(installResult.status)}`);
}
// Clear cache after successful installation
clearVenvStatusCache();
@@ -1237,6 +1371,12 @@ function shouldRetryWithoutLanguageFilters(args: string[], error?: string): bool
return args.includes('--language') && Boolean(error && /Got unexpected extra arguments?\b/i.test(error));
}
function shouldRetryWithLegacySearchArgs(args: string[], error?: string): boolean {
return args[0] === 'search'
&& (args.includes('--limit') || args.includes('--mode') || args.includes('--offset'))
&& Boolean(error && /Got unexpected extra arguments?\b/i.test(error));
}
function stripFlag(args: string[], flag: string): string[] {
return args.filter((arg) => arg !== flag);
}
@@ -1253,6 +1393,29 @@ function stripOptionWithValues(args: string[], option: string): string[] {
return nextArgs;
}
function stripSearchCompatibilityOptions(args: string[]): string[] {
return stripOptionWithValues(
stripOptionWithValues(
stripOptionWithValues(args, '--offset'),
'--mode',
),
'--limit',
);
}
function appendWarning(existing: string | undefined, next: string | undefined): string | undefined {
if (!next) {
return existing;
}
if (!existing) {
return next;
}
if (existing.includes(next)) {
return existing;
}
return `${existing} ${next}`;
}
function shouldRetryWithAstGrepPreference(args: string[], error?: string): boolean {
return !args.includes('--use-astgrep')
&& !args.includes('--no-use-astgrep')
@@ -1334,6 +1497,142 @@ function tryExtractJsonPayload(raw: string): unknown | null {
}
}
function parseLegacySearchPaths(output: string | undefined, cwd: string): string[] {
const lines = stripAnsiCodes(output || '')
.split(/\r?\n/)
.map((line) => line.trim())
.filter(Boolean);
const filePaths: string[] = [];
for (const line of lines) {
if (line.includes('RuntimeWarning:') || line.startsWith('warn(') || line.startsWith('Warning:')) {
continue;
}
const candidate = /^[a-zA-Z]:[\\/]|^\//.test(line)
? line
: resolve(cwd, line);
try {
if (statSync(candidate).isFile()) {
filePaths.push(candidate);
}
} catch {
continue;
}
}
return [...new Set(filePaths)];
}
function buildLegacySearchPayload(query: string, filePaths: string[], limit: number): Record<string, unknown> {
const results = filePaths.slice(0, limit).map((path, index) => ({
path,
score: Math.max(0.1, 1 - index * 0.05),
excerpt: '',
content: '',
source: 'legacy_text_output',
symbol: null,
}));
return {
success: true,
result: {
query,
count: filePaths.length,
results,
},
};
}
function buildLegacySearchFilesPayload(query: string, filePaths: string[], limit: number): Record<string, unknown> {
return {
success: true,
result: {
query,
count: filePaths.length,
files: filePaths.slice(0, limit),
},
};
}
function buildEmptySearchPayload(query: string, filesOnly: boolean): Record<string, unknown> {
return filesOnly
? {
success: true,
result: {
query,
count: 0,
files: [],
},
}
: {
success: true,
result: {
query,
count: 0,
results: [],
},
};
}
function normalizeSearchCommandResult(
result: ExecuteResult,
options: { query: string; cwd: string; limit: number; filesOnly: boolean },
): ExecuteResult {
if (!result.success) {
return result;
}
const { query, cwd, limit, filesOnly } = options;
const rawOutput = typeof result.output === 'string' ? result.output : '';
const parsedPayload = rawOutput ? tryExtractJsonPayload(rawOutput) : null;
if (parsedPayload !== null) {
if (filesOnly) {
result.files = parsedPayload;
} else {
result.results = parsedPayload;
}
delete result.output;
return result;
}
const legacyPaths = parseLegacySearchPaths(rawOutput, cwd);
if (legacyPaths.length > 0) {
const warning = filesOnly
? 'CodexLens CLI returned legacy plain-text file output; synthesized JSON-compatible search_files results.'
: 'CodexLens CLI returned legacy plain-text search output; synthesized JSON-compatible search results.';
if (filesOnly) {
result.files = buildLegacySearchFilesPayload(query, legacyPaths, limit);
} else {
result.results = buildLegacySearchPayload(query, legacyPaths, limit);
}
delete result.output;
result.warning = appendWarning(result.warning, warning);
result.message = appendWarning(result.message, warning);
return result;
}
const warning = rawOutput.trim()
? (filesOnly
? 'CodexLens CLI returned non-JSON search_files output; synthesized an empty JSON-compatible fallback payload.'
: 'CodexLens CLI returned non-JSON search output; synthesized an empty JSON-compatible fallback payload.')
: (filesOnly
? 'CodexLens CLI returned empty stdout in JSON mode for search_files; synthesized an empty JSON-compatible fallback payload.'
: 'CodexLens CLI returned empty stdout in JSON mode for search; synthesized an empty JSON-compatible fallback payload.');
if (filesOnly) {
result.files = buildEmptySearchPayload(query, true);
} else {
result.results = buildEmptySearchPayload(query, false);
}
delete result.output;
result.warning = appendWarning(result.warning, warning);
result.message = appendWarning(result.message, warning);
return result;
}
function extractStructuredError(payload: unknown): string | null {
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
return null;
@@ -1394,6 +1693,11 @@ async function executeCodexLens(args: string[], options: ExecuteOptions = {}): P
transform: (currentArgs: string[]) => stripOptionWithValues(currentArgs, '--language'),
warning: 'CodexLens CLI rejected --language filters; retried without language scoping.',
},
{
shouldRetry: shouldRetryWithLegacySearchArgs,
transform: stripSearchCompatibilityOptions,
warning: 'CodexLens CLI rejected search --limit/--mode compatibility flags; retried with minimal legacy search args.',
},
{
shouldRetry: shouldRetryWithAstGrepPreference,
transform: (currentArgs: string[]) => [...currentArgs, '--use-astgrep'],
@@ -1441,6 +1745,32 @@ async function executeCodexLens(args: string[], options: ExecuteOptions = {}): P
};
}
function buildCodexLensSpawnOptions(cwd: string, timeout: number, overrides: SpawnOptions = {}): SpawnOptions {
const { env, ...rest } = overrides;
return {
cwd,
shell: false,
timeout,
windowsHide: true,
env: { ...process.env, PYTHONIOENCODING: 'utf-8', ...env },
...rest,
};
}
function buildCodexLensSpawnSyncOptions(
overrides: HiddenCodexLensSpawnSyncOptions = {},
): SpawnSyncOptionsWithStringEncoding {
const { env, encoding, ...rest } = overrides;
return {
shell: false,
windowsHide: true,
stdio: ['ignore', 'pipe', 'pipe'],
env: { ...process.env, PYTHONIOENCODING: 'utf-8', ...env },
...rest,
encoding: encoding ?? 'utf8',
};
}
async function executeCodexLensOnce(args: string[], options: ExecuteOptions = {}): Promise<ExecuteResult> {
const { timeout = 300000, cwd = process.cwd(), onProgress } = options; // Default 5 min
@@ -1456,13 +1786,7 @@ async function executeCodexLensOnce(args: string[], options: ExecuteOptions = {}
// spawn's cwd option handles drive changes correctly on Windows
const spawnArgs = ['-m', 'codexlens', ...args];
const child = spawn(getCodexLensPython(), spawnArgs, {
cwd,
shell: false, // CRITICAL: Prevent command injection
timeout,
// Ensure proper encoding on Windows
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
});
const child = spawn(getCodexLensHiddenPython(), spawnArgs, buildCodexLensSpawnOptions(cwd, timeout));
// Track indexing process for cancellation (only for init commands)
const isIndexingCommand = args.includes('init');
@@ -1566,13 +1890,22 @@ async function executeCodexLensOnce(args: string[], options: ExecuteOptions = {}
}
}
const trimmedStdout = stdout.trim();
if (code === 0) {
safeResolve({ success: true, output: stdout.trim() });
const warning = args.includes('--json') && trimmedStdout.length === 0
? `CodexLens CLI exited successfully but produced empty stdout in JSON mode for ${args[0] ?? 'command'}.`
: undefined;
safeResolve({
success: true,
output: trimmedStdout || undefined,
warning,
message: warning,
});
} else {
safeResolve({
success: false,
error: extractCodexLensFailure(stdout, stderr, code),
output: stdout.trim() || undefined,
output: trimmedStdout || undefined,
});
}
});
@@ -1627,18 +1960,12 @@ async function searchCode(params: Params): Promise<ExecuteResult> {
args.push('--enrich');
}
const result = await executeCodexLens(args, { cwd: path });
if (result.success && result.output) {
try {
result.results = JSON.parse(result.output);
delete result.output;
} catch {
// Keep raw output if JSON parse fails
}
}
return result;
return normalizeSearchCommandResult(await executeCodexLens(args, { cwd: path }), {
query,
cwd: path,
limit,
filesOnly: false,
});
}
/**
@@ -1672,18 +1999,12 @@ async function searchFiles(params: Params): Promise<ExecuteResult> {
args.push('--enrich');
}
const result = await executeCodexLens(args, { cwd: path });
if (result.success && result.output) {
try {
result.files = JSON.parse(result.output);
delete result.output;
} catch {
// Keep raw output if JSON parse fails
}
}
return result;
return normalizeSearchCommandResult(await executeCodexLens(args, { cwd: path }), {
query,
cwd: path,
limit,
filesOnly: true,
});
}
/**
@@ -2185,11 +2506,18 @@ export {
// Export Python path for direct spawn usage (e.g., watcher)
export function getVenvPythonPath(): string {
return getCodexLensPython();
return getCodexLensHiddenPython();
}
export type { GpuMode, PythonEnvInfo };
export const __testables = {
normalizeSearchCommandResult,
parseLegacySearchPaths,
buildCodexLensSpawnOptions,
probePythonVersion,
};
// Backward-compatible export for tests
export const codexLensTool = {
name: schema.name,

View File

@@ -12,7 +12,7 @@
import { spawn } from 'child_process';
import { existsSync } from 'fs';
import { join } from 'path';
import { getCodexLensPython, getCodexLensVenvDir } from '../utils/codexlens-path.js';
import { getCodexLensPython, getCodexLensHiddenPython, getCodexLensVenvDir } from '../utils/codexlens-path.js';
export interface LiteLLMConfig {
pythonPath?: string; // Default: CodexLens venv Python
@@ -24,7 +24,7 @@ export interface LiteLLMConfig {
const IS_WINDOWS = process.platform === 'win32';
const CODEXLENS_VENV = getCodexLensVenvDir();
const VENV_BIN_DIR = IS_WINDOWS ? 'Scripts' : 'bin';
const PYTHON_EXECUTABLE = IS_WINDOWS ? 'python.exe' : 'python';
const PYTHON_EXECUTABLE = IS_WINDOWS ? 'pythonw.exe' : 'python';
/**
* Get the Python path from CodexLens venv
@@ -36,6 +36,10 @@ export function getCodexLensVenvPython(): string {
if (existsSync(venvPython)) {
return venvPython;
}
const hiddenPython = getCodexLensHiddenPython();
if (existsSync(hiddenPython)) {
return hiddenPython;
}
// Fallback to system Python if venv not available
return 'python';
}
@@ -46,10 +50,14 @@ export function getCodexLensVenvPython(): string {
* @returns Path to Python executable
*/
export function getCodexLensPythonPath(): string {
const codexLensPython = getCodexLensPython();
const codexLensPython = getCodexLensHiddenPython();
if (existsSync(codexLensPython)) {
return codexLensPython;
}
const fallbackPython = getCodexLensPython();
if (existsSync(fallbackPython)) {
return fallbackPython;
}
// Fallback to system Python if venv not available
return 'python';
}
@@ -100,8 +108,10 @@ export class LiteLLMClient {
return new Promise((resolve, reject) => {
const proc = spawn(this.pythonPath, ['-m', 'ccw_litellm.cli', ...args], {
shell: false,
windowsHide: true,
stdio: ['pipe', 'pipe', 'pipe'],
env: { ...process.env }
env: { ...process.env, PYTHONIOENCODING: 'utf-8' }
});
let stdout = '';

View File

@@ -20,7 +20,7 @@
import { z } from 'zod';
import type { ToolSchema, ToolResult } from '../types/tool.js';
import { spawn, execSync } from 'child_process';
import { spawn, spawnSync, type SpawnOptions } from 'child_process';
import { existsSync, readFileSync, statSync } from 'fs';
import { dirname, join, resolve } from 'path';
import {
@@ -346,8 +346,12 @@ interface SearchMetadata {
api_max_workers?: number;
endpoint_count?: number;
use_gpu?: boolean;
reranker_enabled?: boolean;
reranker_backend?: string;
reranker_model?: string;
cascade_strategy?: string;
staged_stage2_mode?: string;
static_graph_enabled?: boolean;
preset?: string;
}
@@ -474,8 +478,52 @@ const CODEX_LENS_FTS_COMPATIBILITY_PATTERNS = [
];
let codexLensFtsBackendBroken = false;
const autoInitJobs = new Map<string, { startedAt: number; languages?: string[] }>();
const autoEmbedJobs = new Map<string, { startedAt: number; backend?: string; model?: string }>();
type SmartSearchRuntimeOverrides = {
checkSemanticStatus?: typeof checkSemanticStatus;
getVenvPythonPath?: typeof getVenvPythonPath;
spawnProcess?: typeof spawn;
now?: () => number;
};
const runtimeOverrides: SmartSearchRuntimeOverrides = {};
function getSemanticStatusRuntime(): typeof checkSemanticStatus {
return runtimeOverrides.checkSemanticStatus ?? checkSemanticStatus;
}
function getVenvPythonPathRuntime(): typeof getVenvPythonPath {
return runtimeOverrides.getVenvPythonPath ?? getVenvPythonPath;
}
function getSpawnRuntime(): typeof spawn {
return runtimeOverrides.spawnProcess ?? spawn;
}
function getNowRuntime(): number {
return (runtimeOverrides.now ?? Date.now)();
}
function buildSmartSearchSpawnOptions(cwd: string, overrides: SpawnOptions = {}): SpawnOptions {
const { env, ...rest } = overrides;
return {
cwd,
shell: false,
windowsHide: true,
env: { ...process.env, PYTHONIOENCODING: 'utf-8', ...env },
...rest,
};
}
function shouldDetachBackgroundSmartSearchProcess(): boolean {
// On Windows, detached Python children can still create a transient console
// window even when windowsHide is set. Background warmup only needs to outlive
// the current request, not the MCP server process.
return process.platform !== 'win32';
}
/**
* Truncate content to specified length with ellipsis
* @param content - The content to truncate
@@ -523,6 +571,58 @@ interface RipgrepQueryModeResolution {
warning?: string;
}
const GENERATED_QUERY_RE = /(?<!\w)(dist|build|out|coverage|htmlcov|generated|bundle|compiled|artifact|artifacts|\.workflow)(?!\w)/i;
const ENV_STYLE_QUERY_RE = /\b[A-Z][A-Z0-9]+(?:_[A-Z0-9]+)+\b/;
const TOPIC_TOKEN_RE = /[A-Za-z][A-Za-z0-9]*/g;
const LEXICAL_PRIORITY_SURFACE_TOKENS = new Set([
'config',
'configs',
'configuration',
'configurations',
'setting',
'settings',
'backend',
'backends',
'environment',
'env',
'variable',
'variables',
'factory',
'factories',
'override',
'overrides',
'option',
'options',
'flag',
'flags',
'mode',
'modes',
]);
const LEXICAL_PRIORITY_FOCUS_TOKENS = new Set([
'embedding',
'embeddings',
'reranker',
'rerankers',
'onnx',
'api',
'litellm',
'fastembed',
'local',
'legacy',
'stage',
'stage2',
'stage3',
'stage4',
'precomputed',
'realtime',
'static',
'global',
'graph',
'selection',
'model',
'models',
]);
function sanitizeSearchQuery(query: string | undefined): string | undefined {
if (!query) {
return query;
@@ -676,6 +776,18 @@ function noteCodexLensFtsCompatibility(error: string | undefined): boolean {
return true;
}
function shouldSurfaceCodexLensFtsCompatibilityWarning(options: {
compatibilityTriggeredThisQuery: boolean;
skipExactDueToCompatibility: boolean;
ripgrepResultCount: number;
}): boolean {
if (options.ripgrepResultCount > 0) {
return false;
}
return options.compatibilityTriggeredThisQuery || options.skipExactDueToCompatibility;
}
function summarizeBackendError(error: string | undefined): string {
const cleanError = stripAnsi(error || '').trim();
if (!cleanError) {
@@ -765,6 +877,61 @@ function hasCentralizedVectorArtifacts(indexRoot: unknown): boolean {
].every((artifactPath) => existsSync(artifactPath));
}
function asObjectRecord(value: unknown): Record<string, unknown> | undefined {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
return undefined;
}
return value as Record<string, unknown>;
}
function asFiniteNumber(value: unknown): number | undefined {
if (typeof value !== 'number' || !Number.isFinite(value)) {
return undefined;
}
return value;
}
function asBoolean(value: unknown): boolean | undefined {
return typeof value === 'boolean' ? value : undefined;
}
function extractEmbeddingsStatusSummary(embeddingsData: unknown): {
coveragePercent: number;
totalChunks: number;
hasEmbeddings: boolean;
} {
const embeddings = asObjectRecord(embeddingsData) ?? {};
const root = asObjectRecord(embeddings.root) ?? embeddings;
const centralized = asObjectRecord(embeddings.centralized);
const totalIndexes = asFiniteNumber(root.total_indexes)
?? asFiniteNumber(embeddings.total_indexes)
?? 0;
const indexesWithEmbeddings = asFiniteNumber(root.indexes_with_embeddings)
?? asFiniteNumber(embeddings.indexes_with_embeddings)
?? 0;
const totalChunks = asFiniteNumber(root.total_chunks)
?? asFiniteNumber(embeddings.total_chunks)
?? 0;
const coveragePercent = asFiniteNumber(root.coverage_percent)
?? asFiniteNumber(embeddings.coverage_percent)
?? (totalIndexes > 0 ? (indexesWithEmbeddings / totalIndexes) * 100 : 0);
const hasEmbeddings = asBoolean(root.has_embeddings)
?? asBoolean(centralized?.usable)
?? (totalChunks > 0 || indexesWithEmbeddings > 0 || coveragePercent > 0);
return {
coveragePercent,
totalChunks,
hasEmbeddings,
};
}
function selectEmbeddingsStatusPayload(statusData: unknown): Record<string, unknown> {
const status = asObjectRecord(statusData) ?? {};
return asObjectRecord(status.embeddings_status) ?? asObjectRecord(status.embeddings) ?? {};
}
function collectBackendError(
errors: string[],
backendName: string,
@@ -825,8 +992,77 @@ function formatSmartSearchCommand(action: string, pathValue: string, extraParams
return `smart_search(${args.join(', ')})`;
}
function parseOptionalBooleanEnv(raw: string | undefined): boolean | undefined {
const normalized = raw?.trim().toLowerCase();
if (!normalized) {
return undefined;
}
if (['1', 'true', 'on', 'yes'].includes(normalized)) {
return true;
}
if (['0', 'false', 'off', 'no'].includes(normalized)) {
return false;
}
return undefined;
}
function isAutoEmbedMissingEnabled(config: CodexLensConfig | null | undefined): boolean {
return config?.embedding_auto_embed_missing !== false;
const envOverride = parseOptionalBooleanEnv(process.env.CODEXLENS_AUTO_EMBED_MISSING);
if (envOverride !== undefined) {
return envOverride;
}
if (process.platform === 'win32') {
return false;
}
if (typeof config?.embedding_auto_embed_missing === 'boolean') {
return config.embedding_auto_embed_missing;
}
return true;
}
function isAutoInitMissingEnabled(): boolean {
const envOverride = parseOptionalBooleanEnv(process.env.CODEXLENS_AUTO_INIT_MISSING);
if (envOverride !== undefined) {
return envOverride;
}
return process.platform !== 'win32';
}
function getAutoEmbedMissingDisabledReason(config: CodexLensConfig | null | undefined): string {
const envOverride = parseOptionalBooleanEnv(process.env.CODEXLENS_AUTO_EMBED_MISSING);
if (envOverride === false) {
return 'Automatic embedding warmup is disabled by CODEXLENS_AUTO_EMBED_MISSING=false.';
}
if (config?.embedding_auto_embed_missing === false) {
return 'Automatic embedding warmup is disabled by embedding.auto_embed_missing=false.';
}
if (process.platform === 'win32') {
return 'Automatic embedding warmup is disabled by default on Windows even if CodexLens config resolves auto_embed_missing=true. Set CODEXLENS_AUTO_EMBED_MISSING=true to opt in.';
}
return 'Automatic embedding warmup is disabled.';
}
function getAutoInitMissingDisabledReason(): string {
const envOverride = parseOptionalBooleanEnv(process.env.CODEXLENS_AUTO_INIT_MISSING);
if (envOverride === false) {
return 'Automatic static index warmup is disabled by CODEXLENS_AUTO_INIT_MISSING=false.';
}
if (process.platform === 'win32') {
return 'Automatic static index warmup is disabled by default on Windows. Set CODEXLENS_AUTO_INIT_MISSING=true to opt in.';
}
return 'Automatic static index warmup is disabled.';
}
function buildIndexSuggestions(indexStatus: IndexStatus, scope: SearchScope): SearchSuggestion[] | undefined {
@@ -930,29 +1166,24 @@ async function checkIndexStatus(path: string = '.'): Promise<IndexStatus> {
const status = parsed.result || parsed;
// Get embeddings coverage from comprehensive status
const embeddingsData = status.embeddings || {};
const totalIndexes = Number(embeddingsData.total_indexes || 0);
const indexesWithEmbeddings = Number(embeddingsData.indexes_with_embeddings || 0);
const totalChunks = Number(embeddingsData.total_chunks || 0);
const hasCentralizedVectors = hasCentralizedVectorArtifacts(status.index_root);
let embeddingsCoverage = typeof embeddingsData.coverage_percent === 'number'
? embeddingsData.coverage_percent
: (totalIndexes > 0 ? (indexesWithEmbeddings / totalIndexes) * 100 : 0);
if (hasCentralizedVectors) {
embeddingsCoverage = Math.max(embeddingsCoverage, 100);
}
const embeddingsData = selectEmbeddingsStatusPayload(status);
const legacyEmbeddingsData = asObjectRecord(status.embeddings) ?? {};
const embeddingsSummary = extractEmbeddingsStatusSummary(embeddingsData);
const totalIndexes = Number(legacyEmbeddingsData.total_indexes || asObjectRecord(embeddingsData)?.total_indexes || 0);
const embeddingsCoverage = embeddingsSummary.coveragePercent;
const totalChunks = embeddingsSummary.totalChunks;
const indexed = Boolean(status.projects_count > 0 || status.total_files > 0 || status.index_root || totalIndexes > 0 || totalChunks > 0);
const has_embeddings = indexesWithEmbeddings > 0 || embeddingsCoverage > 0 || totalChunks > 0 || hasCentralizedVectors;
const has_embeddings = embeddingsSummary.hasEmbeddings;
// Extract model info if available
const modelInfoData = embeddingsData.model_info;
const modelInfoData = asObjectRecord(embeddingsData.model_info);
const modelInfo: ModelInfo | undefined = modelInfoData ? {
model_profile: modelInfoData.model_profile,
model_name: modelInfoData.model_name,
embedding_dim: modelInfoData.embedding_dim,
backend: modelInfoData.backend,
created_at: modelInfoData.created_at,
updated_at: modelInfoData.updated_at,
model_profile: typeof modelInfoData.model_profile === 'string' ? modelInfoData.model_profile : undefined,
model_name: typeof modelInfoData.model_name === 'string' ? modelInfoData.model_name : undefined,
embedding_dim: typeof modelInfoData.embedding_dim === 'number' ? modelInfoData.embedding_dim : undefined,
backend: typeof modelInfoData.backend === 'string' ? modelInfoData.backend : undefined,
created_at: typeof modelInfoData.created_at === 'string' ? modelInfoData.created_at : undefined,
updated_at: typeof modelInfoData.updated_at === 'string' ? modelInfoData.updated_at : undefined,
} : undefined;
let warning: string | undefined;
@@ -1039,6 +1270,39 @@ function looksLikeCodeQuery(query: string): boolean {
return false;
}
function queryTargetsGeneratedFiles(query: string): boolean {
return GENERATED_QUERY_RE.test(query.trim());
}
function prefersLexicalPriorityQuery(query: string): boolean {
const trimmed = query.trim();
if (!trimmed) return false;
if (ENV_STYLE_QUERY_RE.test(trimmed)) return true;
const tokens = new Set((trimmed.match(TOPIC_TOKEN_RE) ?? []).map((token) => token.toLowerCase()));
if (tokens.size === 0) return false;
if (tokens.has('factory') || tokens.has('factories')) return true;
if ((tokens.has('environment') || tokens.has('env')) && (tokens.has('variable') || tokens.has('variables'))) {
return true;
}
if (
tokens.has('backend') &&
['embedding', 'embeddings', 'reranker', 'rerankers', 'onnx', 'api', 'litellm', 'fastembed', 'local', 'legacy']
.some((token) => tokens.has(token))
) {
return true;
}
let surfaceHit = false;
let focusHit = false;
for (const token of tokens) {
if (LEXICAL_PRIORITY_SURFACE_TOKENS.has(token)) surfaceHit = true;
if (LEXICAL_PRIORITY_FOCUS_TOKENS.has(token)) focusHit = true;
if (surfaceHit && focusHit) return true;
}
return false;
}
/**
* Classify query intent and recommend search mode
* Simple mapping: hybrid (NL + index + embeddings) | exact (index or insufficient embeddings) | ripgrep (no index)
@@ -1051,6 +1315,8 @@ function classifyIntent(query: string, hasIndex: boolean = false, hasSufficientE
const isNaturalLanguage = detectNaturalLanguage(query);
const isCodeQuery = looksLikeCodeQuery(query);
const isRegexPattern = detectRegex(query);
const targetsGeneratedFiles = queryTargetsGeneratedFiles(query);
const prefersLexicalPriority = prefersLexicalPriorityQuery(query);
let mode: string;
let confidence: number;
@@ -1058,9 +1324,9 @@ function classifyIntent(query: string, hasIndex: boolean = false, hasSufficientE
if (!hasIndex) {
mode = 'ripgrep';
confidence = 1.0;
} else if (isCodeQuery || isRegexPattern) {
} else if (targetsGeneratedFiles || prefersLexicalPriority || isCodeQuery || isRegexPattern) {
mode = 'exact';
confidence = 0.95;
confidence = targetsGeneratedFiles ? 0.97 : prefersLexicalPriority ? 0.93 : 0.95;
} else if (isNaturalLanguage && hasSufficientEmbeddings) {
mode = 'hybrid';
confidence = 0.9;
@@ -1075,6 +1341,8 @@ function classifyIntent(query: string, hasIndex: boolean = false, hasSufficientE
if (detectNaturalLanguage(query)) detectedPatterns.push('natural language');
if (detectFilePath(query)) detectedPatterns.push('file path');
if (detectRelationship(query)) detectedPatterns.push('relationship');
if (targetsGeneratedFiles) detectedPatterns.push('generated artifact');
if (prefersLexicalPriority) detectedPatterns.push('lexical priority');
if (isCodeQuery) detectedPatterns.push('code identifier');
const reasoning = `Query classified as ${mode} (confidence: ${confidence.toFixed(2)}, detected: ${detectedPatterns.join(', ')}, index: ${hasIndex ? 'available' : 'not available'}, embeddings: ${hasSufficientEmbeddings ? 'sufficient' : 'insufficient'})`;
@@ -1087,12 +1355,21 @@ function classifyIntent(query: string, hasIndex: boolean = false, hasSufficientE
* @param toolName - Tool executable name
* @returns True if available
*/
function checkToolAvailability(toolName: string): boolean {
function checkToolAvailability(
toolName: string,
lookupRuntime: typeof spawnSync = spawnSync,
): boolean {
try {
const isWindows = process.platform === 'win32';
const command = isWindows ? 'where' : 'which';
execSync(`${command} ${toolName}`, { stdio: 'ignore', timeout: EXEC_TIMEOUTS.SYSTEM_INFO });
return true;
const result = lookupRuntime(command, [toolName], {
shell: false,
windowsHide: true,
stdio: 'ignore',
timeout: EXEC_TIMEOUTS.SYSTEM_INFO,
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
});
return !result.error && result.status === 0;
} catch {
return false;
}
@@ -1330,6 +1607,23 @@ function normalizeEmbeddingBackend(backend?: string): string | undefined {
return normalized;
}
function buildIndexInitArgs(projectPath: string, options: { force?: boolean; languages?: string[]; noEmbeddings?: boolean } = {}): string[] {
const { force = false, languages, noEmbeddings = true } = options;
const args = ['index', 'init', projectPath];
if (noEmbeddings) {
args.push('--no-embeddings');
}
if (force) {
args.push('--force');
}
if (languages && languages.length > 0) {
args.push(...languages.flatMap((language) => ['--language', language]));
}
return args;
}
function resolveEmbeddingSelection(
requestedBackend: string | undefined,
requestedModel: string | undefined,
@@ -1502,17 +1796,17 @@ function spawnBackgroundEmbeddingsViaPython(params: {
}): { success: boolean; error?: string } {
const { projectPath, backend, model } = params;
try {
const child = spawn(getVenvPythonPath(), ['-c', buildEmbeddingPythonCode(params)], {
cwd: projectPath,
shell: false,
detached: true,
stdio: 'ignore',
windowsHide: true,
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
});
const child = getSpawnRuntime()(
getVenvPythonPathRuntime()(),
['-c', buildEmbeddingPythonCode(params)],
buildSmartSearchSpawnOptions(projectPath, {
detached: shouldDetachBackgroundSmartSearchProcess(),
stdio: 'ignore',
}),
);
autoEmbedJobs.set(projectPath, {
startedAt: Date.now(),
startedAt: getNowRuntime(),
backend,
model,
});
@@ -1532,6 +1826,84 @@ function spawnBackgroundEmbeddingsViaPython(params: {
}
}
function spawnBackgroundIndexInit(params: {
projectPath: string;
languages?: string[];
}): { success: boolean; error?: string } {
const { projectPath, languages } = params;
try {
const pythonPath = getVenvPythonPathRuntime()();
if (!existsSync(pythonPath)) {
return {
success: false,
error: 'CodexLens Python environment is not ready yet.',
};
}
const child = getSpawnRuntime()(
pythonPath,
['-m', 'codexlens', ...buildIndexInitArgs(projectPath, { languages })],
buildSmartSearchSpawnOptions(projectPath, {
detached: shouldDetachBackgroundSmartSearchProcess(),
stdio: 'ignore',
}),
);
autoInitJobs.set(projectPath, {
startedAt: getNowRuntime(),
languages,
});
const cleanup = () => {
autoInitJobs.delete(projectPath);
};
child.on('error', cleanup);
child.on('close', cleanup);
child.unref();
return { success: true };
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : String(error),
};
}
}
async function maybeStartBackgroundAutoInit(
scope: SearchScope,
indexStatus: IndexStatus,
): Promise<{ note?: string; warning?: string }> {
if (indexStatus.indexed) {
return {};
}
if (!isAutoInitMissingEnabled()) {
return {
note: getAutoInitMissingDisabledReason(),
};
}
if (autoInitJobs.has(scope.workingDirectory)) {
return {
note: 'Background static index build is already running for this path.',
};
}
const spawned = spawnBackgroundIndexInit({
projectPath: scope.workingDirectory,
});
if (!spawned.success) {
return {
warning: `Automatic static index warmup could not start: ${spawned.error}`,
};
}
return {
note: 'Background static index build started for this path. Re-run search shortly for indexed FTS results.',
};
}
async function maybeStartBackgroundAutoEmbed(
scope: SearchScope,
indexStatus: IndexStatus,
@@ -1542,7 +1914,7 @@ async function maybeStartBackgroundAutoEmbed(
if (!isAutoEmbedMissingEnabled(indexStatus.config)) {
return {
note: 'Automatic embedding warmup is disabled by CODEXLENS_AUTO_EMBED_MISSING=false.',
note: getAutoEmbedMissingDisabledReason(indexStatus.config),
};
}
@@ -1554,7 +1926,7 @@ async function maybeStartBackgroundAutoEmbed(
const backend = normalizeEmbeddingBackend(indexStatus.config?.embedding_backend) ?? 'fastembed';
const model = indexStatus.config?.embedding_model?.trim() || undefined;
const semanticStatus = await checkSemanticStatus();
const semanticStatus = await getSemanticStatusRuntime()();
if (!semanticStatus.available) {
return {
warning: 'Automatic embedding warmup skipped because semantic dependencies are not ready.',
@@ -1604,18 +1976,19 @@ async function executeEmbeddingsViaPython(params: {
const pythonCode = buildEmbeddingPythonCode(params);
return await new Promise((resolve) => {
const child = spawn(getVenvPythonPath(), ['-c', pythonCode], {
cwd: projectPath,
shell: false,
timeout: 1800000,
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
});
const child = getSpawnRuntime()(
getVenvPythonPathRuntime()(),
['-c', pythonCode],
buildSmartSearchSpawnOptions(projectPath, {
timeout: 1800000,
}),
);
let stdout = '';
let stderr = '';
const progressMessages: string[] = [];
child.stdout.on('data', (data: Buffer) => {
child.stdout?.on('data', (data: Buffer) => {
const chunk = data.toString();
stdout += chunk;
for (const line of chunk.split(/\r?\n/)) {
@@ -1625,7 +1998,7 @@ async function executeEmbeddingsViaPython(params: {
}
});
child.stderr.on('data', (data: Buffer) => {
child.stderr?.on('data', (data: Buffer) => {
stderr += data.toString();
});
@@ -1683,13 +2056,7 @@ async function executeInitAction(params: Params, force: boolean = false): Promis
// Build args with --no-embeddings for FTS-only index (faster)
// Use 'index init' subcommand (new CLI structure)
const args = ['index', 'init', scope.workingDirectory, '--no-embeddings'];
if (force) {
args.push('--force'); // Force full rebuild
}
if (languages && languages.length > 0) {
args.push(...languages.flatMap((language) => ['--language', language]));
}
const args = buildIndexInitArgs(scope.workingDirectory, { force, languages });
// Track progress updates
const progressUpdates: ProgressInfo[] = [];
@@ -1805,8 +2172,12 @@ async function executeEmbedAction(params: Params): Promise<SearchResult> {
api_max_workers: normalizedBackend === 'litellm' ? effectiveApiMaxWorkers : undefined,
endpoint_count: endpoints.length,
use_gpu: true,
reranker_enabled: currentStatus.config?.reranker_enabled,
reranker_backend: currentStatus.config?.reranker_backend,
reranker_model: currentStatus.config?.reranker_model,
cascade_strategy: currentStatus.config?.cascade_strategy,
staged_stage2_mode: currentStatus.config?.staged_stage2_mode,
static_graph_enabled: currentStatus.config?.static_graph_enabled,
note: [embeddingSelection.note, progressMessage].filter(Boolean).join(' | ') || undefined,
preset: embeddingSelection.preset,
},
@@ -1856,6 +2227,9 @@ async function executeStatusAction(params: Params): Promise<SearchResult> {
if (cfg.staged_stage2_mode) {
statusParts.push(`Stage2: ${cfg.staged_stage2_mode}`);
}
if (typeof cfg.static_graph_enabled === 'boolean') {
statusParts.push(`Static Graph: ${cfg.static_graph_enabled ? 'on' : 'off'}`);
}
// Reranker info
if (cfg.reranker_enabled) {
@@ -1874,6 +2248,12 @@ async function executeStatusAction(params: Params): Promise<SearchResult> {
action: 'status',
path: scope.workingDirectory,
warning: indexStatus.warning,
reranker_enabled: indexStatus.config?.reranker_enabled,
reranker_backend: indexStatus.config?.reranker_backend,
reranker_model: indexStatus.config?.reranker_model,
cascade_strategy: indexStatus.config?.cascade_strategy,
staged_stage2_mode: indexStatus.config?.staged_stage2_mode,
static_graph_enabled: indexStatus.config?.static_graph_enabled,
suggestions: buildIndexSuggestions(indexStatus, scope),
},
};
@@ -2026,6 +2406,7 @@ async function executeFuzzyMode(params: Params): Promise<SearchResult> {
const ftsWasBroken = codexLensFtsBackendBroken;
const ripgrepQueryMode = resolveRipgrepQueryMode(query, regex, tokenize);
const fuzzyWarnings: string[] = [];
const skipExactDueToCompatibility = ftsWasBroken && !ripgrepQueryMode.literalFallback;
let skipExactReason: string | undefined;
if (ripgrepQueryMode.literalFallback) {
@@ -2043,10 +2424,7 @@ async function executeFuzzyMode(params: Params): Promise<SearchResult> {
]);
timer.mark('parallel_search');
if (!skipExactReason && !ftsWasBroken && codexLensFtsBackendBroken) {
fuzzyWarnings.push('CodexLens FTS backend is incompatible with the current CLI runtime. Falling back to ripgrep results.');
}
if (skipExactReason) {
if (skipExactReason && !skipExactDueToCompatibility) {
fuzzyWarnings.push(skipExactReason);
}
if (ripgrepResult.status === 'fulfilled' && ripgrepResult.value.metadata?.warning) {
@@ -2070,6 +2448,16 @@ async function executeFuzzyMode(params: Params): Promise<SearchResult> {
resultsMap.set('ripgrep', ripgrepResult.value.results as any[]);
}
const ripgrepResultCount = (resultsMap.get('ripgrep') ?? []).length;
const compatibilityTriggeredThisQuery = !skipExactReason && !ftsWasBroken && codexLensFtsBackendBroken;
if (shouldSurfaceCodexLensFtsCompatibilityWarning({
compatibilityTriggeredThisQuery,
skipExactDueToCompatibility,
ripgrepResultCount,
})) {
fuzzyWarnings.push('CodexLens FTS backend is incompatible with the current CLI runtime. Falling back to ripgrep results.');
}
// If both failed, return error
if (resultsMap.size === 0) {
const errors: string[] = [];
@@ -2286,20 +2674,23 @@ async function executeRipgrepMode(params: Params): Promise<SearchResult> {
});
return new Promise((resolve) => {
const child = spawn(command, args, {
cwd: scope.workingDirectory || getProjectRoot(),
stdio: ['ignore', 'pipe', 'pipe'],
});
const child = getSpawnRuntime()(
command,
args,
buildSmartSearchSpawnOptions(scope.workingDirectory || getProjectRoot(), {
stdio: ['ignore', 'pipe', 'pipe'],
}),
);
let stdout = '';
let stderr = '';
let resultLimitReached = false;
child.stdout.on('data', (data) => {
child.stdout?.on('data', (data) => {
stdout += data.toString();
});
child.stderr.on('data', (data) => {
child.stderr?.on('data', (data) => {
stderr += data.toString();
});
@@ -3484,19 +3875,22 @@ async function executeFindFilesAction(params: Params): Promise<SearchResult> {
}
}
const child = spawn('rg', args, {
cwd: scope.workingDirectory || getProjectRoot(),
stdio: ['ignore', 'pipe', 'pipe'],
});
const child = getSpawnRuntime()(
'rg',
args,
buildSmartSearchSpawnOptions(scope.workingDirectory || getProjectRoot(), {
stdio: ['ignore', 'pipe', 'pipe'],
}),
);
let stdout = '';
let stderr = '';
child.stdout.on('data', (data) => {
child.stdout?.on('data', (data) => {
stdout += data.toString();
});
child.stderr.on('data', (data) => {
child.stderr?.on('data', (data) => {
stderr += data.toString();
});
@@ -3800,6 +4194,12 @@ function enrichMetadataWithIndexStatus(
nextMetadata.index_status = indexStatus.indexed
? (indexStatus.has_embeddings ? 'indexed' : 'partial')
: 'not_indexed';
nextMetadata.reranker_enabled = indexStatus.config?.reranker_enabled;
nextMetadata.reranker_backend = indexStatus.config?.reranker_backend;
nextMetadata.reranker_model = indexStatus.config?.reranker_model;
nextMetadata.cascade_strategy = indexStatus.config?.cascade_strategy;
nextMetadata.staged_stage2_mode = indexStatus.config?.staged_stage2_mode;
nextMetadata.static_graph_enabled = indexStatus.config?.static_graph_enabled;
nextMetadata.warning = mergeWarnings(nextMetadata.warning, indexStatus.warning);
nextMetadata.suggestions = mergeSuggestions(nextMetadata.suggestions, buildIndexSuggestions(indexStatus, scope));
return nextMetadata;
@@ -3890,7 +4290,7 @@ export async function handler(params: Record<string, unknown>): Promise<ToolResu
break;
}
let autoEmbedNote: string | undefined;
let backgroundNote: string | undefined;
// Transform output based on output_mode (for search actions only)
if (action === 'search' || action === 'search_files') {
@@ -3898,12 +4298,13 @@ export async function handler(params: Record<string, unknown>): Promise<ToolResu
const indexStatus = await checkIndexStatus(scope.workingDirectory);
result.metadata = enrichMetadataWithIndexStatus(result.metadata, indexStatus, scope);
const autoInitStatus = await maybeStartBackgroundAutoInit(scope, indexStatus);
const autoEmbedStatus = await maybeStartBackgroundAutoEmbed(scope, indexStatus);
autoEmbedNote = autoEmbedStatus.note;
backgroundNote = mergeNotes(autoInitStatus.note, autoEmbedStatus.note);
result.metadata = {
...(result.metadata ?? {}),
note: mergeNotes(result.metadata?.note, autoEmbedStatus.note),
warning: mergeWarnings(result.metadata?.warning, autoEmbedStatus.warning),
note: mergeNotes(result.metadata?.note, autoInitStatus.note, autoEmbedStatus.note),
warning: mergeWarnings(result.metadata?.warning, autoInitStatus.warning, autoEmbedStatus.warning),
};
// Add pagination metadata for search results if not already present
@@ -3935,8 +4336,8 @@ export async function handler(params: Record<string, unknown>): Promise<ToolResu
if (result.metadata?.warning) {
advisoryLines.push('', 'Warnings:', `- ${result.metadata.warning}`);
}
if (autoEmbedNote) {
advisoryLines.push('', 'Notes:', `- ${autoEmbedNote}`);
if (backgroundNote) {
advisoryLines.push('', 'Notes:', `- ${backgroundNote}`);
}
if (result.metadata?.suggestions && result.metadata.suggestions.length > 0) {
advisoryLines.push('', 'Suggestions:');
@@ -3972,13 +4373,40 @@ export async function handler(params: Record<string, unknown>): Promise<ToolResu
*/
export const __testables = {
isCodexLensCliCompatibilityError,
shouldSurfaceCodexLensFtsCompatibilityWarning,
buildSmartSearchSpawnOptions,
shouldDetachBackgroundSmartSearchProcess,
checkToolAvailability,
parseCodexLensJsonOutput,
parsePlainTextFileMatches,
hasCentralizedVectorArtifacts,
extractEmbeddingsStatusSummary,
selectEmbeddingsStatusPayload,
resolveRipgrepQueryMode,
queryTargetsGeneratedFiles,
prefersLexicalPriorityQuery,
classifyIntent,
resolveEmbeddingSelection,
parseOptionalBooleanEnv,
isAutoInitMissingEnabled,
isAutoEmbedMissingEnabled,
getAutoInitMissingDisabledReason,
getAutoEmbedMissingDisabledReason,
buildIndexSuggestions,
maybeStartBackgroundAutoInit,
maybeStartBackgroundAutoEmbed,
__setRuntimeOverrides(overrides: Partial<SmartSearchRuntimeOverrides>) {
Object.assign(runtimeOverrides, overrides);
},
__resetRuntimeOverrides() {
for (const key of Object.keys(runtimeOverrides) as Array<keyof SmartSearchRuntimeOverrides>) {
delete runtimeOverrides[key];
}
},
__resetBackgroundJobs() {
autoInitJobs.clear();
autoEmbedJobs.clear();
},
};
export async function executeInitWithProgress(

View File

@@ -9,6 +9,7 @@
* 2. Default: ~/.codexlens
*/
import { existsSync } from 'fs';
import { join } from 'path';
import { homedir } from 'os';
@@ -47,6 +48,26 @@ export function getCodexLensPython(): string {
: join(venvDir, 'bin', 'python');
}
/**
* Get the preferred Python executable for hidden/windowless CodexLens subprocesses.
* On Windows this prefers pythonw.exe when available to avoid transient console windows.
*
* @returns Path to the preferred hidden-subprocess Python executable
*/
export function getCodexLensHiddenPython(): string {
if (process.platform !== 'win32') {
return getCodexLensPython();
}
const venvDir = getCodexLensVenvDir();
const pythonwPath = join(venvDir, 'Scripts', 'pythonw.exe');
if (existsSync(pythonwPath)) {
return pythonwPath;
}
return getCodexLensPython();
}
/**
* Get the pip executable path in the CodexLens venv.
*

View File

@@ -3,9 +3,19 @@
* Shared module for consistent Python discovery across the application
*/
import { execSync } from 'child_process';
import { spawnSync, type SpawnSyncOptionsWithStringEncoding } from 'child_process';
import { EXEC_TIMEOUTS } from './exec-constants.js';
export interface PythonCommandSpec {
command: string;
args: string[];
display: string;
}
type HiddenPythonProbeOptions = Omit<SpawnSyncOptionsWithStringEncoding, 'encoding'> & {
encoding?: BufferEncoding;
};
function isExecTimeoutError(error: unknown): boolean {
const err = error as { code?: unknown; errno?: unknown; message?: unknown } | null;
const code = err?.code ?? err?.errno;
@@ -14,6 +24,98 @@ function isExecTimeoutError(error: unknown): boolean {
return message.includes('ETIMEDOUT');
}
function quoteCommandPart(value: string): string {
if (!/[\s"]/.test(value)) {
return value;
}
return `"${value.replaceAll('"', '\\"')}"`;
}
function formatPythonCommandDisplay(command: string, args: string[]): string {
return [quoteCommandPart(command), ...args.map(quoteCommandPart)].join(' ');
}
function buildPythonCommandSpec(command: string, args: string[] = []): PythonCommandSpec {
return {
command,
args: [...args],
display: formatPythonCommandDisplay(command, args),
};
}
function tokenizeCommandSpec(raw: string): string[] {
const tokens: string[] = [];
const tokenPattern = /"((?:\\"|[^"])*)"|(\S+)/g;
for (const match of raw.matchAll(tokenPattern)) {
const quoted = match[1];
const plain = match[2];
if (quoted !== undefined) {
tokens.push(quoted.replaceAll('\\"', '"'));
} else if (plain !== undefined) {
tokens.push(plain);
}
}
return tokens;
}
export function parsePythonCommandSpec(raw: string): PythonCommandSpec {
const trimmed = raw.trim();
if (!trimmed) {
throw new Error('Python command cannot be empty');
}
// Unquoted executable paths on Windows commonly contain spaces.
if (!trimmed.includes('"') && /[\\/]/.test(trimmed)) {
return buildPythonCommandSpec(trimmed);
}
const tokens = tokenizeCommandSpec(trimmed);
if (tokens.length === 0) {
return buildPythonCommandSpec(trimmed);
}
return buildPythonCommandSpec(tokens[0], tokens.slice(1));
}
function buildPythonProbeOptions(
overrides: HiddenPythonProbeOptions = {},
): SpawnSyncOptionsWithStringEncoding {
const { env, encoding, ...rest } = overrides;
return {
shell: false,
windowsHide: true,
timeout: EXEC_TIMEOUTS.PYTHON_VERSION,
stdio: ['ignore', 'pipe', 'pipe'],
env: { ...process.env, PYTHONIOENCODING: 'utf-8', ...env },
...rest,
encoding: encoding ?? 'utf8',
};
}
export function probePythonCommandVersion(
pythonCommand: PythonCommandSpec,
runner: typeof spawnSync = spawnSync,
): string {
const result = runner(
pythonCommand.command,
[...pythonCommand.args, '--version'],
buildPythonProbeOptions(),
);
if (result.error) {
throw result.error;
}
const versionOutput = `${result.stdout ?? ''}${result.stderr ?? ''}`.trim();
if (result.status !== 0) {
throw new Error(versionOutput || `Python version probe exited with code ${String(result.status)}`);
}
return versionOutput;
}
/**
* Parse Python version string to major.minor numbers
* @param versionStr - Version string like "Python 3.11.5"
@@ -42,66 +144,72 @@ export function isPythonVersionCompatible(major: number, minor: number): boolean
* Detect available Python 3 executable
* Supports CCW_PYTHON environment variable for custom Python path
* On Windows, uses py launcher to find compatible versions
* @returns Python executable command
* @returns Python executable command spec
*/
export function getSystemPython(): string {
// Check for user-specified Python via environment variable
const customPython = process.env.CCW_PYTHON;
export function getSystemPythonCommand(runner: typeof spawnSync = spawnSync): PythonCommandSpec {
const customPython = process.env.CCW_PYTHON?.trim();
if (customPython) {
const customSpec = parsePythonCommandSpec(customPython);
try {
const version = execSync(`"${customPython}" --version 2>&1`, { encoding: 'utf8', timeout: EXEC_TIMEOUTS.PYTHON_VERSION });
const version = probePythonCommandVersion(customSpec, runner);
if (version.includes('Python 3')) {
const parsed = parsePythonVersion(version);
if (parsed && !isPythonVersionCompatible(parsed.major, parsed.minor)) {
console.warn(`[Python] Warning: CCW_PYTHON points to Python ${parsed.major}.${parsed.minor}, which may not be compatible with onnxruntime (requires 3.9-3.12)`);
console.warn(
`[Python] Warning: CCW_PYTHON points to Python ${parsed.major}.${parsed.minor}, which may not be compatible with onnxruntime (requires 3.9-3.12)`,
);
}
return `"${customPython}"`;
return customSpec;
}
} catch (err: unknown) {
if (isExecTimeoutError(err)) {
console.warn(`[Python] Warning: CCW_PYTHON version check timed out after ${EXEC_TIMEOUTS.PYTHON_VERSION}ms, falling back to system Python`);
console.warn(
`[Python] Warning: CCW_PYTHON version check timed out after ${EXEC_TIMEOUTS.PYTHON_VERSION}ms, falling back to system Python`,
);
} else {
console.warn(`[Python] Warning: CCW_PYTHON="${customPython}" is not a valid Python executable, falling back to system Python`);
console.warn(
`[Python] Warning: CCW_PYTHON="${customPython}" is not a valid Python executable, falling back to system Python`,
);
}
}
}
// On Windows, try py launcher with specific versions first (3.12, 3.11, 3.10, 3.9)
if (process.platform === 'win32') {
const compatibleVersions = ['3.12', '3.11', '3.10', '3.9'];
for (const ver of compatibleVersions) {
const launcherSpec = buildPythonCommandSpec('py', [`-${ver}`]);
try {
const version = execSync(`py -${ver} --version 2>&1`, { encoding: 'utf8', timeout: EXEC_TIMEOUTS.PYTHON_VERSION });
const version = probePythonCommandVersion(launcherSpec, runner);
if (version.includes(`Python ${ver}`)) {
console.log(`[Python] Found compatible Python ${ver} via py launcher`);
return `py -${ver}`;
return launcherSpec;
}
} catch (err: unknown) {
if (isExecTimeoutError(err)) {
console.warn(`[Python] Warning: py -${ver} version check timed out after ${EXEC_TIMEOUTS.PYTHON_VERSION}ms`);
console.warn(
`[Python] Warning: py -${ver} version check timed out after ${EXEC_TIMEOUTS.PYTHON_VERSION}ms`,
);
}
// Version not installed, try next
}
}
}
const commands = process.platform === 'win32' ? ['python', 'py', 'python3'] : ['python3', 'python'];
let fallbackCmd: string | null = null;
let fallbackCmd: PythonCommandSpec | null = null;
let fallbackVersion: { major: number; minor: number } | null = null;
for (const cmd of commands) {
const pythonSpec = buildPythonCommandSpec(cmd);
try {
const version = execSync(`${cmd} --version 2>&1`, { encoding: 'utf8', timeout: EXEC_TIMEOUTS.PYTHON_VERSION });
const version = probePythonCommandVersion(pythonSpec, runner);
if (version.includes('Python 3')) {
const parsed = parsePythonVersion(version);
if (parsed) {
// Prefer compatible version (3.9-3.12)
if (isPythonVersionCompatible(parsed.major, parsed.minor)) {
return cmd;
return pythonSpec;
}
// Keep track of first Python 3 found as fallback
if (!fallbackCmd) {
fallbackCmd = cmd;
fallbackCmd = pythonSpec;
fallbackVersion = parsed;
}
}
@@ -110,13 +218,14 @@ export function getSystemPython(): string {
if (isExecTimeoutError(err)) {
console.warn(`[Python] Warning: ${cmd} --version timed out after ${EXEC_TIMEOUTS.PYTHON_VERSION}ms`);
}
// Try next command
}
}
// If no compatible version found, use fallback with warning
if (fallbackCmd && fallbackVersion) {
console.warn(`[Python] Warning: Only Python ${fallbackVersion.major}.${fallbackVersion.minor} found, which may not be compatible with onnxruntime (requires 3.9-3.12).`);
console.warn(
`[Python] Warning: Only Python ${fallbackVersion.major}.${fallbackVersion.minor} found, which may not be compatible with onnxruntime (requires 3.9-3.12).`,
);
console.warn('[Python] Semantic search may fail with ImportError for onnxruntime.');
console.warn('[Python] To use a specific Python version, set CCW_PYTHON environment variable:');
console.warn(' Windows: set CCW_PYTHON=C:\\path\\to\\python.exe');
console.warn(' Unix: export CCW_PYTHON=/path/to/python3.11');
@@ -124,7 +233,19 @@ export function getSystemPython(): string {
return fallbackCmd;
}
throw new Error('Python 3 not found. Please install Python 3.9-3.12 and ensure it is in PATH, or set CCW_PYTHON environment variable.');
throw new Error(
'Python 3 not found. Please install Python 3.9-3.12 and ensure it is in PATH, or set CCW_PYTHON environment variable.',
);
}
/**
* Detect available Python 3 executable
* Supports CCW_PYTHON environment variable for custom Python path
* On Windows, uses py launcher to find compatible versions
* @returns Python executable command
*/
export function getSystemPython(): string {
return getSystemPythonCommand().display;
}
/**
@@ -135,6 +256,14 @@ export function getPipCommand(): { pythonCmd: string; pipArgs: string[] } {
const pythonCmd = getSystemPython();
return {
pythonCmd,
pipArgs: ['-m', 'pip']
pipArgs: ['-m', 'pip'],
};
}
export const __testables = {
buildPythonCommandSpec,
buildPythonProbeOptions,
formatPythonCommandDisplay,
parsePythonCommandSpec,
probePythonCommandVersion,
};

View File

@@ -9,7 +9,7 @@
* - Support for local project installs with extras
*/
import { execSync, spawn } from 'child_process';
import { spawn, spawnSync, type SpawnOptions, type SpawnSyncOptionsWithStringEncoding } from 'child_process';
import { existsSync, mkdirSync } from 'fs';
import { join, dirname } from 'path';
import { homedir, platform, arch } from 'os';
@@ -52,6 +52,74 @@ const UV_BINARY_NAME = IS_WINDOWS ? 'uv.exe' : 'uv';
const VENV_BIN_DIR = IS_WINDOWS ? 'Scripts' : 'bin';
const PYTHON_EXECUTABLE = IS_WINDOWS ? 'python.exe' : 'python';
type HiddenUvSpawnSyncOptions = Omit<SpawnSyncOptionsWithStringEncoding, 'encoding'> & {
encoding?: BufferEncoding;
};
function buildUvSpawnOptions(overrides: SpawnOptions = {}): SpawnOptions {
const { env, ...rest } = overrides;
return {
shell: false,
windowsHide: true,
env: { ...process.env, PYTHONIOENCODING: 'utf-8', ...env },
...rest,
};
}
function buildUvSpawnSyncOptions(
overrides: HiddenUvSpawnSyncOptions = {},
): SpawnSyncOptionsWithStringEncoding {
const { env, encoding, ...rest } = overrides;
return {
shell: false,
windowsHide: true,
env: { ...process.env, PYTHONIOENCODING: 'utf-8', ...env },
...rest,
encoding: encoding ?? 'utf-8',
};
}
function findExecutableOnPath(executable: string, runner: typeof spawnSync = spawnSync): string | null {
const lookupCommand = IS_WINDOWS ? 'where' : 'which';
const result = runner(
lookupCommand,
[executable],
buildUvSpawnSyncOptions({
timeout: EXEC_TIMEOUTS.SYSTEM_INFO,
stdio: ['ignore', 'pipe', 'pipe'],
}),
);
if (result.error || result.status !== 0) {
return null;
}
const output = `${result.stdout ?? ''}`.trim();
if (!output) {
return null;
}
return output.split(/\r?\n/)[0] || null;
}
function hasWindowsPythonLauncherVersion(version: string, runner: typeof spawnSync = spawnSync): boolean {
const result = runner(
'py',
[`-${version}`, '--version'],
buildUvSpawnSyncOptions({
timeout: EXEC_TIMEOUTS.PYTHON_VERSION,
stdio: ['ignore', 'pipe', 'pipe'],
}),
);
if (result.error || result.status !== 0) {
return false;
}
const output = `${result.stdout ?? ''}${result.stderr ?? ''}`;
return output.includes(`Python ${version}`);
}
/**
* Get the path to the UV binary
* Search order:
@@ -105,15 +173,9 @@ export function getUvBinaryPath(): string {
}
// 4. Try system PATH using 'which' or 'where'
try {
const cmd = IS_WINDOWS ? 'where uv' : 'which uv';
const result = execSync(cmd, { encoding: 'utf-8', timeout: EXEC_TIMEOUTS.SYSTEM_INFO, stdio: ['pipe', 'pipe', 'pipe'] });
const foundPath = result.trim().split('\n')[0];
if (foundPath && existsSync(foundPath)) {
return foundPath;
}
} catch {
// UV not found in PATH
const foundPath = findExecutableOnPath('uv');
if (foundPath && existsSync(foundPath)) {
return foundPath;
}
// Return default path (may not exist)
@@ -135,10 +197,10 @@ export async function isUvAvailable(): Promise<boolean> {
}
return new Promise((resolve) => {
const child = spawn(uvPath, ['--version'], {
const child = spawn(uvPath, ['--version'], buildUvSpawnOptions({
stdio: ['ignore', 'pipe', 'pipe'],
timeout: EXEC_TIMEOUTS.PYTHON_VERSION,
});
}));
child.on('close', (code) => {
resolve(code === 0);
@@ -162,14 +224,14 @@ export async function getUvVersion(): Promise<string | null> {
}
return new Promise((resolve) => {
const child = spawn(uvPath, ['--version'], {
const child = spawn(uvPath, ['--version'], buildUvSpawnOptions({
stdio: ['ignore', 'pipe', 'pipe'],
timeout: EXEC_TIMEOUTS.PYTHON_VERSION,
});
}));
let stdout = '';
child.stdout.on('data', (data) => {
child.stdout?.on('data', (data) => {
stdout += data.toString();
});
@@ -207,19 +269,29 @@ export async function ensureUvInstalled(): Promise<boolean> {
if (IS_WINDOWS) {
// Windows: Use PowerShell to run the install script
const installCmd = 'irm https://astral.sh/uv/install.ps1 | iex';
child = spawn('powershell', ['-ExecutionPolicy', 'ByPass', '-Command', installCmd], {
stdio: 'inherit',
child = spawn('powershell', ['-ExecutionPolicy', 'ByPass', '-Command', installCmd], buildUvSpawnOptions({
stdio: ['pipe', 'pipe', 'pipe'],
timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL,
});
}));
} else {
// Unix: Use curl and sh
const installCmd = 'curl -LsSf https://astral.sh/uv/install.sh | sh';
child = spawn('sh', ['-c', installCmd], {
stdio: 'inherit',
child = spawn('sh', ['-c', installCmd], buildUvSpawnOptions({
stdio: ['pipe', 'pipe', 'pipe'],
timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL,
});
}));
}
child.stdout?.on('data', (data) => {
const line = data.toString().trim();
if (line) console.log(`[UV] ${line}`);
});
child.stderr?.on('data', (data) => {
const line = data.toString().trim();
if (line) console.log(`[UV] ${line}`);
});
child.on('close', (code) => {
if (code === 0) {
console.log('[UV] UV installed successfully');
@@ -315,21 +387,21 @@ export class UvManager {
console.log(`[UV] Python version: ${this.pythonVersion}`);
}
const child = spawn(uvPath, args, {
const child = spawn(uvPath, args, buildUvSpawnOptions({
stdio: ['ignore', 'pipe', 'pipe'],
timeout: EXEC_TIMEOUTS.PROCESS_SPAWN,
});
}));
let stderr = '';
child.stdout.on('data', (data) => {
child.stdout?.on('data', (data) => {
const line = data.toString().trim();
if (line) {
console.log(`[UV] ${line}`);
}
});
child.stderr.on('data', (data) => {
child.stderr?.on('data', (data) => {
stderr += data.toString();
const line = data.toString().trim();
if (line) {
@@ -390,22 +462,22 @@ export class UvManager {
console.log(`[UV] Installing from project: ${installSpec} (editable: ${editable})`);
const child = spawn(uvPath, args, {
const child = spawn(uvPath, args, buildUvSpawnOptions({
stdio: ['ignore', 'pipe', 'pipe'],
timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL,
cwd: projectPath,
});
}));
let stderr = '';
child.stdout.on('data', (data) => {
child.stdout?.on('data', (data) => {
const line = data.toString().trim();
if (line) {
console.log(`[UV] ${line}`);
}
});
child.stderr.on('data', (data) => {
child.stderr?.on('data', (data) => {
stderr += data.toString();
const line = data.toString().trim();
if (line && !line.startsWith('Resolved') && !line.startsWith('Prepared') && !line.startsWith('Installed')) {
@@ -460,21 +532,21 @@ export class UvManager {
console.log(`[UV] Installing packages: ${packages.join(', ')}`);
const child = spawn(uvPath, args, {
const child = spawn(uvPath, args, buildUvSpawnOptions({
stdio: ['ignore', 'pipe', 'pipe'],
timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL,
});
}));
let stderr = '';
child.stdout.on('data', (data) => {
child.stdout?.on('data', (data) => {
const line = data.toString().trim();
if (line) {
console.log(`[UV] ${line}`);
}
});
child.stderr.on('data', (data) => {
child.stderr?.on('data', (data) => {
stderr += data.toString();
});
@@ -524,21 +596,21 @@ export class UvManager {
console.log(`[UV] Uninstalling packages: ${packages.join(', ')}`);
const child = spawn(uvPath, args, {
const child = spawn(uvPath, args, buildUvSpawnOptions({
stdio: ['ignore', 'pipe', 'pipe'],
timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL,
});
}));
let stderr = '';
child.stdout.on('data', (data) => {
child.stdout?.on('data', (data) => {
const line = data.toString().trim();
if (line) {
console.log(`[UV] ${line}`);
}
});
child.stderr.on('data', (data) => {
child.stderr?.on('data', (data) => {
stderr += data.toString();
});
@@ -585,21 +657,21 @@ export class UvManager {
console.log(`[UV] Syncing dependencies from: ${requirementsPath}`);
const child = spawn(uvPath, args, {
const child = spawn(uvPath, args, buildUvSpawnOptions({
stdio: ['ignore', 'pipe', 'pipe'],
timeout: EXEC_TIMEOUTS.PACKAGE_INSTALL,
});
}));
let stderr = '';
child.stdout.on('data', (data) => {
child.stdout?.on('data', (data) => {
const line = data.toString().trim();
if (line) {
console.log(`[UV] ${line}`);
}
});
child.stderr.on('data', (data) => {
child.stderr?.on('data', (data) => {
stderr += data.toString();
});
@@ -640,14 +712,14 @@ export class UvManager {
return new Promise((resolve) => {
const args = ['pip', 'list', '--format', 'json', '--python', this.getVenvPython()];
const child = spawn(uvPath, args, {
const child = spawn(uvPath, args, buildUvSpawnOptions({
stdio: ['ignore', 'pipe', 'pipe'],
timeout: EXEC_TIMEOUTS.PROCESS_SPAWN,
});
}));
let stdout = '';
child.stdout.on('data', (data) => {
child.stdout?.on('data', (data) => {
stdout += data.toString();
});
@@ -704,20 +776,20 @@ export class UvManager {
}
return new Promise((resolve) => {
const child = spawn(pythonPath, args, {
const child = spawn(pythonPath, args, buildUvSpawnOptions({
stdio: ['ignore', 'pipe', 'pipe'],
timeout: options.timeout ?? EXEC_TIMEOUTS.PROCESS_SPAWN,
cwd: options.cwd,
});
}));
let stdout = '';
let stderr = '';
child.stdout.on('data', (data) => {
child.stdout?.on('data', (data) => {
stdout += data.toString();
});
child.stderr.on('data', (data) => {
child.stderr?.on('data', (data) => {
stderr += data.toString();
});
@@ -779,17 +851,8 @@ export function getPreferredCodexLensPythonSpec(): string {
// depend on onnxruntime 1.15.x wheels, which are not consistently available for cp312.
const preferredVersions = ['3.11', '3.10', '3.12'];
for (const version of preferredVersions) {
try {
const output = execSync(`py -${version} --version`, {
encoding: 'utf-8',
timeout: EXEC_TIMEOUTS.PYTHON_VERSION,
stdio: ['pipe', 'pipe', 'pipe'],
});
if (output.includes(`Python ${version}`)) {
return version;
}
} catch {
// Try next installed version
if (hasWindowsPythonLauncherVersion(version)) {
return version;
}
}
@@ -830,3 +893,10 @@ export async function bootstrapUvVenv(
const manager = new UvManager({ venvPath, pythonVersion });
return manager.createVenv();
}
export const __testables = {
buildUvSpawnOptions,
buildUvSpawnSyncOptions,
findExecutableOnPath,
hasWindowsPythonLauncherVersion,
};

View File

@@ -0,0 +1,118 @@
/**
* Cross-project regression coverage for `ccw cli history` and `ccw cli detail`.
*/
import { after, afterEach, before, describe, it, mock } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
const TEST_CCW_HOME = mkdtempSync(join(tmpdir(), 'ccw-cli-history-cross-home-'));
process.env.CCW_DATA_DIR = TEST_CCW_HOME;
const cliCommandPath = new URL('../dist/commands/cli.js', import.meta.url).href;
const cliExecutorPath = new URL('../dist/tools/cli-executor.js', import.meta.url).href;
const historyStorePath = new URL('../dist/tools/cli-history-store.js', import.meta.url).href;
function createConversation({ id, prompt, updatedAt }) {
return {
id,
created_at: updatedAt,
updated_at: updatedAt,
tool: 'gemini',
model: 'default',
mode: 'analysis',
category: 'user',
total_duration_ms: 456,
turn_count: 1,
latest_status: 'success',
turns: [
{
turn: 1,
timestamp: updatedAt,
prompt,
duration_ms: 456,
status: 'success',
exit_code: 0,
output: {
stdout: 'CROSS PROJECT OK',
stderr: '',
truncated: false,
cached: false,
},
},
],
};
}
describe('ccw cli history/detail cross-project', async () => {
let cliModule;
let cliExecutorModule;
let historyStoreModule;
before(async () => {
cliModule = await import(cliCommandPath);
cliExecutorModule = await import(cliExecutorPath);
historyStoreModule = await import(historyStorePath);
});
afterEach(() => {
mock.restoreAll();
try {
historyStoreModule?.closeAllStores?.();
} catch {
// ignore
}
});
after(() => {
try {
historyStoreModule?.closeAllStores?.();
} catch {
// ignore
}
rmSync(TEST_CCW_HOME, { recursive: true, force: true });
});
it('finds history and detail for executions stored in another registered project', async () => {
const projectRoot = mkdtempSync(join(tmpdir(), 'ccw-cli-cross-project-history-'));
const unrelatedCwd = mkdtempSync(join(tmpdir(), 'ccw-cli-cross-project-cwd-'));
const previousCwd = process.cwd();
try {
const store = new historyStoreModule.CliHistoryStore(projectRoot);
store.saveConversation(createConversation({
id: 'CONV-CROSS-PROJECT-1',
prompt: 'Cross project prompt',
updatedAt: new Date('2025-02-01T00:00:01.000Z').toISOString(),
}));
store.close();
const logs = [];
mock.method(console, 'log', (...args) => {
logs.push(args.map(String).join(' '));
});
mock.method(console, 'error', (...args) => {
logs.push(args.map(String).join(' '));
});
process.chdir(unrelatedCwd);
await cliModule.cliCommand('history', [], { limit: '20' });
assert.ok(logs.some((line) => line.includes('CONV-CROSS-PROJECT-1')));
await cliExecutorModule.getExecutionHistoryAsync(projectRoot, { limit: 1 });
logs.length = 0;
await cliModule.cliCommand('detail', ['CONV-CROSS-PROJECT-1'], {});
assert.ok(logs.some((line) => line.includes('Conversation Detail')));
assert.ok(logs.some((line) => line.includes('CONV-CROSS-PROJECT-1')));
assert.ok(logs.some((line) => line.includes('Cross project prompt')));
} finally {
process.chdir(previousCwd);
rmSync(projectRoot, { recursive: true, force: true });
rmSync(unrelatedCwd, { recursive: true, force: true });
}
});
});

View File

@@ -123,6 +123,39 @@ describe('ccw cli output --final', async () => {
}
});
it('loads cached output from another registered project without --project', async () => {
const projectRoot = createTestProjectRoot();
const unrelatedCwd = createTestProjectRoot();
const previousCwd = process.cwd();
const store = new historyStoreModule.CliHistoryStore(projectRoot);
try {
store.saveConversation(createConversation({
id: 'EXEC-CROSS-PROJECT-OUTPUT',
stdoutFull: 'cross project raw output',
parsedOutput: 'cross project parsed output',
finalOutput: 'cross project final output',
}));
process.chdir(unrelatedCwd);
const logs = [];
mock.method(console, 'log', (...args) => {
logs.push(args.map(String).join(' '));
});
mock.method(console, 'error', () => {});
await cliModule.cliCommand('output', ['EXEC-CROSS-PROJECT-OUTPUT'], {});
assert.equal(logs.at(-1), 'cross project final output');
} finally {
process.chdir(previousCwd);
store.close();
rmSync(projectRoot, { recursive: true, force: true });
rmSync(unrelatedCwd, { recursive: true, force: true });
}
});
it('fails fast for explicit --final when no final agent result can be recovered', async () => {
const projectRoot = createTestProjectRoot();
const store = new historyStoreModule.CliHistoryStore(projectRoot);
@@ -159,4 +192,34 @@ describe('ccw cli output --final', async () => {
rmSync(projectRoot, { recursive: true, force: true });
}
});
it('prints CCW execution ID guidance when output cannot find the requested execution', async () => {
const projectRoot = createTestProjectRoot();
const previousCwd = process.cwd();
try {
process.chdir(projectRoot);
const errors = [];
const exitCodes = [];
mock.method(console, 'log', () => {});
mock.method(console, 'error', (...args) => {
errors.push(args.map(String).join(' '));
});
mock.method(process, 'exit', (code) => {
exitCodes.push(code);
});
await cliModule.cliCommand('output', ['rebuttal-structure-analysis'], {});
assert.deepEqual(exitCodes, [1]);
assert.ok(errors.some((line) => line.includes('real CCW execution ID')));
assert.ok(errors.some((line) => line.includes('CCW_EXEC_ID')));
assert.ok(errors.some((line) => line.includes('ccw cli show or ccw cli history')));
} finally {
process.chdir(previousCwd);
rmSync(projectRoot, { recursive: true, force: true });
}
});
});

View File

@@ -163,6 +163,42 @@ describe('ccw cli show running time formatting', async () => {
assert.match(rendered, /1h\.\.\./);
});
it('lists executions from other registered projects in show output', async () => {
const projectRoot = mkdtempSync(join(tmpdir(), 'ccw-cli-show-cross-project-'));
const unrelatedCwd = mkdtempSync(join(tmpdir(), 'ccw-cli-show-cross-cwd-'));
const previousCwd = process.cwd();
try {
process.chdir(unrelatedCwd);
const store = new historyStoreModule.CliHistoryStore(projectRoot);
store.saveConversation(createConversationRecord({
id: 'EXEC-CROSS-PROJECT-SHOW',
prompt: 'cross project show prompt',
updatedAt: new Date('2025-02-02T00:00:00.000Z').toISOString(),
durationMs: 1800,
}));
store.close();
stubActiveExecutionsResponse([]);
const logs = [];
mock.method(console, 'log', (...args) => {
logs.push(args.map(String).join(' '));
});
mock.method(console, 'error', () => {});
await cliModule.cliCommand('show', [], {});
const rendered = logs.join('\n');
assert.match(rendered, /EXEC-CROSS-PROJECT-SHOW/);
assert.match(rendered, /cross project show prompt/);
} finally {
process.chdir(previousCwd);
rmSync(projectRoot, { recursive: true, force: true });
rmSync(unrelatedCwd, { recursive: true, force: true });
}
});
it('suppresses stale running rows when saved history is newer than the active start time', async () => {
const projectRoot = mkdtempSync(join(tmpdir(), 'ccw-cli-show-stale-project-'));
const previousCwd = process.cwd();

View File

@@ -13,6 +13,38 @@ after(() => {
});
describe('CodexLens CLI compatibility retries', () => {
it('builds hidden Python spawn options for CLI invocations', async () => {
const moduleUrl = new URL(`../dist/tools/codex-lens.js?spawn-opts=${Date.now()}`, import.meta.url).href;
const { __testables } = await import(moduleUrl);
const options = __testables.buildCodexLensSpawnOptions(tmpdir(), 12345);
assert.equal(options.cwd, tmpdir());
assert.equal(options.shell, false);
assert.equal(options.timeout, 12345);
assert.equal(options.windowsHide, true);
assert.equal(options.env.PYTHONIOENCODING, 'utf-8');
});
it('probes Python version without a shell-backed console window', async () => {
const moduleUrl = new URL(`../dist/tools/codex-lens.js?python-probe=${Date.now()}`, import.meta.url).href;
const { __testables } = await import(moduleUrl);
const probeCalls = [];
const version = __testables.probePythonVersion({ command: 'python', args: [], display: 'python' }, (command, args, options) => {
probeCalls.push({ command, args, options });
return { status: 0, stdout: '', stderr: 'Python 3.11.9\n' };
});
assert.equal(version, 'Python 3.11.9');
assert.equal(probeCalls.length, 1);
assert.equal(probeCalls[0].command, 'python');
assert.deepEqual(probeCalls[0].args, ['--version']);
assert.equal(probeCalls[0].options.shell, false);
assert.equal(probeCalls[0].options.windowsHide, true);
assert.equal(probeCalls[0].options.env.PYTHONIOENCODING, 'utf-8');
});
it('initializes a tiny index even when CLI emits compatibility conflicts first', async () => {
const moduleUrl = new URL(`../dist/tools/codex-lens.js?compat=${Date.now()}`, import.meta.url).href;
const { checkVenvStatus, executeCodexLens } = await import(moduleUrl);
@@ -32,4 +64,76 @@ describe('CodexLens CLI compatibility retries', () => {
assert.equal(result.success, true, result.error ?? 'Expected init to succeed');
assert.ok((result.output ?? '').length > 0 || (result.warning ?? '').length > 0, 'Expected init output or compatibility warning');
});
it('synthesizes a machine-readable fallback when JSON search output is empty', async () => {
const moduleUrl = new URL(`../dist/tools/codex-lens.js?compat-empty=${Date.now()}`, import.meta.url).href;
const { __testables } = await import(moduleUrl);
const normalized = __testables.normalizeSearchCommandResult(
{ success: true },
{ query: 'missing symbol', cwd: tmpdir(), limit: 5, filesOnly: false },
);
assert.equal(normalized.success, true);
assert.match(normalized.warning ?? '', /empty stdout/i);
assert.deepEqual(normalized.results, {
success: true,
result: {
query: 'missing symbol',
count: 0,
results: [],
},
});
});
it('returns structured semantic search results for a local embedded workspace', async () => {
const codexLensUrl = new URL(`../dist/tools/codex-lens.js?compat-search=${Date.now()}`, import.meta.url).href;
const smartSearchUrl = new URL(`../dist/tools/smart-search.js?compat-search=${Date.now()}`, import.meta.url).href;
const codexLensModule = await import(codexLensUrl);
const smartSearchModule = await import(smartSearchUrl);
const ready = await codexLensModule.checkVenvStatus(true);
if (!ready.ready) {
console.log('Skipping: CodexLens not ready');
return;
}
const semantic = await codexLensModule.checkSemanticStatus();
if (!semantic.available) {
console.log('Skipping: semantic dependencies not ready');
return;
}
const projectDir = mkdtempSync(join(tmpdir(), 'codexlens-search-'));
tempDirs.push(projectDir);
writeFileSync(
join(projectDir, 'sample.ts'),
'export function greet(name) { return `hello ${name}`; }\nexport const sum = (a, b) => a + b;\n',
);
const init = await smartSearchModule.handler({ action: 'init', path: projectDir });
assert.equal(init.success, true, init.error ?? 'Expected smart-search init to succeed');
const embed = await smartSearchModule.handler({
action: 'embed',
path: projectDir,
embeddingBackend: 'local',
force: true,
});
assert.equal(embed.success, true, embed.error ?? 'Expected smart-search embed to succeed');
const result = await codexLensModule.codexLensTool.execute({
action: 'search',
path: projectDir,
query: 'greet function',
mode: 'semantic',
format: 'json',
});
assert.equal(result.success, true, result.error ?? 'Expected semantic search compatibility fallback to succeed');
const payload = result.results?.result ?? result.results;
assert.ok(Array.isArray(payload?.results), 'Expected structured search results payload');
assert.ok(payload.results.length > 0, 'Expected at least one structured semantic search result');
assert.doesNotMatch(result.error ?? '', /unexpected extra arguments/i);
});
});

View File

@@ -0,0 +1,66 @@
import { after, afterEach, describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, rmSync } from 'node:fs';
import { createRequire, syncBuiltinESMExports } from 'node:module';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
const require = createRequire(import.meta.url);
// eslint-disable-next-line @typescript-eslint/no-var-requires
const fs = require('node:fs');
const originalExistsSync = fs.existsSync;
const originalCodexLensDataDir = process.env.CODEXLENS_DATA_DIR;
const tempDirs = [];
afterEach(() => {
fs.existsSync = originalExistsSync;
syncBuiltinESMExports();
if (originalCodexLensDataDir === undefined) {
delete process.env.CODEXLENS_DATA_DIR;
} else {
process.env.CODEXLENS_DATA_DIR = originalCodexLensDataDir;
}
});
after(() => {
while (tempDirs.length > 0) {
rmSync(tempDirs.pop(), { recursive: true, force: true });
}
});
describe('codexlens-path hidden python selection', () => {
it('prefers pythonw.exe for hidden Windows subprocesses when available', async () => {
if (process.platform !== 'win32') {
return;
}
const dataDir = mkdtempSync(join(tmpdir(), 'ccw-codexlens-hidden-python-'));
tempDirs.push(dataDir);
process.env.CODEXLENS_DATA_DIR = dataDir;
const expectedPythonw = join(dataDir, 'venv', 'Scripts', 'pythonw.exe');
fs.existsSync = (path) => String(path) === expectedPythonw;
syncBuiltinESMExports();
const moduleUrl = new URL(`../dist/utils/codexlens-path.js?t=${Date.now()}`, import.meta.url);
const mod = await import(moduleUrl.href);
assert.equal(mod.getCodexLensHiddenPython(), expectedPythonw);
});
it('falls back to python.exe when pythonw.exe is unavailable', async () => {
const dataDir = mkdtempSync(join(tmpdir(), 'ccw-codexlens-hidden-fallback-'));
tempDirs.push(dataDir);
process.env.CODEXLENS_DATA_DIR = dataDir;
fs.existsSync = () => false;
syncBuiltinESMExports();
const moduleUrl = new URL(`../dist/utils/codexlens-path.js?t=${Date.now()}`, import.meta.url);
const mod = await import(moduleUrl.href);
assert.equal(mod.getCodexLensHiddenPython(), mod.getCodexLensPython());
});
});

View File

@@ -105,7 +105,10 @@ describe('memory-embedder-bridge', () => {
assert.equal(spawnCalls.length, 1);
assert.equal(spawnCalls[0].args.at(-2), 'status');
assert.equal(spawnCalls[0].args.at(-1), 'C:\\tmp\\db.sqlite');
assert.equal(spawnCalls[0].options.shell, false);
assert.equal(spawnCalls[0].options.timeout, 30000);
assert.equal(spawnCalls[0].options.windowsHide, true);
assert.equal(spawnCalls[0].options.env.PYTHONIOENCODING, 'utf-8');
});
it('generateEmbeddings builds args for sourceId, batchSize, and force', async () => {
@@ -138,7 +141,10 @@ describe('memory-embedder-bridge', () => {
assert.equal(args[batchSizeIndex + 1], '4');
assert.ok(args.includes('--force'));
assert.equal(spawnCalls[0].options.shell, false);
assert.equal(spawnCalls[0].options.timeout, 300000);
assert.equal(spawnCalls[0].options.windowsHide, true);
assert.equal(spawnCalls[0].options.env.PYTHONIOENCODING, 'utf-8');
spawnCalls.length = 0;
spawnPlan.push({

View File

@@ -103,7 +103,7 @@ describe('LiteLLM client bridge', () => {
assert.equal(available, true);
assert.equal(spawnCalls.length, 1);
assert.equal(spawnCalls[0].command, 'python');
assert.equal(spawnCalls[0].command, mod.getCodexLensVenvPython());
assert.deepEqual(spawnCalls[0].args, ['-m', 'ccw_litellm.cli', 'version']);
});
@@ -117,6 +117,19 @@ describe('LiteLLM client bridge', () => {
assert.equal(spawnCalls[0].command, 'python3');
});
it('spawns LiteLLM Python with hidden window options', async () => {
spawnPlan.push({ type: 'close', code: 0, stdout: '1.2.3\n' });
const client = new mod.LiteLLMClient({ timeout: 10 });
const available = await client.isAvailable();
assert.equal(available, true);
assert.equal(spawnCalls.length, 1);
assert.equal(spawnCalls[0].options.shell, false);
assert.equal(spawnCalls[0].options.windowsHide, true);
assert.equal(spawnCalls[0].options.env.PYTHONIOENCODING, 'utf-8');
});
it('isAvailable returns false on spawn error', async () => {
spawnPlan.push({ type: 'error', error: new Error('ENOENT') });
@@ -154,7 +167,7 @@ describe('LiteLLM client bridge', () => {
assert.deepEqual(cfg, { ok: true });
assert.equal(spawnCalls.length, 1);
assert.deepEqual(spawnCalls[0].args, ['-m', 'ccw_litellm.cli', 'config', '--json']);
assert.deepEqual(spawnCalls[0].args, ['-m', 'ccw_litellm.cli', 'config']);
});
it('getConfig throws on malformed JSON', async () => {

View File

@@ -76,6 +76,26 @@ describe('Smart Search - Query Intent + RRF Weights', async () => {
});
});
describe('classifyIntent lexical routing', () => {
it('routes config/backend queries to exact when index and embeddings are available', () => {
if (!smartSearchModule) return;
const classification = smartSearchModule.__testables.classifyIntent(
'embedding backend fastembed local litellm api config',
true,
true,
);
assert.strictEqual(classification.mode, 'exact');
assert.match(classification.reasoning, /lexical priority/i);
});
it('routes generated artifact queries to exact when index and embeddings are available', () => {
if (!smartSearchModule) return;
const classification = smartSearchModule.__testables.classifyIntent('dist bundle output', true, true);
assert.strictEqual(classification.mode, 'exact');
assert.match(classification.reasoning, /generated artifact/i);
});
});
describe('adjustWeightsByIntent', () => {
it('maps keyword intent to exact-heavy weights', () => {
if (!smartSearchModule) return;
@@ -119,4 +139,3 @@ describe('Smart Search - Query Intent + RRF Weights', async () => {
});
});
});

View File

@@ -1,16 +1,19 @@
import { afterEach, before, describe, it } from 'node:test';
import { after, afterEach, before, describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
const smartSearchPath = new URL('../dist/tools/smart-search.js', import.meta.url).href;
const originalAutoInitMissing = process.env.CODEXLENS_AUTO_INIT_MISSING;
const originalAutoEmbedMissing = process.env.CODEXLENS_AUTO_EMBED_MISSING;
describe('Smart Search MCP usage defaults and path handling', async () => {
let smartSearchModule;
const tempDirs = [];
before(async () => {
process.env.CODEXLENS_AUTO_INIT_MISSING = 'false';
try {
smartSearchModule = await import(smartSearchPath);
} catch (err) {
@@ -18,10 +21,30 @@ describe('Smart Search MCP usage defaults and path handling', async () => {
}
});
after(() => {
if (originalAutoInitMissing === undefined) {
delete process.env.CODEXLENS_AUTO_INIT_MISSING;
} else {
process.env.CODEXLENS_AUTO_INIT_MISSING = originalAutoInitMissing;
}
if (originalAutoEmbedMissing === undefined) {
delete process.env.CODEXLENS_AUTO_EMBED_MISSING;
return;
}
process.env.CODEXLENS_AUTO_EMBED_MISSING = originalAutoEmbedMissing;
});
afterEach(() => {
while (tempDirs.length > 0) {
rmSync(tempDirs.pop(), { recursive: true, force: true });
}
if (smartSearchModule?.__testables) {
smartSearchModule.__testables.__resetRuntimeOverrides();
smartSearchModule.__testables.__resetBackgroundJobs();
}
process.env.CODEXLENS_AUTO_INIT_MISSING = 'false';
delete process.env.CODEXLENS_AUTO_EMBED_MISSING;
});
function createWorkspace() {
@@ -30,6 +53,15 @@ describe('Smart Search MCP usage defaults and path handling', async () => {
return dir;
}
function createDetachedChild() {
return {
on() {
return this;
},
unref() {},
};
}
it('keeps schema defaults aligned with runtime docs', () => {
if (!smartSearchModule) return;
@@ -50,14 +82,202 @@ describe('Smart Search MCP usage defaults and path handling', async () => {
assert.equal(props.output_mode.default, 'ace');
});
it('defaults auto embedding warmup to enabled unless explicitly disabled', () => {
it('defaults auto embedding warmup off on Windows unless explicitly enabled', () => {
if (!smartSearchModule) return;
const { __testables } = smartSearchModule;
assert.equal(__testables.isAutoEmbedMissingEnabled(undefined), true);
assert.equal(__testables.isAutoEmbedMissingEnabled({}), true);
assert.equal(__testables.isAutoEmbedMissingEnabled({ embedding_auto_embed_missing: true }), true);
delete process.env.CODEXLENS_AUTO_EMBED_MISSING;
assert.equal(__testables.isAutoEmbedMissingEnabled(undefined), process.platform !== 'win32');
assert.equal(__testables.isAutoEmbedMissingEnabled({}), process.platform !== 'win32');
assert.equal(
__testables.isAutoEmbedMissingEnabled({ embedding_auto_embed_missing: true }),
process.platform === 'win32' ? false : true,
);
assert.equal(__testables.isAutoEmbedMissingEnabled({ embedding_auto_embed_missing: false }), false);
process.env.CODEXLENS_AUTO_EMBED_MISSING = 'true';
assert.equal(__testables.isAutoEmbedMissingEnabled({ embedding_auto_embed_missing: false }), true);
process.env.CODEXLENS_AUTO_EMBED_MISSING = 'off';
assert.equal(__testables.isAutoEmbedMissingEnabled({ embedding_auto_embed_missing: true }), false);
});
it('defaults auto index warmup off on Windows unless explicitly enabled', () => {
if (!smartSearchModule) return;
const { __testables } = smartSearchModule;
delete process.env.CODEXLENS_AUTO_INIT_MISSING;
assert.equal(__testables.isAutoInitMissingEnabled(), process.platform !== 'win32');
process.env.CODEXLENS_AUTO_INIT_MISSING = 'off';
assert.equal(__testables.isAutoInitMissingEnabled(), false);
process.env.CODEXLENS_AUTO_INIT_MISSING = '1';
assert.equal(__testables.isAutoInitMissingEnabled(), true);
});
it('explains when Windows disables background warmup by default', () => {
if (!smartSearchModule) return;
const { __testables } = smartSearchModule;
delete process.env.CODEXLENS_AUTO_INIT_MISSING;
delete process.env.CODEXLENS_AUTO_EMBED_MISSING;
const initReason = __testables.getAutoInitMissingDisabledReason();
const embedReason = __testables.getAutoEmbedMissingDisabledReason({});
if (process.platform === 'win32') {
assert.match(initReason, /disabled by default on Windows/i);
assert.match(embedReason, /disabled by default on Windows/i);
assert.match(embedReason, /auto_embed_missing=true/i);
} else {
assert.match(initReason, /disabled/i);
assert.match(embedReason, /disabled/i);
}
});
it('builds hidden subprocess options for Smart Search child processes', () => {
if (!smartSearchModule) return;
const options = smartSearchModule.__testables.buildSmartSearchSpawnOptions(tmpdir(), {
detached: true,
stdio: 'ignore',
timeout: 12345,
});
assert.equal(options.cwd, tmpdir());
assert.equal(options.shell, false);
assert.equal(options.windowsHide, true);
assert.equal(options.detached, true);
assert.equal(options.timeout, 12345);
assert.equal(options.env.PYTHONIOENCODING, 'utf-8');
});
it('avoids detached background warmup children on Windows consoles', () => {
if (!smartSearchModule) return;
assert.equal(
smartSearchModule.__testables.shouldDetachBackgroundSmartSearchProcess(),
process.platform !== 'win32',
);
});
it('checks tool availability without shell-based lookup popups', () => {
if (!smartSearchModule) return;
const lookupCalls = [];
const available = smartSearchModule.__testables.checkToolAvailability(
'rg',
(command, args, options) => {
lookupCalls.push({ command, args, options });
return { status: 0, stdout: '', stderr: '' };
},
);
assert.equal(available, true);
assert.equal(lookupCalls.length, 1);
assert.equal(lookupCalls[0].command, process.platform === 'win32' ? 'where' : 'which');
assert.deepEqual(lookupCalls[0].args, ['rg']);
assert.equal(lookupCalls[0].options.shell, false);
assert.equal(lookupCalls[0].options.windowsHide, true);
assert.equal(lookupCalls[0].options.stdio, 'ignore');
assert.equal(lookupCalls[0].options.env.PYTHONIOENCODING, 'utf-8');
});
it('starts background static index build once for unindexed paths', async () => {
if (!smartSearchModule) return;
const { __testables } = smartSearchModule;
const dir = createWorkspace();
const fakePython = join(dir, 'python.exe');
writeFileSync(fakePython, '');
process.env.CODEXLENS_AUTO_INIT_MISSING = 'true';
const spawnCalls = [];
__testables.__setRuntimeOverrides({
getVenvPythonPath: () => fakePython,
now: () => 1234567890,
spawnProcess: (command, args, options) => {
spawnCalls.push({ command, args, options });
return createDetachedChild();
},
});
const scope = { workingDirectory: dir, searchPaths: ['.'] };
const indexStatus = { indexed: false, has_embeddings: false };
const first = await __testables.maybeStartBackgroundAutoInit(scope, indexStatus);
const second = await __testables.maybeStartBackgroundAutoInit(scope, indexStatus);
assert.match(first.note, /started/i);
assert.match(second.note, /already running/i);
assert.equal(spawnCalls.length, 1);
assert.equal(spawnCalls[0].command, fakePython);
assert.deepEqual(spawnCalls[0].args, ['-m', 'codexlens', 'index', 'init', dir, '--no-embeddings']);
assert.equal(spawnCalls[0].options.cwd, dir);
assert.equal(
spawnCalls[0].options.detached,
smartSearchModule.__testables.shouldDetachBackgroundSmartSearchProcess(),
);
assert.equal(spawnCalls[0].options.windowsHide, true);
});
it('starts background embedding build without detached Windows consoles', async () => {
if (!smartSearchModule) return;
const { __testables } = smartSearchModule;
const dir = createWorkspace();
const fakePython = join(dir, 'python.exe');
writeFileSync(fakePython, '');
process.env.CODEXLENS_AUTO_EMBED_MISSING = 'true';
const spawnCalls = [];
__testables.__setRuntimeOverrides({
getVenvPythonPath: () => fakePython,
checkSemanticStatus: async () => ({ available: true, litellmAvailable: true }),
now: () => 1234567890,
spawnProcess: (command, args, options) => {
spawnCalls.push({ command, args, options });
return createDetachedChild();
},
});
const status = await __testables.maybeStartBackgroundAutoEmbed(
{ workingDirectory: dir, searchPaths: ['.'] },
{
indexed: true,
has_embeddings: false,
config: { embedding_backend: 'fastembed' },
},
);
assert.match(status.note, /started/i);
assert.equal(spawnCalls.length, 1);
assert.equal(spawnCalls[0].command, fakePython);
assert.deepEqual(spawnCalls[0].args.slice(0, 1), ['-c']);
assert.equal(spawnCalls[0].options.cwd, dir);
assert.equal(
spawnCalls[0].options.detached,
smartSearchModule.__testables.shouldDetachBackgroundSmartSearchProcess(),
);
assert.equal(spawnCalls[0].options.windowsHide, true);
assert.equal(spawnCalls[0].options.stdio, 'ignore');
});
it('surfaces warnings when background static index warmup cannot start', async () => {
if (!smartSearchModule) return;
const { __testables } = smartSearchModule;
const dir = createWorkspace();
process.env.CODEXLENS_AUTO_INIT_MISSING = 'true';
__testables.__setRuntimeOverrides({
getVenvPythonPath: () => join(dir, 'missing-python.exe'),
});
const status = await __testables.maybeStartBackgroundAutoInit(
{ workingDirectory: dir, searchPaths: ['.'] },
{ indexed: false, has_embeddings: false },
);
assert.match(status.warning, /Automatic static index warmup could not start/i);
assert.match(status.warning, /not ready yet/i);
});
it('honors explicit small limit values', async () => {
@@ -246,15 +466,98 @@ describe('Smart Search MCP usage defaults and path handling', async () => {
assert.match(String(matches[0].file).replace(/\\/g, '/'), /target\.ts$/);
});
it('detects centralized vector artifacts as full embedding coverage evidence', () => {
it('uses root-scoped embedding status instead of subtree artifacts', () => {
if (!smartSearchModule) return;
const dir = createWorkspace();
writeFileSync(join(dir, '_vectors.hnsw'), 'hnsw');
writeFileSync(join(dir, '_vectors_meta.db'), 'meta');
writeFileSync(join(dir, '_binary_vectors.mmap'), 'mmap');
const summary = smartSearchModule.__testables.extractEmbeddingsStatusSummary({
total_indexes: 3,
indexes_with_embeddings: 2,
total_chunks: 24,
coverage_percent: 66.7,
root: {
total_files: 4,
files_with_embeddings: 0,
total_chunks: 0,
coverage_percent: 0,
has_embeddings: false,
},
subtree: {
total_indexes: 3,
indexes_with_embeddings: 2,
total_files: 12,
files_with_embeddings: 8,
total_chunks: 24,
coverage_percent: 66.7,
},
centralized: {
dense_index_exists: true,
binary_index_exists: true,
meta_db_exists: true,
usable: false,
},
});
assert.equal(smartSearchModule.__testables.hasCentralizedVectorArtifacts(dir), true);
assert.equal(summary.coveragePercent, 0);
assert.equal(summary.totalChunks, 0);
assert.equal(summary.hasEmbeddings, false);
});
it('accepts validated root centralized readiness from CLI status payloads', () => {
if (!smartSearchModule) return;
const summary = smartSearchModule.__testables.extractEmbeddingsStatusSummary({
total_indexes: 2,
indexes_with_embeddings: 1,
total_chunks: 10,
coverage_percent: 25,
root: {
total_files: 2,
files_with_embeddings: 1,
total_chunks: 3,
coverage_percent: 50,
has_embeddings: true,
},
centralized: {
usable: true,
dense_ready: true,
chunk_metadata_rows: 3,
},
});
assert.equal(summary.coveragePercent, 50);
assert.equal(summary.totalChunks, 3);
assert.equal(summary.hasEmbeddings, true);
});
it('prefers embeddings_status over legacy embeddings summary payloads', () => {
if (!smartSearchModule) return;
const payload = smartSearchModule.__testables.selectEmbeddingsStatusPayload({
embeddings: {
total_indexes: 7,
indexes_with_embeddings: 4,
total_chunks: 99,
},
embeddings_status: {
total_indexes: 7,
total_chunks: 3,
root: {
total_files: 2,
files_with_embeddings: 1,
total_chunks: 3,
coverage_percent: 50,
has_embeddings: true,
},
centralized: {
usable: true,
dense_ready: true,
chunk_metadata_rows: 3,
},
},
});
assert.equal(payload.root.total_chunks, 3);
assert.equal(payload.centralized.usable, true);
});
it('recognizes CodexLens CLI compatibility failures and invalid regex fallback', () => {
@@ -281,6 +584,37 @@ describe('Smart Search MCP usage defaults and path handling', async () => {
assert.match(resolution.warning, /literal ripgrep matching/i);
});
it('suppresses compatibility-only fuzzy warnings when ripgrep already produced hits', () => {
if (!smartSearchModule) return;
assert.equal(
smartSearchModule.__testables.shouldSurfaceCodexLensFtsCompatibilityWarning({
compatibilityTriggeredThisQuery: true,
skipExactDueToCompatibility: false,
ripgrepResultCount: 2,
}),
false,
);
assert.equal(
smartSearchModule.__testables.shouldSurfaceCodexLensFtsCompatibilityWarning({
compatibilityTriggeredThisQuery: true,
skipExactDueToCompatibility: false,
ripgrepResultCount: 0,
}),
true,
);
assert.equal(
smartSearchModule.__testables.shouldSurfaceCodexLensFtsCompatibilityWarning({
compatibilityTriggeredThisQuery: false,
skipExactDueToCompatibility: true,
ripgrepResultCount: 0,
}),
true,
);
});
it('builds actionable index suggestions for unhealthy index states', () => {
if (!smartSearchModule) return;
@@ -318,4 +652,52 @@ describe('Smart Search MCP usage defaults and path handling', async () => {
assert.match(toolResult.error, /Both search backends failed:/);
assert.match(toolResult.error, /(FTS|Ripgrep)/);
});
it('returns structured semantic results after local init and embed without JSON parse warnings', async () => {
if (!smartSearchModule) return;
const codexLensModule = await import(new URL(`../dist/tools/codex-lens.js?smart-semantic=${Date.now()}`, import.meta.url).href);
const ready = await codexLensModule.checkVenvStatus(true);
if (!ready.ready) {
console.log('Skipping: CodexLens not ready');
return;
}
const semantic = await codexLensModule.checkSemanticStatus();
if (!semantic.available) {
console.log('Skipping: semantic dependencies not ready');
return;
}
const dir = createWorkspace();
writeFileSync(
join(dir, 'sample.ts'),
'export function parseCodexLensOutput() { return stripAnsiOutput(); }\nexport const sum = (a, b) => a + b;\n',
);
const init = await smartSearchModule.handler({ action: 'init', path: dir });
assert.equal(init.success, true, init.error ?? 'Expected init to succeed');
const embed = await smartSearchModule.handler({
action: 'embed',
path: dir,
embeddingBackend: 'local',
force: true,
});
assert.equal(embed.success, true, embed.error ?? 'Expected local embed to succeed');
const search = await smartSearchModule.handler({
action: 'search',
mode: 'semantic',
path: dir,
query: 'parse CodexLens output strip ANSI',
limit: 5,
});
assert.equal(search.success, true, search.error ?? 'Expected semantic search to succeed');
assert.equal(search.result.success, true);
assert.equal(search.result.results.format, 'ace');
assert.ok(search.result.results.total >= 1, 'Expected at least one structured semantic match');
assert.doesNotMatch(search.result.metadata?.warning ?? '', /Failed to parse JSON output/i);
});
});

View File

@@ -0,0 +1,97 @@
import { after, beforeEach, describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { EventEmitter } from 'node:events';
import { createRequire } from 'node:module';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
const require = createRequire(import.meta.url);
// eslint-disable-next-line @typescript-eslint/no-var-requires
const fs = require('node:fs') as typeof import('node:fs');
// eslint-disable-next-line @typescript-eslint/no-var-requires
const childProcess = require('node:child_process') as typeof import('node:child_process');
class FakeChildProcess extends EventEmitter {
stdout = new EventEmitter();
stderr = new EventEmitter();
stdinChunks: string[] = [];
stdin = {
write: (chunk: string | Buffer) => {
this.stdinChunks.push(String(chunk));
return true;
},
end: () => undefined,
};
}
type SpawnCall = {
command: string;
args: string[];
// eslint-disable-next-line @typescript-eslint/no-explicit-any
options: any;
child: FakeChildProcess;
};
const spawnCalls: SpawnCall[] = [];
const tempDirs: string[] = [];
let embedderAvailable = true;
const originalExistsSync = fs.existsSync;
const originalSpawn = childProcess.spawn;
fs.existsSync = ((..._args: unknown[]) => embedderAvailable) as typeof fs.existsSync;
childProcess.spawn = ((command: string, args: string[] = [], options: unknown = {}) => {
const child = new FakeChildProcess();
spawnCalls.push({ command: String(command), args: args.map(String), options, child });
queueMicrotask(() => {
child.stdout.emit('data', JSON.stringify({
success: true,
total_chunks: 4,
hnsw_available: true,
hnsw_count: 4,
dimension: 384,
}));
child.emit('close', 0);
});
return child as unknown as ReturnType<typeof childProcess.spawn>;
}) as typeof childProcess.spawn;
after(() => {
fs.existsSync = originalExistsSync;
childProcess.spawn = originalSpawn;
while (tempDirs.length > 0) {
rmSync(tempDirs.pop() as string, { recursive: true, force: true });
}
});
describe('unified-vector-index', () => {
beforeEach(() => {
embedderAvailable = true;
spawnCalls.length = 0;
});
it('spawns CodexLens venv python with hidden window options', async () => {
const projectDir = mkdtempSync(join(tmpdir(), 'ccw-unified-vector-index-'));
tempDirs.push(projectDir);
const moduleUrl = new URL('../dist/core/unified-vector-index.js', import.meta.url);
moduleUrl.searchParams.set('t', String(Date.now()));
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const mod: any = await import(moduleUrl.href);
const index = new mod.UnifiedVectorIndex(projectDir);
const status = await index.getStatus();
assert.equal(status.success, true);
assert.equal(spawnCalls.length, 1);
assert.equal(spawnCalls[0].options.shell, false);
assert.equal(spawnCalls[0].options.windowsHide, true);
assert.equal(spawnCalls[0].options.env.PYTHONIOENCODING, 'utf-8');
assert.deepEqual(spawnCalls[0].options.stdio, ['pipe', 'pipe', 'pipe']);
assert.match(spawnCalls[0].child.stdinChunks.join(''), /"operation":"status"/);
});
});

View File

@@ -3,13 +3,16 @@ import assert from 'node:assert/strict';
import { execSync } from 'node:child_process';
const uvManagerPath = new URL('../dist/utils/uv-manager.js', import.meta.url).href;
const pythonUtilsPath = new URL('../dist/utils/python-utils.js', import.meta.url).href;
describe('CodexLens UV python preference', async () => {
let mod;
let pythonUtils;
const originalPython = process.env.CCW_PYTHON;
before(async () => {
mod = await import(uvManagerPath);
pythonUtils = await import(pythonUtilsPath);
});
afterEach(() => {
@@ -25,6 +28,73 @@ describe('CodexLens UV python preference', async () => {
assert.equal(mod.getPreferredCodexLensPythonSpec(), 'C:/Custom/Python/python.exe');
});
it('parses py launcher commands into spawn-safe command specs', () => {
const spec = pythonUtils.parsePythonCommandSpec('py -3.11');
assert.equal(spec.command, 'py');
assert.deepEqual(spec.args, ['-3.11']);
assert.equal(spec.display, 'py -3.11');
});
it('treats unquoted Windows-style executable paths as a single command', () => {
const spec = pythonUtils.parsePythonCommandSpec('C:/Program Files/Python311/python.exe');
assert.equal(spec.command, 'C:/Program Files/Python311/python.exe');
assert.deepEqual(spec.args, []);
assert.equal(spec.display, '"C:/Program Files/Python311/python.exe"');
});
it('probes Python launcher versions without opening a shell window', () => {
const probeCalls = [];
const version = pythonUtils.probePythonCommandVersion(
{ command: 'py', args: ['-3.11'], display: 'py -3.11' },
(command, args, options) => {
probeCalls.push({ command, args, options });
return { status: 0, stdout: '', stderr: 'Python 3.11.9\n' };
},
);
assert.equal(version, 'Python 3.11.9');
assert.equal(probeCalls.length, 1);
assert.equal(probeCalls[0].command, 'py');
assert.deepEqual(probeCalls[0].args, ['-3.11', '--version']);
assert.equal(probeCalls[0].options.shell, false);
assert.equal(probeCalls[0].options.windowsHide, true);
assert.equal(probeCalls[0].options.env.PYTHONIOENCODING, 'utf-8');
});
it('looks up uv on PATH without spawning a visible shell window', () => {
const lookupCalls = [];
const found = mod.__testables.findExecutableOnPath('uv', (command, args, options) => {
lookupCalls.push({ command, args, options });
return { status: 0, stdout: 'C:/Tools/uv.exe\n', stderr: '' };
});
assert.equal(found, 'C:/Tools/uv.exe');
assert.equal(lookupCalls.length, 1);
assert.equal(lookupCalls[0].command, process.platform === 'win32' ? 'where' : 'which');
assert.deepEqual(lookupCalls[0].args, ['uv']);
assert.equal(lookupCalls[0].options.shell, false);
assert.equal(lookupCalls[0].options.windowsHide, true);
assert.equal(lookupCalls[0].options.env.PYTHONIOENCODING, 'utf-8');
});
it('checks Windows launcher preferences with hidden subprocess options', () => {
const probeCalls = [];
const available = mod.__testables.hasWindowsPythonLauncherVersion('3.11', (command, args, options) => {
probeCalls.push({ command, args, options });
return { status: 0, stdout: '', stderr: 'Python 3.11.9\n' };
});
assert.equal(available, true);
assert.equal(probeCalls.length, 1);
assert.equal(probeCalls[0].command, 'py');
assert.deepEqual(probeCalls[0].args, ['-3.11', '--version']);
assert.equal(probeCalls[0].options.shell, false);
assert.equal(probeCalls[0].options.windowsHide, true);
assert.equal(probeCalls[0].options.env.PYTHONIOENCODING, 'utf-8');
});
it('prefers Python 3.11 or 3.10 on Windows when available', () => {
if (process.platform !== 'win32') return;
delete process.env.CCW_PYTHON;