Add unit tests for various components and stores in the terminal dashboard

- Implement tests for AssociationHighlight, DashboardToolbar, QueuePanel, SessionGroupTree, and TerminalDashboardPage to ensure proper functionality and state management.
- Create tests for cliSessionStore, issueQueueIntegrationStore, queueExecutionStore, queueSchedulerStore, sessionManagerStore, and terminalGridStore to validate state resets and workspace scoping.
- Mock necessary dependencies and state management hooks to isolate tests and ensure accurate behavior.
This commit is contained in:
catlog22
2026-03-08 21:38:20 +08:00
parent 9aa07e8d01
commit 62d8aa3623
157 changed files with 36544 additions and 71 deletions

View File

@@ -1232,6 +1232,7 @@ export async function handleOrchestratorRoutes(ctx: RouteContext): Promise<boole
flowId: execution.flowId,
status: execution.status,
timestamp,
projectPath: workflowDir,
},
});
} catch {
@@ -1247,6 +1248,7 @@ export async function handleOrchestratorRoutes(ctx: RouteContext): Promise<boole
payload: {
sessionKey,
timestamp,
projectPath: workflowDir,
},
});
} catch {
@@ -1460,7 +1462,8 @@ export async function handleOrchestratorRoutes(ctx: RouteContext): Promise<boole
sessionKey: sessionKey,
stepName: flow.name,
totalSteps: flow.nodes.length,
timestamp: now
timestamp: now,
projectPath: workflowDir,
}
});
@@ -1471,7 +1474,8 @@ export async function handleOrchestratorRoutes(ctx: RouteContext): Promise<boole
sessionKey: sessionKey,
reason: `Executing workflow: ${flow.name}`,
executionId: execId,
timestamp: now
timestamp: now,
projectPath: workflowDir,
}
});
@@ -1731,6 +1735,7 @@ export async function handleOrchestratorRoutes(ctx: RouteContext): Promise<boole
flowId: execution.flowId,
reason: 'User requested stop',
timestamp: now,
projectPath: workflowDir,
},
});

View File

@@ -397,8 +397,9 @@ export class CliSessionManager {
payload: {
sessionKey,
data,
timestamp: nowIso()
} satisfies CliSessionOutputEvent
timestamp: nowIso(),
projectPath: this.projectRoot,
} satisfies CliSessionOutputEvent & { projectPath: string }
});
});
@@ -410,7 +411,8 @@ export class CliSessionManager {
sessionKey,
exitCode,
signal,
timestamp: nowIso()
timestamp: nowIso(),
projectPath: this.projectRoot,
}
});
});
@@ -426,7 +428,11 @@ export class CliSessionManager {
broadcastToClients({
type: 'CLI_SESSION_CREATED',
payload: { session: this.getSession(sessionKey), timestamp: nowIso() }
payload: {
session: this.getSession(sessionKey),
timestamp: nowIso(),
projectPath: this.projectRoot,
}
});
return this.getSession(sessionKey)!;
@@ -464,7 +470,14 @@ export class CliSessionManager {
session.pty.kill();
} finally {
this.sessions.delete(sessionKey);
broadcastToClients({ type: 'CLI_SESSION_CLOSED', payload: { sessionKey, timestamp: nowIso() } });
broadcastToClients({
type: 'CLI_SESSION_CLOSED',
payload: {
sessionKey,
timestamp: nowIso(),
projectPath: this.projectRoot,
}
});
}
}
@@ -486,7 +499,11 @@ export class CliSessionManager {
session.updatedAt = nowIso();
broadcastToClients({
type: 'CLI_SESSION_PAUSED',
payload: { sessionKey, timestamp: nowIso() }
payload: {
sessionKey,
timestamp: nowIso(),
projectPath: this.projectRoot,
}
});
} catch (err) {
throw new Error(`Failed to pause session ${sessionKey}: ${(err as Error).message}`);
@@ -512,7 +529,11 @@ export class CliSessionManager {
session.lastActivityAt = Date.now();
broadcastToClients({
type: 'CLI_SESSION_RESUMED',
payload: { sessionKey, timestamp: nowIso() }
payload: {
sessionKey,
timestamp: nowIso(),
projectPath: this.projectRoot,
}
});
} catch (err) {
throw new Error(`Failed to resume session ${sessionKey}: ${(err as Error).message}`);

View File

@@ -1265,6 +1265,12 @@ function shouldRetryWithStaticGraphPreference(args: string[], error?: string): b
&& Boolean(error && /Options --static-graph and --no-static-graph are mutually exclusive/i.test(error));
}
function shouldRetryWithCentralizedPreference(args: string[], error?: string): boolean {
return !args.includes('--centralized')
&& !args.includes('--distributed')
&& Boolean(error && /Options --centralized and --distributed are mutually exclusive/i.test(error));
}
function stripAnsiCodes(value: string): string {
return value
.replace(/\x1b\[[0-9;]*m/g, '')
@@ -1398,6 +1404,11 @@ async function executeCodexLens(args: string[], options: ExecuteOptions = {}): P
transform: (currentArgs: string[]) => [...currentArgs, '--static-graph'],
warning: 'CodexLens CLI hit a Typer static-graph option conflict; retried with explicit --static-graph.',
},
{
shouldRetry: shouldRetryWithCentralizedPreference,
transform: (currentArgs: string[]) => [...currentArgs, '--centralized'],
warning: 'CodexLens CLI hit a Typer centralized/distributed option conflict; retried with explicit --centralized.',
},
];
for (const retry of compatibilityRetries) {

View File

@@ -10,7 +10,8 @@
* - Multi-backend search routing with RRF ranking
*
* Actions:
* - init: Initialize CodexLens index
* - init: Initialize CodexLens static index
* - embed: Generate semantic/vector embeddings for the index
* - search: Intelligent search with fuzzy (default) or semantic mode
* - status: Check index status
* - update: Incremental index update for changed files
@@ -20,15 +21,20 @@
import { z } from 'zod';
import type { ToolSchema, ToolResult } from '../types/tool.js';
import { spawn, execSync } from 'child_process';
import { statSync } from 'fs';
import { dirname, resolve } from 'path';
import { existsSync, readFileSync, statSync } from 'fs';
import { dirname, join, resolve } from 'path';
import {
ensureReady as ensureCodexLensReady,
ensureLiteLLMEmbedderReady,
executeCodexLens,
getVenvPythonPath,
} from './codex-lens.js';
import type { ProgressInfo } from './codex-lens.js';
import { getProjectRoot } from '../utils/path-validator.js';
import { getCodexLensDataDir } from '../utils/codexlens-path.js';
import { EXEC_TIMEOUTS } from '../utils/exec-constants.js';
import { generateRotationEndpoints } from '../config/litellm-api-config-manager.js';
import type { RotationEndpointConfig } from '../config/litellm-api-config-manager.js';
// Timing utilities for performance analysis
const TIMING_ENABLED = process.env.SMART_SEARCH_TIMING === '1' || process.env.DEBUG?.includes('timing');
@@ -65,10 +71,10 @@ function createTimer(): { mark: (name: string) => void; getTimings: () => Timing
// Define Zod schema for validation
const ParamsSchema = z.object({
// Action: search (content), find_files (path/name pattern), init, init_force, status, update (incremental), watch
// Action: search (content), find_files (path/name pattern), init, init_force, embed, status, update (incremental), watch
// Note: search_files is deprecated, use search with output_mode='files_only'
// init: incremental index (skip existing), init_force: force full rebuild (delete and recreate)
action: z.enum(['init', 'init_force', 'search', 'search_files', 'find_files', 'status', 'update', 'watch']).default('search'),
// init: static FTS index by default, embed: generate semantic/vector embeddings, init_force: force full rebuild (delete and recreate)
action: z.enum(['init', 'init_force', 'embed', 'search', 'search_files', 'find_files', 'status', 'update', 'watch']).default('search'),
query: z.string().optional().describe('Content search query (for action="search")'),
pattern: z.string().optional().describe('Glob pattern for path matching (for action="find_files")'),
mode: z.enum(['fuzzy', 'semantic']).default('fuzzy'),
@@ -79,6 +85,10 @@ const ParamsSchema = z.object({
maxResults: z.number().default(5), // Default 5 with full content
includeHidden: z.boolean().default(false),
languages: z.array(z.string()).optional(),
embeddingBackend: z.string().optional().describe('Embedding backend for action="embed": fastembed/local or litellm/api.'),
embeddingModel: z.string().optional().describe('Embedding model/profile for action="embed". Examples: "code", "fast", "qwen3-embedding-sf".'),
apiMaxWorkers: z.number().int().min(1).optional().describe('Max concurrent API embedding workers for action="embed". Recommended: 8-16 for litellm/api when multiple endpoints are configured.'),
force: z.boolean().default(false).describe('Force regeneration for action="embed".'),
limit: z.number().default(5), // Default 5 with full content
extraFilesCount: z.number().default(10), // Additional file-only results
maxContentLength: z.number().default(200), // Max content length for truncation (50-2000)
@@ -313,6 +323,11 @@ interface SearchMetadata {
totalFiles?: number;
};
progressHistory?: ProgressInfo[];
api_max_workers?: number;
endpoint_count?: number;
use_gpu?: boolean;
cascade_strategy?: string;
staged_stage2_mode?: string;
}
interface SearchResult {
@@ -344,6 +359,11 @@ interface CodexLensConfig {
reranker_backend?: string; // 'onnx' (local) or 'api'
reranker_model?: string;
reranker_top_k?: number;
api_max_workers?: number;
api_batch_size?: number;
cascade_strategy?: string;
staged_stage2_mode?: string;
static_graph_enabled?: boolean;
}
interface IndexStatus {
@@ -357,6 +377,39 @@ interface IndexStatus {
warning?: string;
}
function readCodexLensSettingsSnapshot(): Partial<CodexLensConfig> {
const settingsPath = join(getCodexLensDataDir(), 'settings.json');
if (!existsSync(settingsPath)) {
return {};
}
try {
const parsed = JSON.parse(readFileSync(settingsPath, 'utf-8')) as Record<string, any>;
const embedding = (parsed.embedding ?? {}) as Record<string, any>;
const reranker = (parsed.reranker ?? {}) as Record<string, any>;
const api = (parsed.api ?? {}) as Record<string, any>;
const cascade = (parsed.cascade ?? {}) as Record<string, any>;
const staged = (parsed.staged ?? {}) as Record<string, any>;
const indexing = (parsed.indexing ?? {}) as Record<string, any>;
return {
embedding_backend: normalizeEmbeddingBackend(typeof embedding.backend === 'string' ? embedding.backend : undefined),
embedding_model: typeof embedding.model === 'string' ? embedding.model : undefined,
reranker_enabled: typeof reranker.enabled === 'boolean' ? reranker.enabled : undefined,
reranker_backend: typeof reranker.backend === 'string' ? reranker.backend : undefined,
reranker_model: typeof reranker.model === 'string' ? reranker.model : undefined,
reranker_top_k: typeof reranker.top_k === 'number' ? reranker.top_k : undefined,
api_max_workers: typeof api.max_workers === 'number' ? api.max_workers : undefined,
api_batch_size: typeof api.batch_size === 'number' ? api.batch_size : undefined,
cascade_strategy: typeof cascade.strategy === 'string' ? cascade.strategy : undefined,
staged_stage2_mode: typeof staged.stage2_mode === 'string' ? staged.stage2_mode : undefined,
static_graph_enabled: typeof indexing.static_graph_enabled === 'boolean' ? indexing.static_graph_enabled : undefined,
};
} catch {
return {};
}
}
/**
* Strip ANSI color codes from string (for JSON parsing)
*/
@@ -464,6 +517,99 @@ function filterResultsToTargetFile<T extends { file: string }>(results: T[], sco
return results.filter((result) => normalizeResultFilePath(result.file, scope.workingDirectory) === normalizedTarget);
}
function parseCodexLensJsonOutput(output: string | undefined): any | null {
const cleanOutput = stripAnsi(output || '').trim();
if (!cleanOutput) {
return null;
}
const candidates = [
cleanOutput,
...cleanOutput.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.startsWith('{') || line.startsWith('[')),
];
const firstBrace = cleanOutput.indexOf('{');
const lastBrace = cleanOutput.lastIndexOf('}');
if (firstBrace !== -1 && lastBrace > firstBrace) {
candidates.push(cleanOutput.slice(firstBrace, lastBrace + 1));
}
const firstBracket = cleanOutput.indexOf('[');
const lastBracket = cleanOutput.lastIndexOf(']');
if (firstBracket !== -1 && lastBracket > firstBracket) {
candidates.push(cleanOutput.slice(firstBracket, lastBracket + 1));
}
for (const candidate of candidates) {
try {
return JSON.parse(candidate);
} catch {
continue;
}
}
return null;
}
function mapCodexLensSemanticMatches(data: any[], scope: SearchScope, maxContentLength: number): SemanticMatch[] {
return filterResultsToTargetFile(data.map((item: any) => {
const rawScore = item.score || 0;
const similarityScore = rawScore > 0 ? 1 / (1 + rawScore) : 1;
return {
file: item.path || item.file,
score: similarityScore,
content: truncateContent(item.content || item.excerpt, maxContentLength),
symbol: item.symbol || null,
};
}), scope);
}
function parsePlainTextFileMatches(output: string | undefined, scope: SearchScope): SemanticMatch[] {
const lines = stripAnsi(output || '')
.split(/\r?\n/)
.map((line) => line.trim())
.filter(Boolean);
const fileLines = lines.filter((line) => {
if (line.includes('RuntimeWarning:') || line.startsWith('warn(') || line.startsWith('Warning:')) {
return false;
}
const resolvedPath = /^[a-zA-Z]:[\\/]|^\//.test(line)
? line
: resolve(scope.workingDirectory, line);
try {
return statSync(resolvedPath).isFile();
} catch {
return false;
}
});
return filterResultsToTargetFile(
[...new Set(fileLines)].map((file, index) => ({
file,
score: Math.max(0.1, 1 - index * 0.05),
content: '',
symbol: null,
})),
scope,
);
}
function hasCentralizedVectorArtifacts(indexRoot: unknown): boolean {
if (typeof indexRoot !== 'string' || !indexRoot.trim()) {
return false;
}
const resolvedRoot = resolve(indexRoot);
return [
join(resolvedRoot, '_vectors.hnsw'),
join(resolvedRoot, '_vectors_meta.db'),
join(resolvedRoot, '_binary_vectors.mmap'),
].every((artifactPath) => existsSync(artifactPath));
}
function collectBackendError(
errors: string[],
backendName: string,
@@ -498,18 +644,20 @@ async function checkIndexStatus(path: string = '.'): Promise<IndexStatus> {
try {
// Fetch both status and config in parallel
const [statusResult, configResult] = await Promise.all([
executeCodexLens(['status', '--json'], { cwd: scope.workingDirectory }),
executeCodexLens(['config', 'show', '--json'], { cwd: scope.workingDirectory }),
executeCodexLens(['index', 'status', scope.workingDirectory], { cwd: scope.workingDirectory }),
executeCodexLens(['config', '--json'], { cwd: scope.workingDirectory }),
]);
// Parse config
let config: CodexLensConfig | null = null;
const settingsConfig = readCodexLensSettingsSnapshot();
let config: CodexLensConfig | null = Object.keys(settingsConfig).length > 0 ? { ...settingsConfig } : null;
if (configResult.success && configResult.output) {
try {
const cleanConfigOutput = stripAnsi(configResult.output);
const parsedConfig = JSON.parse(cleanConfigOutput);
const configData = parsedConfig.result || parsedConfig;
config = {
...settingsConfig,
config_file: configData.config_file,
index_dir: configData.index_dir,
embedding_backend: configData.embedding_backend,
@@ -540,13 +688,21 @@ async function checkIndexStatus(path: string = '.'): Promise<IndexStatus> {
const parsed = JSON.parse(cleanOutput);
// Handle both direct and nested response formats (status returns {success, result: {...}})
const status = parsed.result || parsed;
const indexed = status.projects_count > 0 || status.total_files > 0;
// Get embeddings coverage from comprehensive status
const embeddingsData = status.embeddings || {};
const embeddingsCoverage = embeddingsData.coverage_percent || 0;
const has_embeddings = embeddingsCoverage >= 50; // Threshold: 50%
const totalChunks = embeddingsData.total_chunks || 0;
const totalIndexes = Number(embeddingsData.total_indexes || 0);
const indexesWithEmbeddings = Number(embeddingsData.indexes_with_embeddings || 0);
const totalChunks = Number(embeddingsData.total_chunks || 0);
const hasCentralizedVectors = hasCentralizedVectorArtifacts(status.index_root);
let embeddingsCoverage = typeof embeddingsData.coverage_percent === 'number'
? embeddingsData.coverage_percent
: (totalIndexes > 0 ? (indexesWithEmbeddings / totalIndexes) * 100 : 0);
if (hasCentralizedVectors) {
embeddingsCoverage = Math.max(embeddingsCoverage, 100);
}
const indexed = Boolean(status.projects_count > 0 || status.total_files > 0 || status.index_root || totalIndexes > 0 || totalChunks > 0);
const has_embeddings = indexesWithEmbeddings > 0 || embeddingsCoverage > 0 || totalChunks > 0 || hasCentralizedVectors;
// Extract model info if available
const modelInfoData = embeddingsData.model_info;
@@ -563,9 +719,9 @@ async function checkIndexStatus(path: string = '.'): Promise<IndexStatus> {
if (!indexed) {
warning = 'No CodexLens index found. Run smart_search(action="init") to create index for better search results.';
} else if (embeddingsCoverage === 0) {
warning = 'Index exists but no embeddings generated. Run: codexlens embeddings-generate --recursive';
warning = 'Index exists but no embeddings generated. Run smart_search(action="embed") to build the vector index.';
} else if (embeddingsCoverage < 50) {
warning = `Embeddings coverage is ${embeddingsCoverage.toFixed(1)}% (below 50%). Hybrid search will use exact mode. Run: codexlens embeddings-generate --recursive`;
warning = `Embeddings coverage is ${embeddingsCoverage.toFixed(1)}% (below 50%). Hybrid search will degrade. Run smart_search(action="embed") to improve vector coverage.`;
}
return {
@@ -777,9 +933,198 @@ function buildRipgrepCommand(params: {
return { command: 'rg', args, tokens };
}
function normalizeEmbeddingBackend(backend?: string): string | undefined {
if (!backend) {
return undefined;
}
const normalized = backend.trim().toLowerCase();
if (!normalized) {
return undefined;
}
if (normalized === 'api') {
return 'litellm';
}
if (normalized === 'local') {
return 'fastembed';
}
return normalized;
}
const EMBED_PROGRESS_PREFIX = '__CCW_EMBED_PROGRESS__';
function resolveEmbeddingEndpoints(backend?: string): RotationEndpointConfig[] {
if (backend !== 'litellm') {
return [];
}
try {
return generateRotationEndpoints(getProjectRoot()).filter((endpoint) => {
const apiKey = endpoint.api_key?.trim() ?? '';
return Boolean(
apiKey &&
apiKey.length > 8 &&
!/^\*+$/.test(apiKey) &&
endpoint.api_base?.trim() &&
endpoint.model?.trim()
);
});
} catch {
return [];
}
}
function resolveApiWorkerCount(
requestedWorkers: number | undefined,
backend: string | undefined,
endpoints: RotationEndpointConfig[]
): number | undefined {
if (backend !== 'litellm') {
return undefined;
}
if (typeof requestedWorkers === 'number' && Number.isFinite(requestedWorkers)) {
return Math.max(1, Math.floor(requestedWorkers));
}
if (endpoints.length <= 1) {
return 4;
}
return Math.min(16, Math.max(4, endpoints.length * 2));
}
function extractEmbedJsonLine(stdout: string): string | undefined {
const lines = stdout
.split(/\r?\n/)
.map((line) => line.trim())
.filter(Boolean)
.filter((line) => !line.startsWith(EMBED_PROGRESS_PREFIX));
return [...lines].reverse().find((line) => line.startsWith('{') && line.endsWith('}'));
}
async function executeEmbeddingsViaPython(params: {
projectPath: string;
backend?: string;
model?: string;
force: boolean;
maxWorkers?: number;
endpoints?: RotationEndpointConfig[];
}): Promise<{ success: boolean; error?: string; progressMessages?: string[] }> {
const { projectPath, backend, model, force, maxWorkers, endpoints = [] } = params;
const pythonCode = `
import json
import sys
from pathlib import Path
from codexlens.storage.registry import RegistryStore
from codexlens.cli.embedding_manager import generate_dense_embeddings_centralized
target_path = Path(r"__PROJECT_PATH__").expanduser().resolve()
backend = __BACKEND__
model = __MODEL__
force = __FORCE__
max_workers = __MAX_WORKERS__
endpoints = json.loads(r'''__ENDPOINTS_JSON__''')
def progress_update(message: str):
print("__CCW_EMBED_PROGRESS__" + str(message), flush=True)
registry = RegistryStore()
registry.initialize()
try:
project = registry.get_project(target_path)
if project is None:
print(json.dumps({"success": False, "error": f"No index found for: {target_path}"}), flush=True)
sys.exit(1)
index_root = Path(project.index_root)
result = generate_dense_embeddings_centralized(
index_root,
embedding_backend=backend,
model_profile=model,
force=force,
use_gpu=True,
max_workers=max_workers,
endpoints=endpoints if endpoints else None,
progress_callback=progress_update,
)
print(json.dumps(result), flush=True)
if not result.get("success"):
sys.exit(1)
finally:
registry.close()
`
.replace('__PROJECT_PATH__', projectPath.replace(/\\/g, '\\\\'))
.replace('__BACKEND__', backend ? JSON.stringify(backend) : 'None')
.replace('__MODEL__', model ? JSON.stringify(model) : 'None')
.replace('__FORCE__', force ? 'True' : 'False')
.replace('__MAX_WORKERS__', typeof maxWorkers === 'number' ? String(Math.max(1, Math.floor(maxWorkers))) : 'None')
.replace('__ENDPOINTS_JSON__', JSON.stringify(endpoints).replace(/\\/g, '\\\\').replace(/'''/g, "\\'\\'\\'"));
return await new Promise((resolve) => {
const child = spawn(getVenvPythonPath(), ['-c', pythonCode], {
cwd: projectPath,
shell: false,
timeout: 1800000,
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
});
let stdout = '';
let stderr = '';
const progressMessages: string[] = [];
child.stdout.on('data', (data: Buffer) => {
const chunk = data.toString();
stdout += chunk;
for (const line of chunk.split(/\r?\n/)) {
if (line.startsWith(EMBED_PROGRESS_PREFIX)) {
progressMessages.push(line.slice(EMBED_PROGRESS_PREFIX.length).trim());
}
}
});
child.stderr.on('data', (data: Buffer) => {
stderr += data.toString();
});
child.on('error', (err) => {
resolve({ success: false, error: `Failed to start embeddings process: ${err.message}`, progressMessages });
});
child.on('close', (code) => {
const jsonLine = extractEmbedJsonLine(stdout);
if (jsonLine) {
try {
const parsed = JSON.parse(jsonLine) as { success?: boolean; error?: string };
if (parsed.success) {
resolve({ success: true, progressMessages });
return;
}
resolve({
success: false,
error: parsed.error || stderr.trim() || stdout.trim() || `Embeddings process exited with code ${code}`,
progressMessages,
});
return;
} catch {
// Fall through to generic error handling below.
}
}
resolve({
success: code === 0,
error: code === 0 ? undefined : (stderr.trim() || stdout.trim() || `Embeddings process exited with code ${code}`),
progressMessages,
});
});
});
}
/**
* Action: init - Initialize CodexLens index (FTS only, no embeddings)
* For semantic/vector search, use ccw view dashboard or codexlens CLI directly
* For semantic/vector search, follow with action="embed" to generate vectors.
* @param params - Search parameters
* @param force - If true, force full rebuild (delete existing index first)
*/
@@ -853,6 +1198,80 @@ async function executeInitAction(params: Params, force: boolean = false): Promis
};
}
/**
* Action: embed - Generate semantic/vector embeddings for an indexed project
*/
async function executeEmbedAction(params: Params): Promise<SearchResult> {
const { path = '.', embeddingBackend, embeddingModel, apiMaxWorkers, force = false } = params;
const scope = resolveSearchScope(path);
const readyStatus = await ensureCodexLensReady();
if (!readyStatus.ready) {
return {
success: false,
error: `CodexLens not available: ${readyStatus.error}. CodexLens will be auto-installed on first use.`,
};
}
const currentStatus = await checkIndexStatus(scope.workingDirectory);
const normalizedBackend = normalizeEmbeddingBackend(embeddingBackend) || currentStatus.config?.embedding_backend;
const trimmedModel = embeddingModel?.trim() || currentStatus.config?.embedding_model;
const endpoints = resolveEmbeddingEndpoints(normalizedBackend);
const configuredApiMaxWorkers = currentStatus.config?.api_max_workers;
const effectiveApiMaxWorkers = typeof apiMaxWorkers === 'number'
? Math.max(1, Math.floor(apiMaxWorkers))
: (typeof configuredApiMaxWorkers === 'number'
? Math.max(1, Math.floor(configuredApiMaxWorkers))
: resolveApiWorkerCount(undefined, normalizedBackend, endpoints));
if (normalizedBackend === 'litellm') {
const embedderReady = await ensureLiteLLMEmbedderReady();
if (!embedderReady.success) {
return {
success: false,
error: embedderReady.error || 'LiteLLM embedder is not ready.',
};
}
}
const result = await executeEmbeddingsViaPython({
projectPath: scope.workingDirectory,
backend: normalizedBackend,
model: trimmedModel,
force,
maxWorkers: effectiveApiMaxWorkers,
endpoints,
});
const indexStatus = result.success ? await checkIndexStatus(scope.workingDirectory) : currentStatus;
const coverage = indexStatus?.embeddings_coverage_percent;
const coverageText = coverage !== undefined ? ` (${coverage.toFixed(1)}% coverage)` : '';
const progressMessage = result.progressMessages && result.progressMessages.length > 0
? result.progressMessages[result.progressMessages.length - 1]
: undefined;
return {
success: result.success,
error: result.error,
message: result.success
? `Embeddings generated for ${path}${coverageText}`
: undefined,
metadata: {
action: 'embed',
path: scope.workingDirectory,
backend: normalizedBackend || indexStatus?.config?.embedding_backend,
embeddings_coverage_percent: coverage,
api_max_workers: effectiveApiMaxWorkers,
endpoint_count: endpoints.length,
use_gpu: true,
cascade_strategy: currentStatus.config?.cascade_strategy,
staged_stage2_mode: currentStatus.config?.staged_stage2_mode,
note: progressMessage,
},
status: indexStatus,
};
}
/**
* Action: status - Check CodexLens index status
*/
@@ -885,6 +1304,15 @@ async function executeStatusAction(params: Params): Promise<SearchResult> {
// Embedding backend info
const embeddingType = cfg.embedding_backend === 'litellm' ? 'API' : 'Local';
statusParts.push(`Embedding: ${embeddingType} (${cfg.embedding_model || 'default'})`);
if (typeof cfg.api_max_workers === 'number') {
statusParts.push(`API Workers: ${cfg.api_max_workers}`);
}
if (cfg.cascade_strategy) {
statusParts.push(`Cascade: ${cfg.cascade_strategy}`);
}
if (cfg.staged_stage2_mode) {
statusParts.push(`Stage2: ${cfg.staged_stage2_mode}`);
}
// Reranker info
if (cfg.reranker_enabled) {
@@ -1583,8 +2011,11 @@ async function executeHybridMode(params: Params): Promise<SearchResult> {
timer.mark('index_status_check');
// Request more results to support split (full content + extra files)
// NOTE: Current CodexLens search CLI in this environment rejects value-taking options
// like --limit/--offset/--method for search. Keep the invocation minimal and apply
// pagination/selection in CCW after parsing results.
const totalToFetch = maxResults + extraFilesCount;
const args = ['search', query, '--limit', totalToFetch.toString(), '--offset', offset.toString(), '--method', 'dense_rerank', '--json'];
const args = ['search', query, '--json'];
if (enrich) {
args.push('--enrich');
}
@@ -1619,22 +2050,10 @@ async function executeHybridMode(params: Params): Promise<SearchResult> {
let baselineInfo: { score: number; count: number } | null = null;
let initialCount = 0;
try {
const parsed = JSON.parse(stripAnsi(result.output || '{}'));
const data = parsed.result?.results || parsed.results || parsed;
allResults = filterResultsToTargetFile((Array.isArray(data) ? data : []).map((item: any) => {
const rawScore = item.score || 0;
// Hybrid mode returns distance scores (lower is better).
// Convert to similarity scores (higher is better) for consistency.
// Formula: similarity = 1 / (1 + distance)
const similarityScore = rawScore > 0 ? 1 / (1 + rawScore) : 1;
return {
file: item.path || item.file,
score: similarityScore,
content: truncateContent(item.content || item.excerpt, maxContentLength),
symbol: item.symbol || null,
};
}), scope);
const parsedOutput = parseCodexLensJsonOutput(result.output);
const parsedData = parsedOutput?.result?.results || parsedOutput?.results || parsedOutput;
if (Array.isArray(parsedData)) {
allResults = mapCodexLensSemanticMatches(parsedData, scope, maxContentLength);
timer.mark('parse_results');
initialCount = allResults.length;
@@ -1655,19 +2074,24 @@ async function executeHybridMode(params: Params): Promise<SearchResult> {
// 4. Re-sort by adjusted scores
allResults.sort((a, b) => b.score - a.score);
timer.mark('post_processing');
} catch {
return {
success: true,
results: [],
output: result.output,
metadata: {
mode: 'hybrid',
backend: 'codexlens',
count: 0,
query,
warning: mergeWarnings(indexStatus.warning, result.warning, 'Failed to parse JSON output'),
},
};
} else {
allResults = parsePlainTextFileMatches(result.output, scope);
if (allResults.length === 0) {
return {
success: true,
results: [],
output: result.output,
metadata: {
mode: 'hybrid',
backend: 'codexlens',
count: 0,
query,
warning: mergeWarnings(indexStatus.warning, result.warning, 'Failed to parse JSON output'),
},
};
}
timer.mark('parse_results');
initialCount = allResults.length;
}
// Split results: first N with full content, rest as file paths only
@@ -2164,6 +2588,13 @@ Recommended MCP flow: use **action=\"search\"** for lookups, **action=\"init\"**
* **init_force**: Force full rebuild (delete and recreate static index).
* *path* (string): Directory to index (default: current).
* **embed**: Generate semantic/vector embeddings for an indexed project.
* *path* (string): Directory to embed (default: current).
* *embeddingBackend* (string): 'litellm'/'api' for remote API embeddings, 'fastembed'/'local' for local embeddings.
* *embeddingModel* (string): Embedding model/profile to use.
* *apiMaxWorkers* (number): Max concurrent API embedding workers. Defaults to auto-sizing from the configured endpoint pool.
* *force* (boolean): Regenerate embeddings even if they already exist.
* **status**: Check index status. (No required params)
* **update**: Incremental index update.
@@ -2175,16 +2606,17 @@ Recommended MCP flow: use **action=\"search\"** for lookups, **action=\"init\"**
**Examples:**
smart_search(query="authentication logic") # Content search (default action)
smart_search(query="MyClass", mode="semantic") # Semantic search
smart_search(action="find_files", pattern="*.ts") # Find TypeScript files
smart_search(action=\"embed\", path=\"/project\", embeddingBackend=\"api\", apiMaxWorkers=8) # Build API vector index
smart_search(action="init", path="/project") # Build static FTS index
smart_search(action="embed", path="/project", embeddingBackend="api") # Build API vector index
smart_search(query="auth", limit=10, offset=0) # Paginated search`,
inputSchema: {
type: 'object',
properties: {
action: {
type: 'string',
enum: ['init', 'init_force', 'search', 'find_files', 'status', 'update', 'watch', 'search_files'],
description: 'Action: search (content search; default and recommended), find_files (path pattern matching), init (create static FTS index, incremental), init_force (force full rebuild), status (check index), update (incremental refresh), watch (auto-update watcher; opt-in). Note: search_files is deprecated.',
enum: ['init', 'init_force', 'embed', 'search', 'find_files', 'status', 'update', 'watch', 'search_files'],
description: 'Action: search (content search; default and recommended), find_files (path pattern matching), init (create static FTS index, incremental), init_force (force full rebuild), embed (generate semantic/vector embeddings), status (check index), update (incremental refresh), watch (auto-update watcher; opt-in). Note: search_files is deprecated.',
default: 'search',
},
query: {
@@ -2259,6 +2691,23 @@ Recommended MCP flow: use **action=\"search\"** for lookups, **action=\"init\"**
items: { type: 'string' },
description: 'Languages to index (for init action). Example: ["javascript", "typescript"]',
},
embeddingBackend: {
type: 'string',
description: 'Embedding backend for action="embed": litellm/api (remote API) or fastembed/local (local GPU/CPU).',
},
embeddingModel: {
type: 'string',
description: 'Embedding model/profile for action="embed". Examples: "code", "fast", "qwen3-embedding-sf".',
},
apiMaxWorkers: {
type: 'number',
description: 'Max concurrent API embedding workers for action="embed". Defaults to auto-sizing from the configured endpoint pool.',
},
force: {
type: 'boolean',
description: 'Force regeneration for action="embed".',
default: false,
},
enrich: {
type: 'boolean',
description: 'Enrich search results with code graph relationships (calls, imports, called_by, imported_by).',
@@ -2625,6 +3074,10 @@ export async function handler(params: Record<string, unknown>): Promise<ToolResu
result = await executeInitAction(parsed.data, true);
break;
case 'embed':
result = await executeEmbedAction(parsed.data);
break;
case 'status':
result = await executeStatusAction(parsed.data);
break;