mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-02-05 01:50:27 +08:00
Add parallel search mode and index progress bar
Features: - CCW smart_search: Add 'parallel' mode that runs hybrid + exact + ripgrep simultaneously with RRF (Reciprocal Rank Fusion) for result merging - Dashboard: Add real-time progress bar for CodexLens index initialization - MCP: Return progress metadata in init action response - Codex-lens: Auto-detect optimal worker count for parallel indexing Changes: - smart-search.ts: Add parallel mode, RRF fusion, progress tracking - codex-lens.ts: Add onProgress callback support, progress parsing - codexlens-routes.ts: Broadcast index progress via WebSocket - codexlens-manager.js: New index progress modal with real-time updates - notifications.js: Add WebSocket event handler registration system - i18n.js: Add English/Chinese translations for progress UI - index_tree.py: Workers parameter now auto-detects CPU count (max 16) - commands.py: CLI --workers parameter supports auto-detection 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -12,6 +12,7 @@ import {
|
||||
installSemantic,
|
||||
uninstallCodexLens
|
||||
} from '../../tools/codex-lens.js';
|
||||
import type { ProgressInfo } from '../../tools/codex-lens.js';
|
||||
|
||||
export interface RouteContext {
|
||||
pathname: string;
|
||||
@@ -217,9 +218,32 @@ export async function handleCodexLensRoutes(ctx: RouteContext): Promise<boolean>
|
||||
const { path: projectPath } = body;
|
||||
const targetPath = projectPath || initialPath;
|
||||
|
||||
// Broadcast start event
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { stage: 'start', message: 'Starting index...', percent: 0, path: targetPath }
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await executeCodexLens(['init', targetPath, '--json'], { cwd: targetPath });
|
||||
const result = await executeCodexLens(['init', targetPath, '--json'], {
|
||||
cwd: targetPath,
|
||||
timeout: 300000, // 5 minutes
|
||||
onProgress: (progress: ProgressInfo) => {
|
||||
// Broadcast progress to all connected clients
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { ...progress, path: targetPath }
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
if (result.success) {
|
||||
// Broadcast completion
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { stage: 'complete', message: 'Index complete', percent: 100, path: targetPath }
|
||||
});
|
||||
|
||||
try {
|
||||
const parsed = extractJSON(result.output);
|
||||
return { success: true, result: parsed };
|
||||
@@ -227,9 +251,19 @@ export async function handleCodexLensRoutes(ctx: RouteContext): Promise<boolean>
|
||||
return { success: true, output: result.output };
|
||||
}
|
||||
} else {
|
||||
// Broadcast error
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { stage: 'error', message: result.error || 'Unknown error', percent: 0, path: targetPath }
|
||||
});
|
||||
return { success: false, error: result.error, status: 500 };
|
||||
}
|
||||
} catch (err) {
|
||||
// Broadcast error
|
||||
broadcastToClients({
|
||||
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||
payload: { stage: 'error', message: err.message, percent: 0, path: targetPath }
|
||||
});
|
||||
return { success: false, error: err.message, status: 500 };
|
||||
}
|
||||
});
|
||||
|
||||
@@ -87,6 +87,49 @@ let autoRefreshInterval = null;
|
||||
let lastDataHash = null;
|
||||
const AUTO_REFRESH_INTERVAL_MS = 30000; // 30 seconds
|
||||
|
||||
// Custom event handlers registry for components to subscribe to specific events
|
||||
const wsEventHandlers = {};
|
||||
|
||||
/**
|
||||
* Register a custom handler for a specific WebSocket event type
|
||||
* @param {string} eventType - The event type to listen for
|
||||
* @param {Function} handler - The handler function
|
||||
*/
|
||||
function registerWsEventHandler(eventType, handler) {
|
||||
if (!wsEventHandlers[eventType]) {
|
||||
wsEventHandlers[eventType] = [];
|
||||
}
|
||||
wsEventHandlers[eventType].push(handler);
|
||||
}
|
||||
|
||||
/**
|
||||
* Unregister a custom handler for a specific WebSocket event type
|
||||
* @param {string} eventType - The event type
|
||||
* @param {Function} handler - The handler function to remove
|
||||
*/
|
||||
function unregisterWsEventHandler(eventType, handler) {
|
||||
if (wsEventHandlers[eventType]) {
|
||||
wsEventHandlers[eventType] = wsEventHandlers[eventType].filter(h => h !== handler);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch event to registered handlers
|
||||
* @param {string} eventType - The event type
|
||||
* @param {Object} data - The full event data
|
||||
*/
|
||||
function dispatchToEventHandlers(eventType, data) {
|
||||
if (wsEventHandlers[eventType]) {
|
||||
wsEventHandlers[eventType].forEach(handler => {
|
||||
try {
|
||||
handler(data);
|
||||
} catch (e) {
|
||||
console.error('[WS] Error in custom handler for', eventType, e);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ========== WebSocket Connection ==========
|
||||
function initWebSocket() {
|
||||
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
|
||||
@@ -390,6 +433,12 @@ function handleNotification(data) {
|
||||
console.log('[CodexLens] Uninstallation completed:', payload);
|
||||
break;
|
||||
|
||||
case 'CODEXLENS_INDEX_PROGRESS':
|
||||
// Handle CodexLens index progress updates
|
||||
dispatchToEventHandlers('CODEXLENS_INDEX_PROGRESS', data);
|
||||
console.log('[CodexLens] Index progress:', payload.stage, payload.percent + '%');
|
||||
break;
|
||||
|
||||
default:
|
||||
console.log('[WS] Unknown notification type:', type);
|
||||
}
|
||||
|
||||
@@ -278,6 +278,15 @@ const i18n = {
|
||||
'codexlens.modelDeleteFailed': 'Model deletion failed',
|
||||
'codexlens.deleteModelConfirm': 'Are you sure you want to delete model',
|
||||
|
||||
// CodexLens Indexing Progress
|
||||
'codexlens.indexing': 'Indexing',
|
||||
'codexlens.indexingDesc': 'Building code index for workspace',
|
||||
'codexlens.preparingIndex': 'Preparing index...',
|
||||
'codexlens.filesProcessed': 'Files processed',
|
||||
'codexlens.indexComplete': 'Index complete',
|
||||
'codexlens.indexSuccess': 'Index created successfully',
|
||||
'codexlens.indexFailed': 'Indexing failed',
|
||||
|
||||
// Semantic Search Configuration
|
||||
'semantic.settings': 'Semantic Search Settings',
|
||||
'semantic.testSearch': 'Test Semantic Search',
|
||||
@@ -1394,6 +1403,15 @@ const i18n = {
|
||||
'codexlens.modelDeleteFailed': '模型删除失败',
|
||||
'codexlens.deleteModelConfirm': '确定要删除模型',
|
||||
|
||||
// CodexLens 索引进度
|
||||
'codexlens.indexing': '索引中',
|
||||
'codexlens.indexingDesc': '正在为工作区构建代码索引',
|
||||
'codexlens.preparingIndex': '准备索引...',
|
||||
'codexlens.filesProcessed': '已处理文件',
|
||||
'codexlens.indexComplete': '索引完成',
|
||||
'codexlens.indexSuccess': '索引创建成功',
|
||||
'codexlens.indexFailed': '索引失败',
|
||||
|
||||
// Semantic Search 配置
|
||||
'semantic.settings': '语义搜索设置',
|
||||
'semantic.testSearch': '测试语义搜索',
|
||||
|
||||
@@ -542,10 +542,160 @@ async function deleteModel(profile) {
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Initialize CodexLens index
|
||||
* Initialize CodexLens index with progress tracking
|
||||
*/
|
||||
function initCodexLensIndex() {
|
||||
openCliInstallWizard('codexlens');
|
||||
// Create progress modal
|
||||
var modal = document.createElement('div');
|
||||
modal.id = 'codexlensIndexModal';
|
||||
modal.className = 'fixed inset-0 bg-black/50 flex items-center justify-center z-50';
|
||||
modal.innerHTML =
|
||||
'<div class="bg-card rounded-lg shadow-xl w-full max-w-md mx-4 overflow-hidden">' +
|
||||
'<div class="p-6">' +
|
||||
'<div class="flex items-center gap-3 mb-4">' +
|
||||
'<div class="w-10 h-10 rounded-full bg-primary/10 flex items-center justify-center">' +
|
||||
'<i data-lucide="database" class="w-5 h-5 text-primary"></i>' +
|
||||
'</div>' +
|
||||
'<div>' +
|
||||
'<h3 class="text-lg font-semibold">' + t('codexlens.indexing') + '</h3>' +
|
||||
'<p class="text-sm text-muted-foreground">' + t('codexlens.indexingDesc') + '</p>' +
|
||||
'</div>' +
|
||||
'</div>' +
|
||||
'<div class="space-y-4">' +
|
||||
'<div id="codexlensIndexProgress">' +
|
||||
'<div class="flex items-center gap-3">' +
|
||||
'<div class="animate-spin w-5 h-5 border-2 border-primary border-t-transparent rounded-full"></div>' +
|
||||
'<span class="text-sm" id="codexlensIndexStatus">' + t('codexlens.preparingIndex') + '</span>' +
|
||||
'</div>' +
|
||||
'<div class="mt-3 h-2 bg-muted rounded-full overflow-hidden">' +
|
||||
'<div id="codexlensIndexProgressBar" class="h-full bg-primary transition-all duration-300" style="width: 0%"></div>' +
|
||||
'</div>' +
|
||||
'<div class="mt-2 text-xs text-muted-foreground" id="codexlensIndexDetails"></div>' +
|
||||
'</div>' +
|
||||
'</div>' +
|
||||
'</div>' +
|
||||
'<div class="border-t border-border p-4 flex justify-end gap-3 bg-muted/30">' +
|
||||
'<button class="btn-outline px-4 py-2" id="codexlensIndexCancelBtn" onclick="cancelCodexLensIndex()">' + t('common.cancel') + '</button>' +
|
||||
'</div>' +
|
||||
'</div>';
|
||||
|
||||
document.body.appendChild(modal);
|
||||
if (window.lucide) lucide.createIcons();
|
||||
|
||||
// Start indexing
|
||||
startCodexLensIndexing();
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the indexing process
|
||||
*/
|
||||
async function startCodexLensIndexing() {
|
||||
var statusText = document.getElementById('codexlensIndexStatus');
|
||||
var progressBar = document.getElementById('codexlensIndexProgressBar');
|
||||
var detailsText = document.getElementById('codexlensIndexDetails');
|
||||
var cancelBtn = document.getElementById('codexlensIndexCancelBtn');
|
||||
|
||||
// Setup WebSocket listener for progress events
|
||||
window.codexlensIndexProgressHandler = function(event) {
|
||||
if (event.type === 'CODEXLENS_INDEX_PROGRESS') {
|
||||
var payload = event.payload;
|
||||
if (statusText) statusText.textContent = payload.message || t('codexlens.indexing');
|
||||
if (progressBar) progressBar.style.width = (payload.percent || 0) + '%';
|
||||
if (detailsText && payload.filesProcessed !== undefined) {
|
||||
detailsText.textContent = t('codexlens.filesProcessed') + ': ' + payload.filesProcessed +
|
||||
(payload.totalFiles ? ' / ' + payload.totalFiles : '');
|
||||
}
|
||||
|
||||
// Handle completion
|
||||
if (payload.stage === 'complete') {
|
||||
handleIndexComplete(true, payload.message);
|
||||
} else if (payload.stage === 'error') {
|
||||
handleIndexComplete(false, payload.message);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Register with notification system if available
|
||||
if (typeof registerWsEventHandler === 'function') {
|
||||
registerWsEventHandler('CODEXLENS_INDEX_PROGRESS', window.codexlensIndexProgressHandler);
|
||||
}
|
||||
|
||||
try {
|
||||
var response = await fetch('/api/codexlens/init', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ path: projectPath })
|
||||
});
|
||||
|
||||
var result = await response.json();
|
||||
|
||||
if (!result.success && statusText) {
|
||||
// If WebSocket didn't report error, show it now
|
||||
handleIndexComplete(false, result.error || t('common.unknownError'));
|
||||
}
|
||||
} catch (err) {
|
||||
handleIndexComplete(false, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle index completion
|
||||
*/
|
||||
function handleIndexComplete(success, message) {
|
||||
var statusText = document.getElementById('codexlensIndexStatus');
|
||||
var progressBar = document.getElementById('codexlensIndexProgressBar');
|
||||
var cancelBtn = document.getElementById('codexlensIndexCancelBtn');
|
||||
|
||||
// Unregister WebSocket handler
|
||||
if (typeof unregisterWsEventHandler === 'function') {
|
||||
unregisterWsEventHandler('CODEXLENS_INDEX_PROGRESS', window.codexlensIndexProgressHandler);
|
||||
}
|
||||
|
||||
if (success) {
|
||||
if (progressBar) progressBar.style.width = '100%';
|
||||
if (statusText) statusText.textContent = t('codexlens.indexComplete');
|
||||
if (cancelBtn) cancelBtn.textContent = t('common.close');
|
||||
|
||||
showRefreshToast(t('codexlens.indexSuccess'), 'success');
|
||||
|
||||
// Auto-close after 2 seconds
|
||||
setTimeout(function() {
|
||||
closeCodexLensIndexModal();
|
||||
// Refresh status
|
||||
if (typeof loadCodexLensStatus === 'function') {
|
||||
loadCodexLensStatus().then(function() {
|
||||
renderToolsSection();
|
||||
if (window.lucide) lucide.createIcons();
|
||||
});
|
||||
}
|
||||
}, 2000);
|
||||
} else {
|
||||
if (progressBar) progressBar.classList.add('bg-destructive');
|
||||
if (statusText) statusText.textContent = t('codexlens.indexFailed') + ': ' + message;
|
||||
if (cancelBtn) cancelBtn.textContent = t('common.close');
|
||||
|
||||
showRefreshToast(t('codexlens.indexFailed') + ': ' + message, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel indexing
|
||||
*/
|
||||
function cancelCodexLensIndex() {
|
||||
closeCodexLensIndexModal();
|
||||
}
|
||||
|
||||
/**
|
||||
* Close index modal
|
||||
*/
|
||||
function closeCodexLensIndexModal() {
|
||||
var modal = document.getElementById('codexlensIndexModal');
|
||||
if (modal) modal.remove();
|
||||
|
||||
// Unregister WebSocket handler
|
||||
if (typeof unregisterWsEventHandler === 'function' && window.codexlensIndexProgressHandler) {
|
||||
unregisterWsEventHandler('CODEXLENS_INDEX_PROGRESS', window.codexlensIndexProgressHandler);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -91,6 +91,15 @@ interface ExecuteResult {
|
||||
interface ExecuteOptions {
|
||||
timeout?: number;
|
||||
cwd?: string;
|
||||
onProgress?: (progress: ProgressInfo) => void;
|
||||
}
|
||||
|
||||
interface ProgressInfo {
|
||||
stage: string;
|
||||
message: string;
|
||||
percent: number;
|
||||
filesProcessed?: number;
|
||||
totalFiles?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -361,6 +370,57 @@ async function ensureReady(): Promise<ReadyStatus> {
|
||||
return recheck;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse progress info from CodexLens output
|
||||
* @param line - Output line to parse
|
||||
* @returns Progress info or null
|
||||
*/
|
||||
function parseProgressLine(line: string): ProgressInfo | null {
|
||||
// Parse file processing progress: "Processing file X/Y: path"
|
||||
const fileMatch = line.match(/Processing file (\d+)\/(\d+):\s*(.+)/i);
|
||||
if (fileMatch) {
|
||||
const current = parseInt(fileMatch[1], 10);
|
||||
const total = parseInt(fileMatch[2], 10);
|
||||
return {
|
||||
stage: 'indexing',
|
||||
message: `Processing ${fileMatch[3]}`,
|
||||
percent: Math.round((current / total) * 80) + 10, // 10-90%
|
||||
filesProcessed: current,
|
||||
totalFiles: total,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse stage messages
|
||||
if (line.includes('Discovering files')) {
|
||||
return { stage: 'discover', message: 'Discovering files...', percent: 5 };
|
||||
}
|
||||
if (line.includes('Building index')) {
|
||||
return { stage: 'build', message: 'Building index...', percent: 10 };
|
||||
}
|
||||
if (line.includes('Extracting symbols')) {
|
||||
return { stage: 'symbols', message: 'Extracting symbols...', percent: 50 };
|
||||
}
|
||||
if (line.includes('Generating embeddings') || line.includes('Creating embeddings')) {
|
||||
return { stage: 'embeddings', message: 'Generating embeddings...', percent: 70 };
|
||||
}
|
||||
if (line.includes('Finalizing') || line.includes('Complete')) {
|
||||
return { stage: 'complete', message: 'Finalizing...', percent: 95 };
|
||||
}
|
||||
|
||||
// Parse indexed count: "Indexed X files"
|
||||
const indexedMatch = line.match(/Indexed (\d+) files/i);
|
||||
if (indexedMatch) {
|
||||
return {
|
||||
stage: 'complete',
|
||||
message: `Indexed ${indexedMatch[1]} files`,
|
||||
percent: 100,
|
||||
filesProcessed: parseInt(indexedMatch[1], 10),
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute CodexLens CLI command
|
||||
* @param args - CLI arguments
|
||||
@@ -368,7 +428,7 @@ async function ensureReady(): Promise<ReadyStatus> {
|
||||
* @returns Execution result
|
||||
*/
|
||||
async function executeCodexLens(args: string[], options: ExecuteOptions = {}): Promise<ExecuteResult> {
|
||||
const { timeout = 60000, cwd = process.cwd() } = options;
|
||||
const { timeout = 60000, cwd = process.cwd(), onProgress } = options;
|
||||
|
||||
// Ensure ready
|
||||
const readyStatus = await ensureReady();
|
||||
@@ -387,10 +447,35 @@ async function executeCodexLens(args: string[], options: ExecuteOptions = {}): P
|
||||
let timedOut = false;
|
||||
|
||||
child.stdout.on('data', (data) => {
|
||||
stdout += data.toString();
|
||||
const chunk = data.toString();
|
||||
stdout += chunk;
|
||||
|
||||
// Report progress if callback provided
|
||||
if (onProgress) {
|
||||
const lines = chunk.split('\n');
|
||||
for (const line of lines) {
|
||||
const progress = parseProgressLine(line.trim());
|
||||
if (progress) {
|
||||
onProgress(progress);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
child.stderr.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
const chunk = data.toString();
|
||||
stderr += chunk;
|
||||
|
||||
// Also check stderr for progress (some tools output there)
|
||||
if (onProgress) {
|
||||
const lines = chunk.split('\n');
|
||||
for (const line of lines) {
|
||||
const progress = parseProgressLine(line.trim());
|
||||
if (progress) {
|
||||
onProgress(progress);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const timeoutId = setTimeout(() => {
|
||||
@@ -803,6 +888,9 @@ async function uninstallCodexLens(): Promise<BootstrapResult> {
|
||||
}
|
||||
}
|
||||
|
||||
// Export types
|
||||
export type { ProgressInfo, ExecuteOptions };
|
||||
|
||||
// Export for direct usage
|
||||
export { ensureReady, executeCodexLens, checkVenvStatus, bootstrapVenv, checkSemanticStatus, installSemantic, uninstallCodexLens };
|
||||
|
||||
|
||||
@@ -21,12 +21,13 @@ import {
|
||||
ensureReady as ensureCodexLensReady,
|
||||
executeCodexLens,
|
||||
} from './codex-lens.js';
|
||||
import type { ProgressInfo } from './codex-lens.js';
|
||||
|
||||
// Define Zod schema for validation
|
||||
const ParamsSchema = z.object({
|
||||
action: z.enum(['init', 'search', 'search_files', 'status']).default('search'),
|
||||
query: z.string().optional(),
|
||||
mode: z.enum(['auto', 'hybrid', 'exact', 'ripgrep']).default('auto'),
|
||||
mode: z.enum(['auto', 'hybrid', 'exact', 'ripgrep', 'parallel']).default('auto'),
|
||||
output_mode: z.enum(['full', 'files_only', 'count']).default('full'),
|
||||
path: z.string().optional(),
|
||||
paths: z.array(z.string()).default([]),
|
||||
@@ -35,12 +36,17 @@ const ParamsSchema = z.object({
|
||||
includeHidden: z.boolean().default(false),
|
||||
languages: z.array(z.string()).optional(),
|
||||
limit: z.number().default(100),
|
||||
parallelWeights: z.object({
|
||||
hybrid: z.number().default(0.5),
|
||||
exact: z.number().default(0.3),
|
||||
ripgrep: z.number().default(0.2),
|
||||
}).optional(),
|
||||
});
|
||||
|
||||
type Params = z.infer<typeof ParamsSchema>;
|
||||
|
||||
// Search mode constants
|
||||
const SEARCH_MODES = ['auto', 'hybrid', 'exact', 'ripgrep'] as const;
|
||||
const SEARCH_MODES = ['auto', 'hybrid', 'exact', 'ripgrep', 'parallel'] as const;
|
||||
|
||||
// Classification confidence threshold
|
||||
const CONFIDENCE_THRESHOLD = 0.7;
|
||||
@@ -72,10 +78,10 @@ interface GraphMatch {
|
||||
}
|
||||
|
||||
interface SearchMetadata {
|
||||
mode: string;
|
||||
backend: string;
|
||||
count: number;
|
||||
query: string;
|
||||
mode?: string;
|
||||
backend?: string;
|
||||
count?: number;
|
||||
query?: string;
|
||||
classified_as?: string;
|
||||
confidence?: number;
|
||||
reasoning?: string;
|
||||
@@ -83,6 +89,17 @@ interface SearchMetadata {
|
||||
warning?: string;
|
||||
note?: string;
|
||||
index_status?: 'indexed' | 'not_indexed' | 'partial';
|
||||
// Init action specific
|
||||
action?: string;
|
||||
path?: string;
|
||||
progress?: {
|
||||
stage: string;
|
||||
message: string;
|
||||
percent: number;
|
||||
filesProcessed?: number;
|
||||
totalFiles?: number;
|
||||
};
|
||||
progressHistory?: ProgressInfo[];
|
||||
}
|
||||
|
||||
interface SearchResult {
|
||||
@@ -326,7 +343,39 @@ async function executeInitAction(params: Params): Promise<SearchResult> {
|
||||
args.push('--languages', languages.join(','));
|
||||
}
|
||||
|
||||
const result = await executeCodexLens(args, { cwd: path, timeout: 300000 });
|
||||
// Track progress updates
|
||||
const progressUpdates: ProgressInfo[] = [];
|
||||
let lastProgress: ProgressInfo | null = null;
|
||||
|
||||
const result = await executeCodexLens(args, {
|
||||
cwd: path,
|
||||
timeout: 300000,
|
||||
onProgress: (progress: ProgressInfo) => {
|
||||
progressUpdates.push(progress);
|
||||
lastProgress = progress;
|
||||
},
|
||||
});
|
||||
|
||||
// Build metadata with progress info
|
||||
const metadata: SearchMetadata = {
|
||||
action: 'init',
|
||||
path,
|
||||
};
|
||||
|
||||
if (lastProgress !== null) {
|
||||
const p = lastProgress as ProgressInfo;
|
||||
metadata.progress = {
|
||||
stage: p.stage,
|
||||
message: p.message,
|
||||
percent: p.percent,
|
||||
filesProcessed: p.filesProcessed,
|
||||
totalFiles: p.totalFiles,
|
||||
};
|
||||
}
|
||||
|
||||
if (progressUpdates.length > 0) {
|
||||
metadata.progressHistory = progressUpdates.slice(-5); // Keep last 5 progress updates
|
||||
}
|
||||
|
||||
return {
|
||||
success: result.success,
|
||||
@@ -334,6 +383,7 @@ async function executeInitAction(params: Params): Promise<SearchResult> {
|
||||
message: result.success
|
||||
? `CodexLens index created successfully for ${path}`
|
||||
: undefined,
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -726,17 +776,155 @@ async function executeHybridMode(params: Params): Promise<SearchResult> {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* TypeScript implementation of Reciprocal Rank Fusion
|
||||
* Reference: codex-lens/src/codexlens/search/ranking.py
|
||||
* Formula: score(d) = Σ weight_source / (k + rank_source(d))
|
||||
*/
|
||||
function applyRRFFusion(
|
||||
resultsMap: Map<string, any[]>,
|
||||
weights: Record<string, number>,
|
||||
limit: number,
|
||||
k: number = 60,
|
||||
): any[] {
|
||||
const pathScores = new Map<string, { score: number; result: any; sources: string[] }>();
|
||||
|
||||
resultsMap.forEach((results, source) => {
|
||||
const weight = weights[source] || 0;
|
||||
if (weight === 0 || !results) return;
|
||||
|
||||
results.forEach((result, rank) => {
|
||||
const path = result.file || result.path;
|
||||
if (!path) return;
|
||||
|
||||
const rrfContribution = weight / (k + rank + 1);
|
||||
|
||||
if (!pathScores.has(path)) {
|
||||
pathScores.set(path, { score: 0, result, sources: [] });
|
||||
}
|
||||
const entry = pathScores.get(path)!;
|
||||
entry.score += rrfContribution;
|
||||
if (!entry.sources.includes(source)) {
|
||||
entry.sources.push(source);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Sort by fusion score descending
|
||||
return Array.from(pathScores.values())
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, limit)
|
||||
.map(item => ({
|
||||
...item.result,
|
||||
fusion_score: item.score,
|
||||
matched_backends: item.sources,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Mode: parallel - Run all backends simultaneously with RRF fusion
|
||||
* Returns best results from hybrid + exact + ripgrep combined
|
||||
*/
|
||||
async function executeParallelMode(params: Params): Promise<SearchResult> {
|
||||
const { query, path = '.', limit = 100, parallelWeights } = params;
|
||||
|
||||
if (!query) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'Query is required for search',
|
||||
};
|
||||
}
|
||||
|
||||
// Default weights if not provided
|
||||
const weights = parallelWeights || {
|
||||
hybrid: 0.5,
|
||||
exact: 0.3,
|
||||
ripgrep: 0.2,
|
||||
};
|
||||
|
||||
// Run all backends in parallel
|
||||
const [hybridResult, exactResult, ripgrepResult] = await Promise.allSettled([
|
||||
executeHybridMode(params),
|
||||
executeCodexLensExactMode(params),
|
||||
executeRipgrepMode(params),
|
||||
]);
|
||||
|
||||
// Collect successful results
|
||||
const resultsMap = new Map<string, any[]>();
|
||||
const backendStatus: Record<string, string> = {};
|
||||
|
||||
if (hybridResult.status === 'fulfilled' && hybridResult.value.success) {
|
||||
resultsMap.set('hybrid', hybridResult.value.results as any[]);
|
||||
backendStatus.hybrid = 'success';
|
||||
} else {
|
||||
backendStatus.hybrid = hybridResult.status === 'rejected'
|
||||
? `error: ${hybridResult.reason}`
|
||||
: `failed: ${(hybridResult as PromiseFulfilledResult<SearchResult>).value.error}`;
|
||||
}
|
||||
|
||||
if (exactResult.status === 'fulfilled' && exactResult.value.success) {
|
||||
resultsMap.set('exact', exactResult.value.results as any[]);
|
||||
backendStatus.exact = 'success';
|
||||
} else {
|
||||
backendStatus.exact = exactResult.status === 'rejected'
|
||||
? `error: ${exactResult.reason}`
|
||||
: `failed: ${(exactResult as PromiseFulfilledResult<SearchResult>).value.error}`;
|
||||
}
|
||||
|
||||
if (ripgrepResult.status === 'fulfilled' && ripgrepResult.value.success) {
|
||||
resultsMap.set('ripgrep', ripgrepResult.value.results as any[]);
|
||||
backendStatus.ripgrep = 'success';
|
||||
} else {
|
||||
backendStatus.ripgrep = ripgrepResult.status === 'rejected'
|
||||
? `error: ${ripgrepResult.reason}`
|
||||
: `failed: ${(ripgrepResult as PromiseFulfilledResult<SearchResult>).value.error}`;
|
||||
}
|
||||
|
||||
// If no results from any backend
|
||||
if (resultsMap.size === 0) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'All search backends failed',
|
||||
metadata: {
|
||||
mode: 'parallel',
|
||||
backend: 'multi-backend',
|
||||
count: 0,
|
||||
query,
|
||||
backend_status: backendStatus,
|
||||
} as any,
|
||||
};
|
||||
}
|
||||
|
||||
// Apply RRF fusion
|
||||
const fusedResults = applyRRFFusion(resultsMap, weights, limit);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
results: fusedResults,
|
||||
metadata: {
|
||||
mode: 'parallel',
|
||||
backend: 'multi-backend',
|
||||
count: fusedResults.length,
|
||||
query,
|
||||
backends_used: Array.from(resultsMap.keys()),
|
||||
backend_status: backendStatus,
|
||||
weights,
|
||||
note: 'Parallel mode runs hybrid + exact + ripgrep simultaneously with RRF fusion',
|
||||
} as any,
|
||||
};
|
||||
}
|
||||
|
||||
// Tool schema for MCP
|
||||
export const schema: ToolSchema = {
|
||||
name: 'smart_search',
|
||||
description: `Intelligent code search with three optimized modes: hybrid, exact, ripgrep.
|
||||
description: `Intelligent code search with five modes: auto, hybrid, exact, ripgrep, parallel.
|
||||
|
||||
**Quick Start:**
|
||||
smart_search(query="authentication logic") # Auto mode (intelligent routing)
|
||||
smart_search(action="init", path=".") # Initialize index (required for hybrid)
|
||||
smart_search(action="status") # Check index status
|
||||
|
||||
**Three Core Modes:**
|
||||
**Five Modes:**
|
||||
1. auto (default): Intelligent routing based on query and index
|
||||
- Natural language + index → hybrid
|
||||
- Simple query + index → exact
|
||||
@@ -754,6 +942,10 @@ export const schema: ToolSchema = {
|
||||
- Fast, no index required
|
||||
- Literal string matching
|
||||
|
||||
5. parallel: Run all backends simultaneously
|
||||
- Highest recall, runs hybrid + exact + ripgrep in parallel
|
||||
- Results merged using RRF fusion with configurable weights
|
||||
|
||||
**Actions:**
|
||||
- search (default): Intelligent search with auto routing
|
||||
- init: Create CodexLens index (required for hybrid/exact)
|
||||
@@ -780,7 +972,7 @@ export const schema: ToolSchema = {
|
||||
mode: {
|
||||
type: 'string',
|
||||
enum: SEARCH_MODES,
|
||||
description: 'Search mode: auto (default), hybrid (best quality), exact (CodexLens FTS), ripgrep (fast, no index)',
|
||||
description: 'Search mode: auto (default), hybrid (best quality), exact (CodexLens FTS), ripgrep (fast, no index), parallel (all backends with RRF fusion)',
|
||||
default: 'auto',
|
||||
},
|
||||
output_mode: {
|
||||
@@ -826,6 +1018,15 @@ export const schema: ToolSchema = {
|
||||
items: { type: 'string' },
|
||||
description: 'Languages to index (for init action). Example: ["javascript", "typescript"]',
|
||||
},
|
||||
parallelWeights: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
hybrid: { type: 'number', default: 0.5 },
|
||||
exact: { type: 'number', default: 0.3 },
|
||||
ripgrep: { type: 'number', default: 0.2 },
|
||||
},
|
||||
description: 'RRF weights for parallel mode. Weights should sum to 1.0. Default: {hybrid: 0.5, exact: 0.3, ripgrep: 0.2}',
|
||||
},
|
||||
},
|
||||
required: [],
|
||||
},
|
||||
@@ -902,7 +1103,7 @@ export async function handler(params: Record<string, unknown>): Promise<ToolResu
|
||||
|
||||
case 'search':
|
||||
default:
|
||||
// Handle search modes: auto | hybrid | exact | ripgrep
|
||||
// Handle search modes: auto | hybrid | exact | ripgrep | parallel
|
||||
switch (mode) {
|
||||
case 'auto':
|
||||
result = await executeAutoMode(parsed.data);
|
||||
@@ -916,8 +1117,11 @@ export async function handler(params: Record<string, unknown>): Promise<ToolResu
|
||||
case 'ripgrep':
|
||||
result = await executeRipgrepMode(parsed.data);
|
||||
break;
|
||||
case 'parallel':
|
||||
result = await executeParallelMode(parsed.data);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unsupported mode: ${mode}. Use: auto, hybrid, exact, or ripgrep`);
|
||||
throw new Error(`Unsupported mode: ${mode}. Use: auto, hybrid, exact, ripgrep, or parallel`);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -77,7 +77,7 @@ def init(
|
||||
"-l",
|
||||
help="Limit indexing to specific languages (repeat or comma-separated).",
|
||||
),
|
||||
workers: int = typer.Option(4, "--workers", "-w", min=1, max=16, help="Parallel worker processes."),
|
||||
workers: Optional[int] = typer.Option(None, "--workers", "-w", min=1, max=16, help="Parallel worker processes (default: auto-detect based on CPU count, max 16)."),
|
||||
force: bool = typer.Option(False, "--force", "-f", help="Force full reindex (skip incremental mode)."),
|
||||
no_embeddings: bool = typer.Option(False, "--no-embeddings", help="Skip automatic embedding generation (if semantic deps installed)."),
|
||||
embedding_model: str = typer.Option("code", "--embedding-model", help="Embedding model profile: fast, code, multilingual, balanced."),
|
||||
|
||||
@@ -98,7 +98,7 @@ class IndexTreeBuilder:
|
||||
self,
|
||||
source_root: Path,
|
||||
languages: List[str] = None,
|
||||
workers: int = 4,
|
||||
workers: int = None,
|
||||
force_full: bool = False,
|
||||
) -> BuildResult:
|
||||
"""Build complete index tree for a project.
|
||||
@@ -127,6 +127,11 @@ class IndexTreeBuilder:
|
||||
if not source_root.exists():
|
||||
raise ValueError(f"Source root does not exist: {source_root}")
|
||||
|
||||
# Auto-detect optimal worker count if not specified
|
||||
if workers is None:
|
||||
workers = min(os.cpu_count() or 4, 16) # Cap at 16 workers
|
||||
self.logger.debug("Auto-detected %d workers for parallel indexing", workers)
|
||||
|
||||
# Override incremental mode if force_full is True
|
||||
use_incremental = self.incremental and not force_full
|
||||
if force_full:
|
||||
@@ -238,7 +243,7 @@ class IndexTreeBuilder:
|
||||
self,
|
||||
source_path: Path,
|
||||
languages: List[str] = None,
|
||||
workers: int = 4,
|
||||
workers: int = None,
|
||||
) -> BuildResult:
|
||||
"""Incrementally update a subtree.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user