mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-02-12 02:37:45 +08:00
Add parallel search mode and index progress bar
Features: - CCW smart_search: Add 'parallel' mode that runs hybrid + exact + ripgrep simultaneously with RRF (Reciprocal Rank Fusion) for result merging - Dashboard: Add real-time progress bar for CodexLens index initialization - MCP: Return progress metadata in init action response - Codex-lens: Auto-detect optimal worker count for parallel indexing Changes: - smart-search.ts: Add parallel mode, RRF fusion, progress tracking - codex-lens.ts: Add onProgress callback support, progress parsing - codexlens-routes.ts: Broadcast index progress via WebSocket - codexlens-manager.js: New index progress modal with real-time updates - notifications.js: Add WebSocket event handler registration system - i18n.js: Add English/Chinese translations for progress UI - index_tree.py: Workers parameter now auto-detects CPU count (max 16) - commands.py: CLI --workers parameter supports auto-detection 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -12,6 +12,7 @@ import {
|
|||||||
installSemantic,
|
installSemantic,
|
||||||
uninstallCodexLens
|
uninstallCodexLens
|
||||||
} from '../../tools/codex-lens.js';
|
} from '../../tools/codex-lens.js';
|
||||||
|
import type { ProgressInfo } from '../../tools/codex-lens.js';
|
||||||
|
|
||||||
export interface RouteContext {
|
export interface RouteContext {
|
||||||
pathname: string;
|
pathname: string;
|
||||||
@@ -217,9 +218,32 @@ export async function handleCodexLensRoutes(ctx: RouteContext): Promise<boolean>
|
|||||||
const { path: projectPath } = body;
|
const { path: projectPath } = body;
|
||||||
const targetPath = projectPath || initialPath;
|
const targetPath = projectPath || initialPath;
|
||||||
|
|
||||||
|
// Broadcast start event
|
||||||
|
broadcastToClients({
|
||||||
|
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||||
|
payload: { stage: 'start', message: 'Starting index...', percent: 0, path: targetPath }
|
||||||
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await executeCodexLens(['init', targetPath, '--json'], { cwd: targetPath });
|
const result = await executeCodexLens(['init', targetPath, '--json'], {
|
||||||
|
cwd: targetPath,
|
||||||
|
timeout: 300000, // 5 minutes
|
||||||
|
onProgress: (progress: ProgressInfo) => {
|
||||||
|
// Broadcast progress to all connected clients
|
||||||
|
broadcastToClients({
|
||||||
|
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||||
|
payload: { ...progress, path: targetPath }
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
if (result.success) {
|
if (result.success) {
|
||||||
|
// Broadcast completion
|
||||||
|
broadcastToClients({
|
||||||
|
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||||
|
payload: { stage: 'complete', message: 'Index complete', percent: 100, path: targetPath }
|
||||||
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const parsed = extractJSON(result.output);
|
const parsed = extractJSON(result.output);
|
||||||
return { success: true, result: parsed };
|
return { success: true, result: parsed };
|
||||||
@@ -227,9 +251,19 @@ export async function handleCodexLensRoutes(ctx: RouteContext): Promise<boolean>
|
|||||||
return { success: true, output: result.output };
|
return { success: true, output: result.output };
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
// Broadcast error
|
||||||
|
broadcastToClients({
|
||||||
|
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||||
|
payload: { stage: 'error', message: result.error || 'Unknown error', percent: 0, path: targetPath }
|
||||||
|
});
|
||||||
return { success: false, error: result.error, status: 500 };
|
return { success: false, error: result.error, status: 500 };
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
// Broadcast error
|
||||||
|
broadcastToClients({
|
||||||
|
type: 'CODEXLENS_INDEX_PROGRESS',
|
||||||
|
payload: { stage: 'error', message: err.message, percent: 0, path: targetPath }
|
||||||
|
});
|
||||||
return { success: false, error: err.message, status: 500 };
|
return { success: false, error: err.message, status: 500 };
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -87,6 +87,49 @@ let autoRefreshInterval = null;
|
|||||||
let lastDataHash = null;
|
let lastDataHash = null;
|
||||||
const AUTO_REFRESH_INTERVAL_MS = 30000; // 30 seconds
|
const AUTO_REFRESH_INTERVAL_MS = 30000; // 30 seconds
|
||||||
|
|
||||||
|
// Custom event handlers registry for components to subscribe to specific events
|
||||||
|
const wsEventHandlers = {};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Register a custom handler for a specific WebSocket event type
|
||||||
|
* @param {string} eventType - The event type to listen for
|
||||||
|
* @param {Function} handler - The handler function
|
||||||
|
*/
|
||||||
|
function registerWsEventHandler(eventType, handler) {
|
||||||
|
if (!wsEventHandlers[eventType]) {
|
||||||
|
wsEventHandlers[eventType] = [];
|
||||||
|
}
|
||||||
|
wsEventHandlers[eventType].push(handler);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unregister a custom handler for a specific WebSocket event type
|
||||||
|
* @param {string} eventType - The event type
|
||||||
|
* @param {Function} handler - The handler function to remove
|
||||||
|
*/
|
||||||
|
function unregisterWsEventHandler(eventType, handler) {
|
||||||
|
if (wsEventHandlers[eventType]) {
|
||||||
|
wsEventHandlers[eventType] = wsEventHandlers[eventType].filter(h => h !== handler);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dispatch event to registered handlers
|
||||||
|
* @param {string} eventType - The event type
|
||||||
|
* @param {Object} data - The full event data
|
||||||
|
*/
|
||||||
|
function dispatchToEventHandlers(eventType, data) {
|
||||||
|
if (wsEventHandlers[eventType]) {
|
||||||
|
wsEventHandlers[eventType].forEach(handler => {
|
||||||
|
try {
|
||||||
|
handler(data);
|
||||||
|
} catch (e) {
|
||||||
|
console.error('[WS] Error in custom handler for', eventType, e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ========== WebSocket Connection ==========
|
// ========== WebSocket Connection ==========
|
||||||
function initWebSocket() {
|
function initWebSocket() {
|
||||||
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
|
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
|
||||||
@@ -390,6 +433,12 @@ function handleNotification(data) {
|
|||||||
console.log('[CodexLens] Uninstallation completed:', payload);
|
console.log('[CodexLens] Uninstallation completed:', payload);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
case 'CODEXLENS_INDEX_PROGRESS':
|
||||||
|
// Handle CodexLens index progress updates
|
||||||
|
dispatchToEventHandlers('CODEXLENS_INDEX_PROGRESS', data);
|
||||||
|
console.log('[CodexLens] Index progress:', payload.stage, payload.percent + '%');
|
||||||
|
break;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
console.log('[WS] Unknown notification type:', type);
|
console.log('[WS] Unknown notification type:', type);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -278,6 +278,15 @@ const i18n = {
|
|||||||
'codexlens.modelDeleteFailed': 'Model deletion failed',
|
'codexlens.modelDeleteFailed': 'Model deletion failed',
|
||||||
'codexlens.deleteModelConfirm': 'Are you sure you want to delete model',
|
'codexlens.deleteModelConfirm': 'Are you sure you want to delete model',
|
||||||
|
|
||||||
|
// CodexLens Indexing Progress
|
||||||
|
'codexlens.indexing': 'Indexing',
|
||||||
|
'codexlens.indexingDesc': 'Building code index for workspace',
|
||||||
|
'codexlens.preparingIndex': 'Preparing index...',
|
||||||
|
'codexlens.filesProcessed': 'Files processed',
|
||||||
|
'codexlens.indexComplete': 'Index complete',
|
||||||
|
'codexlens.indexSuccess': 'Index created successfully',
|
||||||
|
'codexlens.indexFailed': 'Indexing failed',
|
||||||
|
|
||||||
// Semantic Search Configuration
|
// Semantic Search Configuration
|
||||||
'semantic.settings': 'Semantic Search Settings',
|
'semantic.settings': 'Semantic Search Settings',
|
||||||
'semantic.testSearch': 'Test Semantic Search',
|
'semantic.testSearch': 'Test Semantic Search',
|
||||||
@@ -1394,6 +1403,15 @@ const i18n = {
|
|||||||
'codexlens.modelDeleteFailed': '模型删除失败',
|
'codexlens.modelDeleteFailed': '模型删除失败',
|
||||||
'codexlens.deleteModelConfirm': '确定要删除模型',
|
'codexlens.deleteModelConfirm': '确定要删除模型',
|
||||||
|
|
||||||
|
// CodexLens 索引进度
|
||||||
|
'codexlens.indexing': '索引中',
|
||||||
|
'codexlens.indexingDesc': '正在为工作区构建代码索引',
|
||||||
|
'codexlens.preparingIndex': '准备索引...',
|
||||||
|
'codexlens.filesProcessed': '已处理文件',
|
||||||
|
'codexlens.indexComplete': '索引完成',
|
||||||
|
'codexlens.indexSuccess': '索引创建成功',
|
||||||
|
'codexlens.indexFailed': '索引失败',
|
||||||
|
|
||||||
// Semantic Search 配置
|
// Semantic Search 配置
|
||||||
'semantic.settings': '语义搜索设置',
|
'semantic.settings': '语义搜索设置',
|
||||||
'semantic.testSearch': '测试语义搜索',
|
'semantic.testSearch': '测试语义搜索',
|
||||||
|
|||||||
@@ -542,10 +542,160 @@ async function deleteModel(profile) {
|
|||||||
// ============================================================
|
// ============================================================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize CodexLens index
|
* Initialize CodexLens index with progress tracking
|
||||||
*/
|
*/
|
||||||
function initCodexLensIndex() {
|
function initCodexLensIndex() {
|
||||||
openCliInstallWizard('codexlens');
|
// Create progress modal
|
||||||
|
var modal = document.createElement('div');
|
||||||
|
modal.id = 'codexlensIndexModal';
|
||||||
|
modal.className = 'fixed inset-0 bg-black/50 flex items-center justify-center z-50';
|
||||||
|
modal.innerHTML =
|
||||||
|
'<div class="bg-card rounded-lg shadow-xl w-full max-w-md mx-4 overflow-hidden">' +
|
||||||
|
'<div class="p-6">' +
|
||||||
|
'<div class="flex items-center gap-3 mb-4">' +
|
||||||
|
'<div class="w-10 h-10 rounded-full bg-primary/10 flex items-center justify-center">' +
|
||||||
|
'<i data-lucide="database" class="w-5 h-5 text-primary"></i>' +
|
||||||
|
'</div>' +
|
||||||
|
'<div>' +
|
||||||
|
'<h3 class="text-lg font-semibold">' + t('codexlens.indexing') + '</h3>' +
|
||||||
|
'<p class="text-sm text-muted-foreground">' + t('codexlens.indexingDesc') + '</p>' +
|
||||||
|
'</div>' +
|
||||||
|
'</div>' +
|
||||||
|
'<div class="space-y-4">' +
|
||||||
|
'<div id="codexlensIndexProgress">' +
|
||||||
|
'<div class="flex items-center gap-3">' +
|
||||||
|
'<div class="animate-spin w-5 h-5 border-2 border-primary border-t-transparent rounded-full"></div>' +
|
||||||
|
'<span class="text-sm" id="codexlensIndexStatus">' + t('codexlens.preparingIndex') + '</span>' +
|
||||||
|
'</div>' +
|
||||||
|
'<div class="mt-3 h-2 bg-muted rounded-full overflow-hidden">' +
|
||||||
|
'<div id="codexlensIndexProgressBar" class="h-full bg-primary transition-all duration-300" style="width: 0%"></div>' +
|
||||||
|
'</div>' +
|
||||||
|
'<div class="mt-2 text-xs text-muted-foreground" id="codexlensIndexDetails"></div>' +
|
||||||
|
'</div>' +
|
||||||
|
'</div>' +
|
||||||
|
'</div>' +
|
||||||
|
'<div class="border-t border-border p-4 flex justify-end gap-3 bg-muted/30">' +
|
||||||
|
'<button class="btn-outline px-4 py-2" id="codexlensIndexCancelBtn" onclick="cancelCodexLensIndex()">' + t('common.cancel') + '</button>' +
|
||||||
|
'</div>' +
|
||||||
|
'</div>';
|
||||||
|
|
||||||
|
document.body.appendChild(modal);
|
||||||
|
if (window.lucide) lucide.createIcons();
|
||||||
|
|
||||||
|
// Start indexing
|
||||||
|
startCodexLensIndexing();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start the indexing process
|
||||||
|
*/
|
||||||
|
async function startCodexLensIndexing() {
|
||||||
|
var statusText = document.getElementById('codexlensIndexStatus');
|
||||||
|
var progressBar = document.getElementById('codexlensIndexProgressBar');
|
||||||
|
var detailsText = document.getElementById('codexlensIndexDetails');
|
||||||
|
var cancelBtn = document.getElementById('codexlensIndexCancelBtn');
|
||||||
|
|
||||||
|
// Setup WebSocket listener for progress events
|
||||||
|
window.codexlensIndexProgressHandler = function(event) {
|
||||||
|
if (event.type === 'CODEXLENS_INDEX_PROGRESS') {
|
||||||
|
var payload = event.payload;
|
||||||
|
if (statusText) statusText.textContent = payload.message || t('codexlens.indexing');
|
||||||
|
if (progressBar) progressBar.style.width = (payload.percent || 0) + '%';
|
||||||
|
if (detailsText && payload.filesProcessed !== undefined) {
|
||||||
|
detailsText.textContent = t('codexlens.filesProcessed') + ': ' + payload.filesProcessed +
|
||||||
|
(payload.totalFiles ? ' / ' + payload.totalFiles : '');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle completion
|
||||||
|
if (payload.stage === 'complete') {
|
||||||
|
handleIndexComplete(true, payload.message);
|
||||||
|
} else if (payload.stage === 'error') {
|
||||||
|
handleIndexComplete(false, payload.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Register with notification system if available
|
||||||
|
if (typeof registerWsEventHandler === 'function') {
|
||||||
|
registerWsEventHandler('CODEXLENS_INDEX_PROGRESS', window.codexlensIndexProgressHandler);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
var response = await fetch('/api/codexlens/init', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ path: projectPath })
|
||||||
|
});
|
||||||
|
|
||||||
|
var result = await response.json();
|
||||||
|
|
||||||
|
if (!result.success && statusText) {
|
||||||
|
// If WebSocket didn't report error, show it now
|
||||||
|
handleIndexComplete(false, result.error || t('common.unknownError'));
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
handleIndexComplete(false, err.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle index completion
|
||||||
|
*/
|
||||||
|
function handleIndexComplete(success, message) {
|
||||||
|
var statusText = document.getElementById('codexlensIndexStatus');
|
||||||
|
var progressBar = document.getElementById('codexlensIndexProgressBar');
|
||||||
|
var cancelBtn = document.getElementById('codexlensIndexCancelBtn');
|
||||||
|
|
||||||
|
// Unregister WebSocket handler
|
||||||
|
if (typeof unregisterWsEventHandler === 'function') {
|
||||||
|
unregisterWsEventHandler('CODEXLENS_INDEX_PROGRESS', window.codexlensIndexProgressHandler);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (success) {
|
||||||
|
if (progressBar) progressBar.style.width = '100%';
|
||||||
|
if (statusText) statusText.textContent = t('codexlens.indexComplete');
|
||||||
|
if (cancelBtn) cancelBtn.textContent = t('common.close');
|
||||||
|
|
||||||
|
showRefreshToast(t('codexlens.indexSuccess'), 'success');
|
||||||
|
|
||||||
|
// Auto-close after 2 seconds
|
||||||
|
setTimeout(function() {
|
||||||
|
closeCodexLensIndexModal();
|
||||||
|
// Refresh status
|
||||||
|
if (typeof loadCodexLensStatus === 'function') {
|
||||||
|
loadCodexLensStatus().then(function() {
|
||||||
|
renderToolsSection();
|
||||||
|
if (window.lucide) lucide.createIcons();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}, 2000);
|
||||||
|
} else {
|
||||||
|
if (progressBar) progressBar.classList.add('bg-destructive');
|
||||||
|
if (statusText) statusText.textContent = t('codexlens.indexFailed') + ': ' + message;
|
||||||
|
if (cancelBtn) cancelBtn.textContent = t('common.close');
|
||||||
|
|
||||||
|
showRefreshToast(t('codexlens.indexFailed') + ': ' + message, 'error');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cancel indexing
|
||||||
|
*/
|
||||||
|
function cancelCodexLensIndex() {
|
||||||
|
closeCodexLensIndexModal();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Close index modal
|
||||||
|
*/
|
||||||
|
function closeCodexLensIndexModal() {
|
||||||
|
var modal = document.getElementById('codexlensIndexModal');
|
||||||
|
if (modal) modal.remove();
|
||||||
|
|
||||||
|
// Unregister WebSocket handler
|
||||||
|
if (typeof unregisterWsEventHandler === 'function' && window.codexlensIndexProgressHandler) {
|
||||||
|
unregisterWsEventHandler('CODEXLENS_INDEX_PROGRESS', window.codexlensIndexProgressHandler);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -91,6 +91,15 @@ interface ExecuteResult {
|
|||||||
interface ExecuteOptions {
|
interface ExecuteOptions {
|
||||||
timeout?: number;
|
timeout?: number;
|
||||||
cwd?: string;
|
cwd?: string;
|
||||||
|
onProgress?: (progress: ProgressInfo) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ProgressInfo {
|
||||||
|
stage: string;
|
||||||
|
message: string;
|
||||||
|
percent: number;
|
||||||
|
filesProcessed?: number;
|
||||||
|
totalFiles?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -361,6 +370,57 @@ async function ensureReady(): Promise<ReadyStatus> {
|
|||||||
return recheck;
|
return recheck;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse progress info from CodexLens output
|
||||||
|
* @param line - Output line to parse
|
||||||
|
* @returns Progress info or null
|
||||||
|
*/
|
||||||
|
function parseProgressLine(line: string): ProgressInfo | null {
|
||||||
|
// Parse file processing progress: "Processing file X/Y: path"
|
||||||
|
const fileMatch = line.match(/Processing file (\d+)\/(\d+):\s*(.+)/i);
|
||||||
|
if (fileMatch) {
|
||||||
|
const current = parseInt(fileMatch[1], 10);
|
||||||
|
const total = parseInt(fileMatch[2], 10);
|
||||||
|
return {
|
||||||
|
stage: 'indexing',
|
||||||
|
message: `Processing ${fileMatch[3]}`,
|
||||||
|
percent: Math.round((current / total) * 80) + 10, // 10-90%
|
||||||
|
filesProcessed: current,
|
||||||
|
totalFiles: total,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse stage messages
|
||||||
|
if (line.includes('Discovering files')) {
|
||||||
|
return { stage: 'discover', message: 'Discovering files...', percent: 5 };
|
||||||
|
}
|
||||||
|
if (line.includes('Building index')) {
|
||||||
|
return { stage: 'build', message: 'Building index...', percent: 10 };
|
||||||
|
}
|
||||||
|
if (line.includes('Extracting symbols')) {
|
||||||
|
return { stage: 'symbols', message: 'Extracting symbols...', percent: 50 };
|
||||||
|
}
|
||||||
|
if (line.includes('Generating embeddings') || line.includes('Creating embeddings')) {
|
||||||
|
return { stage: 'embeddings', message: 'Generating embeddings...', percent: 70 };
|
||||||
|
}
|
||||||
|
if (line.includes('Finalizing') || line.includes('Complete')) {
|
||||||
|
return { stage: 'complete', message: 'Finalizing...', percent: 95 };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse indexed count: "Indexed X files"
|
||||||
|
const indexedMatch = line.match(/Indexed (\d+) files/i);
|
||||||
|
if (indexedMatch) {
|
||||||
|
return {
|
||||||
|
stage: 'complete',
|
||||||
|
message: `Indexed ${indexedMatch[1]} files`,
|
||||||
|
percent: 100,
|
||||||
|
filesProcessed: parseInt(indexedMatch[1], 10),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Execute CodexLens CLI command
|
* Execute CodexLens CLI command
|
||||||
* @param args - CLI arguments
|
* @param args - CLI arguments
|
||||||
@@ -368,7 +428,7 @@ async function ensureReady(): Promise<ReadyStatus> {
|
|||||||
* @returns Execution result
|
* @returns Execution result
|
||||||
*/
|
*/
|
||||||
async function executeCodexLens(args: string[], options: ExecuteOptions = {}): Promise<ExecuteResult> {
|
async function executeCodexLens(args: string[], options: ExecuteOptions = {}): Promise<ExecuteResult> {
|
||||||
const { timeout = 60000, cwd = process.cwd() } = options;
|
const { timeout = 60000, cwd = process.cwd(), onProgress } = options;
|
||||||
|
|
||||||
// Ensure ready
|
// Ensure ready
|
||||||
const readyStatus = await ensureReady();
|
const readyStatus = await ensureReady();
|
||||||
@@ -387,10 +447,35 @@ async function executeCodexLens(args: string[], options: ExecuteOptions = {}): P
|
|||||||
let timedOut = false;
|
let timedOut = false;
|
||||||
|
|
||||||
child.stdout.on('data', (data) => {
|
child.stdout.on('data', (data) => {
|
||||||
stdout += data.toString();
|
const chunk = data.toString();
|
||||||
|
stdout += chunk;
|
||||||
|
|
||||||
|
// Report progress if callback provided
|
||||||
|
if (onProgress) {
|
||||||
|
const lines = chunk.split('\n');
|
||||||
|
for (const line of lines) {
|
||||||
|
const progress = parseProgressLine(line.trim());
|
||||||
|
if (progress) {
|
||||||
|
onProgress(progress);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
child.stderr.on('data', (data) => {
|
child.stderr.on('data', (data) => {
|
||||||
stderr += data.toString();
|
const chunk = data.toString();
|
||||||
|
stderr += chunk;
|
||||||
|
|
||||||
|
// Also check stderr for progress (some tools output there)
|
||||||
|
if (onProgress) {
|
||||||
|
const lines = chunk.split('\n');
|
||||||
|
for (const line of lines) {
|
||||||
|
const progress = parseProgressLine(line.trim());
|
||||||
|
if (progress) {
|
||||||
|
onProgress(progress);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
const timeoutId = setTimeout(() => {
|
const timeoutId = setTimeout(() => {
|
||||||
@@ -803,6 +888,9 @@ async function uninstallCodexLens(): Promise<BootstrapResult> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Export types
|
||||||
|
export type { ProgressInfo, ExecuteOptions };
|
||||||
|
|
||||||
// Export for direct usage
|
// Export for direct usage
|
||||||
export { ensureReady, executeCodexLens, checkVenvStatus, bootstrapVenv, checkSemanticStatus, installSemantic, uninstallCodexLens };
|
export { ensureReady, executeCodexLens, checkVenvStatus, bootstrapVenv, checkSemanticStatus, installSemantic, uninstallCodexLens };
|
||||||
|
|
||||||
|
|||||||
@@ -21,12 +21,13 @@ import {
|
|||||||
ensureReady as ensureCodexLensReady,
|
ensureReady as ensureCodexLensReady,
|
||||||
executeCodexLens,
|
executeCodexLens,
|
||||||
} from './codex-lens.js';
|
} from './codex-lens.js';
|
||||||
|
import type { ProgressInfo } from './codex-lens.js';
|
||||||
|
|
||||||
// Define Zod schema for validation
|
// Define Zod schema for validation
|
||||||
const ParamsSchema = z.object({
|
const ParamsSchema = z.object({
|
||||||
action: z.enum(['init', 'search', 'search_files', 'status']).default('search'),
|
action: z.enum(['init', 'search', 'search_files', 'status']).default('search'),
|
||||||
query: z.string().optional(),
|
query: z.string().optional(),
|
||||||
mode: z.enum(['auto', 'hybrid', 'exact', 'ripgrep']).default('auto'),
|
mode: z.enum(['auto', 'hybrid', 'exact', 'ripgrep', 'parallel']).default('auto'),
|
||||||
output_mode: z.enum(['full', 'files_only', 'count']).default('full'),
|
output_mode: z.enum(['full', 'files_only', 'count']).default('full'),
|
||||||
path: z.string().optional(),
|
path: z.string().optional(),
|
||||||
paths: z.array(z.string()).default([]),
|
paths: z.array(z.string()).default([]),
|
||||||
@@ -35,12 +36,17 @@ const ParamsSchema = z.object({
|
|||||||
includeHidden: z.boolean().default(false),
|
includeHidden: z.boolean().default(false),
|
||||||
languages: z.array(z.string()).optional(),
|
languages: z.array(z.string()).optional(),
|
||||||
limit: z.number().default(100),
|
limit: z.number().default(100),
|
||||||
|
parallelWeights: z.object({
|
||||||
|
hybrid: z.number().default(0.5),
|
||||||
|
exact: z.number().default(0.3),
|
||||||
|
ripgrep: z.number().default(0.2),
|
||||||
|
}).optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
type Params = z.infer<typeof ParamsSchema>;
|
type Params = z.infer<typeof ParamsSchema>;
|
||||||
|
|
||||||
// Search mode constants
|
// Search mode constants
|
||||||
const SEARCH_MODES = ['auto', 'hybrid', 'exact', 'ripgrep'] as const;
|
const SEARCH_MODES = ['auto', 'hybrid', 'exact', 'ripgrep', 'parallel'] as const;
|
||||||
|
|
||||||
// Classification confidence threshold
|
// Classification confidence threshold
|
||||||
const CONFIDENCE_THRESHOLD = 0.7;
|
const CONFIDENCE_THRESHOLD = 0.7;
|
||||||
@@ -72,10 +78,10 @@ interface GraphMatch {
|
|||||||
}
|
}
|
||||||
|
|
||||||
interface SearchMetadata {
|
interface SearchMetadata {
|
||||||
mode: string;
|
mode?: string;
|
||||||
backend: string;
|
backend?: string;
|
||||||
count: number;
|
count?: number;
|
||||||
query: string;
|
query?: string;
|
||||||
classified_as?: string;
|
classified_as?: string;
|
||||||
confidence?: number;
|
confidence?: number;
|
||||||
reasoning?: string;
|
reasoning?: string;
|
||||||
@@ -83,6 +89,17 @@ interface SearchMetadata {
|
|||||||
warning?: string;
|
warning?: string;
|
||||||
note?: string;
|
note?: string;
|
||||||
index_status?: 'indexed' | 'not_indexed' | 'partial';
|
index_status?: 'indexed' | 'not_indexed' | 'partial';
|
||||||
|
// Init action specific
|
||||||
|
action?: string;
|
||||||
|
path?: string;
|
||||||
|
progress?: {
|
||||||
|
stage: string;
|
||||||
|
message: string;
|
||||||
|
percent: number;
|
||||||
|
filesProcessed?: number;
|
||||||
|
totalFiles?: number;
|
||||||
|
};
|
||||||
|
progressHistory?: ProgressInfo[];
|
||||||
}
|
}
|
||||||
|
|
||||||
interface SearchResult {
|
interface SearchResult {
|
||||||
@@ -326,7 +343,39 @@ async function executeInitAction(params: Params): Promise<SearchResult> {
|
|||||||
args.push('--languages', languages.join(','));
|
args.push('--languages', languages.join(','));
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await executeCodexLens(args, { cwd: path, timeout: 300000 });
|
// Track progress updates
|
||||||
|
const progressUpdates: ProgressInfo[] = [];
|
||||||
|
let lastProgress: ProgressInfo | null = null;
|
||||||
|
|
||||||
|
const result = await executeCodexLens(args, {
|
||||||
|
cwd: path,
|
||||||
|
timeout: 300000,
|
||||||
|
onProgress: (progress: ProgressInfo) => {
|
||||||
|
progressUpdates.push(progress);
|
||||||
|
lastProgress = progress;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// Build metadata with progress info
|
||||||
|
const metadata: SearchMetadata = {
|
||||||
|
action: 'init',
|
||||||
|
path,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (lastProgress !== null) {
|
||||||
|
const p = lastProgress as ProgressInfo;
|
||||||
|
metadata.progress = {
|
||||||
|
stage: p.stage,
|
||||||
|
message: p.message,
|
||||||
|
percent: p.percent,
|
||||||
|
filesProcessed: p.filesProcessed,
|
||||||
|
totalFiles: p.totalFiles,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (progressUpdates.length > 0) {
|
||||||
|
metadata.progressHistory = progressUpdates.slice(-5); // Keep last 5 progress updates
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
success: result.success,
|
success: result.success,
|
||||||
@@ -334,6 +383,7 @@ async function executeInitAction(params: Params): Promise<SearchResult> {
|
|||||||
message: result.success
|
message: result.success
|
||||||
? `CodexLens index created successfully for ${path}`
|
? `CodexLens index created successfully for ${path}`
|
||||||
: undefined,
|
: undefined,
|
||||||
|
metadata,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -726,17 +776,155 @@ async function executeHybridMode(params: Params): Promise<SearchResult> {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TypeScript implementation of Reciprocal Rank Fusion
|
||||||
|
* Reference: codex-lens/src/codexlens/search/ranking.py
|
||||||
|
* Formula: score(d) = Σ weight_source / (k + rank_source(d))
|
||||||
|
*/
|
||||||
|
function applyRRFFusion(
|
||||||
|
resultsMap: Map<string, any[]>,
|
||||||
|
weights: Record<string, number>,
|
||||||
|
limit: number,
|
||||||
|
k: number = 60,
|
||||||
|
): any[] {
|
||||||
|
const pathScores = new Map<string, { score: number; result: any; sources: string[] }>();
|
||||||
|
|
||||||
|
resultsMap.forEach((results, source) => {
|
||||||
|
const weight = weights[source] || 0;
|
||||||
|
if (weight === 0 || !results) return;
|
||||||
|
|
||||||
|
results.forEach((result, rank) => {
|
||||||
|
const path = result.file || result.path;
|
||||||
|
if (!path) return;
|
||||||
|
|
||||||
|
const rrfContribution = weight / (k + rank + 1);
|
||||||
|
|
||||||
|
if (!pathScores.has(path)) {
|
||||||
|
pathScores.set(path, { score: 0, result, sources: [] });
|
||||||
|
}
|
||||||
|
const entry = pathScores.get(path)!;
|
||||||
|
entry.score += rrfContribution;
|
||||||
|
if (!entry.sources.includes(source)) {
|
||||||
|
entry.sources.push(source);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Sort by fusion score descending
|
||||||
|
return Array.from(pathScores.values())
|
||||||
|
.sort((a, b) => b.score - a.score)
|
||||||
|
.slice(0, limit)
|
||||||
|
.map(item => ({
|
||||||
|
...item.result,
|
||||||
|
fusion_score: item.score,
|
||||||
|
matched_backends: item.sources,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mode: parallel - Run all backends simultaneously with RRF fusion
|
||||||
|
* Returns best results from hybrid + exact + ripgrep combined
|
||||||
|
*/
|
||||||
|
async function executeParallelMode(params: Params): Promise<SearchResult> {
|
||||||
|
const { query, path = '.', limit = 100, parallelWeights } = params;
|
||||||
|
|
||||||
|
if (!query) {
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
error: 'Query is required for search',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default weights if not provided
|
||||||
|
const weights = parallelWeights || {
|
||||||
|
hybrid: 0.5,
|
||||||
|
exact: 0.3,
|
||||||
|
ripgrep: 0.2,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Run all backends in parallel
|
||||||
|
const [hybridResult, exactResult, ripgrepResult] = await Promise.allSettled([
|
||||||
|
executeHybridMode(params),
|
||||||
|
executeCodexLensExactMode(params),
|
||||||
|
executeRipgrepMode(params),
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Collect successful results
|
||||||
|
const resultsMap = new Map<string, any[]>();
|
||||||
|
const backendStatus: Record<string, string> = {};
|
||||||
|
|
||||||
|
if (hybridResult.status === 'fulfilled' && hybridResult.value.success) {
|
||||||
|
resultsMap.set('hybrid', hybridResult.value.results as any[]);
|
||||||
|
backendStatus.hybrid = 'success';
|
||||||
|
} else {
|
||||||
|
backendStatus.hybrid = hybridResult.status === 'rejected'
|
||||||
|
? `error: ${hybridResult.reason}`
|
||||||
|
: `failed: ${(hybridResult as PromiseFulfilledResult<SearchResult>).value.error}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (exactResult.status === 'fulfilled' && exactResult.value.success) {
|
||||||
|
resultsMap.set('exact', exactResult.value.results as any[]);
|
||||||
|
backendStatus.exact = 'success';
|
||||||
|
} else {
|
||||||
|
backendStatus.exact = exactResult.status === 'rejected'
|
||||||
|
? `error: ${exactResult.reason}`
|
||||||
|
: `failed: ${(exactResult as PromiseFulfilledResult<SearchResult>).value.error}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ripgrepResult.status === 'fulfilled' && ripgrepResult.value.success) {
|
||||||
|
resultsMap.set('ripgrep', ripgrepResult.value.results as any[]);
|
||||||
|
backendStatus.ripgrep = 'success';
|
||||||
|
} else {
|
||||||
|
backendStatus.ripgrep = ripgrepResult.status === 'rejected'
|
||||||
|
? `error: ${ripgrepResult.reason}`
|
||||||
|
: `failed: ${(ripgrepResult as PromiseFulfilledResult<SearchResult>).value.error}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// If no results from any backend
|
||||||
|
if (resultsMap.size === 0) {
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
error: 'All search backends failed',
|
||||||
|
metadata: {
|
||||||
|
mode: 'parallel',
|
||||||
|
backend: 'multi-backend',
|
||||||
|
count: 0,
|
||||||
|
query,
|
||||||
|
backend_status: backendStatus,
|
||||||
|
} as any,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply RRF fusion
|
||||||
|
const fusedResults = applyRRFFusion(resultsMap, weights, limit);
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
results: fusedResults,
|
||||||
|
metadata: {
|
||||||
|
mode: 'parallel',
|
||||||
|
backend: 'multi-backend',
|
||||||
|
count: fusedResults.length,
|
||||||
|
query,
|
||||||
|
backends_used: Array.from(resultsMap.keys()),
|
||||||
|
backend_status: backendStatus,
|
||||||
|
weights,
|
||||||
|
note: 'Parallel mode runs hybrid + exact + ripgrep simultaneously with RRF fusion',
|
||||||
|
} as any,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
// Tool schema for MCP
|
// Tool schema for MCP
|
||||||
export const schema: ToolSchema = {
|
export const schema: ToolSchema = {
|
||||||
name: 'smart_search',
|
name: 'smart_search',
|
||||||
description: `Intelligent code search with three optimized modes: hybrid, exact, ripgrep.
|
description: `Intelligent code search with five modes: auto, hybrid, exact, ripgrep, parallel.
|
||||||
|
|
||||||
**Quick Start:**
|
**Quick Start:**
|
||||||
smart_search(query="authentication logic") # Auto mode (intelligent routing)
|
smart_search(query="authentication logic") # Auto mode (intelligent routing)
|
||||||
smart_search(action="init", path=".") # Initialize index (required for hybrid)
|
smart_search(action="init", path=".") # Initialize index (required for hybrid)
|
||||||
smart_search(action="status") # Check index status
|
smart_search(action="status") # Check index status
|
||||||
|
|
||||||
**Three Core Modes:**
|
**Five Modes:**
|
||||||
1. auto (default): Intelligent routing based on query and index
|
1. auto (default): Intelligent routing based on query and index
|
||||||
- Natural language + index → hybrid
|
- Natural language + index → hybrid
|
||||||
- Simple query + index → exact
|
- Simple query + index → exact
|
||||||
@@ -754,6 +942,10 @@ export const schema: ToolSchema = {
|
|||||||
- Fast, no index required
|
- Fast, no index required
|
||||||
- Literal string matching
|
- Literal string matching
|
||||||
|
|
||||||
|
5. parallel: Run all backends simultaneously
|
||||||
|
- Highest recall, runs hybrid + exact + ripgrep in parallel
|
||||||
|
- Results merged using RRF fusion with configurable weights
|
||||||
|
|
||||||
**Actions:**
|
**Actions:**
|
||||||
- search (default): Intelligent search with auto routing
|
- search (default): Intelligent search with auto routing
|
||||||
- init: Create CodexLens index (required for hybrid/exact)
|
- init: Create CodexLens index (required for hybrid/exact)
|
||||||
@@ -780,7 +972,7 @@ export const schema: ToolSchema = {
|
|||||||
mode: {
|
mode: {
|
||||||
type: 'string',
|
type: 'string',
|
||||||
enum: SEARCH_MODES,
|
enum: SEARCH_MODES,
|
||||||
description: 'Search mode: auto (default), hybrid (best quality), exact (CodexLens FTS), ripgrep (fast, no index)',
|
description: 'Search mode: auto (default), hybrid (best quality), exact (CodexLens FTS), ripgrep (fast, no index), parallel (all backends with RRF fusion)',
|
||||||
default: 'auto',
|
default: 'auto',
|
||||||
},
|
},
|
||||||
output_mode: {
|
output_mode: {
|
||||||
@@ -826,6 +1018,15 @@ export const schema: ToolSchema = {
|
|||||||
items: { type: 'string' },
|
items: { type: 'string' },
|
||||||
description: 'Languages to index (for init action). Example: ["javascript", "typescript"]',
|
description: 'Languages to index (for init action). Example: ["javascript", "typescript"]',
|
||||||
},
|
},
|
||||||
|
parallelWeights: {
|
||||||
|
type: 'object',
|
||||||
|
properties: {
|
||||||
|
hybrid: { type: 'number', default: 0.5 },
|
||||||
|
exact: { type: 'number', default: 0.3 },
|
||||||
|
ripgrep: { type: 'number', default: 0.2 },
|
||||||
|
},
|
||||||
|
description: 'RRF weights for parallel mode. Weights should sum to 1.0. Default: {hybrid: 0.5, exact: 0.3, ripgrep: 0.2}',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
required: [],
|
required: [],
|
||||||
},
|
},
|
||||||
@@ -902,7 +1103,7 @@ export async function handler(params: Record<string, unknown>): Promise<ToolResu
|
|||||||
|
|
||||||
case 'search':
|
case 'search':
|
||||||
default:
|
default:
|
||||||
// Handle search modes: auto | hybrid | exact | ripgrep
|
// Handle search modes: auto | hybrid | exact | ripgrep | parallel
|
||||||
switch (mode) {
|
switch (mode) {
|
||||||
case 'auto':
|
case 'auto':
|
||||||
result = await executeAutoMode(parsed.data);
|
result = await executeAutoMode(parsed.data);
|
||||||
@@ -916,8 +1117,11 @@ export async function handler(params: Record<string, unknown>): Promise<ToolResu
|
|||||||
case 'ripgrep':
|
case 'ripgrep':
|
||||||
result = await executeRipgrepMode(parsed.data);
|
result = await executeRipgrepMode(parsed.data);
|
||||||
break;
|
break;
|
||||||
|
case 'parallel':
|
||||||
|
result = await executeParallelMode(parsed.data);
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
throw new Error(`Unsupported mode: ${mode}. Use: auto, hybrid, exact, or ripgrep`);
|
throw new Error(`Unsupported mode: ${mode}. Use: auto, hybrid, exact, ripgrep, or parallel`);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ def init(
|
|||||||
"-l",
|
"-l",
|
||||||
help="Limit indexing to specific languages (repeat or comma-separated).",
|
help="Limit indexing to specific languages (repeat or comma-separated).",
|
||||||
),
|
),
|
||||||
workers: int = typer.Option(4, "--workers", "-w", min=1, max=16, help="Parallel worker processes."),
|
workers: Optional[int] = typer.Option(None, "--workers", "-w", min=1, max=16, help="Parallel worker processes (default: auto-detect based on CPU count, max 16)."),
|
||||||
force: bool = typer.Option(False, "--force", "-f", help="Force full reindex (skip incremental mode)."),
|
force: bool = typer.Option(False, "--force", "-f", help="Force full reindex (skip incremental mode)."),
|
||||||
no_embeddings: bool = typer.Option(False, "--no-embeddings", help="Skip automatic embedding generation (if semantic deps installed)."),
|
no_embeddings: bool = typer.Option(False, "--no-embeddings", help="Skip automatic embedding generation (if semantic deps installed)."),
|
||||||
embedding_model: str = typer.Option("code", "--embedding-model", help="Embedding model profile: fast, code, multilingual, balanced."),
|
embedding_model: str = typer.Option("code", "--embedding-model", help="Embedding model profile: fast, code, multilingual, balanced."),
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ class IndexTreeBuilder:
|
|||||||
self,
|
self,
|
||||||
source_root: Path,
|
source_root: Path,
|
||||||
languages: List[str] = None,
|
languages: List[str] = None,
|
||||||
workers: int = 4,
|
workers: int = None,
|
||||||
force_full: bool = False,
|
force_full: bool = False,
|
||||||
) -> BuildResult:
|
) -> BuildResult:
|
||||||
"""Build complete index tree for a project.
|
"""Build complete index tree for a project.
|
||||||
@@ -127,6 +127,11 @@ class IndexTreeBuilder:
|
|||||||
if not source_root.exists():
|
if not source_root.exists():
|
||||||
raise ValueError(f"Source root does not exist: {source_root}")
|
raise ValueError(f"Source root does not exist: {source_root}")
|
||||||
|
|
||||||
|
# Auto-detect optimal worker count if not specified
|
||||||
|
if workers is None:
|
||||||
|
workers = min(os.cpu_count() or 4, 16) # Cap at 16 workers
|
||||||
|
self.logger.debug("Auto-detected %d workers for parallel indexing", workers)
|
||||||
|
|
||||||
# Override incremental mode if force_full is True
|
# Override incremental mode if force_full is True
|
||||||
use_incremental = self.incremental and not force_full
|
use_incremental = self.incremental and not force_full
|
||||||
if force_full:
|
if force_full:
|
||||||
@@ -238,7 +243,7 @@ class IndexTreeBuilder:
|
|||||||
self,
|
self,
|
||||||
source_path: Path,
|
source_path: Path,
|
||||||
languages: List[str] = None,
|
languages: List[str] = None,
|
||||||
workers: int = 4,
|
workers: int = None,
|
||||||
) -> BuildResult:
|
) -> BuildResult:
|
||||||
"""Incrementally update a subtree.
|
"""Incrementally update a subtree.
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user