Add comprehensive tests for tokenizer, performance benchmarks, and TreeSitter parser functionality

- Implemented unit tests for the Tokenizer class, covering various text inputs, edge cases, and fallback mechanisms.
- Created performance benchmarks comparing tiktoken and pure Python implementations for token counting.
- Developed extensive tests for TreeSitterSymbolParser across Python, JavaScript, and TypeScript, ensuring accurate symbol extraction and parsing.
- Added configuration documentation for MCP integration and custom prompts, enhancing usability and flexibility.
- Introduced a refactor script for GraphAnalyzer to streamline future improvements.
This commit is contained in:
catlog22
2025-12-15 14:36:09 +08:00
parent 82dcafff00
commit 0fe16963cd
49 changed files with 9307 additions and 438 deletions

View File

@@ -15,6 +15,11 @@ let mcpCurrentProjectServers = {};
let mcpConfigSources = [];
let mcpCreateMode = 'form'; // 'form' or 'json'
// ========== CLI Toggle State (Claude / Codex) ==========
let currentCliMode = 'claude'; // 'claude' or 'codex'
let codexMcpConfig = null;
let codexMcpServers = {};
// ========== Initialization ==========
function initMcpManager() {
// Initialize MCP navigation
@@ -44,6 +49,12 @@ async function loadMcpConfig() {
mcpEnterpriseServers = data.enterpriseServers || {};
mcpConfigSources = data.configSources || [];
// Load Codex MCP config
if (data.codex) {
codexMcpConfig = data.codex;
codexMcpServers = data.codex.servers || {};
}
// Get current project servers
const currentPath = projectPath.replace(/\//g, '\\');
mcpCurrentProjectServers = mcpAllProjects[currentPath]?.mcpServers || {};
@@ -58,6 +69,135 @@ async function loadMcpConfig() {
}
}
// ========== CLI Mode Toggle ==========
function setCliMode(mode) {
currentCliMode = mode;
renderMcpManager();
}
function getCliMode() {
return currentCliMode;
}
// ========== Codex MCP Functions ==========
/**
* Add MCP server to Codex config.toml
*/
async function addCodexMcpServer(serverName, serverConfig) {
try {
const response = await fetch('/api/codex-mcp-add', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
serverName: serverName,
serverConfig: serverConfig
})
});
if (!response.ok) throw new Error('Failed to add Codex MCP server');
const result = await response.json();
if (result.success) {
await loadMcpConfig();
renderMcpManager();
showRefreshToast(t('mcp.codex.serverAdded', { name: serverName }), 'success');
} else {
showRefreshToast(result.error || t('mcp.codex.addFailed'), 'error');
}
return result;
} catch (err) {
console.error('Failed to add Codex MCP server:', err);
showRefreshToast(t('mcp.codex.addFailed') + ': ' + err.message, 'error');
return null;
}
}
/**
* Remove MCP server from Codex config.toml
*/
async function removeCodexMcpServer(serverName) {
try {
const response = await fetch('/api/codex-mcp-remove', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ serverName })
});
if (!response.ok) throw new Error('Failed to remove Codex MCP server');
const result = await response.json();
if (result.success) {
await loadMcpConfig();
renderMcpManager();
showRefreshToast(t('mcp.codex.serverRemoved', { name: serverName }), 'success');
} else {
showRefreshToast(result.error || t('mcp.codex.removeFailed'), 'error');
}
return result;
} catch (err) {
console.error('Failed to remove Codex MCP server:', err);
showRefreshToast(t('mcp.codex.removeFailed') + ': ' + err.message, 'error');
return null;
}
}
/**
* Toggle Codex MCP server enabled state
*/
async function toggleCodexMcpServer(serverName, enabled) {
try {
const response = await fetch('/api/codex-mcp-toggle', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ serverName, enabled })
});
if (!response.ok) throw new Error('Failed to toggle Codex MCP server');
const result = await response.json();
if (result.success) {
await loadMcpConfig();
renderMcpManager();
showRefreshToast(t('mcp.codex.serverToggled', { name: serverName, state: enabled ? 'enabled' : 'disabled' }), 'success');
}
return result;
} catch (err) {
console.error('Failed to toggle Codex MCP server:', err);
showRefreshToast(t('mcp.codex.toggleFailed') + ': ' + err.message, 'error');
return null;
}
}
/**
* Copy Claude MCP server to Codex
*/
async function copyClaudeServerToCodex(serverName, serverConfig) {
return await addCodexMcpServer(serverName, serverConfig);
}
/**
* Copy Codex MCP server to Claude (global)
*/
async function copyCodexServerToClaude(serverName, serverConfig) {
// Convert Codex format to Claude format
const claudeConfig = {
command: serverConfig.command,
args: serverConfig.args || [],
};
if (serverConfig.env) {
claudeConfig.env = serverConfig.env;
}
// If it's an HTTP server
if (serverConfig.url) {
claudeConfig.url = serverConfig.url;
}
return await addGlobalMcpServer(serverName, claudeConfig);
}
async function toggleMcpServer(serverName, enable) {
try {
const response = await fetch('/api/mcp-toggle', {
@@ -255,7 +395,7 @@ async function removeGlobalMcpServer(serverName) {
function updateMcpBadge() {
const badge = document.getElementById('badgeMcpServers');
if (badge) {
const currentPath = projectPath.replace(/\//g, '\\');
const currentPath = projectPath; // Keep original format (forward slash)
const projectData = mcpAllProjects[currentPath];
const servers = projectData?.mcpServers || {};
const disabledServers = projectData?.disabledMcpServers || [];
@@ -702,7 +842,20 @@ async function createMcpServerWithConfig(name, serverConfig, scope = 'project')
// Submit to API
try {
let response;
if (scope === 'global') {
let scopeLabel;
if (scope === 'codex') {
// Create in Codex config.toml
response = await fetch('/api/codex-mcp-add', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
serverName: name,
serverConfig: serverConfig
})
});
scopeLabel = 'Codex';
} else if (scope === 'global') {
response = await fetch('/api/mcp-add-global-server', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
@@ -711,6 +864,7 @@ async function createMcpServerWithConfig(name, serverConfig, scope = 'project')
serverConfig: serverConfig
})
});
scopeLabel = 'global';
} else {
response = await fetch('/api/mcp-copy-server', {
method: 'POST',
@@ -721,6 +875,7 @@ async function createMcpServerWithConfig(name, serverConfig, scope = 'project')
serverConfig: serverConfig
})
});
scopeLabel = 'project';
}
if (!response.ok) throw new Error('Failed to create MCP server');
@@ -730,7 +885,6 @@ async function createMcpServerWithConfig(name, serverConfig, scope = 'project')
closeMcpCreateModal();
await loadMcpConfig();
renderMcpManager();
const scopeLabel = scope === 'global' ? 'global' : 'project';
showRefreshToast(`MCP server "${name}" created in ${scopeLabel} scope`, 'success');
} else {
showRefreshToast(result.error || 'Failed to create MCP server', 'error');
@@ -787,7 +941,7 @@ function buildCcwToolsConfig(selectedTools) {
return config;
}
async function installCcwToolsMcp() {
async function installCcwToolsMcp(scope = 'workspace') {
const selectedTools = getSelectedCcwTools();
if (selectedTools.length === 0) {
@@ -798,27 +952,52 @@ async function installCcwToolsMcp() {
const ccwToolsConfig = buildCcwToolsConfig(selectedTools);
try {
showRefreshToast('Installing CCW Tools MCP...', 'info');
const scopeLabel = scope === 'global' ? 'globally' : 'to workspace';
showRefreshToast(`Installing CCW Tools MCP ${scopeLabel}...`, 'info');
const response = await fetch('/api/mcp-copy-server', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
projectPath: projectPath,
serverName: 'ccw-tools',
serverConfig: ccwToolsConfig
})
});
if (scope === 'global') {
// Install to global (~/.claude.json mcpServers)
const response = await fetch('/api/mcp-add-global', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
serverName: 'ccw-tools',
serverConfig: ccwToolsConfig
})
});
if (!response.ok) throw new Error('Failed to install CCW Tools MCP');
if (!response.ok) throw new Error('Failed to install CCW Tools MCP globally');
const result = await response.json();
if (result.success) {
await loadMcpConfig();
renderMcpManager();
showRefreshToast(`CCW Tools installed (${selectedTools.length} tools)`, 'success');
const result = await response.json();
if (result.success) {
await loadMcpConfig();
renderMcpManager();
showRefreshToast(`CCW Tools installed globally (${selectedTools.length} tools)`, 'success');
} else {
showRefreshToast(result.error || 'Failed to install CCW Tools MCP globally', 'error');
}
} else {
showRefreshToast(result.error || 'Failed to install CCW Tools MCP', 'error');
// Install to workspace (.mcp.json)
const response = await fetch('/api/mcp-copy-server', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
projectPath: projectPath,
serverName: 'ccw-tools',
serverConfig: ccwToolsConfig
})
});
if (!response.ok) throw new Error('Failed to install CCW Tools MCP to workspace');
const result = await response.json();
if (result.success) {
await loadMcpConfig();
renderMcpManager();
showRefreshToast(`CCW Tools installed to workspace (${selectedTools.length} tools)`, 'success');
} else {
showRefreshToast(result.error || 'Failed to install CCW Tools MCP to workspace', 'error');
}
}
} catch (err) {
console.error('Failed to install CCW Tools MCP:', err);
@@ -826,7 +1005,7 @@ async function installCcwToolsMcp() {
}
}
async function updateCcwToolsMcp() {
async function updateCcwToolsMcp(scope = 'workspace') {
const selectedTools = getSelectedCcwTools();
if (selectedTools.length === 0) {
@@ -837,27 +1016,52 @@ async function updateCcwToolsMcp() {
const ccwToolsConfig = buildCcwToolsConfig(selectedTools);
try {
showRefreshToast('Updating CCW Tools MCP...', 'info');
const scopeLabel = scope === 'global' ? 'globally' : 'in workspace';
showRefreshToast(`Updating CCW Tools MCP ${scopeLabel}...`, 'info');
const response = await fetch('/api/mcp-copy-server', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
projectPath: projectPath,
serverName: 'ccw-tools',
serverConfig: ccwToolsConfig
})
});
if (scope === 'global') {
// Update global (~/.claude.json mcpServers)
const response = await fetch('/api/mcp-add-global', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
serverName: 'ccw-tools',
serverConfig: ccwToolsConfig
})
});
if (!response.ok) throw new Error('Failed to update CCW Tools MCP');
if (!response.ok) throw new Error('Failed to update CCW Tools MCP globally');
const result = await response.json();
if (result.success) {
await loadMcpConfig();
renderMcpManager();
showRefreshToast(`CCW Tools updated (${selectedTools.length} tools)`, 'success');
const result = await response.json();
if (result.success) {
await loadMcpConfig();
renderMcpManager();
showRefreshToast(`CCW Tools updated globally (${selectedTools.length} tools)`, 'success');
} else {
showRefreshToast(result.error || 'Failed to update CCW Tools MCP globally', 'error');
}
} else {
showRefreshToast(result.error || 'Failed to update CCW Tools MCP', 'error');
// Update workspace (.mcp.json)
const response = await fetch('/api/mcp-copy-server', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
projectPath: projectPath,
serverName: 'ccw-tools',
serverConfig: ccwToolsConfig
})
});
if (!response.ok) throw new Error('Failed to update CCW Tools MCP in workspace');
const result = await response.json();
if (result.success) {
await loadMcpConfig();
renderMcpManager();
showRefreshToast(`CCW Tools updated in workspace (${selectedTools.length} tools)`, 'success');
} else {
showRefreshToast(result.error || 'Failed to update CCW Tools MCP in workspace', 'error');
}
}
} catch (err) {
console.error('Failed to update CCW Tools MCP:', err);

View File

@@ -96,7 +96,7 @@ function renderImplPlanContent(implPlan) {
// Lite Context Tab Rendering
// ==========================================
function renderLiteContextContent(context, explorations, session) {
function renderLiteContextContent(context, explorations, session, diagnoses) {
const plan = session.plan || {};
let sections = [];
@@ -105,6 +105,11 @@ function renderLiteContextContent(context, explorations, session) {
sections.push(renderExplorationContext(explorations));
}
// Render diagnoses if available (from diagnosis-*.json files)
if (diagnoses && diagnoses.manifest) {
sections.push(renderDiagnosisContext(diagnoses));
}
// If we have context from context-package.json
if (context) {
sections.push(`
@@ -153,7 +158,7 @@ function renderLiteContextContent(context, explorations, session) {
<div class="tab-empty-state">
<div class="empty-icon"><i data-lucide="package" class="w-12 h-12"></i></div>
<div class="empty-title">No Context Data</div>
<div class="empty-text">No context-package.json or exploration files found for this session.</div>
<div class="empty-text">No context-package.json, exploration files, or diagnosis files found for this session.</div>
</div>
`;
}
@@ -185,15 +190,19 @@ function renderExplorationContext(explorations) {
`);
// Render each exploration angle as collapsible section
const explorationOrder = ['architecture', 'dependencies', 'patterns', 'integration-points'];
const explorationOrder = ['architecture', 'dependencies', 'patterns', 'integration-points', 'testing'];
const explorationTitles = {
'architecture': '<i data-lucide="blocks" class="w-4 h-4 inline mr-1"></i>Architecture',
'dependencies': '<i data-lucide="package" class="w-4 h-4 inline mr-1"></i>Dependencies',
'patterns': '<i data-lucide="git-branch" class="w-4 h-4 inline mr-1"></i>Patterns',
'integration-points': '<i data-lucide="plug" class="w-4 h-4 inline mr-1"></i>Integration Points'
'integration-points': '<i data-lucide="plug" class="w-4 h-4 inline mr-1"></i>Integration Points',
'testing': '<i data-lucide="flask-conical" class="w-4 h-4 inline mr-1"></i>Testing'
};
for (const angle of explorationOrder) {
// Collect all angles from data (in case there are exploration angles not in our predefined list)
const allAngles = [...new Set([...explorationOrder, ...Object.keys(data)])];
for (const angle of allAngles) {
const expData = data[angle];
if (!expData) {
continue;
@@ -205,7 +214,7 @@ function renderExplorationContext(explorations) {
<div class="exploration-section collapsible-section">
<div class="collapsible-header">
<span class="collapse-icon">▶</span>
<span class="section-label">${explorationTitles[angle] || angle}</span>
<span class="section-label">${explorationTitles[angle] || ('<i data-lucide="file-search" class="w-4 h-4 inline mr-1"></i>' + escapeHtml(angle.toUpperCase()))}</span>
</div>
<div class="collapsible-content collapsed">
${angleContent}
@@ -271,3 +280,145 @@ function renderExplorationAngle(angle, data) {
return content.join('') || '<p>No data available</p>';
}
// ==========================================
// Diagnosis Context Rendering
// ==========================================
function renderDiagnosisContext(diagnoses) {
if (!diagnoses || !diagnoses.manifest) {
return '';
}
const manifest = diagnoses.manifest;
const data = diagnoses.data || {};
let sections = [];
// Header with manifest info
sections.push(`
<div class="diagnosis-header">
<h4><i data-lucide="stethoscope" class="w-4 h-4 inline mr-1"></i> ${escapeHtml(manifest.task_description || 'Diagnosis Context')}</h4>
<div class="diagnosis-meta">
<span class="meta-item">Diagnoses: <strong>${manifest.diagnosis_count || 0}</strong></span>
</div>
</div>
`);
// Render each diagnosis angle as collapsible section
const diagnosisOrder = ['root-cause', 'api-contracts', 'dataflow', 'performance', 'security', 'error-handling'];
const diagnosisTitles = {
'root-cause': '<i data-lucide="search" class="w-4 h-4 inline mr-1"></i>Root Cause',
'api-contracts': '<i data-lucide="plug" class="w-4 h-4 inline mr-1"></i>API Contracts',
'dataflow': '<i data-lucide="git-merge" class="w-4 h-4 inline mr-1"></i>Data Flow',
'performance': '<i data-lucide="zap" class="w-4 h-4 inline mr-1"></i>Performance',
'security': '<i data-lucide="shield" class="w-4 h-4 inline mr-1"></i>Security',
'error-handling': '<i data-lucide="alert-circle" class="w-4 h-4 inline mr-1"></i>Error Handling'
};
// Collect all angles from data (in case there are diagnosis angles not in our predefined list)
const allAngles = [...new Set([...diagnosisOrder, ...Object.keys(data)])];
for (const angle of allAngles) {
const diagData = data[angle];
if (!diagData) {
continue;
}
const angleContent = renderDiagnosisAngle(angle, diagData);
sections.push(`
<div class="diagnosis-section collapsible-section">
<div class="collapsible-header">
<span class="collapse-icon">▶</span>
<span class="section-label">${diagnosisTitles[angle] || ('<i data-lucide="file-search" class="w-4 h-4 inline mr-1"></i>' + angle)}</span>
</div>
<div class="collapsible-content collapsed">
${angleContent}
</div>
</div>
`);
}
return `<div class="diagnosis-context">${sections.join('')}</div>`;
}
function renderDiagnosisAngle(angle, data) {
let content = [];
// Summary/Overview
if (data.summary || data.overview) {
content.push(renderExpField('Summary', data.summary || data.overview));
}
// Root cause analysis
if (data.root_cause || data.root_cause_analysis) {
content.push(renderExpField('Root Cause', data.root_cause || data.root_cause_analysis));
}
// Issues/Findings
if (data.issues && Array.isArray(data.issues)) {
content.push(`
<div class="exp-field">
<label>Issues Found (${data.issues.length})</label>
<div class="issues-list">
${data.issues.map(issue => {
if (typeof issue === 'string') {
return `<div class="issue-item">${escapeHtml(issue)}</div>`;
} else {
return `
<div class="issue-item">
<div class="issue-title">${escapeHtml(issue.title || issue.description || 'Unknown')}</div>
${issue.location ? `<div class="issue-location"><code>${escapeHtml(issue.location)}</code></div>` : ''}
${issue.severity ? `<span class="severity-badge ${escapeHtml(issue.severity)}">${escapeHtml(issue.severity)}</span>` : ''}
</div>
`;
}
}).join('')}
</div>
</div>
`);
}
// Affected files
if (data.affected_files && Array.isArray(data.affected_files)) {
content.push(`
<div class="exp-field">
<label>Affected Files (${data.affected_files.length})</label>
<div class="path-tags">
${data.affected_files.map(f => {
const filePath = typeof f === 'string' ? f : (f.path || f.file || '');
return `<span class="path-tag">${escapeHtml(filePath)}</span>`;
}).join('')}
</div>
</div>
`);
}
// Recommendations
if (data.recommendations && Array.isArray(data.recommendations)) {
content.push(`
<div class="exp-field">
<label>Recommendations</label>
<ol class="recommendations-list">
${data.recommendations.map(rec => {
const recText = typeof rec === 'string' ? rec : (rec.description || rec.action || '');
return `<li>${escapeHtml(recText)}</li>`;
}).join('')}
</ol>
</div>
`);
}
// API Contracts (specific to api-contracts diagnosis)
if (data.contracts && Array.isArray(data.contracts)) {
content.push(renderExpField('API Contracts', data.contracts));
}
// Data flow (specific to dataflow diagnosis)
if (data.dataflow || data.data_flow) {
content.push(renderExpField('Data Flow', data.dataflow || data.data_flow));
}
return content.join('') || '<p>No diagnosis data available</p>';
}