catlog22/Claude-Code-Workflow
1
0
Fork 0
mirror of https://github.com/catlog22/Claude-Code-Workflow.git synced 2026-03-10 17:11:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: Implement DeepWiki generator and CLI integration

Browse Source 
- Added `deepwiki_generator.py` for generating documentation from source code.
- Integrated symbol extraction and markdown generation for supported file types.
- Implemented database migration for legacy timestamp formats in DeepWikiStore.
- Enhanced debug logging for better traceability during conversation and store operations.
- Updated dependencies in `PKG-INFO` and `requires.txt` for compatibility.
- Added new tests for the DeepWiki generator and storage functionalities.
- Refactored existing code for improved readability and maintainability.
This commit is contained in:
catlog22
2026-03-07 17:05:49 +08:00
parent 75d5f7f230
commit ece4afcac8
17 changed files with 816 additions and 25221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
.ccw/workflows/cli-tools-usage.md
View File

@@ -216,7 +216,7 @@ rg "export.*Component" --files-with-matches --type ts
CONTEXT: @components/Auth.tsx @types/auth.d.ts | Memory: Previous type refactoring CONTEXT: @components/Auth.tsx @types/auth.d.ts | Memory: Previous type refactoring
# Step 3: Execute CLI # Step 3: Execute CLI
ccw cli -p "..." --tool <tool-id> --mode analysis --cd src ccw cli -p "..." --tool <tool-id> --mode analysis --cd "src"
``` ```
### --rule Configuration ### --rule Configuration
@@ -313,12 +313,12 @@ ccw cli -p "..." --tool gemini --mode analysis --rule analysis-review-architectu
- Description: Model override - Description: Model override
- Default: Tool's primaryModel from config - Default: Tool's primaryModel from config
- **`--cd <path>`** - **`--cd "<path>"`**
- Description: Working directory - Description: Working directory (quote if path contains spaces)
- Default: current - Default: current
- **`--includeDirs <dirs>`** - **`--includeDirs "<dirs>"`**
- Description: Additional directories (comma-separated) - Description: Additional directories (comma-separated, quote if paths contain spaces)
- Default: none - Default: none
- **`--resume [id]`** - **`--resume [id]`**
@@ -347,10 +347,10 @@ When using `--cd`:
```bash ```bash
# Single directory # Single directory
ccw cli -p "CONTEXT: @**/* @../shared/**/*" --tool <tool-id> --mode analysis --cd src/auth --includeDirs ../shared ccw cli -p "CONTEXT: @**/* @../shared/**/*" --tool <tool-id> --mode analysis --cd "src/auth" --includeDirs "../shared"
# Multiple directories # Multiple directories
ccw cli -p "..." --tool <tool-id> --mode analysis --cd src/auth --includeDirs ../shared,../types,../utils ccw cli -p "..." --tool <tool-id> --mode analysis --cd "src/auth" --includeDirs "../shared,../types,../utils"
``` ```
**Rule**: If CONTEXT contains `@../dir/**/*`, MUST include `--includeDirs ../dir` **Rule**: If CONTEXT contains `@../dir/**/*`, MUST include `--includeDirs ../dir`
@@ -397,7 +397,7 @@ MODE: analysis
CONTEXT: @src/auth/**/* @src/middleware/auth.ts | Memory: Using bcrypt for passwords, JWT for sessions CONTEXT: @src/auth/**/* @src/middleware/auth.ts | Memory: Using bcrypt for passwords, JWT for sessions
EXPECTED: Security report with: severity matrix, file:line references, CVE mappings where applicable, remediation code snippets prioritized by risk EXPECTED: Security report with: severity matrix, file:line references, CVE mappings where applicable, remediation code snippets prioritized by risk
CONSTRAINTS: Focus on authentication | Ignore test files CONSTRAINTS: Focus on authentication | Ignore test files
" --tool gemini --mode analysis --rule analysis-assess-security-risks --cd src/auth " --tool gemini --mode analysis --rule analysis-assess-security-risks --cd "src/auth"
``` ```
**Implementation Task** (New Feature): **Implementation Task** (New Feature):
@@ -419,7 +419,7 @@ MODE: analysis
CONTEXT: @src/websocket/**/* @src/services/connection-manager.ts | Memory: Using ws library, ~5000 concurrent connections in production CONTEXT: @src/websocket/**/* @src/services/connection-manager.ts | Memory: Using ws library, ~5000 concurrent connections in production
EXPECTED: Root cause analysis with: memory profile, leak source (file:line), fix recommendation with code, verification steps EXPECTED: Root cause analysis with: memory profile, leak source (file:line), fix recommendation with code, verification steps
CONSTRAINTS: Focus on resource cleanup CONSTRAINTS: Focus on resource cleanup
" --tool gemini --mode analysis --rule analysis-diagnose-bug-root-cause --cd src " --tool gemini --mode analysis --rule analysis-diagnose-bug-root-cause --cd "src"
``` ```
**Refactoring Task**: **Refactoring Task**:

24749
ccw/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

7
ccw/src/cli.ts
View File

@@ -201,7 +201,7 @@ export function run(argv: string[]): void {
.option('--status <status>', 'Filter by status') .option('--status <status>', 'Filter by status')
.option('--category <category>', 'Execution category: user, internal, insight', 'user') .option('--category <category>', 'Execution category: user, internal, insight', 'user')
.option('--resume [id]', 'Resume previous session (empty=last, or execution ID, or comma-separated IDs for merge)') .option('--resume [id]', 'Resume previous session (empty=last, or execution ID, or comma-separated IDs for merge)')
.option('--id <id>', 'Custom execution ID (e.g., IMPL-001-step1)') .option('--id <id>', 'Execution ID (recommended, auto-generated if omitted)')
.option('--no-native', 'Force prompt concatenation instead of native resume') .option('--no-native', 'Force prompt concatenation instead of native resume')
.option('--cache [items]', 'Cache: comma-separated @patterns and text content') .option('--cache [items]', 'Cache: comma-separated @patterns and text content')
.option('--inject-mode <mode>', 'Inject mode: none, full, progressive (default: codex=full, others=none)') .option('--inject-mode <mode>', 'Inject mode: none, full, progressive (default: codex=full, others=none)')
@@ -226,7 +226,10 @@ export function run(argv: string[]): void {
.option('--output-type <type>', 'Output type: stdout, stderr, both', 'both') .option('--output-type <type>', 'Output type: stdout, stderr, both', 'both')
.option('--turn <n>', 'Turn number for cache (default: latest)') .option('--turn <n>', 'Turn number for cache (default: latest)')
.option('--raw', 'Raw output only (no formatting)') .option('--raw', 'Raw output only (no formatting)')
.option('--final', 'Output final result only with usage hint') .option('--final', 'Output final result only (legacy, now default)')
.option('--verbose', 'Show full metadata in output view')
.option('--timeout <seconds>', 'Timeout for watch command')
.option('--all', 'Show all executions in show command')
.option('--to-file <path>', 'Save output to file') .option('--to-file <path>', 'Save output to file')
.action((subcommand, args, options) => cliCommand(subcommand, args, options)); .action((subcommand, args, options) => cliCommand(subcommand, args, options));

275
ccw/src/commands/cli.ts
View File

@@ -10,6 +10,7 @@ import type { CliOutputUnit } from '../tools/cli-output-converter.js';
import { SmartContentFormatter } from '../tools/cli-output-converter.js'; import { SmartContentFormatter } from '../tools/cli-output-converter.js';
import { import {
cliExecutorTool, cliExecutorTool,
generateExecutionId,
getCliToolsStatus, getCliToolsStatus,
getExecutionHistory, getExecutionHistory,
getExecutionHistoryAsync, getExecutionHistoryAsync,
@@ -181,6 +182,7 @@ interface OutputViewOptions {
turn?: string; turn?: string;
raw?: boolean; raw?: boolean;
final?: boolean; // Only output final result with usage hint final?: boolean; // Only output final result with usage hint
verbose?: boolean; // Show full metadata (original default behavior)
project?: string; // Optional project path for lookup project?: string; // Optional project path for lookup
} }
@@ -432,25 +434,8 @@ async function outputAction(conversationId: string | undefined, options: OutputV
return; return;
} }
if (options.final) { if (options.verbose) {
// Final result only with usage hint // Verbose: full metadata + output (original default behavior)
// Prefer finalOutput (agent_message only) > parsedOutput (filtered) > raw stdout
const outputContent = result.finalOutput?.content || result.parsedOutput?.content || result.stdout?.content;
if (outputContent) {
console.log(outputContent);
}
console.log();
console.log(chalk.gray('─'.repeat(60)));
console.log(chalk.dim(`Usage: ccw cli output ${conversationId} [options]`));
console.log(chalk.dim(' --raw Raw output (no formatting)'));
console.log(chalk.dim(' --offset <n> Start from byte offset'));
console.log(chalk.dim(' --limit <n> Limit output bytes'));
console.log(chalk.dim(' --project <p> Specify project path explicitly'));
console.log(chalk.dim(` --resume ccw cli -p "..." --resume ${conversationId}`));
return;
}
// Formatted output
console.log(chalk.bold.cyan('Execution Output\n')); console.log(chalk.bold.cyan('Execution Output\n'));
console.log(` ${chalk.gray('ID:')} ${result.conversationId}`); console.log(` ${chalk.gray('ID:')} ${result.conversationId}`);
console.log(` ${chalk.gray('Turn:')} ${result.turnNumber}`); console.log(` ${chalk.gray('Turn:')} ${result.turnNumber}`);
@@ -482,6 +467,24 @@ async function outputAction(conversationId: string | undefined, options: OutputV
} }
console.log(); console.log();
} }
return;
}
// Default: final result only (equivalent to --final)
// Prefer finalOutput (agent_message only) > parsedOutput (filtered) > raw stdout
const outputContent = result.finalOutput?.content || result.parsedOutput?.content || result.stdout?.content;
if (outputContent) {
console.log(outputContent);
}
console.log();
console.log(chalk.gray('\u2500'.repeat(60)));
console.log(chalk.dim(`Usage: ccw cli output ${conversationId} [options]`));
console.log(chalk.dim(' --verbose Show full metadata'));
console.log(chalk.dim(' --raw Raw output (no formatting)'));
console.log(chalk.dim(' --offset <n> Start from byte offset'));
console.log(chalk.dim(' --limit <n> Limit output bytes'));
console.log(chalk.dim(' --project <p> Specify project path explicitly'));
console.log(chalk.dim(` --resume ccw cli -p "..." --resume ${conversationId}`));
} }
/** /**
@@ -922,8 +925,8 @@ async function execAction(positionalPrompt: string | undefined, options: CliExec
console.log(); console.log();
} }
// Generate execution ID for streaming (use custom ID or timestamp-based) // Generate execution ID for streaming (use custom ID or auto-generated readable ID)
const executionId = id || `${Date.now()}-${tool}`; const executionId = id || generateExecutionId(tool);
const startTime = Date.now(); const startTime = Date.now();
const modelInfo = model ? ` @${model}` : ''; const modelInfo = model ? ` @${model}` : '';
const spinnerBaseText = `Executing ${tool}${modelInfo} (${mode} mode${resumeInfo}${nativeMode})${idInfo}...`; const spinnerBaseText = `Executing ${tool}${modelInfo} (${mode} mode${resumeInfo}${nativeMode})${idInfo}...`;
@@ -989,9 +992,9 @@ async function execAction(positionalPrompt: string | undefined, options: CliExec
mode mode
}); });
if (process.env.DEBUG) { // Always output execution ID to stderr for programmatic capture
console.error(`[CLI] Generated executionId: ${executionId}`); // Callers can: ccw cli -p "..." 2>&1 | grep CCW_EXEC_ID
} console.error(`[CCW_EXEC_ID=${executionId}]`);
// Buffer to accumulate output when both --stream and --to-file are specified // Buffer to accumulate output when both --stream and --to-file are specified
let streamBuffer = ''; let streamBuffer = '';
@@ -1306,6 +1309,214 @@ async function execAction(positionalPrompt: string | undefined, options: CliExec
} }
} }
/**
* Show all executions — active (running) + recent completed
* Combines live dashboard state with SQLite history
*/
async function showAction(options: { all?: boolean }): Promise<void> {
console.log(chalk.bold.cyan('\n CLI Executions\n'));
// 1. Try to fetch active executions from dashboard
let activeExecs: Array<{
id: string; tool: string; mode: string; status: string;
prompt: string; startTime: number; isComplete?: boolean;
}> = [];
try {
const data = await new Promise<string>((resolve, reject) => {
const req = http.request({
hostname: 'localhost',
port: Number(DASHBOARD_PORT),
path: '/api/cli/active',
method: 'GET',
timeout: 2000,
agent: false,
headers: { 'Connection': 'close' }
}, (res) => {
let body = '';
res.on('data', (chunk: Buffer) => { body += chunk.toString(); });
res.on('end', () => resolve(body));
});
req.on('error', reject);
req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
req.end();
});
const parsed = JSON.parse(data);
activeExecs = Array.isArray(parsed) ? parsed : (parsed.executions || []);
} catch {
// Dashboard not available — show only history
}
// 2. Get recent history from SQLite
const historyLimit = options.all ? 100 : 20;
const history = await getExecutionHistoryAsync(process.cwd(), { limit: historyLimit, recursive: true });
// 3. Build unified list: active first, then history (de-duped)
const seenIds = new Set<string>();
const rows: Array<{
id: string; tool: string; mode: string; status: string;
prompt: string; time: string; duration: string;
}> = [];
// Active executions (running)
for (const exec of activeExecs) {
if (exec.status === 'running') {
seenIds.add(exec.id);
const elapsed = Math.floor((Date.now() - exec.startTime) / 1000);
rows.push({
id: exec.id,
tool: exec.tool,
mode: exec.mode,
status: 'running',
prompt: (exec.prompt || '').replace(/\n/g, ' ').substring(0, 50),
time: `${elapsed}s ago`,
duration: `${elapsed}s...`,
});
}
}
// History executions
for (const exec of history.executions) {
if (seenIds.has(exec.id)) continue;
seenIds.add(exec.id);
const duration = exec.duration_ms >= 1000
? `${(exec.duration_ms / 1000).toFixed(1)}s`
: `${exec.duration_ms}ms`;
const timeAgo = getTimeAgo(new Date(exec.updated_at || exec.timestamp));
rows.push({
id: exec.id,
tool: exec.tool,
mode: exec.mode || 'analysis',
status: exec.status,
prompt: exec.prompt_preview.replace(/\n/g, ' ').substring(0, 50),
time: timeAgo,
duration,
});
}
if (rows.length === 0) {
console.log(chalk.gray(' No executions found.\n'));
return;
}
// 4. Render table
console.log(chalk.gray(' Status Tool Mode Duration Time ID'));
console.log(chalk.gray(' ' + '\u2500'.repeat(80)));
for (const row of rows) {
const statusIcon = row.status === 'running' ? chalk.blue('\u25CF') :
row.status === 'success' || row.status === 'completed' ? chalk.green('\u25CF') :
row.status === 'timeout' ? chalk.yellow('\u25CF') : chalk.red('\u25CF');
console.log(` ${statusIcon} ${chalk.bold.white(row.tool.padEnd(8))} ${chalk.gray(row.mode.padEnd(9))} ${chalk.gray(row.duration.padEnd(9))} ${chalk.gray(row.time.padEnd(11))} ${chalk.dim(row.id)}`);
if (row.prompt) {
console.log(chalk.gray(` ${row.prompt}${row.prompt.length >= 50 ? '...' : ''}`));
}
}
console.log();
console.log(chalk.gray(' ' + '\u2500'.repeat(80)));
console.log(chalk.dim(' Output: ccw cli output <id>'));
console.log(chalk.dim(' Watch: ccw cli watch <id>'));
console.log(chalk.dim(' Detail: ccw cli detail <id>'));
console.log();
}
/**
* Watch a running execution — stream output to stderr until completion
* Exits with code 0 (success), 1 (error), or 2 (timeout)
*/
async function watchAction(watchId: string | undefined, options: { timeout?: string }): Promise<void> {
if (!watchId) {
console.error(chalk.red('Error: Execution ID is required'));
console.error(chalk.gray('Usage: ccw cli watch <id> [--timeout 120]'));
process.exit(1);
}
const timeoutMs = options.timeout ? parseInt(options.timeout, 10) * 1000 : 0;
const startTime = Date.now();
process.stderr.write(chalk.cyan(`Watching execution: ${watchId}\n`));
// Track output position for incremental display
let lastOutputLen = 0;
const poll = async (): Promise<number> => {
// Check timeout
if (timeoutMs > 0 && (Date.now() - startTime) > timeoutMs) {
process.stderr.write(chalk.yellow('\nWatch timed out.\n'));
return 2;
}
try {
// Fetch active execution state from dashboard
const data = await new Promise<string>((resolve, reject) => {
const req = http.request({
hostname: 'localhost',
port: Number(DASHBOARD_PORT),
path: '/api/cli/active',
method: 'GET',
timeout: 3000,
agent: false,
headers: { 'Connection': 'close' }
}, (res) => {
let body = '';
res.on('data', (chunk: Buffer) => { body += chunk.toString(); });
res.on('end', () => resolve(body));
});
req.on('error', reject);
req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
req.end();
});
const parsed = JSON.parse(data);
const executions = Array.isArray(parsed) ? parsed : (parsed.executions || []);
const exec = executions.find((e: { id: string }) => e.id === watchId);
if (exec) {
// Show incremental output
const fullOutput = exec.output || '';
if (fullOutput.length > lastOutputLen) {
process.stderr.write(fullOutput.slice(lastOutputLen));
lastOutputLen = fullOutput.length;
}
if (exec.status === 'running') {
// Still running — wait and poll again
await new Promise(r => setTimeout(r, 1000));
return poll();
}
// Completed
process.stderr.write(chalk.green(`\nExecution ${exec.status === 'completed' || exec.status === 'success' ? 'completed' : 'failed'}.\n`));
return (exec.status === 'completed' || exec.status === 'success') ? 0 : 1;
}
} catch {
// Dashboard not available
}
// Not found in active — check SQLite history
const store = getHistoryStore(process.cwd());
const result = store.getCachedOutput(watchId);
if (result) {
process.stderr.write(chalk.gray(`\nExecution already completed (status: ${result.status}).\n`));
process.stderr.write(chalk.dim(`Use: ccw cli output ${watchId}\n`));
return result.status === 'success' ? 0 : 1;
}
// Not found anywhere — may still be starting, wait and retry a few times
if ((Date.now() - startTime) < 10000) {
await new Promise(r => setTimeout(r, 1000));
return poll();
}
process.stderr.write(chalk.red(`\nExecution not found: ${watchId}\n`));
return 1;
};
const exitCode = await poll();
process.exit(exitCode);
}
/** /**
* Show execution history * Show execution history
* @param {Object} options - CLI options * @param {Object} options - CLI options
@@ -1455,6 +1666,14 @@ export async function cliCommand(
await historyAction(options as HistoryOptions); await historyAction(options as HistoryOptions);
break; break;
case 'show':
await showAction(options as unknown as { all?: boolean });
break;
case 'watch':
await watchAction(argsArray[0], options as unknown as { timeout?: string });
break;
case 'detail': case 'detail':
await detailAction(argsArray[0]); await detailAction(argsArray[0]);
break; break;
@@ -1506,11 +1725,13 @@ export async function cliCommand(
console.log(chalk.gray(' echo "prompt" | ccw cli --tool <tool> Execute from stdin (pipe)')); console.log(chalk.gray(' echo "prompt" | ccw cli --tool <tool> Execute from stdin (pipe)'));
console.log(); console.log();
console.log(' Subcommands:'); console.log(' Subcommands:');
console.log(chalk.gray(' show List all executions (active + recent)'));
console.log(chalk.gray(' watch <id> Stream execution output (stderr, exits on completion)'));
console.log(chalk.gray(' output <id> Get final execution result'));
console.log(chalk.gray(' status Check CLI tools availability')); console.log(chalk.gray(' status Check CLI tools availability'));
console.log(chalk.gray(' storage [cmd] Manage CCW storage (info/clean/config)')); console.log(chalk.gray(' storage [cmd] Manage CCW storage (info/clean/config)'));
console.log(chalk.gray(' history Show execution history')); console.log(chalk.gray(' history Show execution history'));
console.log(chalk.gray(' detail <id> Show execution detail')); console.log(chalk.gray(' detail <id> Show execution detail (legacy, use show/output)'));
console.log(chalk.gray(' output <id> Show execution output with pagination'));
console.log(chalk.gray(' test-parse [args] Debug CLI argument parsing')); console.log(chalk.gray(' test-parse [args] Debug CLI argument parsing'));
console.log(); console.log();
console.log(' Options:'); console.log(' Options:');
@@ -1523,7 +1744,7 @@ export async function cliCommand(
console.log(chalk.gray(' --effort <level> Effort level for claude (low, medium, high)')); console.log(chalk.gray(' --effort <level> Effort level for claude (low, medium, high)'));
console.log(chalk.gray(' --cd <path> Working directory')); console.log(chalk.gray(' --cd <path> Working directory'));
console.log(chalk.gray(' --includeDirs <dirs> Additional directories')); console.log(chalk.gray(' --includeDirs <dirs> Additional directories'));
// --timeout removed - controlled by external caller (bash timeout) console.log(chalk.gray(' --id <id> Execution ID (recommended, auto-generated if omitted)'));
console.log(chalk.gray(' --resume [id] Resume previous session')); console.log(chalk.gray(' --resume [id] Resume previous session'));
console.log(chalk.gray(' --cache <items> Cache: comma-separated @patterns and text')); console.log(chalk.gray(' --cache <items> Cache: comma-separated @patterns and text'));
console.log(chalk.gray(' --inject-mode <m> Inject mode: none, full, progressive')); console.log(chalk.gray(' --inject-mode <m> Inject mode: none, full, progressive'));

3
ccw/src/core/routes/cli-routes.ts
View File

@@ -18,6 +18,7 @@ import {
deleteExecutionAsync, deleteExecutionAsync,
batchDeleteExecutionsAsync, batchDeleteExecutionsAsync,
executeCliTool, executeCliTool,
generateExecutionId,
getNativeSessionContent, getNativeSessionContent,
getFormattedNativeConversation, getFormattedNativeConversation,
getEnrichedConversation, getEnrichedConversation,
@@ -940,7 +941,7 @@ export async function handleCliRoutes(ctx: RouteContext): Promise<boolean> {
} }
} }
const executionId = `${Date.now()}-${tool}`; const executionId = generateExecutionId(tool);
// Store active execution for state recovery // Store active execution for state recovery
// Check map size limit before creating new execution // Check map size limit before creating new execution

11
ccw/src/core/routes/deepwiki-routes.ts
View File

@@ -11,13 +11,22 @@
import type { RouteContext } from './types.js'; import type { RouteContext } from './types.js';
import { getDeepWikiService } from '../../services/deepwiki-service.js'; import { getDeepWikiService } from '../../services/deepwiki-service.js';
import { handleDeepWikiRoutes as handleDeepWikiPostRoutes } from '../../services/deepwiki-service.js';
/** /**
* Handle DeepWiki routes * Handle DeepWiki routes
* @returns true if route was handled, false otherwise * @returns true if route was handled, false otherwise
*/ */
export async function handleDeepWikiRoutes(ctx: RouteContext): Promise<boolean> { export async function handleDeepWikiRoutes(ctx: RouteContext): Promise<boolean> {
const { pathname, url, res } = ctx; const { pathname, url, res, req } = ctx;
// POST endpoints implemented in the DeepWiki service module.
if (
(pathname === '/api/deepwiki/symbols-for-paths' || pathname === '/api/deepwiki/stale-files') &&
req.method === 'POST'
) {
return handleDeepWikiPostRoutes({ pathname, req, res });
}
// GET /api/deepwiki/files - List all documented files // GET /api/deepwiki/files - List all documented files
if (pathname === '/api/deepwiki/files') { if (pathname === '/api/deepwiki/files') {

3
ccw/src/core/services/cli-session-manager.ts
View File

@@ -17,6 +17,7 @@ import { getLaunchConfig } from './cli-launch-registry.js';
import { assembleInstruction, type InstructionType } from './cli-instruction-assembler.js'; import { assembleInstruction, type InstructionType } from './cli-instruction-assembler.js';
import { loadEndpointSettings } from '../../config/cli-settings-manager.js'; import { loadEndpointSettings } from '../../config/cli-settings-manager.js';
import { getToolConfig } from '../../tools/claude-cli-tools.js'; import { getToolConfig } from '../../tools/claude-cli-tools.js';
import { generateExecutionId } from '../../tools/cli-executor.js';
export interface CliSession { export interface CliSession {
sessionKey: string; sessionKey: string;
@@ -532,7 +533,7 @@ export class CliSessionManager {
const executionId = resumeKey const executionId = resumeKey
? `${resumeKey}-${Date.now()}` ? `${resumeKey}-${Date.now()}`
: `exec-${Date.now()}-${randomBytes(3).toString('hex')}`; : generateExecutionId(options.tool);
let command: string; let command: string;

77
ccw/src/tools/cli-executor-core.ts
View File

@@ -35,22 +35,31 @@ import {
saveConversation saveConversation
} from './cli-executor-state.js'; } from './cli-executor-state.js';
// Track all running child processes for cleanup on interruption (multi-process support) // Debug logging for history save investigation (Iteration 4)
const runningChildProcesses = new Set<ChildProcess>();
// Debug logging for parallel execution testing
const DEBUG_SESSION_ID = 'DBG-parallel-ccw-cli-test-2026-03-07'; const DEBUG_SESSION_ID = 'DBG-parallel-ccw-cli-test-2026-03-07';
const DEBUG_LOG_PATH = path.join(process.cwd(), '.workflow', '.debug', DEBUG_SESSION_ID, 'debug.log'); const DEBUG_LOG_PATH = path.join(process.cwd(), '.workflow', '.debug', DEBUG_SESSION_ID, 'debug-save.log');
// Ensure debug log directory exists
try {
const debugDir = path.dirname(DEBUG_LOG_PATH);
if (!fs.existsSync(debugDir)) {
fs.mkdirSync(debugDir, { recursive: true });
}
} catch (err) {
// Ignore directory creation errors
}
function writeDebugLog(event: string, data: Record<string, any>): void { function writeDebugLog(event: string, data: Record<string, any>): void {
try { try {
const logEntry = JSON.stringify({ event, ...data, timestamp: new Date().toISOString() }) + '\n'; const logEntry = JSON.stringify({ event, ...data, timestamp: new Date().toISOString() }) + '\n';
fs.appendFileSync(DEBUG_LOG_PATH, logEntry, 'utf8'); fs.appendFileSync(DEBUG_LOG_PATH, logEntry, 'utf8');
} catch (err) { } catch (err) {
// Silently ignore logging errors to avoid disrupting execution // Silently ignore logging errors
} }
} }
// Track all running child processes for cleanup on interruption (multi-process support)
const runningChildProcesses = new Set<ChildProcess>();
/** /**
* Kill all running CLI child processes * Kill all running CLI child processes
@@ -58,16 +67,14 @@ function writeDebugLog(event: string, data: Record<string, any>): void {
*/ */
export function killAllCliProcesses(): boolean { export function killAllCliProcesses(): boolean {
if (runningChildProcesses.size === 0) return false; if (runningChildProcesses.size === 0) return false;
writeDebugLog('KILL_ALL_START', { initial_set_size: runningChildProcesses.size });
const processesToKill = Array.from(runningChildProcesses); const processesToKill = Array.from(runningChildProcesses);
debugLog('KILL', `Killing ${processesToKill.length} child process(es)`, { pids: processesToKill.map(p => p.pid) }); debugLog('KILL', `Killing ${processesToKill.length} child process(es)`, { pids: processesToKill.map(p => p.pid) });
writeDebugLog('KILL_ALL_COPY', { pids_to_kill: processesToKill.map(p => p.pid) });
// 1. SIGTERM for graceful shutdown // 1. SIGTERM for graceful shutdown
for (const child of processesToKill) { for (const child of processesToKill) {
if (!child.killed) { if (!child.killed) {
try { child.kill('SIGTERM'); } catch (e: any) { writeDebugLog('KILL_SIGTERM_ERROR', { pid: child.pid, error: e.message }); } try { child.kill('SIGTERM'); } catch { /* ignore */ }
} }
} }
@@ -75,13 +82,12 @@ export function killAllCliProcesses(): boolean {
const killTimeout = setTimeout(() => { const killTimeout = setTimeout(() => {
for (const child of processesToKill) { for (const child of processesToKill) {
if (!child.killed) { if (!child.killed) {
try { child.kill('SIGKILL'); } catch (e: any) { writeDebugLog('KILL_SIGKILL_ERROR', { pid: child.pid, error: e.message }); } try { child.kill('SIGKILL'); } catch { /* ignore */ }
} }
} }
}, 2000); }, 2000);
killTimeout.unref(); killTimeout.unref();
writeDebugLog('KILL_ALL_CLEAR', { set_size_before: runningChildProcesses.size, pids_in_set: Array.from(runningChildProcesses).map(p => p.pid) });
runningChildProcesses.clear(); runningChildProcesses.clear();
return true; return true;
} }
@@ -257,7 +263,6 @@ async function executeClaudeWithSettings(params: ClaudeWithSettingsParams): Prom
// Track child process for cleanup (multi-process support) // Track child process for cleanup (multi-process support)
runningChildProcesses.add(child); runningChildProcesses.add(child);
writeDebugLog('PROCESS_ADD', { pid: child.pid, set_size_after: runningChildProcesses.size, function: 'executeClaudeWithSettings' });
let stdout = ''; let stdout = '';
let stderr = ''; let stderr = '';
@@ -297,7 +302,6 @@ async function executeClaudeWithSettings(params: ClaudeWithSettingsParams): Prom
child.on('close', (code) => { child.on('close', (code) => {
runningChildProcesses.delete(child); runningChildProcesses.delete(child);
writeDebugLog('PROCESS_DELETE', { pid: child.pid, exit_code: code, set_size_after: runningChildProcesses.size, function: 'executeClaudeWithSettings', handler: 'close' });
const endTime = Date.now(); const endTime = Date.now();
const duration = endTime - startTime; const duration = endTime - startTime;
@@ -338,10 +342,8 @@ async function executeClaudeWithSettings(params: ClaudeWithSettingsParams): Prom
// Save to history // Save to history
try { try {
writeDebugLog('SAVE_CONVERSATION_START', { conversationId: conversation.id, pid: child.pid, function: 'executeClaudeWithSettings' });
saveConversation(workingDir, conversation); saveConversation(workingDir, conversation);
} catch (err) { } catch (err) {
writeDebugLog('SAVE_CONVERSATION_ERROR', { conversationId: conversation.id, pid: child.pid, error: (err as Error).message, stack: (err as Error).stack, function: 'executeClaudeWithSettings' });
console.error('[CLI Executor] Failed to save CLI封装 history:', (err as Error).message); console.error('[CLI Executor] Failed to save CLI封装 history:', (err as Error).message);
} }
@@ -356,7 +358,6 @@ async function executeClaudeWithSettings(params: ClaudeWithSettingsParams): Prom
child.on('error', (error) => { child.on('error', (error) => {
runningChildProcesses.delete(child); runningChildProcesses.delete(child);
writeDebugLog('PROCESS_DELETE', { pid: child.pid, set_size_after: runningChildProcesses.size, function: 'executeClaudeWithSettings', handler: 'error' });
reject(new Error(`Failed to spawn claude: ${error.message}`)); reject(new Error(`Failed to spawn claude: ${error.message}`));
}); });
}); });
@@ -391,6 +392,30 @@ type BuiltinCliTool = typeof BUILTIN_CLI_TOOLS[number];
*/ */
export type TransactionId = string; export type TransactionId = string;
/**
* Generate a readable execution ID for CLI executions
* Format: {prefix}-{HHmmss}-{rand4} → e.g. gem-143022-x7k2
* @param tool - CLI tool name (gemini, qwen, codex, claude, opencode, litellm, etc.)
* @returns Short, human-readable execution ID
*/
export function generateExecutionId(tool: string): string {
const prefixMap: Record<string, string> = {
gemini: 'gem',
qwen: 'qwn',
codex: 'cdx',
claude: 'cld',
opencode: 'opc',
litellm: 'llm',
};
const prefix = prefixMap[tool] || tool.slice(0, 3);
const now = new Date();
const time = [now.getHours(), now.getMinutes(), now.getSeconds()]
.map(n => String(n).padStart(2, '0'))
.join('');
const rand = Math.random().toString(36).slice(2, 6);
return `${prefix}-${time}-${rand}`;
}
/** /**
* Generate a unique transaction ID for the current execution * Generate a unique transaction ID for the current execution
* @param conversationId - CCW conversation ID * @param conversationId - CCW conversation ID
@@ -526,7 +551,7 @@ async function executeCliTool(
const duration = endTime - startTime; const duration = endTime - startTime;
const execution: ExecutionRecord = { const execution: ExecutionRecord = {
id: customId || `${Date.now()}-litellm`, id: customId || generateExecutionId('litellm'),
timestamp: new Date(startTime).toISOString(), timestamp: new Date(startTime).toISOString(),
tool: 'litellm', tool: 'litellm',
model: result.model, model: result.model,
@@ -676,7 +701,7 @@ async function executeCliTool(
const duration = endTime - startTime; const duration = endTime - startTime;
const execution: ExecutionRecord = { const execution: ExecutionRecord = {
id: customId || `${Date.now()}-litellm`, id: customId || generateExecutionId('litellm'),
timestamp: new Date(startTime).toISOString(), timestamp: new Date(startTime).toISOString(),
tool: toolName, tool: toolName,
model: result.model, // Use effective model from result (reflects any override) model: result.model, // Use effective model from result (reflects any override)
@@ -789,11 +814,11 @@ async function executeCliTool(
existingConversation = loadConversation(workingDir, conversationId); existingConversation = loadConversation(workingDir, conversationId);
} else { } else {
// No previous conversation, create new // No previous conversation, create new
conversationId = `${Date.now()}-${tool}`; conversationId = generateExecutionId(tool);
} }
} else { } else {
// New conversation with auto-generated ID // New conversation with auto-generated ID
conversationId = `${Date.now()}-${tool}`; conversationId = generateExecutionId(tool);
} }
// Generate transaction ID for concurrent session disambiguation // Generate transaction ID for concurrent session disambiguation
@@ -1019,7 +1044,6 @@ async function executeCliTool(
// Track child process for cleanup on interruption (multi-process support) // Track child process for cleanup on interruption (multi-process support)
runningChildProcesses.add(child); runningChildProcesses.add(child);
writeDebugLog('PROCESS_ADD', { pid: child.pid, set_size_after: runningChildProcesses.size, function: 'executeCliTool', tool });
debugLog('SPAWN', `Process spawned`, { pid: child.pid }); debugLog('SPAWN', `Process spawned`, { pid: child.pid });
@@ -1071,7 +1095,6 @@ async function executeCliTool(
child.on('close', async (code) => { child.on('close', async (code) => {
// Remove from running processes // Remove from running processes
runningChildProcesses.delete(child); runningChildProcesses.delete(child);
writeDebugLog('PROCESS_DELETE', { pid: child.pid, exit_code: code, set_size_after: runningChildProcesses.size, function: 'executeCliTool', handler: 'close', tool });
// Flush remaining buffer from parser // Flush remaining buffer from parser
const remainingUnits = parser.flush(); const remainingUnits = parser.flush();
@@ -1200,11 +1223,9 @@ async function executeCliTool(
// Save all source conversations // Save all source conversations
try { try {
for (const conv of savedConversations) { for (const conv of savedConversations) {
writeDebugLog('SAVE_CONVERSATION_START', { conversationId: conv.id, pid: child.pid, function: 'executeCliTool', context: 'merge-loop', tool });
saveConversation(workingDir, conv); saveConversation(workingDir, conv);
} }
} catch (err) { } catch (err) {
writeDebugLog('SAVE_CONVERSATION_ERROR', { pid: child.pid, error: (err as Error).message, stack: (err as Error).stack, function: 'executeCliTool', context: 'merge-loop', tool });
console.error('[CLI Executor] Failed to save merged histories:', (err as Error).message); console.error('[CLI Executor] Failed to save merged histories:', (err as Error).message);
} }
} else if (isMerge && mergeResult && customId) { } else if (isMerge && mergeResult && customId) {
@@ -1244,10 +1265,8 @@ async function executeCliTool(
}; };
// Save merged conversation // Save merged conversation
try { try {
writeDebugLog('SAVE_CONVERSATION_START', { conversationId: conversation.id, pid: child.pid, function: 'executeCliTool', context: 'merge-with-id', tool });
saveConversation(workingDir, conversation); saveConversation(workingDir, conversation);
} catch (err) { } catch (err) {
writeDebugLog('SAVE_CONVERSATION_ERROR', { conversationId: conversation.id, pid: child.pid, error: (err as Error).message, stack: (err as Error).stack, function: 'executeCliTool', context: 'merge-with-id', tool });
console.error('[CLI Executor] Failed to save merged conversation:', (err as Error).message); console.error('[CLI Executor] Failed to save merged conversation:', (err as Error).message);
} }
} else { } else {
@@ -1277,10 +1296,11 @@ async function executeCliTool(
}; };
// Try to save conversation to history // Try to save conversation to history
try { try {
writeDebugLog('SAVE_CONVERSATION_START', { conversationId: conversation.id, pid: child.pid, function: 'executeCliTool', context: 'normal', tool }); writeDebugLog('BEFORE_SAVE_CONV', { conversationId: conversation.id, workingDir, tool });
saveConversation(workingDir, conversation); saveConversation(workingDir, conversation);
writeDebugLog('AFTER_SAVE_CONV', { conversationId: conversation.id, workingDir, tool });
} catch (err) { } catch (err) {
writeDebugLog('SAVE_CONVERSATION_ERROR', { conversationId: conversation.id, pid: child.pid, error: (err as Error).message, stack: (err as Error).stack, function: 'executeCliTool', context: 'normal', tool }); writeDebugLog('SAVE_CONV_OUTER_ERROR', { conversationId: conversation.id, workingDir, tool, error: (err as Error).message, stack: (err as Error).stack });
// Non-fatal: continue even if history save fails // Non-fatal: continue even if history save fails
console.error('[CLI Executor] Failed to save history:', (err as Error).message); console.error('[CLI Executor] Failed to save history:', (err as Error).message);
} }
@@ -1341,7 +1361,6 @@ async function executeCliTool(
child.on('error', (error) => { child.on('error', (error) => {
// Remove from running processes // Remove from running processes
runningChildProcesses.delete(child); runningChildProcesses.delete(child);
writeDebugLog('PROCESS_DELETE', { pid: child.pid, set_size_after: runningChildProcesses.size, function: 'executeCliTool', handler: 'error', tool });
errorLog('SPAWN', `Failed to spawn process`, error, { errorLog('SPAWN', `Failed to spawn process`, error, {
tool, tool,

34
ccw/src/tools/cli-executor-state.ts
View File

@@ -6,6 +6,31 @@
import type { HistoryIndexEntry } from './cli-history-store.js'; import type { HistoryIndexEntry } from './cli-history-store.js';
import { StoragePaths, ensureStorageDir } from '../config/storage-paths.js'; import { StoragePaths, ensureStorageDir } from '../config/storage-paths.js';
import type { CliOutputUnit } from './cli-output-converter.js'; import type { CliOutputUnit } from './cli-output-converter.js';
import * as fs from 'fs';
import * as path from 'path';
// Debug logging for history save investigation (Iteration 4)
const DEBUG_SESSION_ID = 'DBG-parallel-ccw-cli-test-2026-03-07';
const DEBUG_LOG_PATH = path.join(process.cwd(), '.workflow', '.debug', DEBUG_SESSION_ID, 'debug-save.log');
// Ensure debug log directory exists
try {
const debugDir = path.dirname(DEBUG_LOG_PATH);
if (!fs.existsSync(debugDir)) {
fs.mkdirSync(debugDir, { recursive: true });
}
} catch (err) {
// Ignore directory creation errors
}
function writeDebugLog(event: string, data: Record<string, any>): void {
try {
const logEntry = JSON.stringify({ event, ...data, timestamp: new Date().toISOString() }) + '\n';
fs.appendFileSync(DEBUG_LOG_PATH, logEntry, 'utf8');
} catch (err) {
// Silently ignore logging errors
}
}
// Lazy-loaded SQLite store module // Lazy-loaded SQLite store module
let sqliteStoreModule: typeof import('./cli-history-store.js') | null = null; let sqliteStoreModule: typeof import('./cli-history-store.js') | null = null;
@@ -14,8 +39,10 @@ let sqliteStoreModule: typeof import('./cli-history-store.js') | null = null;
* Get or initialize SQLite store (async) * Get or initialize SQLite store (async)
*/ */
export async function getSqliteStore(baseDir: string) { export async function getSqliteStore(baseDir: string) {
writeDebugLog('GET_STORE', { baseDir, baseDirType: typeof baseDir, moduleInitialized: sqliteStoreModule !== null });
if (!sqliteStoreModule) { if (!sqliteStoreModule) {
sqliteStoreModule = await import('./cli-history-store.js'); sqliteStoreModule = await import('./cli-history-store.js');
writeDebugLog('MODULE_LOADED', { baseDir });
} }
return sqliteStoreModule.getHistoryStore(baseDir); return sqliteStoreModule.getHistoryStore(baseDir);
} }
@@ -136,15 +163,20 @@ async function saveConversationAsync(baseDir: string, conversation: Conversation
* @param baseDir - Project base directory (NOT historyDir) * @param baseDir - Project base directory (NOT historyDir)
*/ */
export function saveConversation(baseDir: string, conversation: ConversationRecord): void { export function saveConversation(baseDir: string, conversation: ConversationRecord): void {
writeDebugLog('SAVE_CONV_START', { baseDir, conversationId: conversation.id, moduleInitialized: sqliteStoreModule !== null });
try { try {
const store = getSqliteStoreSync(baseDir); const store = getSqliteStoreSync(baseDir);
writeDebugLog('SAVE_CONV_SYNC', { baseDir, conversationId: conversation.id });
// Fire and forget - don't block on async save in sync context // Fire and forget - don't block on async save in sync context
store.saveConversation(conversation).catch(err => { store.saveConversation(conversation).catch(err => {
writeDebugLog('SAVE_CONV_ERROR', { baseDir, conversationId: conversation.id, error: err.message, stack: err.stack });
console.error('[CLI Executor] Failed to save conversation:', err.message); console.error('[CLI Executor] Failed to save conversation:', err.message);
}); });
} catch { } catch (err) {
writeDebugLog('SAVE_CONV_FALLBACK_ASYNC', { baseDir, conversationId: conversation.id, error: (err as Error).message });
// If sync not available, queue for async save // If sync not available, queue for async save
saveConversationAsync(baseDir, conversation).catch(err => { saveConversationAsync(baseDir, conversation).catch(err => {
writeDebugLog('SAVE_CONV_ASYNC_ERROR', { baseDir, conversationId: conversation.id, error: err.message, stack: err.stack });
console.error('[CLI Executor] Failed to save conversation:', err.message); console.error('[CLI Executor] Failed to save conversation:', err.message);
}); });
} }

38
ccw/src/tools/cli-history-store.ts
View File

@@ -4,13 +4,36 @@
*/ */
import Database from 'better-sqlite3'; import Database from 'better-sqlite3';
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, unlinkSync, rmdirSync } from 'fs'; import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, unlinkSync, rmdirSync, appendFileSync } from 'fs';
import { join, dirname, resolve } from 'path'; import { join, dirname, resolve } from 'path';
import { parseSessionFile, formatConversation, extractConversationPairs, type ParsedSession, type ParsedTurn } from './session-content-parser.js'; import { parseSessionFile, formatConversation, extractConversationPairs, type ParsedSession, type ParsedTurn } from './session-content-parser.js';
import { getDiscoverer, getNativeSessions } from './native-session-discovery.js'; import { getDiscoverer, getNativeSessions } from './native-session-discovery.js';
import { StoragePaths, ensureStorageDir, getProjectId, getCCWHome } from '../config/storage-paths.js'; import { StoragePaths, ensureStorageDir, getProjectId, getCCWHome } from '../config/storage-paths.js';
import type { CliOutputUnit } from './cli-output-converter.js'; import type { CliOutputUnit } from './cli-output-converter.js';
// Debug logging for history save investigation (Iteration 4)
const DEBUG_SESSION_ID = 'DBG-parallel-ccw-cli-test-2026-03-07';
const DEBUG_LOG_PATH = join(process.cwd(), '.workflow', '.debug', DEBUG_SESSION_ID, 'debug-save.log');
// Ensure debug log directory exists
try {
const debugDir = dirname(DEBUG_LOG_PATH);
if (!existsSync(debugDir)) {
mkdirSync(debugDir, { recursive: true });
}
} catch (err) {
// Ignore directory creation errors
}
function writeDebugLog(event: string, data: Record<string, any>): void {
try {
const logEntry = JSON.stringify({ event, ...data, timestamp: new Date().toISOString() }) + '\n';
appendFileSync(DEBUG_LOG_PATH, logEntry, 'utf8');
} catch (err) {
// Silently ignore logging errors
}
}
// Types // Types
export interface ConversationTurn { export interface ConversationTurn {
turn: number; turn: number;
@@ -110,22 +133,29 @@ export class CliHistoryStore {
private projectPath: string; private projectPath: string;
constructor(baseDir: string) { constructor(baseDir: string) {
writeDebugLog('STORE_CONSTRUCT_START', { baseDir });
this.projectPath = baseDir; this.projectPath = baseDir;
// Use centralized storage path // Use centralized storage path
const paths = StoragePaths.project(baseDir); const paths = StoragePaths.project(baseDir);
const historyDir = paths.cliHistory; const historyDir = paths.cliHistory;
writeDebugLog('STORAGE_PATHS', { baseDir, historyDir, historyDb: paths.historyDb });
ensureStorageDir(historyDir); ensureStorageDir(historyDir);
this.dbPath = paths.historyDb; this.dbPath = paths.historyDb;
writeDebugLog('DB_INSTANCE_CREATE', { dbPath: this.dbPath });
this.db = new Database(this.dbPath); this.db = new Database(this.dbPath);
writeDebugLog('DB_INSTANCE_CREATED', { dbPath: this.dbPath });
this.db.pragma('journal_mode = WAL'); this.db.pragma('journal_mode = WAL');
this.db.pragma('synchronous = NORMAL'); this.db.pragma('synchronous = NORMAL');
this.db.pragma('busy_timeout = 10000'); // Wait up to 10 seconds for locks (increased for write-heavy scenarios) this.db.pragma('busy_timeout = 10000'); // Wait up to 10 seconds for locks (increased for write-heavy scenarios)
this.db.pragma('wal_autocheckpoint = 1000'); // Optimize WAL checkpointing this.db.pragma('wal_autocheckpoint = 1000'); // Optimize WAL checkpointing
writeDebugLog('INIT_SCHEMA_START', { dbPath: this.dbPath });
this.initSchema(); this.initSchema();
writeDebugLog('INIT_SCHEMA_COMPLETE', { dbPath: this.dbPath });
this.migrateFromJson(historyDir); this.migrateFromJson(historyDir);
writeDebugLog('STORE_CONSTRUCT_COMPLETE', { baseDir, dbPath: this.dbPath });
} }
/** /**
@@ -936,11 +966,7 @@ export class CliHistoryStore {
const stmt = this.db.prepare(` const stmt = this.db.prepare(`
INSERT INTO native_session_mapping (ccw_id, tool, native_session_id, native_session_path, project_hash, transaction_id, created_at) INSERT INTO native_session_mapping (ccw_id, tool, native_session_id, native_session_path, project_hash, transaction_id, created_at)
VALUES (@ccw_id, @tool, @native_session_id, @native_session_path, @project_hash, @transaction_id, @created_at) VALUES (@ccw_id, @tool, @native_session_id, @native_session_path, @project_hash, @transaction_id, @created_at)
ON CONFLICT(ccw_id) DO UPDATE SET ON CONFLICT(tool, native_session_id) DO NOTHING
native_session_id = @native_session_id,
native_session_path = @native_session_path,
project_hash = @project_hash,
transaction_id = @transaction_id
`); `);
await this.withRetry(() => stmt.run({ await this.withRetry(() => stmt.run({

9
codex-lens/src/codex_lens.egg-info/PKG-INFO
View File

@@ -9,8 +9,9 @@ Requires-Python: >=3.10
Description-Content-Type: text/markdown Description-Content-Type: text/markdown
License-File: LICENSE License-File: LICENSE
Requires-Dist: typer~=0.9.0 Requires-Dist: typer~=0.9.0
Requires-Dist: click<9,>=8.0.0
Requires-Dist: rich~=13.0.0 Requires-Dist: rich~=13.0.0
Requires-Dist: pydantic~=2.0.0 Requires-Dist: pydantic>=2.5.0
Requires-Dist: tree-sitter~=0.20.0 Requires-Dist: tree-sitter~=0.20.0
Requires-Dist: tree-sitter-python~=0.25.0 Requires-Dist: tree-sitter-python~=0.25.0
Requires-Dist: tree-sitter-javascript~=0.25.0 Requires-Dist: tree-sitter-javascript~=0.25.0
@@ -20,16 +21,16 @@ Requires-Dist: watchdog~=3.0.0
Requires-Dist: ast-grep-py~=0.40.0 Requires-Dist: ast-grep-py~=0.40.0
Provides-Extra: semantic Provides-Extra: semantic
Requires-Dist: numpy~=1.26.0; extra == "semantic" Requires-Dist: numpy~=1.26.0; extra == "semantic"
Requires-Dist: fastembed~=0.2.0; extra == "semantic" Requires-Dist: fastembed~=0.2.1; extra == "semantic"
Requires-Dist: hnswlib~=0.8.0; extra == "semantic" Requires-Dist: hnswlib~=0.8.0; extra == "semantic"
Provides-Extra: semantic-gpu Provides-Extra: semantic-gpu
Requires-Dist: numpy~=1.26.0; extra == "semantic-gpu" Requires-Dist: numpy~=1.26.0; extra == "semantic-gpu"
Requires-Dist: fastembed~=0.2.0; extra == "semantic-gpu" Requires-Dist: fastembed~=0.2.1; extra == "semantic-gpu"
Requires-Dist: hnswlib~=0.8.0; extra == "semantic-gpu" Requires-Dist: hnswlib~=0.8.0; extra == "semantic-gpu"
Requires-Dist: onnxruntime-gpu~=1.15.0; extra == "semantic-gpu" Requires-Dist: onnxruntime-gpu~=1.15.0; extra == "semantic-gpu"
Provides-Extra: semantic-directml Provides-Extra: semantic-directml
Requires-Dist: numpy~=1.26.0; extra == "semantic-directml" Requires-Dist: numpy~=1.26.0; extra == "semantic-directml"
Requires-Dist: fastembed~=0.2.0; extra == "semantic-directml" Requires-Dist: fastembed~=0.2.1; extra == "semantic-directml"
Requires-Dist: hnswlib~=0.8.0; extra == "semantic-directml" Requires-Dist: hnswlib~=0.8.0; extra == "semantic-directml"
Requires-Dist: onnxruntime-directml~=1.15.0; extra == "semantic-directml" Requires-Dist: onnxruntime-directml~=1.15.0; extra == "semantic-directml"
Provides-Extra: reranker-onnx Provides-Extra: reranker-onnx

8
codex-lens/src/codex_lens.egg-info/SOURCES.txt
View File

@@ -36,6 +36,7 @@ src/codexlens/indexing/symbol_extractor.py
src/codexlens/lsp/__init__.py src/codexlens/lsp/__init__.py
src/codexlens/lsp/handlers.py src/codexlens/lsp/handlers.py
src/codexlens/lsp/keepalive_bridge.py src/codexlens/lsp/keepalive_bridge.py
src/codexlens/lsp/lsp-servers.json
src/codexlens/lsp/lsp_bridge.py src/codexlens/lsp/lsp_bridge.py
src/codexlens/lsp/lsp_graph_builder.py src/codexlens/lsp/lsp_graph_builder.py
src/codexlens/lsp/providers.py src/codexlens/lsp/providers.py
@@ -97,6 +98,8 @@ src/codexlens/semantic/reranker/legacy.py
src/codexlens/semantic/reranker/litellm_reranker.py src/codexlens/semantic/reranker/litellm_reranker.py
src/codexlens/semantic/reranker/onnx_reranker.py src/codexlens/semantic/reranker/onnx_reranker.py
src/codexlens/storage/__init__.py src/codexlens/storage/__init__.py
src/codexlens/storage/deepwiki_models.py
src/codexlens/storage/deepwiki_store.py
src/codexlens/storage/dir_index.py src/codexlens/storage/dir_index.py
src/codexlens/storage/file_cache.py src/codexlens/storage/file_cache.py
src/codexlens/storage/global_index.py src/codexlens/storage/global_index.py
@@ -117,6 +120,8 @@ src/codexlens/storage/migrations/migration_006_enhance_relationships.py
src/codexlens/storage/migrations/migration_007_add_graph_neighbors.py src/codexlens/storage/migrations/migration_007_add_graph_neighbors.py
src/codexlens/storage/migrations/migration_008_add_merkle_hashes.py src/codexlens/storage/migrations/migration_008_add_merkle_hashes.py
src/codexlens/storage/migrations/migration_010_add_multi_vector_chunks.py src/codexlens/storage/migrations/migration_010_add_multi_vector_chunks.py
src/codexlens/tools/__init__.py
src/codexlens/tools/deepwiki_generator.py
src/codexlens/watcher/__init__.py src/codexlens/watcher/__init__.py
src/codexlens/watcher/events.py src/codexlens/watcher/events.py
src/codexlens/watcher/file_watcher.py src/codexlens/watcher/file_watcher.py
@@ -129,6 +134,7 @@ tests/test_astgrep_binding.py
tests/test_binary_searcher.py tests/test_binary_searcher.py
tests/test_cascade_strategies.py tests/test_cascade_strategies.py
tests/test_chain_search.py tests/test_chain_search.py
tests/test_cli_help.py
tests/test_cli_hybrid_search.py tests/test_cli_hybrid_search.py
tests/test_cli_output.py tests/test_cli_output.py
tests/test_clustering_strategies.py tests/test_clustering_strategies.py
@@ -136,6 +142,8 @@ tests/test_code_extractor.py
tests/test_config.py tests/test_config.py
tests/test_config_cascade.py tests/test_config_cascade.py
tests/test_config_staged_env_overrides.py tests/test_config_staged_env_overrides.py
tests/test_deepwiki_store.py
tests/test_deepwiki_types.py
tests/test_dual_fts.py tests/test_dual_fts.py
tests/test_embedder.py tests/test_embedder.py
tests/test_embedding_backend_availability.py tests/test_embedding_backend_availability.py

2
codex-lens/src/codex_lens.egg-info/entry_points.txt
View File

@@ -1,2 +1,2 @@
[console_scripts] [console_scripts]
codexlens-lsp = codexlens.lsp:main codexlens-lsp = codexlens.lsp.server:main

9
codex-lens/src/codex_lens.egg-info/requires.txt
View File

@@ -1,6 +1,7 @@
typer~=0.9.0 typer~=0.9.0
click<9,>=8.0.0
rich~=13.0.0 rich~=13.0.0
pydantic~=2.0.0 pydantic>=2.5.0
tree-sitter~=0.20.0 tree-sitter~=0.20.0
tree-sitter-python~=0.25.0 tree-sitter-python~=0.25.0
tree-sitter-javascript~=0.25.0 tree-sitter-javascript~=0.25.0
@@ -43,17 +44,17 @@ transformers~=4.36.0
[semantic] [semantic]
numpy~=1.26.0 numpy~=1.26.0
fastembed~=0.2.0 fastembed~=0.2.1
hnswlib~=0.8.0 hnswlib~=0.8.0
[semantic-directml] [semantic-directml]
numpy~=1.26.0 numpy~=1.26.0
fastembed~=0.2.0 fastembed~=0.2.1
hnswlib~=0.8.0 hnswlib~=0.8.0
onnxruntime-directml~=1.15.0 onnxruntime-directml~=1.15.0
[semantic-gpu] [semantic-gpu]
numpy~=1.26.0 numpy~=1.26.0
fastembed~=0.2.0 fastembed~=0.2.1
hnswlib~=0.8.0 hnswlib~=0.8.0
onnxruntime-gpu~=1.15.0 onnxruntime-gpu~=1.15.0

192
codex-lens/src/codexlens/storage/deepwiki_store.py
View File

@@ -251,6 +251,11 @@ class DeepWikiStore:
except sqlite3.OperationalError: except sqlite3.OperationalError:
pass # Column already exists pass # Column already exists
# Legacy migration: some earlier DeepWiki DBs stored timestamps as TEXT (ISO strings).
# better-sqlite3 + JS code expects numeric (REAL) seconds, so ensure timestamp columns
# have REAL affinity by rebuilding affected tables when needed.
self._migrate_text_timestamps_to_real(conn)
conn.commit() conn.commit()
except sqlite3.DatabaseError as exc: except sqlite3.DatabaseError as exc:
raise StorageError( raise StorageError(
@@ -270,6 +275,193 @@ class DeepWikiStore:
""" """
return str(Path(path).resolve()).replace("\\", "/") return str(Path(path).resolve()).replace("\\", "/")
def _migrate_text_timestamps_to_real(self, conn: sqlite3.Connection) -> None:
"""Migrate legacy TEXT timestamp columns to REAL affinity.
SQLite's type system is dynamic, but column affinity influences how values are stored and
returned. Older DeepWiki databases used TEXT timestamps (often ISO strings). The current
schema uses REAL epoch seconds. When we detect TEXT affinity on timestamp columns, we
rebuild the table with REAL columns and convert existing values during copy.
"""
self._rebuild_table_with_timestamp_conversion(
conn,
table="deepwiki_files",
create_sql="""
CREATE TABLE deepwiki_files (
id INTEGER PRIMARY KEY,
path TEXT UNIQUE NOT NULL,
content_hash TEXT NOT NULL,
last_indexed REAL NOT NULL,
symbols_count INTEGER DEFAULT 0,
docs_generated INTEGER DEFAULT 0,
staleness_score REAL DEFAULT 0.0,
last_checked_commit TEXT,
last_checked_at REAL,
staleness_factors TEXT
)
""",
timestamp_columns={"last_indexed", "last_checked_at"},
required_timestamp_columns={"last_indexed"},
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_deepwiki_files_path ON deepwiki_files(path)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_deepwiki_files_hash ON deepwiki_files(content_hash)"
)
self._rebuild_table_with_timestamp_conversion(
conn,
table="deepwiki_docs",
create_sql="""
CREATE TABLE deepwiki_docs (
id INTEGER PRIMARY KEY,
path TEXT UNIQUE NOT NULL,
content_hash TEXT NOT NULL,
symbols TEXT DEFAULT '[]',
generated_at REAL NOT NULL,
llm_tool TEXT
)
""",
timestamp_columns={"generated_at"},
required_timestamp_columns={"generated_at"},
)
conn.execute("CREATE INDEX IF NOT EXISTS idx_deepwiki_docs_path ON deepwiki_docs(path)")
self._rebuild_table_with_timestamp_conversion(
conn,
table="deepwiki_symbols",
create_sql="""
CREATE TABLE deepwiki_symbols (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
type TEXT NOT NULL,
source_file TEXT NOT NULL,
doc_file TEXT NOT NULL,
anchor TEXT NOT NULL,
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
created_at REAL,
updated_at REAL,
staleness_score REAL DEFAULT 0.0,
last_checked_commit TEXT,
last_checked_at REAL,
staleness_factors TEXT,
UNIQUE(name, source_file)
)
""",
timestamp_columns={"created_at", "updated_at", "last_checked_at"},
required_timestamp_columns=set(),
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_deepwiki_symbols_name ON deepwiki_symbols(name)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_deepwiki_symbols_source ON deepwiki_symbols(source_file)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_deepwiki_symbols_doc ON deepwiki_symbols(doc_file)"
)
self._rebuild_table_with_timestamp_conversion(
conn,
table="generation_progress",
create_sql="""
CREATE TABLE generation_progress (
id INTEGER PRIMARY KEY AUTOINCREMENT,
symbol_key TEXT NOT NULL UNIQUE,
file_path TEXT NOT NULL,
symbol_name TEXT NOT NULL,
symbol_type TEXT NOT NULL,
layer INTEGER NOT NULL,
source_hash TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
attempts INTEGER DEFAULT 0,
last_tool TEXT,
last_error TEXT,
generated_at REAL,
created_at REAL,
updated_at REAL
)
""",
timestamp_columns={"generated_at", "created_at", "updated_at"},
required_timestamp_columns=set(),
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_progress_status ON generation_progress(status)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_progress_file ON generation_progress(file_path)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_progress_hash ON generation_progress(source_hash)"
)
def _rebuild_table_with_timestamp_conversion(
self,
conn: sqlite3.Connection,
*,
table: str,
create_sql: str,
timestamp_columns: set[str],
required_timestamp_columns: set[str],
) -> None:
info = conn.execute(f"PRAGMA table_info({table})").fetchall()
if not info:
return
declared_types = {
row["name"]: str(row["type"] or "").strip().upper() for row in info
}
needs_migration = any(
declared_types.get(col) == "TEXT" for col in timestamp_columns if col in declared_types
)
if not needs_migration:
return
old_table = f"{table}__old_ts"
conn.execute(f"ALTER TABLE {table} RENAME TO {old_table}")
conn.execute(create_sql)
old_cols = [
r["name"]
for r in conn.execute(f"PRAGMA table_info({old_table})").fetchall()
]
new_cols = [r["name"] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()]
common_cols = [c for c in new_cols if c in old_cols]
select_exprs: list[str] = []
for col in common_cols:
if col in timestamp_columns:
expr = self._sql_timestamp_to_real(col)
if col in required_timestamp_columns:
expr = f"COALESCE({expr}, CAST(strftime('%s','now') AS REAL))"
select_exprs.append(f"{expr} AS {col}")
else:
select_exprs.append(col)
cols_sql = ", ".join(common_cols)
select_sql = ", ".join(select_exprs)
conn.execute(
f"INSERT INTO {table} ({cols_sql}) SELECT {select_sql} FROM {old_table}"
)
conn.execute(f"DROP TABLE {old_table}")
def _sql_timestamp_to_real(self, col: str) -> str:
# Convert various timestamp representations to epoch seconds (REAL).
# - numeric types: keep as REAL
# - numeric strings: CAST to REAL
# - ISO datetime strings: strftime('%s', ...) to epoch seconds
return f"""(
CASE
WHEN {col} IS NULL THEN NULL
WHEN typeof({col}) IN ('integer', 'real') THEN CAST({col} AS REAL)
WHEN trim({col}) GLOB '[0-9]*' THEN CAST({col} AS REAL)
ELSE CAST(strftime('%s', replace(substr({col}, 1, 19), 'T', ' ')) AS REAL)
END
)"""
# === File Operations === # === File Operations ===
def add_file( def add_file(

475
codex-lens/src/codexlens/tools/__init__.py
View File

@@ -1,6 +1,5 @@
"""DeepWiki document generation tools. """DeepWiki document generation tools.
This module provides tools for generating documentation from source code. This module provides tools for generating documentation from source code.
""" """
@@ -8,15 +7,15 @@ from __future__ import annotations
import hashlib import hashlib
import logging import logging
import re import os
from pathlib import Path from pathlib import Path
from typing import List, Dict, Optional, Protocol from typing import Dict, List, Optional, Protocol, Any
from codexlens.storage.deepwiki_store import DeepWikiStore from codexlens.errors import StorageError
from codexlens.storage.deepwiki_models import DeepWikiSymbol
from codexlens.indexing.symbol_extractor import SymbolExtractor from codexlens.indexing.symbol_extractor import SymbolExtractor
from codexlens.parsers.factory import ParserFactory from codexlens.parsers.factory import ParserFactory
from codexlens.errors import StorageError from codexlens.storage.deepwiki_models import DeepWikiSymbol
from codexlens.storage.deepwiki_store import DeepWikiStore
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -24,7 +23,7 @@ logger = logging.getLogger(__name__)
# Default timeout for AI generation (30 seconds) # Default timeout for AI generation (30 seconds)
AI_TIMEOUT = 30 AI_TIMEOUT = 30
# HTML metadata markers for documentation # HTML metadata markers for documentation
SYMBOL_START_MARKER = "<!-- deepwiki-symbol-start name=\"symbol_name}\" -->" SYMBOL_START_MARKER = '<!-- deepwiki-symbol-start name="{symbol_name}" -->'
SYMBOL_END_MARKER = "<!-- deepwiki-symbol-end -->" SYMBOL_END_MARKER = "<!-- deepwiki-symbol-end -->"
@@ -49,7 +48,7 @@ class MockMarkdownGenerator(MarkdownGenerator):
def generate(self, symbol: DeepWikiSymbol, source_code: str) -> str: def generate(self, symbol: DeepWikiSymbol, source_code: str) -> str:
"""Generate mock Markdown documentation.""" """Generate mock Markdown documentation."""
return f"# {symbol.name}\n\n## {symbol.type}\n\n{source_code}\n```\n``` return f"# {symbol.name}\\n\\n## {symbol.type}\\n\\n```\\n{source_code}\\n```"
class DeepWikiGenerator: class DeepWikiGenerator:
@@ -60,89 +59,66 @@ class DeepWikiGenerator:
""" """
DEFAULT_DB_PATH = DeepWikiStore.DEFAULT_DB_PATH DEFAULT_DB_PATH = DeepWikiStore.DEFAULT_DB_PATH
SUPPORT_extensions = [".py", ".ts", ".tsx", ".js", ".jsx", ".java", ".go", ".rs", ".swift"] SUPPORTED_EXTENSIONS = [
".py",
".ts",
".tsx",
".js",
".jsx",
".java",
".go",
".rs",
".swift",
]
AI_TIMEOUT: int = 30 # Timeout for AI generation AI_TIMEOUT: int = 30 # Timeout for AI generation
MAX_SYMBOLS_PER_FILE: int = 100 # Batch size for processing large files MAX_SYMBOLS_PER_FILE: int = 100 # Batch size for processing large files
def __init__( def __init__(
self, self,
db_path: Path | None = None, db_path: Path | None = None,
store: DeepWikiStore = markdown_generator: MarkdownGenerator | None, None, store: DeepWikiStore | None = None,
markdown_generator: MarkdownGenerator | None = None,
max_symbols_per_file: int = 100, max_symbols_per_file: int = 100,
ai_timeout: int = 30, ai_timeout: int = 30,
) -> None: ) -> None:
self.markdown_generator = MockMarkdownGenerator() """
Initializes the DeepWikiGenerator.
"""
if store:
self.store = store self.store = store
self._extractor = Symbol_extractor() else:
self.store = DeepWikiStore(db_path or self.DEFAULT_DB_PATH)
if markdown_generator:
self.markdown_generator = markdown_generator
else: else:
logger.debug("No markdown generator provided, using mock")
self.markdown_generator = MockMarkdownGenerator()
self._extractor = SymbolExtractor() self._extractor = SymbolExtractor()
if file_path not in _should_process_file: self.max_symbols_per_file = max_symbols_per_file
self._extractor.extract_symbols(file_path) self.ai_timeout = ai_timeout
if symbols: self._docs_dir = Path("docs") # Default docs directory
logger.debug(f"Found {len(symbols)} symbols in {file_path}")
else: def _calculate_file_hash(self, file_path: Path) -> str:
logger.debug(f"No symbols found in {file_path}") """Calculate SHA256 hash of file content."""
return []
# Extract symbols from the file
for symbol in symbols:
try: try:
file_type = Parser_factory.get_parser(file_path.suffix) content = file_path.read_bytes()
if file_type is None: hash_obj = hashlib.sha256(content)
logger.warning(f"Unsupported file type: {file_path}")
continue
symbols.append(symbols)
doc_path = self._generate_docs(symbol)
doc_path.mkdir(doc_path, exist_ok=True)
for symbol in symbols:
doc_path = self._generate_markdown(symbol, source_code)
doc.write(doc(doc_id)
logger.debug(f"Generated docs for {len(symbols)} symbols in {file_path}")
self._store.save_symbol(symbol, doc_path, doc_content, doc_path)
self._store.update_file_stats(existing_file.path, symbols_count)
self._store.update_file_stats(
existing_file.path,
symbols_count=len(existing_file.symbols),
new_symbols_count=len(symbols),
docs_generated += 1
)
else:
# Skip unchanged files (skip update)
logger.debug(f"Skipped {len(unchanged_files)} unchanged symbols")
logger.debug(f"No symbols found in {file_path}, skipping update")
except Exception as e:
logger.error(f"Error extracting symbols from {file_path}: {e}")
raise StorageError(f"Failed to extract symbols from {file_path}")
try:
symbol_extractor = SymbolExtractor()
symbols = []
continue
except Exception as e:
logger.error(f"Failed to initialize symbol extractor: {e}")
raise StorageError(f"Failed to initialize symbol extractor for {file_path}")
# Return empty list
doc_paths = []
for doc_path in doc_paths:
try:
doc_path.mkdir(doc_path, parents=True, exist_ok=True)
for file in files:
if not file_path.endswith in support_extensions:
continue
source_file = file_path
source_content = file_path.read_bytes()
content_hash = self._calculate_file_hash(file_path)
return hash_obj.hexdigest() return hash_obj.hexdigest()
file_hash = existing_hash except IOError as e:
if existing_hash == new_hash: logger.error(f"Error reading file for hash calculation: {file_path}: {e}")
logger.debug( return ""
f"File unchanged: {file_path}. Skipping (hash match)"
) def _get_language(self, file_path: Path) -> str | None:
return existing_file """Determine language from file extension."""
# Get language from file path ext = file_path.suffix.lower()
language = self._get_language(file_path) if ext not in self.SUPPORTED_EXTENSIONS:
if language is None: logger.debug(f"Unsupported file extension: {file_path}, skipping file")
language = file_path.suffix return None
# Default to Python if it is other extension
language_map = { language_map = {
".py": "Python",
".ts": "TypeScript", ".ts": "TypeScript",
".tsx": "TypeScript React", ".tsx": "TypeScript React",
".js": "JavaScript", ".js": "JavaScript",
@@ -152,290 +128,99 @@ class DeepWikiGenerator:
".rs": "Rust", ".rs": "Rust",
".swift": "Swift", ".swift": "Swift",
} }
return language return language_map.get(ext)
file_type = None
except ValueError("Unsupported file type: {file_path}")
logger.warning(f"Unsupported file type: {file_path}, skipping")
continue
source_file = file_path
source_code = file.read_text()
if source_code:
try:
source_code = file.read_bytes(). hash_obj = hashlib.sha256(source_code.encode("utf-8")
return hash_obj.hexdigest()
else:
return ""
# Determine language from file extension
file_ext = file_extension.lower().find(f".py, ..ts, .tsx)
if file_ext in SUPPORT_extensions:
for ext in self.Suffix_lower():
logger.debug(f"Unsupported file extension: {file_path}, skipping file")
return None
except Exception as e: def _should_process_file(self, file_path: Path, force: bool) -> bool:
logger.warning(f"Error determining language for {file_path}: {e}") """Check if a file should be processed based on hash."""
return None, else: if force:
return self.suffix_lower() if ext == SUPPORT_extensions: return True
else: new_hash = self._calculate_file_hash(file_path)
return None if not new_hash:
else: return False
# Check if it is markdown generator exists
if markdown_generator:
logger.debug("No markdown generator provided, using mock")
return None
# Check if tool exists
if tool:
logger.debug(f"Tool not available for {tool}")
return None
# Extract symbols using regex for tree-sitter
language_map = self.Language_map
return language_map
# Read all symbols from the database file existing_file = self.store.get_file(str(file_path))
file_path = path if existing_file and existing_file.content_hash == new_hash:
# Get parser factory logger.debug(f"File unchanged: {file_path}. Skipping (hash match)")
if file_path not in support_extensions: return False
logger.debug(f"Unsupported file type: {file_path}, skipping") return True
return []
else:
logger.debug(f"Extracted {len(symbols)} symbols from {file_path}")
return symbols
def _generate_markdown(self, symbol: DeepWikiSymbol, source_code: str) -> str: def _generate_markdown_for_symbol(self, symbol: DeepWikiSymbol, source_code: str) -> str:
"""Generate Markdown documentation for a symbol. """Generate markdown and wrap it with markers."""
markdown_content = self.markdown_generator.generate(symbol, source_code)
return f"{SYMBOL_START_MARKER.format(symbol_name=symbol.name)}\\n{markdown_content}\\n{SYMBOL_END_MARKER}"
Args: def run(self, path: str, output_dir: Optional[str] = None, force: bool = False) -> Dict[str, Any]:
symbol: The symbol information
source_code: The source code content
Returns:
Generated Markdown documentation
"""
def _generate_markdown(
self, symbol: DeepWikiSymbol, source_code: str
) -> str:
"""Generate mock Markdown documentation."""
return f"# {symbol.name}\n\n## {symbol.type}\n\n{source_code}\n```\n```
doc_path.mkdir(self.docs_dir, parents=True, exist_ok=True)
for file in files:
if not file_path.endswith in support_extensions:
continue
source_content = file.read_bytes()
doc_content = f.read_text()
# Add content to markdown
markdown = f"<!-- deepwiki-symbol-start name=\"{symbol.name}\" -->\n{markdown_content}\n{markdown}
# Calculate anchor ( generate a_anchor(symbol)
anchor_line = symbol.line_range[0]
doc_path = self._docs_dir / docs_path
source_file = os.path.join(source_file, relative_path,)
return line_range
elif markdown is None:
anchor = ""
{markdown}
{markdown}
# Add anchor link to the from doc file
# Calculate doc file hash
file_hash = hashlib.sha256(file_content.encode("utf-8")
content_hash = existing_hash
file_path = source_file
if existing_file is None:
return None
source_file = source_file
file_path = str(source_file)
for f in symbols:
if file_changed
logger.info(
f"Generated docs for {len(symbols)} symbols in {file_path}"
)
logger.debug(
f"Updated {len(changed_files)} files - {len(changed_symbols)} "
)
logger.debug(
f"Updated {len(unchanged_files)} files: {len(unchanged_symbols)} "
)
logger.debug(
f"unchanged files: {len(unchanged_files)} (unchanged)"
)
else:
logger.debug(
f"Processed {len(files)} files, {len(files)} changed symbols, {len(changed_symbols)}"
)
logger.debug(f"Processed {len(files)} files in {len(files)} changes:")
f"Total files changed: {len(changed_files)}, "
f" file changes: {len(changed_files)}", "len(changed_symbols)} symbols, {len(changed_symbols)}, new_docs_generated: {len(changed_symbols)}"
)
)
)
# Save stats
stats["total_files"] = total_files
stats["total_symbols"] = total_symbols
stats["total_changed_symbols"] = changed_symbols_count
stats["unchanged_files"] = unchanged_files_count
stats["total_changed_files"] = changed_files
logger.info(
f"Generation complete - {len(files)} files, {len(symbols)} symbols, {len(changed_files)} changed symbols: files_changed}"
f" file changes ({len(changed_files)} changed symbols count} symbols"
}
f"unchanged files: {len(unchanged_files)} (unchanged_files_count}")
stats["unchanged_files"] = unchanged_files
stats["unchanged_files"] = unchanged_files
logger.info(
f"generation complete - {len(files)} files, {len(symbols)} symbols, {len(changed_files)} changed symbols, {len(changed_symbols)} docs generated"
}
else:
stats["unchanged_files"] = len(unchanged_files)
stats["unchanged_symbols"] = len(unchanged_symbols)
stats["total_symbols"] = total_symbols
stats["total_docs_generated"] = total_docs_generated
stats["total_changed_files"] = changed_files_count
stats["total_changed_files"] = unchanged_files_count
return stats
}
finally:
return self.close()
def run(self, path: str, output_dir: Optional[str] = None, db_path: Optional[Path] = None, force: bool = False,
max_symbols_per_file: int = 100,
ai_timeout: int = AI_TIMEOUT,
backend: str = "fastembed",
model: str = "code",
max_workers: int = 1,
json_mode: bool = False,
verbose: bool = False,
) -> None:
""" """
Initialize DeepWiki store and generator, and scan the source. Initialize DeepWiki store and generator, and scan the source.
Args:
path: Path to the source directory
db_path: Optional database path ( defaults to DEFAULT_DB_PATH)
force: Force full reindex ( ignoring file hashes
markdown_generator: Optional generator for markdown. If None, use Mock.
backend: backend or "fastembed"
model: model = "code"
max_workers: Maximum concurrent API calls for AI generation
max_symbols_per_file: maximum symbols to process per file (batch processing)
ai_timeout: timeout for AI generation
max_file_size: maximum file size to read in MB before processing ( chunks
Returns:
Generator result with stats dict[str, Any]:
""" """
source_root = Path(path)
if output_dir:
self._docs_dir = Path(output_dir)
<system_warning> stats = {
This task has subtasks - please focus on the current work. You start by reading the task files and completing summaries. "total_files": 0,
"total_symbols": 0,
"total_changed_files": 0,
"total_changed_symbols": 0,
"total_docs_generated": 0,
"total_unchanged_files": 0,
}
* Reading the `workflow/.lite-plan/implement-deepwiki-2026-03-05/TODO_LIST.md` for I'll the plan file and get started. files_to_process = [p for p in source_root.rglob("*") if p.is_file() and p.suffix in self.SUPPORTED_EXTENSIONS]
stats["total_files"] = len(files_to_process)
* Mark TASK 003 as completed. changed_files_count = 0
* Update TODO_list by checking the off the "Done when" checkboxes and completed sections unchanged_files_count = 0
* Generate completion summary with links to relevant files
* Update main task JSON status to "completed"
* * Read more context from previous tasks and understand what was completed
* Read plan.json to get tech stack info ( verify implementation approach
* * Now I'll implement the deepWiki generator. in `codex-lens/src/codexlens/tools/` directory. add CLI commands. and generate commands to. for file_path in files_to_process:
if not self._should_process_file(file_path, force):
unchanged_files_count += 1
continue
I'll write the file `deepwiki_generator.py` with the generator implementation. changed_files_count += 1
I'll add the `deepwiki` command group to the CLI module.
I'll test the implementation after
update the TODO list accordingly to the instructions.
* * Generate a completion summary in the `.summaries` directory
* Let me know if you wants to context or questions about the implementation.* I'll adjust the plan as necessary.* * Now, let me read the plan.json file to check the current plan structure: if it exists: need to create it. * let me check the completion status in the TODO list. Let me update the completion time and check if there's a status history to and update it task JSON status.
* Finally, I'll create a summary file and documenting the completion.I need to create the tools directory first. then create the generator file. Here's the full implementation: Now let me add the CLI commands to and test the implementation. Let me proceed with the tests.
I I'll verify that `deepwiki generate` command completes successfully
The `deepwiki_index` table contains symbol entries after the first run
A second run with unchanged source results in 0 new database writes.
Finally, I'll generate a summary file, document the implementation.
* Generate a completion summary in the summaries directory
* Update the TODO list to I progress tracking
* Mark the task as completed
* Update the main task JSON status to "completed" (if applicable, set completion timestamps)
Let me start by creating the tools directory and `__init__.py` file: and read the existing `deepwiki_store.py` file to understand the database structure and models, and methods available from the store. The as properties as the file tracking, symbol extraction, and documentation generation.Then it will integrate the AI service for generating the actual markdown. for each symbol. Finally, I'll update the stats in the store to track progress, display progress information in the console, and and table output, and log the completion status for each file.
total_symbols = len(symbols)
total_changed_files = len(changed_files)
total_unchanged_files = len(unchanged_files)
total_docs_generated = len(docs)
total_changed_symbols += len(changed_symbols)
total_docs_generated += docs
# Clean up removed symbols
for symbol in removed_symbols:
self.store.delete_symbols_for_file(file_path)
for doc in docs:
self.store.delete_doc(doc_id)
# Remove dangling references
for doc in docs:
self.store.delete_symbols_for_file(file_path)
self.store.delete_file(file_path)
# Remove empty docs directory if needed
docs_dir.mkdir(self.docs_dir, exist_ok=True)
os.makedirs(doc_path, parents=True, exist_ok=True)
# Generate markdown for each symbol
for symbol in symbols:
markdown = self._generate_markdown(symbol, source_code)
doc_path = self._docs_dir / docs_path
doc_content = f"# {symbol.name}\n\n{markdown_content}\n\n # write to database
try: try:
self.store.save_symbol(symbol, doc_path, doc_content) source_code = file_path.read_text("utf-8")
doc_id = doc.id symbols = self._extractor.extract_symbols(source_code, file_path.suffix, str(file_path))
logger.debug(f"Generated documentation for symbol: {symbol.name}")
total_generated += 1
total_symbols += 1
total_changed_files.append(file_path)
else:
logger.debug(f"Skipped {len(unchanged_files)} unchanged symbols")
# Clean up removed symbols if not symbols:
for file_path in removed_files: logger.debug(f"No symbols found in {file_path}")
for doc in docs: continue
self.store.delete_symbols_for_file(file_path)
# Delete the doc files for removed files
self._cleanup_removed_docs()
for doc in docs
doc_path.unlink(missing=True)
logger.debug(f"Found {len(symbols)} symbols in {file_path}")
stats["total_symbols"] += len(symbols)
docs_generated_count = 0
for symbol in symbols:
# Generate documentation
doc_content = self._generate_markdown_for_symbol(symbol, source_code)
# Define doc path
relative_path = file_path.relative_to(source_root)
doc_path = (self._docs_dir / relative_path).with_suffix(".md")
doc_path.parent.mkdir(parents=True, exist_ok=True)
# Save symbol and doc
self.store.save_symbol(symbol, str(doc_path), doc_content)
docs_generated_count += 1
stats["total_docs_generated"] += docs_generated_count
stats["total_changed_symbols"] += len(symbols)
# Update file stats in DB
content_hash = self._calculate_file_hash(file_path)
self.store.update_file_stats(str(file_path), len(symbols), content_hash)
logger.debug(f"Generated docs for {len(symbols)} symbols in {file_path}")
except Exception as e:
logger.error(f"Error processing file {file_path}: {e}")
raise StorageError(f"Failed to process {file_path}") from e
stats["total_changed_files"] = changed_files_count
stats["total_unchanged_files"] = unchanged_files_count
logger.info(f"Generation complete. Stats: {stats}")
return stats return stats
return total_symbols, total_changed_files, total_changed_symbols, total_docs_generated, total_unchanged_files, len(unchanged_files) def close(self):
"""Close the store connection."""
}
def _cleanup_removed_docs(self) -> None:
for doc in docs:
doc_path.unlink(missing=True)
try:
os.remove(doc_path)
except OSError:
pass
else:
logger.warning(f"Error removing doc file: {doc_path}: {e}")
continue
self.close()
logger.info(
f"DeepWiki generation complete - {len(files)} files, {len(symbols)} symbols"
)
self.store.close() self.store.close()
return {
"total_files": total_files,
"total_symbols": total_symbols,
"total_changed_files": total_changed_files,
"total_changed_symbols": total_changed_symbols,
"total_docs_generated": total_docs_generated,
"total_unchanged_files": total_unchanged_files,
}

21
codex-lens/src/codexlens/tools/deepwiki_generator.py
View File

@@ -40,12 +40,13 @@ class MockMarkdownGenerator:
def generate(self, symbol: DeepWikiSymbol, source_code: str) -> str: def generate(self, symbol: DeepWikiSymbol, source_code: str) -> str:
"""Generate mock Markdown documentation.""" """Generate mock Markdown documentation."""
return f"""{SYMBOL_START_TEMPLATE.format(name=symbol.name, type=symbol.symbol_type)} start_line, end_line = symbol.line_range
return f"""{SYMBOL_START_TEMPLATE.format(name=symbol.name, type=symbol.type)}
## `{symbol.name}` ## `{symbol.name}`
**Type**: {symbol.symbol_type} **Type**: {symbol.type}
**Location**: `{symbol.source_file}:{symbol.line_start}-{symbol.line_end}` **Location**: `{symbol.source_file}:{start_line}-{end_line}`
```{symbol.source_file.split('.')[-1] if '.' in symbol.source_file else 'text'} ```{symbol.source_file.split('.')[-1] if '.' in symbol.source_file else 'text'}
{source_code} {source_code}
@@ -190,12 +191,11 @@ class DeepWikiGenerator:
# Create symbol record # Create symbol record
symbol = DeepWikiSymbol( symbol = DeepWikiSymbol(
name=sym["name"], name=sym["name"],
symbol_type=sym["type"], type=sym["type"],
source_file=str(file_path), source_file=str(file_path),
doc_file=f".deepwiki/{file_path.stem}.md", doc_file=f".deepwiki/{file_path.stem}.md",
anchor=f"#{sym['name'].lower()}", anchor=f"#{sym['name'].lower()}",
line_start=sym["line_start"], line_range=(sym["line_start"], sym["line_end"]),
line_end=sym["line_end"],
) )
# Generate markdown # Generate markdown
@@ -205,8 +205,13 @@ class DeepWikiGenerator:
self.store.add_symbol(symbol) self.store.add_symbol(symbol)
docs_generated += 1 docs_generated += 1
# Update file hash # Track file hash + metadata for incremental updates and staleness checks.
self.store.update_file_hash(str(file_path), current_hash) self.store.add_file(
file_path=str(file_path),
content_hash=current_hash,
symbols_count=len(raw_symbols),
docs_generated=docs_generated > 0,
)
logger.info(f"Generated docs for {docs_generated} symbols in {file_path}") logger.info(f"Generated docs for {docs_generated} symbols in {file_path}")
return { return {