mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-03-12 17:21:19 +08:00
feat: Implement DeepWiki documentation generation tools
- Added `__init__.py` in `codexlens/tools` for documentation generation. - Created `deepwiki_generator.py` to handle symbol extraction and markdown generation. - Introduced `MockMarkdownGenerator` for testing purposes. - Implemented `DeepWikiGenerator` class for managing documentation generation and file processing. - Added unit tests for `DeepWikiStore` to ensure proper functionality and error handling. - Created tests for DeepWiki TypeScript types matching.
This commit is contained in:
catlog22
143
ccw/src/core/routes/deepwiki-routes.ts
Normal file
143
ccw/src/core/routes/deepwiki-routes.ts
Normal file
@@ -0,0 +1,143 @@
|
||||
/**
|
||||
* DeepWiki Routes Module
|
||||
* Handles all DeepWiki documentation API endpoints.
|
||||
*
|
||||
* Endpoints:
|
||||
* - GET /api/deepwiki/files - List all documented files
|
||||
* - GET /api/deepwiki/doc?path=<filePath> - Get document with symbols
|
||||
* - GET /api/deepwiki/stats - Get storage statistics
|
||||
* - GET /api/deepwiki/search?q=<query> - Search symbols
|
||||
*/
|
||||
|
||||
import type { RouteContext } from './types.js';
|
||||
import { getDeepWikiService } from '../../services/deepwiki-service.js';
|
||||
|
||||
/**
|
||||
* Handle DeepWiki routes
|
||||
* @returns true if route was handled, false otherwise
|
||||
*/
|
||||
export async function handleDeepWikiRoutes(ctx: RouteContext): Promise<boolean> {
|
||||
const { pathname, url, res } = ctx;
|
||||
|
||||
// GET /api/deepwiki/files - List all documented files
|
||||
if (pathname === '/api/deepwiki/files') {
|
||||
try {
|
||||
const service = getDeepWikiService();
|
||||
|
||||
// Return empty array if database not available (not an error)
|
||||
if (!service.isAvailable()) {
|
||||
console.log('[DeepWiki] Database not available, returning empty files list');
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify([]));
|
||||
return true;
|
||||
}
|
||||
|
||||
const files = service.listDocumentedFiles();
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(files));
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error('[DeepWiki] Error listing files:', message);
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: message }));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// GET /api/deepwiki/doc?path=<filePath> - Get document with symbols
|
||||
if (pathname === '/api/deepwiki/doc') {
|
||||
const filePath = url.searchParams.get('path');
|
||||
|
||||
if (!filePath) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'path parameter is required' }));
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
const service = getDeepWikiService();
|
||||
|
||||
// Return 404 if database not available
|
||||
if (!service.isAvailable()) {
|
||||
console.log('[DeepWiki] Database not available');
|
||||
res.writeHead(404, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'DeepWiki database not available' }));
|
||||
return true;
|
||||
}
|
||||
|
||||
const doc = service.getDocumentByPath(filePath);
|
||||
|
||||
if (!doc) {
|
||||
res.writeHead(404, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Document not found', path: filePath }));
|
||||
return true;
|
||||
}
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(doc));
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error('[DeepWiki] Error getting document:', message);
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: message }));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// GET /api/deepwiki/stats - Get storage statistics
|
||||
if (pathname === '/api/deepwiki/stats') {
|
||||
try {
|
||||
const service = getDeepWikiService();
|
||||
|
||||
if (!service.isAvailable()) {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ available: false, files: 0, symbols: 0, docs: 0 }));
|
||||
return true;
|
||||
}
|
||||
|
||||
const stats = service.getStats();
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ available: true, ...stats }));
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error('[DeepWiki] Error getting stats:', message);
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: message }));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// GET /api/deepwiki/search?q=<query> - Search symbols
|
||||
if (pathname === '/api/deepwiki/search') {
|
||||
const query = url.searchParams.get('q');
|
||||
const limit = parseInt(url.searchParams.get('limit') || '50', 10);
|
||||
|
||||
if (!query) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'q parameter is required' }));
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
const service = getDeepWikiService();
|
||||
|
||||
if (!service.isAvailable()) {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify([]));
|
||||
return true;
|
||||
}
|
||||
|
||||
const symbols = service.searchSymbols(query, limit);
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(symbols));
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error('[DeepWiki] Error searching symbols:', message);
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: message }));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
@@ -46,6 +46,7 @@ import { handleTeamRoutes } from './routes/team-routes.js';
|
||||
import { handleNotificationRoutes } from './routes/notification-routes.js';
|
||||
import { handleAnalysisRoutes } from './routes/analysis-routes.js';
|
||||
import { handleSpecRoutes } from './routes/spec-routes.js';
|
||||
import { handleDeepWikiRoutes } from './routes/deepwiki-routes.js';
|
||||
|
||||
// Import WebSocket handling
|
||||
import { handleWebSocketUpgrade, broadcastToClients, extractSessionIdFromPath } from './websocket.js';
|
||||
|
||||
266
ccw/src/services/deepwiki-service.ts
Normal file
266
ccw/src/services/deepwiki-service.ts
Normal file
@@ -0,0 +1,266 @@
|
||||
/**
|
||||
* DeepWiki Service
|
||||
* Read-only SQLite service for DeepWiki documentation index.
|
||||
*
|
||||
* Connects to codex-lens database at ~/.codexlens/deepwiki_index.db
|
||||
*/
|
||||
|
||||
import { homedir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
import Database from 'better-sqlite3';
|
||||
|
||||
// Default database path (same as Python DeepWikiStore)
|
||||
const DEFAULT_DB_PATH = join(homedir(), '.codexlens', 'deepwiki_index.db');
|
||||
|
||||
/**
|
||||
* Symbol information from deepwiki_symbols table
|
||||
*/
|
||||
export interface DeepWikiSymbol {
|
||||
id: number;
|
||||
name: string;
|
||||
type: string;
|
||||
source_file: string;
|
||||
doc_file: string;
|
||||
anchor: string;
|
||||
start_line: number;
|
||||
end_line: number;
|
||||
created_at: number | null;
|
||||
updated_at: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Document information from deepwiki_docs table
|
||||
*/
|
||||
export interface DeepWikiDoc {
|
||||
id: number;
|
||||
path: string;
|
||||
content_hash: string;
|
||||
symbols: string[];
|
||||
generated_at: number;
|
||||
llm_tool: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* File information from deepwiki_files table
|
||||
*/
|
||||
export interface DeepWikiFile {
|
||||
id: number;
|
||||
path: string;
|
||||
content_hash: string;
|
||||
last_indexed: number;
|
||||
symbols_count: number;
|
||||
docs_generated: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Document with symbols for API response
|
||||
*/
|
||||
export interface DocumentWithSymbols {
|
||||
path: string;
|
||||
symbols: Array<{
|
||||
name: string;
|
||||
type: string;
|
||||
anchor: string;
|
||||
start_line: number;
|
||||
end_line: number;
|
||||
}>;
|
||||
generated_at: string | null;
|
||||
llm_tool: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* DeepWiki Service - Read-only SQLite access
|
||||
*/
|
||||
export class DeepWikiService {
|
||||
private dbPath: string;
|
||||
private db: Database.Database | null = null;
|
||||
|
||||
constructor(dbPath: string = DEFAULT_DB_PATH) {
|
||||
this.dbPath = dbPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create database connection
|
||||
*/
|
||||
private getConnection(): Database.Database | null {
|
||||
if (this.db) {
|
||||
return this.db;
|
||||
}
|
||||
|
||||
// Check if database exists
|
||||
if (!existsSync(this.dbPath)) {
|
||||
console.log(`[DeepWiki] Database not found at ${this.dbPath}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
// Open in read-only mode
|
||||
this.db = new Database(this.dbPath, { readonly: true, fileMustExist: true });
|
||||
console.log(`[DeepWiki] Connected to database at ${this.dbPath}`);
|
||||
return this.db;
|
||||
} catch (error) {
|
||||
console.error(`[DeepWiki] Failed to connect to database:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close database connection
|
||||
*/
|
||||
public close(): void {
|
||||
if (this.db) {
|
||||
this.db.close();
|
||||
this.db = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if database is available
|
||||
*/
|
||||
public isAvailable(): boolean {
|
||||
return existsSync(this.dbPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* List all documented files (source files with symbols)
|
||||
* @returns Array of file paths that have documentation
|
||||
*/
|
||||
public listDocumentedFiles(): string[] {
|
||||
const db = this.getConnection();
|
||||
if (!db) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
// Get distinct source files that have symbols documented
|
||||
const rows = db.prepare(`
|
||||
SELECT DISTINCT source_file
|
||||
FROM deepwiki_symbols
|
||||
ORDER BY source_file
|
||||
`).all() as Array<{ source_file: string }>;
|
||||
|
||||
return rows.map(row => row.source_file);
|
||||
} catch (error) {
|
||||
console.error('[DeepWiki] Error listing documented files:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get document information by source file path
|
||||
* @param filePath - Source file path
|
||||
* @returns Document with symbols or null if not found
|
||||
*/
|
||||
public getDocumentByPath(filePath: string): DocumentWithSymbols | null {
|
||||
const db = this.getConnection();
|
||||
if (!db) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
// Normalize path (forward slashes)
|
||||
const normalizedPath = filePath.replace(/\\/g, '/');
|
||||
|
||||
// Get symbols for this source file
|
||||
const symbols = db.prepare(`
|
||||
SELECT name, type, anchor, start_line, end_line
|
||||
FROM deepwiki_symbols
|
||||
WHERE source_file = ?
|
||||
ORDER BY start_line
|
||||
`).all(normalizedPath) as Array<{
|
||||
name: string;
|
||||
type: string;
|
||||
anchor: string;
|
||||
start_line: number;
|
||||
end_line: number;
|
||||
}>;
|
||||
|
||||
if (symbols.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Get the doc file path (from first symbol)
|
||||
const docFile = db.prepare(`
|
||||
SELECT doc_file FROM deepwiki_symbols WHERE source_file = ? LIMIT 1
|
||||
`).get(normalizedPath) as { doc_file: string } | undefined;
|
||||
|
||||
// Get document metadata if available
|
||||
let generatedAt: string | null = null;
|
||||
let llmTool: string | null = null;
|
||||
|
||||
if (docFile) {
|
||||
const doc = db.prepare(`
|
||||
SELECT generated_at, llm_tool
|
||||
FROM deepwiki_docs
|
||||
WHERE path = ?
|
||||
`).get(docFile.doc_file) as { generated_at: number; llm_tool: string | null } | undefined;
|
||||
|
||||
if (doc) {
|
||||
generatedAt = doc.generated_at ? new Date(doc.generated_at * 1000).toISOString() : null;
|
||||
llmTool = doc.llm_tool;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
path: normalizedPath,
|
||||
symbols: symbols.map(s => ({
|
||||
name: s.name,
|
||||
type: s.type,
|
||||
anchor: s.anchor,
|
||||
start_line: s.start_line,
|
||||
end_line: s.end_line
|
||||
})),
|
||||
generated_at: generatedAt,
|
||||
llm_tool: llmTool
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('[DeepWiki] Error getting document by path:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Search symbols by name pattern
|
||||
* @param query - Search query (supports LIKE pattern)
|
||||
* @param limit - Maximum results
|
||||
* @returns Array of matching symbols
|
||||
*/
|
||||
public searchSymbols(query: string, limit: number = 50): DeepWikiSymbol[] {
|
||||
const db = this.getConnection();
|
||||
if (!db) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const pattern = `%${query}%`;
|
||||
const rows = db.prepare(`
|
||||
SELECT id, name, type, source_file, doc_file, anchor, start_line, end_line, created_at, updated_at
|
||||
FROM deepwiki_symbols
|
||||
WHERE name LIKE ?
|
||||
ORDER BY name
|
||||
LIMIT ?
|
||||
`).all(pattern, limit) as DeepWikiSymbol[];
|
||||
|
||||
return rows;
|
||||
} catch (error) {
|
||||
console.error('[DeepWiki] Error searching symbols:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Singleton instance
|
||||
let deepWikiService: DeepWikiService | null = null;
|
||||
|
||||
/**
|
||||
* Get the singleton DeepWiki service instance
|
||||
*/
|
||||
export function getDeepWikiService(): DeepWikiService {
|
||||
if (!deepWikiService) {
|
||||
deepWikiService = new DeepWikiService();
|
||||
}
|
||||
return deepWikiService;
|
||||
}
|
||||
103
ccw/src/types/deepwiki.ts
Normal file
103
ccw/src/types/deepwiki.ts
Normal file
@@ -0,0 +1,103 @@
|
||||
/**
|
||||
* DeepWiki Type Definitions
|
||||
*
|
||||
* Types for DeepWiki documentation index storage.
|
||||
* These types mirror the Python Pydantic models in codex-lens.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A symbol record in the DeepWiki index.
|
||||
* Maps a code symbol to its generated documentation file and anchor.
|
||||
*/
|
||||
export interface DeepWikiSymbol {
|
||||
/** Database row ID */
|
||||
id?: number;
|
||||
/** Symbol name (function, class, etc.) */
|
||||
name: string;
|
||||
/** Symbol type (function, class, method, variable) */
|
||||
type: string;
|
||||
/** Path to source file containing the symbol */
|
||||
sourceFile: string;
|
||||
/** Path to generated documentation file */
|
||||
docFile: string;
|
||||
/** HTML anchor ID for linking to specific section */
|
||||
anchor: string;
|
||||
/** (start_line, end_line) in source file, 1-based inclusive */
|
||||
lineRange: [number, number];
|
||||
/** Record creation timestamp */
|
||||
createdAt?: string;
|
||||
/** Record update timestamp */
|
||||
updatedAt?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A documentation file record in the DeepWiki index.
|
||||
* Tracks generated documentation files and their associated symbols.
|
||||
*/
|
||||
export interface DeepWikiDoc {
|
||||
/** Database row ID */
|
||||
id?: number;
|
||||
/** Path to documentation file */
|
||||
path: string;
|
||||
/** SHA256 hash of file content for change detection */
|
||||
contentHash: string;
|
||||
/** List of symbol names documented in this file */
|
||||
symbols: string[];
|
||||
/** Timestamp when documentation was generated (ISO string) */
|
||||
generatedAt: string;
|
||||
/** LLM tool used to generate documentation (gemini/qwen) */
|
||||
llmTool?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A source file record in the DeepWiki index.
|
||||
* Tracks indexed source files and their content hashes for incremental updates.
|
||||
*/
|
||||
export interface DeepWikiFile {
|
||||
/** Database row ID */
|
||||
id?: number;
|
||||
/** Path to source file */
|
||||
path: string;
|
||||
/** SHA256 hash of file content */
|
||||
contentHash: string;
|
||||
/** Timestamp when file was last indexed (ISO string) */
|
||||
lastIndexed: string;
|
||||
/** Number of symbols indexed from this file */
|
||||
symbolsCount: number;
|
||||
/** Whether documentation has been generated */
|
||||
docsGenerated: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage statistics for DeepWiki index.
|
||||
*/
|
||||
export interface DeepWikiStats {
|
||||
/** Total number of tracked source files */
|
||||
files: number;
|
||||
/** Total number of indexed symbols */
|
||||
symbols: number;
|
||||
/** Total number of documentation files */
|
||||
docs: number;
|
||||
/** Files that need documentation generated */
|
||||
filesNeedingDocs: number;
|
||||
/** Path to the database file */
|
||||
dbPath: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for listing files in DeepWiki index.
|
||||
*/
|
||||
export interface ListFilesOptions {
|
||||
/** Only return files that need documentation generated */
|
||||
needsDocs?: boolean;
|
||||
/** Maximum number of files to return */
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for searching symbols.
|
||||
*/
|
||||
export interface SearchSymbolsOptions {
|
||||
/** Maximum number of results to return */
|
||||
limit?: number;
|
||||
}
|
||||
Reference in New Issue
Block a user