Add comprehensive tests for tokenizer, performance benchmarks, and TreeSitter parser functionality

- Implemented unit tests for the Tokenizer class, covering various text inputs, edge cases, and fallback mechanisms.
- Created performance benchmarks comparing tiktoken and pure Python implementations for token counting.
- Developed extensive tests for TreeSitterSymbolParser across Python, JavaScript, and TypeScript, ensuring accurate symbol extraction and parsing.
- Added configuration documentation for MCP integration and custom prompts, enhancing usability and flexibility.
- Introduced a refactor script for GraphAnalyzer to streamline future improvements.
This commit is contained in:
catlog22
2025-12-15 14:36:09 +08:00
parent 82dcafff00
commit 0fe16963cd
49 changed files with 9307 additions and 438 deletions

View File

@@ -1,5 +1,6 @@
import { existsSync, mkdirSync, readFileSync, writeFileSync, statSync } from 'fs';
import { join, dirname } from 'path';
import { StoragePaths, ensureStorageDir } from '../config/storage-paths.js';
interface CacheEntry<T> {
data: T;
@@ -265,6 +266,16 @@ export class CacheManager<T> {
}
}
/**
* Extract project path from workflow directory
* @param workflowDir - Path to .workflow directory (e.g., /project/.workflow)
* @returns Project root path
*/
function extractProjectPath(workflowDir: string): string {
// workflowDir is typically {projectPath}/.workflow
return workflowDir.replace(/[\/\\]\.workflow$/, '') || workflowDir;
}
/**
* Create a cache manager for dashboard data
* @param workflowDir - Path to .workflow directory
@@ -272,6 +283,9 @@ export class CacheManager<T> {
* @returns CacheManager instance
*/
export function createDashboardCache(workflowDir: string, ttl?: number): CacheManager<any> {
const cacheDir = join(workflowDir, '.ccw-cache');
// Use centralized storage path
const projectPath = extractProjectPath(workflowDir);
const cacheDir = StoragePaths.project(projectPath).cache;
ensureStorageDir(cacheDir);
return new CacheManager('dashboard-data', { cacheDir, ttl });
}