mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-02-28 09:23:08 +08:00
Add comprehensive tests for ast-grep and tree-sitter relationship extraction
- Introduced test suite for AstGrepPythonProcessor covering pattern definitions, parsing, and relationship extraction. - Added comparison tests between tree-sitter and ast-grep for consistency in relationship extraction. - Implemented tests for ast-grep binding module to verify functionality and availability. - Ensured tests cover various scenarios including inheritance, function calls, and imports.
This commit is contained in:
@@ -263,6 +263,10 @@ export function run(argv: string[]): void {
|
||||
.option('--output <file>', 'Output file path for export')
|
||||
.option('--overwrite', 'Overwrite existing memories when importing')
|
||||
.option('--prefix <prefix>', 'Add prefix to imported memory IDs')
|
||||
.option('--unified', 'Use unified vector+FTS search (for search subcommand)')
|
||||
.option('--topK <n>', 'Max results for unified search', '20')
|
||||
.option('--minScore <n>', 'Min relevance score for unified search', '0')
|
||||
.option('--category <cat>', 'Filter by category for unified search')
|
||||
.action((subcommand, args, options) => coreMemoryCommand(subcommand, args, options));
|
||||
|
||||
// Hook command - CLI endpoint for Claude Code hooks
|
||||
|
||||
@@ -35,6 +35,10 @@ interface CommandOptions {
|
||||
delete?: boolean;
|
||||
merge?: string;
|
||||
dedup?: boolean;
|
||||
unified?: boolean;
|
||||
topK?: string;
|
||||
minScore?: string;
|
||||
category?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -844,6 +848,114 @@ async function jobsAction(options: CommandOptions): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unified vector+FTS search across all memory stores
|
||||
*/
|
||||
async function unifiedSearchAction(keyword: string, options: CommandOptions): Promise<void> {
|
||||
if (!keyword || keyword.trim() === '') {
|
||||
console.error(chalk.red('Error: Query is required'));
|
||||
console.error(chalk.gray('Usage: ccw core-memory search --unified <query> [--topK 20] [--minScore 0] [--category <cat>]'));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
const { UnifiedMemoryService } = await import('../core/unified-memory-service.js');
|
||||
const service = new UnifiedMemoryService(getProjectPath());
|
||||
|
||||
const topK = parseInt(options.topK || '20', 10);
|
||||
const minScore = parseFloat(options.minScore || '0');
|
||||
const category = options.category || undefined;
|
||||
|
||||
console.log(chalk.cyan(`\n Unified search: "${keyword}" (topK=${topK}, minScore=${minScore})\n`));
|
||||
|
||||
const results = await service.search(keyword, {
|
||||
limit: topK,
|
||||
minScore,
|
||||
category: category as any,
|
||||
});
|
||||
|
||||
if (results.length === 0) {
|
||||
console.log(chalk.yellow(' No results found.\n'));
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.json) {
|
||||
console.log(JSON.stringify({ query: keyword, total: results.length, results }, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(chalk.gray(' -----------------------------------------------------------------------'));
|
||||
|
||||
for (const result of results) {
|
||||
const sources: string[] = [];
|
||||
if (result.rank_sources.vector_rank) sources.push(`vec:#${result.rank_sources.vector_rank}`);
|
||||
if (result.rank_sources.fts_rank) sources.push(`fts:#${result.rank_sources.fts_rank}`);
|
||||
if (result.rank_sources.heat_score) sources.push(`heat:${result.rank_sources.heat_score.toFixed(1)}`);
|
||||
|
||||
const snippet = result.content.substring(0, 120).replace(/\n/g, ' ');
|
||||
|
||||
console.log(
|
||||
chalk.cyan(` ${result.source_id}`) +
|
||||
chalk.gray(` [${result.source_type}/${result.category}]`) +
|
||||
chalk.white(` score=${result.score.toFixed(4)}`)
|
||||
);
|
||||
console.log(chalk.gray(` Sources: ${sources.join(' | ')}`));
|
||||
console.log(chalk.white(` ${snippet}${result.content.length > 120 ? '...' : ''}`));
|
||||
console.log(chalk.gray(' -----------------------------------------------------------------------'));
|
||||
}
|
||||
|
||||
console.log(chalk.gray(`\n Total: ${results.length}\n`));
|
||||
|
||||
} catch (error) {
|
||||
console.error(chalk.red(`Error: ${(error as Error).message}`));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild the unified HNSW vector index from scratch
|
||||
*/
|
||||
async function reindexAction(options: CommandOptions): Promise<void> {
|
||||
try {
|
||||
const { UnifiedVectorIndex, isUnifiedEmbedderAvailable } = await import('../core/unified-vector-index.js');
|
||||
|
||||
if (!isUnifiedEmbedderAvailable()) {
|
||||
console.error(chalk.red('Error: Unified embedder is not available.'));
|
||||
console.error(chalk.gray('Ensure Python venv and embedder script are set up.'));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const index = new UnifiedVectorIndex(getProjectPath());
|
||||
|
||||
console.log(chalk.cyan('\n Rebuilding unified vector index...\n'));
|
||||
|
||||
const result = await index.reindexAll();
|
||||
|
||||
if (!result.success) {
|
||||
console.error(chalk.red(` Reindex failed: ${result.error}\n`));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (options.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(chalk.green(' Reindex complete.'));
|
||||
if (result.hnsw_count !== undefined) {
|
||||
console.log(chalk.white(` HNSW vectors: ${result.hnsw_count}`));
|
||||
}
|
||||
if (result.elapsed_time !== undefined) {
|
||||
console.log(chalk.white(` Elapsed: ${result.elapsed_time.toFixed(2)}s`));
|
||||
}
|
||||
console.log();
|
||||
|
||||
} catch (error) {
|
||||
console.error(chalk.red(`Error: ${(error as Error).message}`));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Core Memory command entry point
|
||||
*/
|
||||
@@ -889,7 +1001,11 @@ export async function coreMemoryCommand(
|
||||
break;
|
||||
|
||||
case 'search':
|
||||
await searchAction(textArg, options);
|
||||
if (options.unified) {
|
||||
await unifiedSearchAction(textArg, options);
|
||||
} else {
|
||||
await searchAction(textArg, options);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'projects':
|
||||
@@ -921,6 +1037,10 @@ export async function coreMemoryCommand(
|
||||
await jobsAction(options);
|
||||
break;
|
||||
|
||||
case 'reindex':
|
||||
await reindexAction(options);
|
||||
break;
|
||||
|
||||
default:
|
||||
console.log(chalk.bold.cyan('\n CCW Core Memory\n'));
|
||||
console.log(' Manage core memory entries and session clusters.\n');
|
||||
@@ -945,12 +1065,14 @@ export async function coreMemoryCommand(
|
||||
console.log(chalk.white(' context ') + chalk.gray('Get progressive index'));
|
||||
console.log(chalk.white(' load-cluster <id> ') + chalk.gray('Load cluster context'));
|
||||
console.log(chalk.white(' search <keyword> ') + chalk.gray('Search sessions'));
|
||||
console.log(chalk.white(' search --unified <query> ') + chalk.gray('Unified vector+FTS search'));
|
||||
console.log();
|
||||
console.log(chalk.bold(' Memory V2 Pipeline:'));
|
||||
console.log(chalk.white(' extract ') + chalk.gray('Run batch memory extraction'));
|
||||
console.log(chalk.white(' extract-status ') + chalk.gray('Show extraction pipeline status'));
|
||||
console.log(chalk.white(' consolidate ') + chalk.gray('Run memory consolidation'));
|
||||
console.log(chalk.white(' jobs ') + chalk.gray('List all pipeline jobs'));
|
||||
console.log(chalk.white(' reindex ') + chalk.gray('Rebuild unified vector index'));
|
||||
console.log();
|
||||
console.log(chalk.bold(' Options:'));
|
||||
console.log(chalk.gray(' --id <id> Memory ID (for export/summary)'));
|
||||
|
||||
@@ -12,7 +12,7 @@ interface HookOptions {
|
||||
stdin?: boolean;
|
||||
sessionId?: string;
|
||||
prompt?: string;
|
||||
type?: 'session-start' | 'context';
|
||||
type?: 'session-start' | 'context' | 'session-end';
|
||||
path?: string;
|
||||
}
|
||||
|
||||
@@ -95,10 +95,32 @@ function getProjectPath(hookCwd?: string): string {
|
||||
return hookCwd || process.cwd();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if UnifiedContextBuilder is available (embedder dependencies present).
|
||||
* Returns the builder instance or null if not available.
|
||||
*/
|
||||
async function tryCreateContextBuilder(projectPath: string): Promise<any | null> {
|
||||
try {
|
||||
const { isUnifiedEmbedderAvailable } = await import('../core/unified-vector-index.js');
|
||||
if (!isUnifiedEmbedderAvailable()) {
|
||||
return null;
|
||||
}
|
||||
const { UnifiedContextBuilder } = await import('../core/unified-context-builder.js');
|
||||
return new UnifiedContextBuilder(projectPath);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Session context action - provides progressive context loading
|
||||
* First prompt: returns session overview with clusters
|
||||
* Subsequent prompts: returns intent-matched sessions
|
||||
*
|
||||
* Uses UnifiedContextBuilder when available (embedder present):
|
||||
* - session-start: MEMORY.md summary + clusters + hot entities + patterns
|
||||
* - per-prompt: vector search across all memory categories
|
||||
*
|
||||
* Falls back to SessionClusteringService.getProgressiveIndex() when
|
||||
* the embedder is unavailable, preserving backward compatibility.
|
||||
*/
|
||||
async function sessionContextAction(options: HookOptions): Promise<void> {
|
||||
let { stdin, sessionId, prompt } = options;
|
||||
@@ -154,29 +176,43 @@ async function sessionContextAction(options: HookOptions): Promise<void> {
|
||||
let contextType: 'session-start' | 'context';
|
||||
let content = '';
|
||||
|
||||
// Dynamic import to avoid circular dependencies
|
||||
const { SessionClusteringService } = await import('../core/session-clustering-service.js');
|
||||
const clusteringService = new SessionClusteringService(projectPath);
|
||||
// Try UnifiedContextBuilder first; fall back to getProgressiveIndex
|
||||
const contextBuilder = await tryCreateContextBuilder(projectPath);
|
||||
|
||||
if (isFirstPrompt) {
|
||||
// First prompt: return session overview with clusters
|
||||
contextType = 'session-start';
|
||||
content = await clusteringService.getProgressiveIndex({
|
||||
type: 'session-start',
|
||||
sessionId
|
||||
});
|
||||
} else if (prompt && prompt.trim().length > 0) {
|
||||
// Subsequent prompts with content: return intent-matched sessions
|
||||
contextType = 'context';
|
||||
content = await clusteringService.getProgressiveIndex({
|
||||
type: 'context',
|
||||
sessionId,
|
||||
prompt
|
||||
});
|
||||
if (contextBuilder) {
|
||||
// Use UnifiedContextBuilder
|
||||
if (isFirstPrompt) {
|
||||
contextType = 'session-start';
|
||||
content = await contextBuilder.buildSessionStartContext();
|
||||
} else if (prompt && prompt.trim().length > 0) {
|
||||
contextType = 'context';
|
||||
content = await contextBuilder.buildPromptContext(prompt);
|
||||
} else {
|
||||
contextType = 'context';
|
||||
content = '';
|
||||
}
|
||||
} else {
|
||||
// Subsequent prompts without content: return minimal context
|
||||
contextType = 'context';
|
||||
content = ''; // No context needed for empty prompts
|
||||
// Fallback: use legacy SessionClusteringService.getProgressiveIndex()
|
||||
const { SessionClusteringService } = await import('../core/session-clustering-service.js');
|
||||
const clusteringService = new SessionClusteringService(projectPath);
|
||||
|
||||
if (isFirstPrompt) {
|
||||
contextType = 'session-start';
|
||||
content = await clusteringService.getProgressiveIndex({
|
||||
type: 'session-start',
|
||||
sessionId
|
||||
});
|
||||
} else if (prompt && prompt.trim().length > 0) {
|
||||
contextType = 'context';
|
||||
content = await clusteringService.getProgressiveIndex({
|
||||
type: 'context',
|
||||
sessionId,
|
||||
prompt
|
||||
});
|
||||
} else {
|
||||
contextType = 'context';
|
||||
content = '';
|
||||
}
|
||||
}
|
||||
|
||||
if (stdin) {
|
||||
@@ -194,6 +230,7 @@ async function sessionContextAction(options: HookOptions): Promise<void> {
|
||||
console.log(chalk.cyan('Type:'), contextType);
|
||||
console.log(chalk.cyan('First Prompt:'), isFirstPrompt ? 'Yes' : 'No');
|
||||
console.log(chalk.cyan('Load Count:'), newState.loadCount);
|
||||
console.log(chalk.cyan('Builder:'), contextBuilder ? 'UnifiedContextBuilder' : 'Legacy (getProgressiveIndex)');
|
||||
console.log(chalk.gray('─'.repeat(40)));
|
||||
if (content) {
|
||||
console.log(content);
|
||||
@@ -210,6 +247,81 @@ async function sessionContextAction(options: HookOptions): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Session end action - triggers async background tasks for memory maintenance.
|
||||
*
|
||||
* Tasks executed:
|
||||
* 1. Incremental vector embedding (index new/updated content)
|
||||
* 2. Incremental clustering (cluster unclustered sessions)
|
||||
* 3. Heat score updates (recalculate entity heat scores)
|
||||
*
|
||||
* All tasks run best-effort; failures are logged but do not affect exit code.
|
||||
*/
|
||||
async function sessionEndAction(options: HookOptions): Promise<void> {
|
||||
let { stdin, sessionId } = options;
|
||||
let hookCwd: string | undefined;
|
||||
|
||||
if (stdin) {
|
||||
try {
|
||||
const stdinData = await readStdin();
|
||||
if (stdinData) {
|
||||
const hookData = JSON.parse(stdinData) as HookData;
|
||||
sessionId = hookData.session_id || sessionId;
|
||||
hookCwd = hookData.cwd;
|
||||
}
|
||||
} catch {
|
||||
// Silently continue if stdin parsing fails
|
||||
}
|
||||
}
|
||||
|
||||
if (!sessionId) {
|
||||
if (!stdin) {
|
||||
console.error(chalk.red('Error: --session-id is required'));
|
||||
}
|
||||
process.exit(stdin ? 0 : 1);
|
||||
}
|
||||
|
||||
try {
|
||||
const projectPath = getProjectPath(hookCwd);
|
||||
const contextBuilder = await tryCreateContextBuilder(projectPath);
|
||||
|
||||
if (!contextBuilder) {
|
||||
// UnifiedContextBuilder not available - skip session-end tasks
|
||||
if (!stdin) {
|
||||
console.log(chalk.gray('(UnifiedContextBuilder not available, skipping session-end tasks)'));
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const tasks: Array<{ name: string; execute: () => Promise<void> }> = contextBuilder.buildSessionEndTasks(sessionId);
|
||||
|
||||
if (!stdin) {
|
||||
console.log(chalk.green(`Session End: executing ${tasks.length} background tasks...`));
|
||||
}
|
||||
|
||||
// Execute all tasks concurrently (best-effort)
|
||||
const results = await Promise.allSettled(
|
||||
tasks.map((task: { name: string; execute: () => Promise<void> }) => task.execute())
|
||||
);
|
||||
|
||||
if (!stdin) {
|
||||
for (let i = 0; i < tasks.length; i++) {
|
||||
const status = results[i].status === 'fulfilled' ? 'OK' : 'FAIL';
|
||||
const color = status === 'OK' ? chalk.green : chalk.yellow;
|
||||
console.log(color(` [${status}] ${tasks[i].name}`));
|
||||
}
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
if (stdin) {
|
||||
process.exit(0);
|
||||
}
|
||||
console.error(chalk.red(`Error: ${(error as Error).message}`));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse CCW status.json and output formatted status
|
||||
*/
|
||||
@@ -311,6 +423,7 @@ ${chalk.bold('USAGE')}
|
||||
${chalk.bold('SUBCOMMANDS')}
|
||||
parse-status Parse CCW status.json and display current/next command
|
||||
session-context Progressive session context loading (replaces curl/bash hook)
|
||||
session-end Trigger background memory maintenance tasks
|
||||
notify Send notification to ccw view dashboard
|
||||
|
||||
${chalk.bold('OPTIONS')}
|
||||
@@ -363,6 +476,9 @@ export async function hookCommand(
|
||||
case 'context':
|
||||
await sessionContextAction(options);
|
||||
break;
|
||||
case 'session-end':
|
||||
await sessionEndAction(options);
|
||||
break;
|
||||
case 'notify':
|
||||
await notifyAction(options);
|
||||
break;
|
||||
|
||||
154
ccw/src/config/remote-notification-config.ts
Normal file
154
ccw/src/config/remote-notification-config.ts
Normal file
@@ -0,0 +1,154 @@
|
||||
// ========================================
|
||||
// Remote Notification Configuration Manager
|
||||
// ========================================
|
||||
// Manages persistent storage of remote notification settings
|
||||
// Storage: ~/.ccw/config/remote-notification.json
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { getCCWHome, ensureStorageDir } from './storage-paths.js';
|
||||
import type {
|
||||
RemoteNotificationConfig,
|
||||
DEFAULT_REMOTE_NOTIFICATION_CONFIG,
|
||||
} from '../types/remote-notification.js';
|
||||
import { DeepPartial, deepMerge } from '../types/util.js';
|
||||
|
||||
/**
|
||||
* Configuration file path
|
||||
*/
|
||||
function getConfigFilePath(): string {
|
||||
return join(getCCWHome(), 'config', 'remote-notification.json');
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure configuration directory exists
|
||||
*/
|
||||
function ensureConfigDir(): void {
|
||||
const configDir = join(getCCWHome(), 'config');
|
||||
ensureStorageDir(configDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Default configuration factory
|
||||
*/
|
||||
export function getDefaultConfig(): RemoteNotificationConfig {
|
||||
return {
|
||||
enabled: false,
|
||||
platforms: {},
|
||||
events: [
|
||||
{ event: 'ask-user-question', platforms: ['discord', 'telegram'], enabled: true },
|
||||
{ event: 'session-start', platforms: [], enabled: false },
|
||||
{ event: 'session-end', platforms: [], enabled: false },
|
||||
{ event: 'task-completed', platforms: [], enabled: false },
|
||||
{ event: 'task-failed', platforms: ['discord', 'telegram'], enabled: true },
|
||||
],
|
||||
timeout: 10000,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Load remote notification configuration
|
||||
* Returns default config if file doesn't exist
|
||||
*/
|
||||
export function loadConfig(): RemoteNotificationConfig {
|
||||
const configPath = getConfigFilePath();
|
||||
|
||||
if (!existsSync(configPath)) {
|
||||
return getDefaultConfig();
|
||||
}
|
||||
|
||||
try {
|
||||
const data = readFileSync(configPath, 'utf-8');
|
||||
const parsed = JSON.parse(data);
|
||||
|
||||
// Merge with defaults to ensure all fields exist
|
||||
return deepMerge(getDefaultConfig(), parsed);
|
||||
} catch (error) {
|
||||
console.error('[RemoteNotificationConfig] Failed to load config:', error);
|
||||
return getDefaultConfig();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save remote notification configuration
|
||||
*/
|
||||
export function saveConfig(config: RemoteNotificationConfig): void {
|
||||
ensureConfigDir();
|
||||
const configPath = getConfigFilePath();
|
||||
|
||||
try {
|
||||
writeFileSync(configPath, JSON.stringify(config, null, 2), 'utf-8');
|
||||
} catch (error) {
|
||||
console.error('[RemoteNotificationConfig] Failed to save config:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update configuration with partial changes
|
||||
*/
|
||||
export function updateConfig(
|
||||
updates: DeepPartial<RemoteNotificationConfig>
|
||||
): RemoteNotificationConfig {
|
||||
const current = loadConfig();
|
||||
const updated = deepMerge(current, updates);
|
||||
saveConfig(updated);
|
||||
return updated;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset configuration to defaults
|
||||
*/
|
||||
export function resetConfig(): RemoteNotificationConfig {
|
||||
const defaultConfig = getDefaultConfig();
|
||||
saveConfig(defaultConfig);
|
||||
return defaultConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if any platform is configured and enabled
|
||||
*/
|
||||
export function hasEnabledPlatform(config: RemoteNotificationConfig): boolean {
|
||||
if (!config.enabled) return false;
|
||||
|
||||
const { discord, telegram, webhook } = config.platforms;
|
||||
|
||||
return (
|
||||
(discord?.enabled && !!discord.webhookUrl) ||
|
||||
(telegram?.enabled && !!telegram.botToken && !!telegram.chatId) ||
|
||||
(webhook?.enabled && !!webhook.url)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get enabled platforms for a specific event
|
||||
*/
|
||||
export function getEnabledPlatformsForEvent(
|
||||
config: RemoteNotificationConfig,
|
||||
eventType: string
|
||||
): string[] {
|
||||
if (!config.enabled) return [];
|
||||
|
||||
const eventConfig = config.events.find((e) => e.event === eventType);
|
||||
if (!eventConfig || !eventConfig.enabled) return [];
|
||||
|
||||
return eventConfig.platforms.filter((platform) => {
|
||||
const platformConfig = config.platforms[platform as keyof typeof config.platforms];
|
||||
if (!platformConfig) return false;
|
||||
|
||||
switch (platform) {
|
||||
case 'discord':
|
||||
return (platformConfig as { enabled: boolean; webhookUrl?: string }).enabled &&
|
||||
!!(platformConfig as { webhookUrl?: string }).webhookUrl;
|
||||
case 'telegram':
|
||||
return (platformConfig as { enabled: boolean; botToken?: string; chatId?: string }).enabled &&
|
||||
!!(platformConfig as { botToken?: string }).botToken &&
|
||||
!!(platformConfig as { chatId?: string }).chatId;
|
||||
case 'webhook':
|
||||
return (platformConfig as { enabled: boolean; url?: string }).enabled &&
|
||||
!!(platformConfig as { url?: string }).url;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -388,6 +388,15 @@ export interface ProjectPaths {
|
||||
/** Skills directory */
|
||||
skills: string;
|
||||
};
|
||||
/** Unified vector index paths (HNSW-backed) */
|
||||
unifiedVectors: {
|
||||
/** Root: <projectRoot>/unified-vectors/ */
|
||||
root: string;
|
||||
/** SQLite database for vector metadata */
|
||||
vectorsDb: string;
|
||||
/** HNSW index file */
|
||||
hnswIndex: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -454,6 +463,11 @@ export function getProjectPaths(projectPath: string): ProjectPaths {
|
||||
memoryMd: join(projectDir, 'core-memory', 'v2', 'MEMORY.md'),
|
||||
skills: join(projectDir, 'core-memory', 'v2', 'skills'),
|
||||
},
|
||||
unifiedVectors: {
|
||||
root: join(projectDir, 'unified-vectors'),
|
||||
vectorsDb: join(projectDir, 'unified-vectors', 'vectors.db'),
|
||||
hnswIndex: join(projectDir, 'unified-vectors', 'vectors.hnsw'),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -483,6 +497,11 @@ export function getProjectPathsById(projectId: string): ProjectPaths {
|
||||
memoryMd: join(projectDir, 'core-memory', 'v2', 'MEMORY.md'),
|
||||
skills: join(projectDir, 'core-memory', 'v2', 'skills'),
|
||||
},
|
||||
unifiedVectors: {
|
||||
root: join(projectDir, 'unified-vectors'),
|
||||
vectorsDb: join(projectDir, 'unified-vectors', 'vectors.db'),
|
||||
hnswIndex: join(projectDir, 'unified-vectors', 'vectors.hnsw'),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ import Database from 'better-sqlite3';
|
||||
import { existsSync, mkdirSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { StoragePaths, ensureStorageDir } from '../config/storage-paths.js';
|
||||
import { UnifiedVectorIndex, isUnifiedEmbedderAvailable } from './unified-vector-index.js';
|
||||
import type { ChunkMetadata } from './unified-vector-index.js';
|
||||
|
||||
// Types
|
||||
export interface CoreMemory {
|
||||
@@ -101,6 +103,7 @@ export class CoreMemoryStore {
|
||||
private db: Database.Database;
|
||||
private dbPath: string;
|
||||
private projectPath: string;
|
||||
private vectorIndex: UnifiedVectorIndex | null = null;
|
||||
|
||||
constructor(projectPath: string) {
|
||||
this.projectPath = projectPath;
|
||||
@@ -328,6 +331,38 @@ export class CoreMemoryStore {
|
||||
return this.db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create the UnifiedVectorIndex instance (lazy initialization).
|
||||
* Returns null if the embedder is not available.
|
||||
*/
|
||||
private getVectorIndex(): UnifiedVectorIndex | null {
|
||||
if (this.vectorIndex) return this.vectorIndex;
|
||||
if (!isUnifiedEmbedderAvailable()) return null;
|
||||
this.vectorIndex = new UnifiedVectorIndex(this.projectPath);
|
||||
return this.vectorIndex;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fire-and-forget: sync content to the vector index.
|
||||
* Logs errors but never throws, to avoid disrupting the synchronous write path.
|
||||
*/
|
||||
private syncToVectorIndex(content: string, sourceId: string): void {
|
||||
const idx = this.getVectorIndex();
|
||||
if (!idx) return;
|
||||
|
||||
const metadata: ChunkMetadata = {
|
||||
source_id: sourceId,
|
||||
source_type: 'core_memory',
|
||||
category: 'core_memory',
|
||||
};
|
||||
|
||||
idx.indexContent(content, metadata).catch((err) => {
|
||||
if (process.env.DEBUG) {
|
||||
console.error(`[CoreMemoryStore] Vector index sync failed for ${sourceId}:`, (err as Error).message);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate timestamp-based ID for core memory
|
||||
*/
|
||||
@@ -387,6 +422,9 @@ export class CoreMemoryStore {
|
||||
id
|
||||
);
|
||||
|
||||
// Sync updated content to vector index
|
||||
this.syncToVectorIndex(memory.content, id);
|
||||
|
||||
return this.getMemory(id)!;
|
||||
} else {
|
||||
// Insert new memory
|
||||
@@ -406,6 +444,9 @@ export class CoreMemoryStore {
|
||||
memory.metadata || null
|
||||
);
|
||||
|
||||
// Sync new content to vector index
|
||||
this.syncToVectorIndex(memory.content, id);
|
||||
|
||||
return this.getMemory(id)!;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,10 @@ import type { ConversationRecord } from '../tools/cli-history-store.js';
|
||||
import { getHistoryStore } from '../tools/cli-history-store.js';
|
||||
import { getCoreMemoryStore, type Stage1Output } from './core-memory-store.js';
|
||||
import { MemoryJobScheduler } from './memory-job-scheduler.js';
|
||||
import { UnifiedVectorIndex, isUnifiedEmbedderAvailable } from './unified-vector-index.js';
|
||||
import type { ChunkMetadata } from './unified-vector-index.js';
|
||||
import { SessionClusteringService } from './session-clustering-service.js';
|
||||
import { PatternDetector } from './pattern-detector.js';
|
||||
import {
|
||||
MAX_SESSION_AGE_DAYS,
|
||||
MIN_IDLE_HOURS,
|
||||
@@ -384,9 +388,38 @@ export class MemoryExtractionPipeline {
|
||||
const store = getCoreMemoryStore(this.projectPath);
|
||||
store.upsertStage1Output(output);
|
||||
|
||||
// Sync extracted content to vector index (fire-and-forget)
|
||||
this.syncExtractionToVectorIndex(output);
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync extraction output to the vector index.
|
||||
* Indexes both raw_memory and rollout_summary with category='cli_history'.
|
||||
* Fire-and-forget: errors are logged but never thrown.
|
||||
*/
|
||||
private syncExtractionToVectorIndex(output: Stage1Output): void {
|
||||
if (!isUnifiedEmbedderAvailable()) return;
|
||||
|
||||
const vectorIndex = new UnifiedVectorIndex(this.projectPath);
|
||||
const combinedContent = `${output.raw_memory}\n\n---\n\n${output.rollout_summary}`;
|
||||
const metadata: ChunkMetadata = {
|
||||
source_id: output.thread_id,
|
||||
source_type: 'cli_history',
|
||||
category: 'cli_history',
|
||||
};
|
||||
|
||||
vectorIndex.indexContent(combinedContent, metadata).catch((err) => {
|
||||
if (process.env.DEBUG) {
|
||||
console.error(
|
||||
`[MemoryExtractionPipeline] Vector index sync failed for ${output.thread_id}:`,
|
||||
(err as Error).message
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Batch orchestration
|
||||
// ========================================================================
|
||||
@@ -461,6 +494,76 @@ export class MemoryExtractionPipeline {
|
||||
await Promise.all(promises);
|
||||
}
|
||||
|
||||
// Post-extraction: trigger incremental clustering and pattern detection
|
||||
// These are fire-and-forget to avoid blocking the main extraction flow.
|
||||
if (result.succeeded > 0) {
|
||||
this.triggerPostExtractionHooks(
|
||||
eligibleSessions.filter((_, i) => i < result.processed).map(s => s.id)
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fire-and-forget: trigger incremental clustering and pattern detection
|
||||
* after Phase 1 extraction completes.
|
||||
*
|
||||
* - incrementalCluster: processes each newly extracted session
|
||||
* - detectPatterns: runs pattern detection across all chunks
|
||||
*
|
||||
* Errors are logged but never thrown, to avoid disrupting the caller.
|
||||
*/
|
||||
private triggerPostExtractionHooks(extractedSessionIds: string[]): void {
|
||||
const clusteringService = new SessionClusteringService(this.projectPath);
|
||||
const patternDetector = new PatternDetector(this.projectPath);
|
||||
|
||||
// Incremental clustering for each extracted session (fire-and-forget)
|
||||
(async () => {
|
||||
try {
|
||||
// Check frequency control before running clustering
|
||||
const shouldCluster = await clusteringService.shouldRunClustering();
|
||||
if (!shouldCluster) {
|
||||
if (process.env.DEBUG) {
|
||||
console.log('[PostExtraction] Clustering skipped: frequency control not met');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for (const sessionId of extractedSessionIds) {
|
||||
try {
|
||||
await clusteringService.incrementalCluster(sessionId);
|
||||
} catch (err) {
|
||||
if (process.env.DEBUG) {
|
||||
console.warn(
|
||||
`[PostExtraction] Incremental clustering failed for ${sessionId}:`,
|
||||
(err as Error).message
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (process.env.DEBUG) {
|
||||
console.warn('[PostExtraction] Clustering hook failed:', (err as Error).message);
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
// Pattern detection (fire-and-forget)
|
||||
(async () => {
|
||||
try {
|
||||
const result = await patternDetector.detectPatterns();
|
||||
if (result.patterns.length > 0) {
|
||||
console.log(
|
||||
`[PostExtraction] Pattern detection: ${result.patterns.length} patterns found, ` +
|
||||
`${result.solidified.length} solidified (${result.elapsedMs}ms)`
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
if (process.env.DEBUG) {
|
||||
console.warn('[PostExtraction] Pattern detection failed:', (err as Error).message);
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
}
|
||||
|
||||
485
ccw/src/core/pattern-detector.ts
Normal file
485
ccw/src/core/pattern-detector.ts
Normal file
@@ -0,0 +1,485 @@
|
||||
/**
|
||||
* Pattern Detector - Detects recurring content patterns across sessions
|
||||
*
|
||||
* Uses vector clustering (cosine similarity > 0.85) to group semantically similar
|
||||
* chunks into patterns. Patterns appearing in N>=3 distinct sessions are flagged
|
||||
* as candidates. High-confidence patterns (>=0.8) are solidified into CoreMemory
|
||||
* and skills/*.md files.
|
||||
*/
|
||||
|
||||
import { CoreMemoryStore, getCoreMemoryStore } from './core-memory-store.js';
|
||||
import { UnifiedVectorIndex, isUnifiedEmbedderAvailable } from './unified-vector-index.js';
|
||||
import type { VectorSearchMatch } from './unified-vector-index.js';
|
||||
import { existsSync, mkdirSync, writeFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
// -- Constants --
|
||||
|
||||
/** Minimum cosine similarity to group chunks into the same pattern */
|
||||
const PATTERN_SIMILARITY_THRESHOLD = 0.85;
|
||||
|
||||
/** Minimum number of distinct sessions a pattern must appear in */
|
||||
const MIN_SESSION_FREQUENCY = 3;
|
||||
|
||||
/** Confidence threshold for auto-solidification */
|
||||
const SOLIDIFY_CONFIDENCE_THRESHOLD = 0.8;
|
||||
|
||||
/** Maximum number of chunks to analyze per detection run */
|
||||
const MAX_CHUNKS_TO_ANALYZE = 200;
|
||||
|
||||
/** Top-K neighbors to search per chunk during clustering */
|
||||
const NEIGHBOR_TOP_K = 15;
|
||||
|
||||
// -- Types --
|
||||
|
||||
export interface DetectedPattern {
|
||||
/** Unique pattern identifier */
|
||||
id: string;
|
||||
/** Human-readable pattern name derived from content */
|
||||
name: string;
|
||||
/** Representative content snippet */
|
||||
representative: string;
|
||||
/** Source IDs (sessions) where this pattern appears */
|
||||
sourceIds: string[];
|
||||
/** Number of distinct sessions */
|
||||
sessionCount: number;
|
||||
/** Average similarity score within the pattern group */
|
||||
avgSimilarity: number;
|
||||
/** Confidence score (0-1), based on frequency and similarity */
|
||||
confidence: number;
|
||||
/** Category of the chunks in this pattern */
|
||||
category: string;
|
||||
}
|
||||
|
||||
export interface PatternDetectionResult {
|
||||
/** All detected patterns */
|
||||
patterns: DetectedPattern[];
|
||||
/** Number of chunks analyzed */
|
||||
chunksAnalyzed: number;
|
||||
/** Patterns that were solidified (written to CoreMemory + skills) */
|
||||
solidified: string[];
|
||||
/** Elapsed time in ms */
|
||||
elapsedMs: number;
|
||||
}
|
||||
|
||||
export interface SolidifyResult {
|
||||
memoryId: string;
|
||||
skillPath: string | null;
|
||||
}
|
||||
|
||||
// -- PatternDetector --
|
||||
|
||||
export class PatternDetector {
|
||||
private projectPath: string;
|
||||
private coreMemoryStore: CoreMemoryStore;
|
||||
private vectorIndex: UnifiedVectorIndex | null = null;
|
||||
|
||||
constructor(projectPath: string) {
|
||||
this.projectPath = projectPath;
|
||||
this.coreMemoryStore = getCoreMemoryStore(projectPath);
|
||||
|
||||
if (isUnifiedEmbedderAvailable()) {
|
||||
this.vectorIndex = new UnifiedVectorIndex(projectPath);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect recurring patterns across sessions by vector clustering.
|
||||
*
|
||||
* Algorithm:
|
||||
* 1. Get representative chunks from VectorStore (via search with broad queries)
|
||||
* 2. For each chunk, search HNSW for nearest neighbors (cosine > PATTERN_SIMILARITY_THRESHOLD)
|
||||
* 3. Group chunks with high mutual similarity into pattern clusters
|
||||
* 4. Count distinct source_ids per cluster (session frequency)
|
||||
* 5. Patterns with sessionCount >= MIN_SESSION_FREQUENCY become candidates
|
||||
*
|
||||
* @returns Detection result with candidate patterns
|
||||
*/
|
||||
async detectPatterns(): Promise<PatternDetectionResult> {
|
||||
const startTime = Date.now();
|
||||
const result: PatternDetectionResult = {
|
||||
patterns: [],
|
||||
chunksAnalyzed: 0,
|
||||
solidified: [],
|
||||
elapsedMs: 0,
|
||||
};
|
||||
|
||||
if (!this.vectorIndex) {
|
||||
result.elapsedMs = Date.now() - startTime;
|
||||
return result;
|
||||
}
|
||||
|
||||
// Step 1: Gather chunks from the vector store via broad category searches
|
||||
const allChunks = await this.gatherChunksForAnalysis();
|
||||
result.chunksAnalyzed = allChunks.length;
|
||||
|
||||
if (allChunks.length < MIN_SESSION_FREQUENCY) {
|
||||
result.elapsedMs = Date.now() - startTime;
|
||||
return result;
|
||||
}
|
||||
|
||||
// Step 2: Cluster chunks by vector similarity
|
||||
const patternGroups = await this.clusterChunksByVector(allChunks);
|
||||
|
||||
// Step 3: Filter by session frequency and build DetectedPattern objects
|
||||
for (const group of patternGroups) {
|
||||
const uniqueSources = new Set(group.map(c => c.source_id));
|
||||
if (uniqueSources.size < MIN_SESSION_FREQUENCY) continue;
|
||||
|
||||
const avgSim = group.reduce((sum, c) => sum + c.score, 0) / group.length;
|
||||
|
||||
// Confidence: combines frequency (normalized) and avg similarity
|
||||
const frequencyScore = Math.min(uniqueSources.size / 10, 1.0);
|
||||
const confidence = avgSim * 0.6 + frequencyScore * 0.4;
|
||||
|
||||
const representative = group[0]; // Highest scoring chunk
|
||||
const patternName = this.derivePatternName(group);
|
||||
const patternId = `PAT-${Date.now()}-${Math.random().toString(36).substring(2, 6)}`;
|
||||
|
||||
result.patterns.push({
|
||||
id: patternId,
|
||||
name: patternName,
|
||||
representative: representative.content.substring(0, 500),
|
||||
sourceIds: Array.from(uniqueSources),
|
||||
sessionCount: uniqueSources.size,
|
||||
avgSimilarity: Math.round(avgSim * 1000) / 1000,
|
||||
confidence: Math.round(confidence * 1000) / 1000,
|
||||
category: representative.category || 'unknown',
|
||||
});
|
||||
}
|
||||
|
||||
// Sort by confidence descending
|
||||
result.patterns.sort((a, b) => b.confidence - a.confidence);
|
||||
|
||||
// Step 4: Auto-solidify high-confidence patterns (fire-and-forget)
|
||||
for (const pattern of result.patterns) {
|
||||
if (pattern.confidence >= SOLIDIFY_CONFIDENCE_THRESHOLD) {
|
||||
try {
|
||||
await this.solidifyPattern(pattern);
|
||||
result.solidified.push(pattern.id);
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
`[PatternDetector] Failed to solidify pattern ${pattern.id}:`,
|
||||
(err as Error).message
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.elapsedMs = Date.now() - startTime;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gather a representative set of chunks for pattern analysis.
|
||||
* Uses broad search queries across categories to collect diverse chunks.
|
||||
*/
|
||||
private async gatherChunksForAnalysis(): Promise<VectorSearchMatch[]> {
|
||||
if (!this.vectorIndex) return [];
|
||||
|
||||
const allChunks: VectorSearchMatch[] = [];
|
||||
const seenContent = new Set<string>();
|
||||
|
||||
// Search across common categories with broad queries
|
||||
const broadQueries = [
|
||||
'implementation pattern',
|
||||
'configuration setup',
|
||||
'error handling',
|
||||
'testing approach',
|
||||
'workflow process',
|
||||
];
|
||||
|
||||
const categories = ['core_memory', 'cli_history', 'workflow'] as const;
|
||||
|
||||
for (const category of categories) {
|
||||
for (const query of broadQueries) {
|
||||
if (allChunks.length >= MAX_CHUNKS_TO_ANALYZE) break;
|
||||
|
||||
try {
|
||||
const result = await this.vectorIndex.search(query, {
|
||||
topK: Math.ceil(MAX_CHUNKS_TO_ANALYZE / (broadQueries.length * categories.length)),
|
||||
minScore: 0.1,
|
||||
category,
|
||||
});
|
||||
|
||||
if (result.success) {
|
||||
for (const match of result.matches) {
|
||||
// Deduplicate by content hash (first 100 chars)
|
||||
const contentKey = match.content.substring(0, 100);
|
||||
if (!seenContent.has(contentKey)) {
|
||||
seenContent.add(contentKey);
|
||||
allChunks.push(match);
|
||||
}
|
||||
if (allChunks.length >= MAX_CHUNKS_TO_ANALYZE) break;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Search failed for this query/category, continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return allChunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cluster chunks by vector similarity using HNSW neighbor search.
|
||||
*
|
||||
* For each unprocessed chunk, search for its nearest neighbors.
|
||||
* Chunks with cosine similarity > PATTERN_SIMILARITY_THRESHOLD are grouped together.
|
||||
* Uses a union-find-like approach via visited tracking.
|
||||
*/
|
||||
private async clusterChunksByVector(
|
||||
chunks: VectorSearchMatch[]
|
||||
): Promise<VectorSearchMatch[][]> {
|
||||
if (!this.vectorIndex) return [];
|
||||
|
||||
const groups: VectorSearchMatch[][] = [];
|
||||
const processed = new Set<number>();
|
||||
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
if (processed.has(i)) continue;
|
||||
|
||||
const seedChunk = chunks[i];
|
||||
const group: VectorSearchMatch[] = [seedChunk];
|
||||
processed.add(i);
|
||||
|
||||
// Search for neighbors of this chunk's content
|
||||
try {
|
||||
const neighbors = await this.vectorIndex.search(seedChunk.content, {
|
||||
topK: NEIGHBOR_TOP_K,
|
||||
minScore: PATTERN_SIMILARITY_THRESHOLD,
|
||||
});
|
||||
|
||||
if (neighbors.success) {
|
||||
for (const neighbor of neighbors.matches) {
|
||||
// Skip self-matches
|
||||
if (neighbor.content === seedChunk.content) continue;
|
||||
|
||||
// Find this neighbor in our chunk list
|
||||
for (let j = 0; j < chunks.length; j++) {
|
||||
if (processed.has(j)) continue;
|
||||
if (
|
||||
chunks[j].source_id === neighbor.source_id &&
|
||||
chunks[j].chunk_index === neighbor.chunk_index
|
||||
) {
|
||||
group.push({ ...chunks[j], score: neighbor.score });
|
||||
processed.add(j);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Also include neighbors not in our original list
|
||||
if (neighbor.source_id && neighbor.source_id !== seedChunk.source_id) {
|
||||
// Check if already in group by source_id
|
||||
const alreadyInGroup = group.some(
|
||||
g => g.source_id === neighbor.source_id && g.chunk_index === neighbor.chunk_index
|
||||
);
|
||||
if (!alreadyInGroup) {
|
||||
group.push(neighbor);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// HNSW search failed, skip this chunk's neighborhood
|
||||
}
|
||||
|
||||
// Only keep groups with chunks from multiple sources
|
||||
const uniqueSources = new Set(group.map(c => c.source_id));
|
||||
if (uniqueSources.size >= 2) {
|
||||
groups.push(group);
|
||||
}
|
||||
}
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive a human-readable pattern name from a group of similar chunks.
|
||||
* Extracts common keywords/phrases from the representative content.
|
||||
*/
|
||||
private derivePatternName(group: VectorSearchMatch[]): string {
|
||||
// Extended stopwords including generic tech terms
|
||||
const stopwords = new Set([
|
||||
'the', 'and', 'for', 'that', 'this', 'with', 'from', 'have', 'will',
|
||||
'are', 'was', 'were', 'been', 'what', 'when', 'where', 'which',
|
||||
'there', 'their', 'they', 'them', 'then', 'than', 'into', 'some',
|
||||
'code', 'file', 'function', 'class', 'import', 'export', 'const',
|
||||
'async', 'await', 'return', 'type', 'interface', 'string', 'number',
|
||||
'true', 'false', 'null', 'undefined', 'object', 'array', 'value',
|
||||
'data', 'result', 'error', 'name', 'path', 'index', 'item', 'list',
|
||||
'should', 'would', 'could', 'does', 'make', 'like', 'just', 'also',
|
||||
'used', 'using', 'each', 'other', 'more', 'only', 'need', 'very',
|
||||
]);
|
||||
|
||||
const isSignificant = (w: string) => w.length >= 4 && !stopwords.has(w);
|
||||
|
||||
// Count word and bigram frequency across all chunks
|
||||
const wordFreq = new Map<string, number>();
|
||||
const bigramFreq = new Map<string, number>();
|
||||
|
||||
for (const chunk of group) {
|
||||
const words = chunk.content.toLowerCase().split(/[\s\W]+/).filter(isSignificant);
|
||||
const uniqueWords = new Set(words);
|
||||
for (const word of uniqueWords) {
|
||||
wordFreq.set(word, (wordFreq.get(word) || 0) + 1);
|
||||
}
|
||||
|
||||
// Extract bigrams from consecutive significant words
|
||||
for (let i = 0; i < words.length - 1; i++) {
|
||||
const bigram = `${words[i]}-${words[i + 1]}`;
|
||||
bigramFreq.set(bigram, (bigramFreq.get(bigram) || 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
// Prefer bigrams that appear in multiple chunks
|
||||
const topBigrams = Array.from(bigramFreq.entries())
|
||||
.filter(([, count]) => count >= 2)
|
||||
.sort((a, b) => b[1] - a[1]);
|
||||
|
||||
if (topBigrams.length > 0) {
|
||||
// Use top bigram, optionally append a distinguishing single word
|
||||
const name = topBigrams[0][0];
|
||||
const bigramWords = new Set(name.split('-'));
|
||||
const extra = Array.from(wordFreq.entries())
|
||||
.filter(([w, count]) => count >= 2 && !bigramWords.has(w))
|
||||
.sort((a, b) => b[1] - a[1]);
|
||||
if (extra.length > 0) {
|
||||
const candidate = `${name}-${extra[0][0]}`;
|
||||
return candidate.length <= 50 ? candidate : name;
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
// Fallback to top single words
|
||||
const topWords = Array.from(wordFreq.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 3)
|
||||
.map(([w]) => w);
|
||||
|
||||
if (topWords.length >= 2) {
|
||||
const name = topWords.join('-');
|
||||
return name.length <= 50 ? name : topWords.slice(0, 2).join('-');
|
||||
} else if (topWords.length === 1) {
|
||||
return topWords[0];
|
||||
}
|
||||
|
||||
return 'unnamed-pattern';
|
||||
}
|
||||
|
||||
/**
|
||||
* Solidify a detected pattern by writing it to CoreMemory and skills/*.md.
|
||||
*
|
||||
* Creates:
|
||||
* 1. A CoreMemory entry with the pattern content and metadata
|
||||
* 2. A skills/{pattern_slug}.md file with the pattern documentation
|
||||
*
|
||||
* This method is fire-and-forget - errors are logged but not propagated.
|
||||
*
|
||||
* @param pattern - The detected pattern to solidify
|
||||
* @returns Result with memory ID and skill file path
|
||||
*/
|
||||
async solidifyPattern(pattern: DetectedPattern): Promise<SolidifyResult> {
|
||||
// 1. Create CoreMemory entry
|
||||
const memoryContent = this.buildPatternMemoryContent(pattern);
|
||||
const memory = this.coreMemoryStore.upsertMemory({
|
||||
content: memoryContent,
|
||||
summary: `Detected pattern: ${pattern.name} (${pattern.sessionCount} sessions, confidence: ${pattern.confidence})`,
|
||||
metadata: JSON.stringify({
|
||||
type: 'detected_pattern',
|
||||
pattern_id: pattern.id,
|
||||
pattern_name: pattern.name,
|
||||
session_count: pattern.sessionCount,
|
||||
confidence: pattern.confidence,
|
||||
source_ids: pattern.sourceIds,
|
||||
detected_at: new Date().toISOString(),
|
||||
}),
|
||||
});
|
||||
|
||||
// 2. Write skills file
|
||||
let skillPath: string | null = null;
|
||||
try {
|
||||
const slug = pattern.name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-|-$/g, '')
|
||||
.substring(0, 50);
|
||||
|
||||
const skillsDir = join(this.projectPath, '.claude', 'skills');
|
||||
if (!existsSync(skillsDir)) {
|
||||
mkdirSync(skillsDir, { recursive: true });
|
||||
}
|
||||
|
||||
skillPath = join(skillsDir, `${slug}.md`);
|
||||
const skillContent = this.buildSkillContent(pattern);
|
||||
writeFileSync(skillPath, skillContent, 'utf-8');
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
`[PatternDetector] Failed to write skill file for ${pattern.name}:`,
|
||||
(err as Error).message
|
||||
);
|
||||
skillPath = null;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[PatternDetector] Solidified pattern '${pattern.name}' -> memory=${memory.id}, skill=${skillPath || 'none'}`
|
||||
);
|
||||
|
||||
return { memoryId: memory.id, skillPath };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build CoreMemory content for a detected pattern.
|
||||
*/
|
||||
private buildPatternMemoryContent(pattern: DetectedPattern): string {
|
||||
const lines: string[] = [
|
||||
`# Detected Pattern: ${pattern.name}`,
|
||||
'',
|
||||
`**Confidence**: ${pattern.confidence}`,
|
||||
`**Sessions**: ${pattern.sessionCount} (${pattern.sourceIds.join(', ')})`,
|
||||
`**Category**: ${pattern.category}`,
|
||||
`**Avg Similarity**: ${pattern.avgSimilarity}`,
|
||||
'',
|
||||
'## Representative Content',
|
||||
'',
|
||||
pattern.representative,
|
||||
'',
|
||||
'## Usage',
|
||||
'',
|
||||
'This pattern was automatically detected across multiple sessions.',
|
||||
'It represents a recurring approach or concept in this project.',
|
||||
];
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Build skill file content for a detected pattern.
|
||||
*/
|
||||
private buildSkillContent(pattern: DetectedPattern): string {
|
||||
const lines: string[] = [
|
||||
`# ${pattern.name}`,
|
||||
'',
|
||||
`> Auto-detected pattern (confidence: ${pattern.confidence}, sessions: ${pattern.sessionCount})`,
|
||||
'',
|
||||
'## Description',
|
||||
'',
|
||||
pattern.representative,
|
||||
'',
|
||||
'## Context',
|
||||
'',
|
||||
`This pattern was detected across ${pattern.sessionCount} sessions:`,
|
||||
...pattern.sourceIds.map(id => `- ${id}`),
|
||||
'',
|
||||
'## When to Apply',
|
||||
'',
|
||||
'Apply this pattern when working on similar tasks or encountering related concepts.',
|
||||
'',
|
||||
`---`,
|
||||
`*Auto-generated by PatternDetector on ${new Date().toISOString()}*`,
|
||||
];
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
}
|
||||
357
ccw/src/core/routes/notification-routes.ts
Normal file
357
ccw/src/core/routes/notification-routes.ts
Normal file
@@ -0,0 +1,357 @@
|
||||
// ========================================
|
||||
// Remote Notification Routes
|
||||
// ========================================
|
||||
// API endpoints for remote notification configuration
|
||||
|
||||
import type { IncomingMessage, ServerResponse } from 'http';
|
||||
import { URL } from 'url';
|
||||
import {
|
||||
loadConfig,
|
||||
saveConfig,
|
||||
resetConfig,
|
||||
} from '../../config/remote-notification-config.js';
|
||||
import {
|
||||
remoteNotificationService,
|
||||
} from '../../services/remote-notification-service.js';
|
||||
import {
|
||||
maskSensitiveConfig,
|
||||
type RemoteNotificationConfig,
|
||||
type TestNotificationRequest,
|
||||
type NotificationPlatform,
|
||||
type DiscordConfig,
|
||||
type TelegramConfig,
|
||||
type WebhookConfig,
|
||||
} from '../../types/remote-notification.js';
|
||||
import { deepMerge } from '../../types/util.js';
|
||||
|
||||
// ========== Input Validation ==========
|
||||
|
||||
/**
|
||||
* Validate URL format (must be http or https)
|
||||
*/
|
||||
function isValidUrl(url: string): boolean {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return ['http:', 'https:'].includes(parsed.protocol);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate Discord webhook URL format
|
||||
*/
|
||||
function isValidDiscordWebhookUrl(url: string): boolean {
|
||||
if (!isValidUrl(url)) return false;
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
// Discord webhooks are typically: discord.com/api/webhooks/{id}/{token}
|
||||
return (
|
||||
(parsed.hostname === 'discord.com' || parsed.hostname === 'discordapp.com') &&
|
||||
parsed.pathname.startsWith('/api/webhooks/')
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate Telegram bot token format (typically: 123456789:ABCdef...)
|
||||
*/
|
||||
function isValidTelegramBotToken(token: string): boolean {
|
||||
// Telegram bot tokens are in format: {bot_id}:{token}
|
||||
// Bot ID is a number, token is alphanumeric with underscores and hyphens
|
||||
return /^\d{8,15}:[A-Za-z0-9_-]{30,50}$/.test(token);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate Telegram chat ID format
|
||||
*/
|
||||
function isValidTelegramChatId(chatId: string): boolean {
|
||||
// Chat IDs are numeric, optionally negative (for groups)
|
||||
return /^-?\d{1,20}$/.test(chatId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate webhook headers (must be valid JSON object)
|
||||
*/
|
||||
function isValidHeaders(headers: unknown): { valid: boolean; error?: string } {
|
||||
if (headers === undefined || headers === null) {
|
||||
return { valid: true }; // Optional field
|
||||
}
|
||||
|
||||
if (typeof headers !== 'object' || Array.isArray(headers)) {
|
||||
return { valid: false, error: 'Headers must be an object' };
|
||||
}
|
||||
|
||||
const headerObj = headers as Record<string, unknown>;
|
||||
|
||||
// Check for reasonable size limit (10KB)
|
||||
const serialized = JSON.stringify(headers);
|
||||
if (serialized.length > 10240) {
|
||||
return { valid: false, error: 'Headers too large (max 10KB)' };
|
||||
}
|
||||
|
||||
// Validate each header key and value
|
||||
for (const [key, value] of Object.entries(headerObj)) {
|
||||
if (typeof key !== 'string' || key.length === 0) {
|
||||
return { valid: false, error: 'Header keys must be non-empty strings' };
|
||||
}
|
||||
if (typeof value !== 'string') {
|
||||
return { valid: false, error: `Header '${key}' value must be a string` };
|
||||
}
|
||||
// Block potentially dangerous headers
|
||||
const lowerKey = key.toLowerCase();
|
||||
if (['host', 'content-length', 'connection'].includes(lowerKey)) {
|
||||
return { valid: false, error: `Header '${key}' is not allowed` };
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate configuration updates
|
||||
*/
|
||||
function validateConfigUpdates(updates: Partial<RemoteNotificationConfig>): { valid: boolean; error?: string } {
|
||||
// Validate platforms if present
|
||||
if (updates.platforms) {
|
||||
const { discord, telegram, webhook } = updates.platforms;
|
||||
|
||||
// Validate Discord config
|
||||
if (discord) {
|
||||
if (discord.webhookUrl !== undefined && discord.webhookUrl !== '') {
|
||||
if (!isValidUrl(discord.webhookUrl)) {
|
||||
return { valid: false, error: 'Invalid Discord webhook URL format' };
|
||||
}
|
||||
// Warning: we allow non-Discord URLs for flexibility, but log it
|
||||
if (!isValidDiscordWebhookUrl(discord.webhookUrl)) {
|
||||
console.warn('[RemoteNotification] Webhook URL does not match Discord format');
|
||||
}
|
||||
}
|
||||
if (discord.username !== undefined && discord.username.length > 80) {
|
||||
return { valid: false, error: 'Discord username too long (max 80 chars)' };
|
||||
}
|
||||
}
|
||||
|
||||
// Validate Telegram config
|
||||
if (telegram) {
|
||||
if (telegram.botToken !== undefined && telegram.botToken !== '') {
|
||||
if (!isValidTelegramBotToken(telegram.botToken)) {
|
||||
return { valid: false, error: 'Invalid Telegram bot token format' };
|
||||
}
|
||||
}
|
||||
if (telegram.chatId !== undefined && telegram.chatId !== '') {
|
||||
if (!isValidTelegramChatId(telegram.chatId)) {
|
||||
return { valid: false, error: 'Invalid Telegram chat ID format' };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate Webhook config
|
||||
if (webhook) {
|
||||
if (webhook.url !== undefined && webhook.url !== '') {
|
||||
if (!isValidUrl(webhook.url)) {
|
||||
return { valid: false, error: 'Invalid webhook URL format' };
|
||||
}
|
||||
}
|
||||
if (webhook.headers !== undefined) {
|
||||
const headerValidation = isValidHeaders(webhook.headers);
|
||||
if (!headerValidation.valid) {
|
||||
return { valid: false, error: headerValidation.error };
|
||||
}
|
||||
}
|
||||
if (webhook.timeout !== undefined && (webhook.timeout < 1000 || webhook.timeout > 60000)) {
|
||||
return { valid: false, error: 'Webhook timeout must be between 1000ms and 60000ms' };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate timeout
|
||||
if (updates.timeout !== undefined && (updates.timeout < 1000 || updates.timeout > 60000)) {
|
||||
return { valid: false, error: 'Timeout must be between 1000ms and 60000ms' };
|
||||
}
|
||||
|
||||
return { valid: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate test notification request
|
||||
*/
|
||||
function validateTestRequest(request: TestNotificationRequest): { valid: boolean; error?: string } {
|
||||
if (!request.platform) {
|
||||
return { valid: false, error: 'Missing platform' };
|
||||
}
|
||||
|
||||
const validPlatforms: NotificationPlatform[] = ['discord', 'telegram', 'webhook'];
|
||||
if (!validPlatforms.includes(request.platform as NotificationPlatform)) {
|
||||
return { valid: false, error: `Invalid platform: ${request.platform}` };
|
||||
}
|
||||
|
||||
if (!request.config) {
|
||||
return { valid: false, error: 'Missing config' };
|
||||
}
|
||||
|
||||
// Platform-specific validation
|
||||
switch (request.platform) {
|
||||
case 'discord': {
|
||||
const config = request.config as Partial<DiscordConfig>;
|
||||
if (!config.webhookUrl) {
|
||||
return { valid: false, error: 'Discord webhook URL is required' };
|
||||
}
|
||||
if (!isValidUrl(config.webhookUrl)) {
|
||||
return { valid: false, error: 'Invalid Discord webhook URL format' };
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'telegram': {
|
||||
const config = request.config as Partial<TelegramConfig>;
|
||||
if (!config.botToken) {
|
||||
return { valid: false, error: 'Telegram bot token is required' };
|
||||
}
|
||||
if (!config.chatId) {
|
||||
return { valid: false, error: 'Telegram chat ID is required' };
|
||||
}
|
||||
if (!isValidTelegramBotToken(config.botToken)) {
|
||||
return { valid: false, error: 'Invalid Telegram bot token format' };
|
||||
}
|
||||
if (!isValidTelegramChatId(config.chatId)) {
|
||||
return { valid: false, error: 'Invalid Telegram chat ID format' };
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'webhook': {
|
||||
const config = request.config as Partial<WebhookConfig>;
|
||||
if (!config.url) {
|
||||
return { valid: false, error: 'Webhook URL is required' };
|
||||
}
|
||||
if (!isValidUrl(config.url)) {
|
||||
return { valid: false, error: 'Invalid webhook URL format' };
|
||||
}
|
||||
if (config.headers) {
|
||||
const headerValidation = isValidHeaders(config.headers);
|
||||
if (!headerValidation.valid) {
|
||||
return { valid: false, error: headerValidation.error };
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle remote notification routes
|
||||
* GET /api/notifications/remote/config - Get current config
|
||||
* POST /api/notifications/remote/config - Update config
|
||||
* POST /api/notifications/remote/test - Test notification
|
||||
* POST /api/notifications/remote/reset - Reset to defaults
|
||||
*/
|
||||
export async function handleNotificationRoutes(
|
||||
req: IncomingMessage,
|
||||
res: ServerResponse,
|
||||
pathname: string
|
||||
): Promise<boolean> {
|
||||
// GET /api/notifications/remote/config
|
||||
if (pathname === '/api/notifications/remote/config' && req.method === 'GET') {
|
||||
const config = loadConfig();
|
||||
const masked = maskSensitiveConfig(config);
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(masked));
|
||||
return true;
|
||||
}
|
||||
|
||||
// POST /api/notifications/remote/config
|
||||
if (pathname === '/api/notifications/remote/config' && req.method === 'POST') {
|
||||
const body = await readBody(req);
|
||||
|
||||
try {
|
||||
const updates = JSON.parse(body) as Partial<RemoteNotificationConfig>;
|
||||
|
||||
// Validate input
|
||||
const validation = validateConfigUpdates(updates);
|
||||
if (!validation.valid) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: validation.error }));
|
||||
return true;
|
||||
}
|
||||
|
||||
const current = loadConfig();
|
||||
const updated = deepMerge(current, updates);
|
||||
|
||||
saveConfig(updated);
|
||||
|
||||
// Reload service config
|
||||
remoteNotificationService.reloadConfig();
|
||||
|
||||
const masked = maskSensitiveConfig(updated);
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ success: true, config: masked }));
|
||||
} catch (error) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
error: error instanceof Error ? error.message : 'Invalid configuration',
|
||||
}));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// POST /api/notifications/remote/test
|
||||
if (pathname === '/api/notifications/remote/test' && req.method === 'POST') {
|
||||
const body = await readBody(req);
|
||||
|
||||
try {
|
||||
const request = JSON.parse(body) as TestNotificationRequest;
|
||||
|
||||
// Validate input
|
||||
const validation = validateTestRequest(request);
|
||||
if (!validation.valid) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ success: false, error: validation.error }));
|
||||
return true;
|
||||
}
|
||||
|
||||
const result = await remoteNotificationService.testPlatform(
|
||||
request.platform as NotificationPlatform,
|
||||
request.config
|
||||
);
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(result));
|
||||
} catch (error) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Invalid request',
|
||||
}));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// POST /api/notifications/remote/reset
|
||||
if (pathname === '/api/notifications/remote/reset' && req.method === 'POST') {
|
||||
const config = resetConfig();
|
||||
remoteNotificationService.reloadConfig();
|
||||
|
||||
const masked = maskSensitiveConfig(config);
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ success: true, config: masked }));
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read request body as string
|
||||
*/
|
||||
async function readBody(req: IncomingMessage): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let body = '';
|
||||
req.on('data', (chunk) => { body += chunk; });
|
||||
req.on('end', () => resolve(body));
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
151
ccw/src/core/routes/unified-memory-routes.ts
Normal file
151
ccw/src/core/routes/unified-memory-routes.ts
Normal file
@@ -0,0 +1,151 @@
|
||||
/**
|
||||
* Unified Memory API Routes
|
||||
*
|
||||
* Provides HTTP endpoints for the unified memory system:
|
||||
* - GET /api/unified-memory/search - RRF fusion search (vector + FTS5)
|
||||
* - GET /api/unified-memory/stats - Aggregated statistics
|
||||
* - POST /api/unified-memory/reindex - Rebuild HNSW vector index
|
||||
* - GET /api/unified-memory/recommendations/:id - KNN recommendations
|
||||
*/
|
||||
|
||||
import type { RouteContext } from './types.js';
|
||||
|
||||
/**
|
||||
* Handle Unified Memory API routes.
|
||||
* @returns true if route was handled, false otherwise
|
||||
*/
|
||||
export async function handleUnifiedMemoryRoutes(ctx: RouteContext): Promise<boolean> {
|
||||
const { pathname, url, req, res, initialPath, handlePostRequest } = ctx;
|
||||
|
||||
// =========================================================================
|
||||
// GET /api/unified-memory/search
|
||||
// Query params: q (required), categories, topK, minScore
|
||||
// =========================================================================
|
||||
if (pathname === '/api/unified-memory/search' && req.method === 'GET') {
|
||||
const query = url.searchParams.get('q');
|
||||
const projectPath = url.searchParams.get('path') || initialPath;
|
||||
const topK = parseInt(url.searchParams.get('topK') || '20', 10);
|
||||
const minScore = parseFloat(url.searchParams.get('minScore') || '0');
|
||||
const category = url.searchParams.get('category') || undefined;
|
||||
|
||||
if (!query) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Query parameter q is required' }));
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
const { UnifiedMemoryService } = await import('../unified-memory-service.js');
|
||||
const service = new UnifiedMemoryService(projectPath);
|
||||
|
||||
const results = await service.search(query, {
|
||||
limit: topK,
|
||||
minScore,
|
||||
category: category as any,
|
||||
});
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
success: true,
|
||||
query,
|
||||
total: results.length,
|
||||
results,
|
||||
}));
|
||||
} catch (error: unknown) {
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: (error as Error).message }));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// GET /api/unified-memory/stats
|
||||
// =========================================================================
|
||||
if (pathname === '/api/unified-memory/stats' && req.method === 'GET') {
|
||||
const projectPath = url.searchParams.get('path') || initialPath;
|
||||
|
||||
try {
|
||||
const { UnifiedMemoryService } = await import('../unified-memory-service.js');
|
||||
const service = new UnifiedMemoryService(projectPath);
|
||||
const stats = await service.getStats();
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ success: true, stats }));
|
||||
} catch (error: unknown) {
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: (error as Error).message }));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// POST /api/unified-memory/reindex
|
||||
// Body (optional): { path: string }
|
||||
// =========================================================================
|
||||
if (pathname === '/api/unified-memory/reindex' && req.method === 'POST') {
|
||||
handlePostRequest(req, res, async (body: any) => {
|
||||
const { path: projectPath } = body || {};
|
||||
const basePath = projectPath || initialPath;
|
||||
|
||||
try {
|
||||
const { UnifiedVectorIndex, isUnifiedEmbedderAvailable } = await import('../unified-vector-index.js');
|
||||
|
||||
if (!isUnifiedEmbedderAvailable()) {
|
||||
return {
|
||||
error: 'Unified embedder is not available. Ensure Python venv and embedder script are set up.',
|
||||
status: 503,
|
||||
};
|
||||
}
|
||||
|
||||
const index = new UnifiedVectorIndex(basePath);
|
||||
const result = await index.reindexAll();
|
||||
|
||||
return {
|
||||
success: result.success,
|
||||
hnsw_count: result.hnsw_count,
|
||||
elapsed_time: result.elapsed_time,
|
||||
error: result.error,
|
||||
};
|
||||
} catch (error: unknown) {
|
||||
return { error: (error as Error).message, status: 500 };
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// GET /api/unified-memory/recommendations/:id
|
||||
// Query params: limit (optional, default 5)
|
||||
// =========================================================================
|
||||
if (pathname.startsWith('/api/unified-memory/recommendations/') && req.method === 'GET') {
|
||||
const memoryId = pathname.replace('/api/unified-memory/recommendations/', '');
|
||||
const projectPath = url.searchParams.get('path') || initialPath;
|
||||
const limit = parseInt(url.searchParams.get('limit') || '5', 10);
|
||||
|
||||
if (!memoryId) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Memory ID is required' }));
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
const { UnifiedMemoryService } = await import('../unified-memory-service.js');
|
||||
const service = new UnifiedMemoryService(projectPath);
|
||||
const recommendations = await service.getRecommendations(memoryId, limit);
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
success: true,
|
||||
memory_id: memoryId,
|
||||
total: recommendations.length,
|
||||
recommendations,
|
||||
}));
|
||||
} catch (error: unknown) {
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: (error as Error).message }));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import { handleAuditRoutes } from './routes/audit-routes.js';
|
||||
import { handleProviderRoutes } from './routes/provider-routes.js';
|
||||
import { handleMemoryRoutes } from './routes/memory-routes.js';
|
||||
import { handleCoreMemoryRoutes } from './routes/core-memory-routes.js';
|
||||
import { handleUnifiedMemoryRoutes } from './routes/unified-memory-routes.js';
|
||||
import { handleMcpRoutes } from './routes/mcp-routes.js';
|
||||
import { handleHooksRoutes } from './routes/hooks-routes.js';
|
||||
import { handleUnsplashRoutes, handleBackgroundRoutes } from './routes/unsplash-routes.js';
|
||||
@@ -37,6 +38,7 @@ import { handleDashboardRoutes } from './routes/dashboard-routes.js';
|
||||
import { handleOrchestratorRoutes } from './routes/orchestrator-routes.js';
|
||||
import { handleConfigRoutes } from './routes/config-routes.js';
|
||||
import { handleTeamRoutes } from './routes/team-routes.js';
|
||||
import { handleNotificationRoutes } from './routes/notification-routes.js';
|
||||
|
||||
// Import WebSocket handling
|
||||
import { handleWebSocketUpgrade, broadcastToClients, extractSessionIdFromPath } from './websocket.js';
|
||||
@@ -462,6 +464,11 @@ export async function startServer(options: ServerOptions = {}): Promise<http.Ser
|
||||
if (await handleCoreMemoryRoutes(routeContext)) return;
|
||||
}
|
||||
|
||||
// Unified Memory routes (/api/unified-memory/*)
|
||||
if (pathname.startsWith('/api/unified-memory/')) {
|
||||
if (await handleUnifiedMemoryRoutes(routeContext)) return;
|
||||
}
|
||||
|
||||
|
||||
// MCP routes (/api/mcp*, /api/codex-mcp*)
|
||||
if (pathname.startsWith('/api/mcp') || pathname.startsWith('/api/codex-mcp')) {
|
||||
@@ -533,6 +540,11 @@ export async function startServer(options: ServerOptions = {}): Promise<http.Ser
|
||||
if (await handleTeamRoutes(routeContext)) return;
|
||||
}
|
||||
|
||||
// Remote notification routes (/api/notifications/remote/*)
|
||||
if (pathname.startsWith('/api/notifications/remote')) {
|
||||
if (await handleNotificationRoutes(req, res, pathname)) return;
|
||||
}
|
||||
|
||||
// Task routes (/api/tasks)
|
||||
if (pathname.startsWith('/api/tasks')) {
|
||||
if (await handleTaskRoutes(routeContext)) return;
|
||||
|
||||
592
ccw/src/core/services/remote-notification-service.ts
Normal file
592
ccw/src/core/services/remote-notification-service.ts
Normal file
@@ -0,0 +1,592 @@
|
||||
// ========================================
|
||||
// Remote Notification Service
|
||||
// ========================================
|
||||
// Core service for dispatching notifications to external platforms
|
||||
// Non-blocking, best-effort delivery with parallel dispatch
|
||||
|
||||
import http from 'http';
|
||||
import https from 'https';
|
||||
import { URL } from 'url';
|
||||
import type {
|
||||
RemoteNotificationConfig,
|
||||
NotificationContext,
|
||||
NotificationDispatchResult,
|
||||
PlatformNotificationResult,
|
||||
NotificationPlatform,
|
||||
DiscordConfig,
|
||||
TelegramConfig,
|
||||
WebhookConfig,
|
||||
} from '../../types/remote-notification.js';
|
||||
import {
|
||||
loadConfig,
|
||||
getEnabledPlatformsForEvent,
|
||||
hasEnabledPlatform,
|
||||
} from '../../config/remote-notification-config.js';
|
||||
|
||||
/**
|
||||
* Remote Notification Service
|
||||
* Handles dispatching notifications to configured platforms
|
||||
*/
|
||||
class RemoteNotificationService {
|
||||
private config: RemoteNotificationConfig | null = null;
|
||||
private configLoadedAt: number = 0;
|
||||
private readonly CONFIG_TTL = 30000; // Reload config every 30 seconds
|
||||
|
||||
/**
|
||||
* Get current config (with auto-reload)
|
||||
*/
|
||||
private getConfig(): RemoteNotificationConfig {
|
||||
const now = Date.now();
|
||||
if (!this.config || now - this.configLoadedAt > this.CONFIG_TTL) {
|
||||
this.config = loadConfig();
|
||||
this.configLoadedAt = now;
|
||||
}
|
||||
return this.config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Force reload configuration
|
||||
*/
|
||||
reloadConfig(): void {
|
||||
this.config = loadConfig();
|
||||
this.configLoadedAt = Date.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if notifications are enabled for a given event
|
||||
*/
|
||||
shouldNotify(eventType: string): boolean {
|
||||
const config = this.getConfig();
|
||||
if (!config.enabled) return false;
|
||||
|
||||
const enabledPlatforms = getEnabledPlatformsForEvent(config, eventType);
|
||||
return enabledPlatforms.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send notification to all configured platforms for an event
|
||||
* Non-blocking: returns immediately, actual dispatch is async
|
||||
*/
|
||||
sendNotification(
|
||||
eventType: string,
|
||||
context: Omit<NotificationContext, 'eventType' | 'timestamp'>
|
||||
): void {
|
||||
const config = this.getConfig();
|
||||
|
||||
// Quick check before async dispatch
|
||||
if (!config.enabled) return;
|
||||
|
||||
const enabledPlatforms = getEnabledPlatformsForEvent(config, eventType);
|
||||
if (enabledPlatforms.length === 0) return;
|
||||
|
||||
const fullContext: NotificationContext = {
|
||||
...context,
|
||||
eventType: eventType as NotificationContext['eventType'],
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
// Fire-and-forget dispatch
|
||||
this.dispatchToPlatforms(enabledPlatforms, fullContext, config).catch((error) => {
|
||||
// Silent failure - log only
|
||||
console.error('[RemoteNotification] Dispatch failed:', error);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Send notification and wait for results (for testing)
|
||||
*/
|
||||
async sendNotificationAsync(
|
||||
eventType: string,
|
||||
context: Omit<NotificationContext, 'eventType' | 'timestamp'>
|
||||
): Promise<NotificationDispatchResult> {
|
||||
const config = this.getConfig();
|
||||
const startTime = Date.now();
|
||||
|
||||
if (!config.enabled) {
|
||||
return { success: false, results: [], totalTime: 0 };
|
||||
}
|
||||
|
||||
const enabledPlatforms = getEnabledPlatformsForEvent(config, eventType);
|
||||
if (enabledPlatforms.length === 0) {
|
||||
return { success: false, results: [], totalTime: Date.now() - startTime };
|
||||
}
|
||||
|
||||
const fullContext: NotificationContext = {
|
||||
...context,
|
||||
eventType: eventType as NotificationContext['eventType'],
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const results = await this.dispatchToPlatforms(enabledPlatforms, fullContext, config);
|
||||
|
||||
return {
|
||||
success: results.some((r) => r.success),
|
||||
results,
|
||||
totalTime: Date.now() - startTime,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch to multiple platforms in parallel
|
||||
*/
|
||||
private async dispatchToPlatforms(
|
||||
platforms: string[],
|
||||
context: NotificationContext,
|
||||
config: RemoteNotificationConfig
|
||||
): Promise<PlatformNotificationResult[]> {
|
||||
const promises = platforms.map((platform) =>
|
||||
this.dispatchToPlatform(platform as NotificationPlatform, context, config)
|
||||
);
|
||||
|
||||
const results = await Promise.allSettled(promises);
|
||||
|
||||
return results.map((result, index) => {
|
||||
if (result.status === 'fulfilled') {
|
||||
return result.value;
|
||||
}
|
||||
return {
|
||||
platform: platforms[index] as NotificationPlatform,
|
||||
success: false,
|
||||
error: result.reason?.message || 'Unknown error',
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch to a single platform
|
||||
*/
|
||||
private async dispatchToPlatform(
|
||||
platform: NotificationPlatform,
|
||||
context: NotificationContext,
|
||||
config: RemoteNotificationConfig
|
||||
): Promise<PlatformNotificationResult> {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
switch (platform) {
|
||||
case 'discord':
|
||||
return await this.sendDiscord(context, config.platforms.discord!, config.timeout);
|
||||
case 'telegram':
|
||||
return await this.sendTelegram(context, config.platforms.telegram!, config.timeout);
|
||||
case 'webhook':
|
||||
return await this.sendWebhook(context, config.platforms.webhook!, config.timeout);
|
||||
default:
|
||||
return {
|
||||
platform,
|
||||
success: false,
|
||||
error: `Unknown platform: ${platform}`,
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
platform,
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
responseTime: Date.now() - startTime,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send Discord notification via webhook
|
||||
*/
|
||||
private async sendDiscord(
|
||||
context: NotificationContext,
|
||||
config: DiscordConfig,
|
||||
timeout: number
|
||||
): Promise<PlatformNotificationResult> {
|
||||
const startTime = Date.now();
|
||||
|
||||
if (!config.webhookUrl) {
|
||||
return { platform: 'discord', success: false, error: 'Webhook URL not configured' };
|
||||
}
|
||||
|
||||
const embed = this.buildDiscordEmbed(context);
|
||||
const body = {
|
||||
username: config.username || 'CCW Notification',
|
||||
avatar_url: config.avatarUrl,
|
||||
embeds: [embed],
|
||||
};
|
||||
|
||||
try {
|
||||
await this.httpRequest(config.webhookUrl, body, timeout);
|
||||
return {
|
||||
platform: 'discord',
|
||||
success: true,
|
||||
responseTime: Date.now() - startTime,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
platform: 'discord',
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
responseTime: Date.now() - startTime,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build Discord embed from context
|
||||
*/
|
||||
private buildDiscordEmbed(context: NotificationContext): Record<string, unknown> {
|
||||
const eventEmoji: Record<string, string> = {
|
||||
'ask-user-question': '❓',
|
||||
'session-start': '▶️',
|
||||
'session-end': '⏹️',
|
||||
'task-completed': '✅',
|
||||
'task-failed': '❌',
|
||||
};
|
||||
|
||||
const eventColors: Record<string, number> = {
|
||||
'ask-user-question': 0x3498db, // Blue
|
||||
'session-start': 0x2ecc71, // Green
|
||||
'session-end': 0x95a5a6, // Gray
|
||||
'task-completed': 0x27ae60, // Dark Green
|
||||
'task-failed': 0xe74c3c, // Red
|
||||
};
|
||||
|
||||
const fields: Array<{ name: string; value: string; inline?: boolean }> = [];
|
||||
|
||||
if (context.sessionId) {
|
||||
fields.push({ name: 'Session', value: context.sessionId.slice(0, 16) + '...', inline: true });
|
||||
}
|
||||
|
||||
if (context.questionText) {
|
||||
const truncated = context.questionText.length > 200
|
||||
? context.questionText.slice(0, 200) + '...'
|
||||
: context.questionText;
|
||||
fields.push({ name: 'Question', value: truncated, inline: false });
|
||||
}
|
||||
|
||||
if (context.taskDescription) {
|
||||
const truncated = context.taskDescription.length > 200
|
||||
? context.taskDescription.slice(0, 200) + '...'
|
||||
: context.taskDescription;
|
||||
fields.push({ name: 'Task', value: truncated, inline: false });
|
||||
}
|
||||
|
||||
if (context.errorMessage) {
|
||||
const truncated = context.errorMessage.length > 200
|
||||
? context.errorMessage.slice(0, 200) + '...'
|
||||
: context.errorMessage;
|
||||
fields.push({ name: 'Error', value: truncated, inline: false });
|
||||
}
|
||||
|
||||
return {
|
||||
title: `${eventEmoji[context.eventType] || '📢'} ${this.formatEventName(context.eventType)}`,
|
||||
color: eventColors[context.eventType] || 0x9b59b6,
|
||||
fields,
|
||||
timestamp: context.timestamp,
|
||||
footer: { text: 'CCW Remote Notification' },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Send Telegram notification via Bot API
|
||||
*/
|
||||
private async sendTelegram(
|
||||
context: NotificationContext,
|
||||
config: TelegramConfig,
|
||||
timeout: number
|
||||
): Promise<PlatformNotificationResult> {
|
||||
const startTime = Date.now();
|
||||
|
||||
if (!config.botToken || !config.chatId) {
|
||||
return { platform: 'telegram', success: false, error: 'Bot token or chat ID not configured' };
|
||||
}
|
||||
|
||||
const text = this.buildTelegramMessage(context);
|
||||
const url = `https://api.telegram.org/bot${config.botToken}/sendMessage`;
|
||||
const body = {
|
||||
chat_id: config.chatId,
|
||||
text,
|
||||
parse_mode: config.parseMode || 'HTML',
|
||||
};
|
||||
|
||||
try {
|
||||
await this.httpRequest(url, body, timeout);
|
||||
return {
|
||||
platform: 'telegram',
|
||||
success: true,
|
||||
responseTime: Date.now() - startTime,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
platform: 'telegram',
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
responseTime: Date.now() - startTime,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build Telegram message from context
|
||||
*/
|
||||
private buildTelegramMessage(context: NotificationContext): string {
|
||||
const eventEmoji: Record<string, string> = {
|
||||
'ask-user-question': '❓',
|
||||
'session-start': '▶️',
|
||||
'session-end': '⏹️',
|
||||
'task-completed': '✅',
|
||||
'task-failed': '❌',
|
||||
};
|
||||
|
||||
const lines: string[] = [];
|
||||
lines.push(`<b>${eventEmoji[context.eventType] || '📢'} ${this.formatEventName(context.eventType)}</b>`);
|
||||
lines.push('');
|
||||
|
||||
if (context.sessionId) {
|
||||
lines.push(`<b>Session:</b> <code>${context.sessionId.slice(0, 16)}...</code>`);
|
||||
}
|
||||
|
||||
if (context.questionText) {
|
||||
const truncated = context.questionText.length > 300
|
||||
? context.questionText.slice(0, 300) + '...'
|
||||
: context.questionText;
|
||||
lines.push(`<b>Question:</b> ${this.escapeHtml(truncated)}`);
|
||||
}
|
||||
|
||||
if (context.taskDescription) {
|
||||
const truncated = context.taskDescription.length > 300
|
||||
? context.taskDescription.slice(0, 300) + '...'
|
||||
: context.taskDescription;
|
||||
lines.push(`<b>Task:</b> ${this.escapeHtml(truncated)}`);
|
||||
}
|
||||
|
||||
if (context.errorMessage) {
|
||||
const truncated = context.errorMessage.length > 300
|
||||
? context.errorMessage.slice(0, 300) + '...'
|
||||
: context.errorMessage;
|
||||
lines.push(`<b>Error:</b> <code>${this.escapeHtml(truncated)}</code>`);
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
lines.push(`<i>📅 ${new Date(context.timestamp).toLocaleString()}</i>`);
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Send generic webhook notification
|
||||
*/
|
||||
private async sendWebhook(
|
||||
context: NotificationContext,
|
||||
config: WebhookConfig,
|
||||
timeout: number
|
||||
): Promise<PlatformNotificationResult> {
|
||||
const startTime = Date.now();
|
||||
|
||||
if (!config.url) {
|
||||
return { platform: 'webhook', success: false, error: 'Webhook URL not configured' };
|
||||
}
|
||||
|
||||
const body = {
|
||||
event: context.eventType,
|
||||
timestamp: context.timestamp,
|
||||
sessionId: context.sessionId,
|
||||
questionText: context.questionText,
|
||||
taskDescription: context.taskDescription,
|
||||
errorMessage: context.errorMessage,
|
||||
metadata: context.metadata,
|
||||
};
|
||||
|
||||
try {
|
||||
await this.httpRequest(config.url, body, config.timeout || timeout, config.method, config.headers);
|
||||
return {
|
||||
platform: 'webhook',
|
||||
success: true,
|
||||
responseTime: Date.now() - startTime,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
platform: 'webhook',
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
responseTime: Date.now() - startTime,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a URL is safe from SSRF attacks
|
||||
* Blocks private IP ranges, loopback, and link-local addresses
|
||||
*/
|
||||
private isUrlSafe(urlString: string): { safe: boolean; error?: string } {
|
||||
try {
|
||||
const parsedUrl = new URL(urlString);
|
||||
|
||||
// Only allow http and https protocols
|
||||
if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
|
||||
return { safe: false, error: 'Only http and https protocols are allowed' };
|
||||
}
|
||||
|
||||
const hostname = parsedUrl.hostname.toLowerCase();
|
||||
|
||||
// Block localhost variants
|
||||
if (hostname === 'localhost' || hostname === 'localhost.localdomain' || hostname === '0.0.0.0') {
|
||||
return { safe: false, error: 'Localhost addresses are not allowed' };
|
||||
}
|
||||
|
||||
// Block IPv4 loopback (127.0.0.0/8)
|
||||
if (/^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(hostname)) {
|
||||
return { safe: false, error: 'Loopback addresses are not allowed' };
|
||||
}
|
||||
|
||||
// Block IPv4 private ranges
|
||||
// 10.0.0.0/8
|
||||
if (/^10\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(hostname)) {
|
||||
return { safe: false, error: 'Private IP addresses are not allowed' };
|
||||
}
|
||||
// 172.16.0.0/12
|
||||
if (/^172\.(1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3}$/.test(hostname)) {
|
||||
return { safe: false, error: 'Private IP addresses are not allowed' };
|
||||
}
|
||||
// 192.168.0.0/16
|
||||
if (/^192\.168\.\d{1,3}\.\d{1,3}$/.test(hostname)) {
|
||||
return { safe: false, error: 'Private IP addresses are not allowed' };
|
||||
}
|
||||
|
||||
// Block link-local addresses (169.254.0.0/16)
|
||||
if (/^169\.254\.\d{1,3}\.\d{1,3}$/.test(hostname)) {
|
||||
return { safe: false, error: 'Link-local addresses are not allowed' };
|
||||
}
|
||||
|
||||
// Block IPv6 loopback and private
|
||||
if (hostname === '::1' || hostname.startsWith('fc') || hostname.startsWith('fd') || hostname === '::') {
|
||||
return { safe: false, error: 'IPv6 private/loopback addresses are not allowed' };
|
||||
}
|
||||
|
||||
// Block hostnames that look like IP addresses in various formats
|
||||
// (e.g., 0x7f.0.0.1, 2130706433, etc.)
|
||||
if (/^0x[0-9a-f]+/i.test(hostname) || /^\d{8,}$/.test(hostname)) {
|
||||
return { safe: false, error: 'Suspicious hostname format' };
|
||||
}
|
||||
|
||||
// Block cloud metadata endpoints
|
||||
if (hostname === '169.254.169.254' || hostname === 'metadata.google.internal' || hostname === 'metadata.azure.internal') {
|
||||
return { safe: false, error: 'Cloud metadata endpoints are not allowed' };
|
||||
}
|
||||
|
||||
return { safe: true };
|
||||
} catch (error) {
|
||||
return { safe: false, error: 'Invalid URL format' };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic HTTP request helper
|
||||
*/
|
||||
private httpRequest(
|
||||
url: string,
|
||||
body: unknown,
|
||||
timeout: number,
|
||||
method: 'POST' | 'PUT' = 'POST',
|
||||
headers: Record<string, string> = {}
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// SSRF protection: validate URL before making request
|
||||
const urlSafety = this.isUrlSafe(url);
|
||||
if (!urlSafety.safe) {
|
||||
reject(new Error(`URL validation failed: ${urlSafety.error}`));
|
||||
return;
|
||||
}
|
||||
|
||||
const parsedUrl = new URL(url);
|
||||
const isHttps = parsedUrl.protocol === 'https:';
|
||||
const client = isHttps ? https : http;
|
||||
|
||||
const requestOptions: http.RequestOptions = {
|
||||
hostname: parsedUrl.hostname,
|
||||
port: parsedUrl.port || (isHttps ? 443 : 80),
|
||||
path: parsedUrl.pathname + parsedUrl.search,
|
||||
method,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...headers,
|
||||
},
|
||||
timeout,
|
||||
};
|
||||
|
||||
const req = client.request(requestOptions, (res) => {
|
||||
let data = '';
|
||||
res.on('data', (chunk) => { data += chunk; });
|
||||
res.on('end', () => {
|
||||
if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`HTTP ${res.statusCode}: ${data.slice(0, 200)}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
req.on('error', reject);
|
||||
req.on('timeout', () => {
|
||||
req.destroy();
|
||||
reject(new Error('Request timeout'));
|
||||
});
|
||||
|
||||
req.write(JSON.stringify(body));
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Format event name for display
|
||||
*/
|
||||
private formatEventName(eventType: string): string {
|
||||
return eventType
|
||||
.split('-')
|
||||
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
|
||||
.join(' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape HTML for Telegram messages
|
||||
*/
|
||||
private escapeHtml(text: string): string {
|
||||
return text
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>');
|
||||
}
|
||||
|
||||
/**
|
||||
* Test a platform configuration
|
||||
*/
|
||||
async testPlatform(
|
||||
platform: NotificationPlatform,
|
||||
config: DiscordConfig | TelegramConfig | WebhookConfig
|
||||
): Promise<{ success: boolean; error?: string; responseTime?: number }> {
|
||||
const testContext: NotificationContext = {
|
||||
eventType: 'task-completed',
|
||||
sessionId: 'test-session',
|
||||
taskDescription: 'This is a test notification from CCW',
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
switch (platform) {
|
||||
case 'discord':
|
||||
return await this.sendDiscord(testContext, config as DiscordConfig, 10000);
|
||||
case 'telegram':
|
||||
return await this.sendTelegram(testContext, config as TelegramConfig, 10000);
|
||||
case 'webhook':
|
||||
return await this.sendWebhook(testContext, config as WebhookConfig, 10000);
|
||||
default:
|
||||
return { success: false, error: `Unknown platform: ${platform}` };
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
responseTime: Date.now() - startTime,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Singleton instance
|
||||
export const remoteNotificationService = new RemoteNotificationService();
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
import { CoreMemoryStore, SessionCluster, ClusterMember, SessionMetadataCache } from './core-memory-store.js';
|
||||
import { CliHistoryStore } from '../tools/cli-history-store.js';
|
||||
import { UnifiedVectorIndex, isUnifiedEmbedderAvailable } from './unified-vector-index.js';
|
||||
import { StoragePaths } from '../config/storage-paths.js';
|
||||
import { readdirSync, readFileSync, statSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
@@ -21,6 +22,10 @@ const WEIGHTS = {
|
||||
// Clustering threshold (0.4 = moderate similarity required)
|
||||
const CLUSTER_THRESHOLD = 0.4;
|
||||
|
||||
// Incremental clustering frequency control
|
||||
const MIN_CLUSTER_INTERVAL_HOURS = 6;
|
||||
const MIN_NEW_SESSIONS_FOR_CLUSTER = 5;
|
||||
|
||||
export interface ClusteringOptions {
|
||||
scope?: 'all' | 'recent' | 'unclustered';
|
||||
timeRange?: { start: string; end: string };
|
||||
@@ -33,15 +38,29 @@ export interface ClusteringResult {
|
||||
sessionsClustered: number;
|
||||
}
|
||||
|
||||
export interface IncrementalClusterResult {
|
||||
sessionId: string;
|
||||
clusterId: string | null;
|
||||
action: 'joined_existing' | 'created_new' | 'skipped';
|
||||
}
|
||||
|
||||
export class SessionClusteringService {
|
||||
private coreMemoryStore: CoreMemoryStore;
|
||||
private cliHistoryStore: CliHistoryStore;
|
||||
private projectPath: string;
|
||||
private vectorIndex: UnifiedVectorIndex | null = null;
|
||||
/** Cache: sessionId -> list of nearby session source_ids from HNSW search */
|
||||
private vectorNeighborCache: Map<string, Map<string, number>> = new Map();
|
||||
|
||||
constructor(projectPath: string) {
|
||||
this.projectPath = projectPath;
|
||||
this.coreMemoryStore = new CoreMemoryStore(projectPath);
|
||||
this.cliHistoryStore = new CliHistoryStore(projectPath);
|
||||
|
||||
// Initialize vector index if available
|
||||
if (isUnifiedEmbedderAvailable()) {
|
||||
this.vectorIndex = new UnifiedVectorIndex(projectPath);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -331,14 +350,36 @@ export class SessionClusteringService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate vector similarity using pre-computed embeddings from memory_chunks
|
||||
* Returns average cosine similarity of chunk embeddings
|
||||
* Calculate vector similarity using HNSW index when available.
|
||||
* Falls back to direct cosine similarity on pre-computed embeddings from memory_chunks.
|
||||
*
|
||||
* HNSW path: Uses cached neighbor lookup from vectorNeighborCache (populated by
|
||||
* preloadVectorNeighbors). This replaces the O(N) full-table scan with O(1) cache lookup.
|
||||
*
|
||||
* Fallback path: Averages chunk embeddings from SQLite and computes cosine similarity directly.
|
||||
*/
|
||||
private calculateVectorSimilarity(s1: SessionMetadataCache, s2: SessionMetadataCache): number {
|
||||
// HNSW path: check if we have pre-loaded neighbor scores
|
||||
const neighbors1 = this.vectorNeighborCache.get(s1.session_id);
|
||||
if (neighbors1) {
|
||||
const score = neighbors1.get(s2.session_id);
|
||||
if (score !== undefined) return score;
|
||||
// s2 is not a neighbor of s1 via HNSW - low similarity
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Also check reverse direction
|
||||
const neighbors2 = this.vectorNeighborCache.get(s2.session_id);
|
||||
if (neighbors2) {
|
||||
const score = neighbors2.get(s1.session_id);
|
||||
if (score !== undefined) return score;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Fallback: direct cosine similarity on chunk embeddings
|
||||
const embedding1 = this.getSessionEmbedding(s1.session_id);
|
||||
const embedding2 = this.getSessionEmbedding(s2.session_id);
|
||||
|
||||
// Graceful fallback if no embeddings available
|
||||
if (!embedding1 || !embedding2) {
|
||||
return 0;
|
||||
}
|
||||
@@ -346,6 +387,55 @@ export class SessionClusteringService {
|
||||
return this.cosineSimilarity(embedding1, embedding2);
|
||||
}
|
||||
|
||||
/**
|
||||
* Preload vector neighbors for a set of sessions using HNSW search.
|
||||
* For each session, gets its average embedding and searches for nearby chunks,
|
||||
* then aggregates scores by source_id to get session-level similarity scores.
|
||||
*
|
||||
* This replaces the O(N^2) full-table scan with O(N * topK) HNSW lookups.
|
||||
*/
|
||||
async preloadVectorNeighbors(sessionIds: string[], topK: number = 20): Promise<void> {
|
||||
if (!this.vectorIndex) return;
|
||||
|
||||
this.vectorNeighborCache.clear();
|
||||
|
||||
for (const sessionId of sessionIds) {
|
||||
const avgEmbedding = this.getSessionEmbedding(sessionId);
|
||||
if (!avgEmbedding) continue;
|
||||
|
||||
try {
|
||||
const result = await this.vectorIndex.searchByVector(avgEmbedding, {
|
||||
topK,
|
||||
minScore: 0.1,
|
||||
});
|
||||
|
||||
if (!result.success || !result.matches.length) continue;
|
||||
|
||||
// Aggregate scores by source_id (session-level similarity)
|
||||
const neighborScores = new Map<string, number[]>();
|
||||
for (const match of result.matches) {
|
||||
const sourceId = match.source_id;
|
||||
if (sourceId === sessionId) continue; // skip self
|
||||
if (!neighborScores.has(sourceId)) {
|
||||
neighborScores.set(sourceId, []);
|
||||
}
|
||||
neighborScores.get(sourceId)!.push(match.score);
|
||||
}
|
||||
|
||||
// Average scores per neighbor session
|
||||
const avgScores = new Map<string, number>();
|
||||
for (const [neighborId, scores] of neighborScores) {
|
||||
const avg = scores.reduce((sum, s) => sum + s, 0) / scores.length;
|
||||
avgScores.set(neighborId, avg);
|
||||
}
|
||||
|
||||
this.vectorNeighborCache.set(sessionId, avgScores);
|
||||
} catch {
|
||||
// HNSW search failed for this session, skip
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get session embedding by averaging all chunk embeddings
|
||||
*/
|
||||
@@ -494,11 +584,16 @@ export class SessionClusteringService {
|
||||
this.coreMemoryStore.upsertSessionMetadata(session);
|
||||
}
|
||||
|
||||
// 4. Calculate relevance matrix
|
||||
const n = sessions.length;
|
||||
const relevanceMatrix: number[][] = Array(n).fill(0).map(() => Array(n).fill(0));
|
||||
// 4. Preload HNSW vector neighbors for efficient similarity calculation
|
||||
if (this.vectorIndex) {
|
||||
const sessionIds = sessions.map(s => s.session_id);
|
||||
await this.preloadVectorNeighbors(sessionIds);
|
||||
console.log(`[Clustering] Preloaded HNSW vector neighbors for ${sessionIds.length} sessions`);
|
||||
}
|
||||
|
||||
let maxScore = 0;
|
||||
// 5. Calculate relevance matrix
|
||||
const n = sessions.length;
|
||||
const relevanceMatrix: number[][] = Array(n).fill(0).map(() => Array(n).fill(0)); let maxScore = 0;
|
||||
let avgScore = 0;
|
||||
let pairCount = 0;
|
||||
|
||||
@@ -519,7 +614,7 @@ export class SessionClusteringService {
|
||||
console.log(`[Clustering] Relevance stats: max=${maxScore.toFixed(3)}, avg=${avgScore.toFixed(3)}, pairs=${pairCount}, threshold=${CLUSTER_THRESHOLD}`);
|
||||
}
|
||||
|
||||
// 5. Agglomerative clustering
|
||||
// 6. Agglomerative clustering
|
||||
const minClusterSize = options?.minClusterSize || 2;
|
||||
|
||||
// Early return if not enough sessions
|
||||
@@ -531,7 +626,7 @@ export class SessionClusteringService {
|
||||
const newPotentialClusters = this.agglomerativeClustering(sessions, relevanceMatrix, CLUSTER_THRESHOLD);
|
||||
console.log(`[Clustering] Generated ${newPotentialClusters.length} potential clusters`);
|
||||
|
||||
// 6. Process clusters: create new or merge with existing
|
||||
// 7. Process clusters: create new or merge with existing
|
||||
let clustersCreated = 0;
|
||||
let clustersMerged = 0;
|
||||
let sessionsClustered = 0;
|
||||
@@ -716,6 +811,145 @@ export class SessionClusteringService {
|
||||
return { merged, deleted, remaining };
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether clustering should run based on frequency control.
|
||||
* Conditions: last clustering > MIN_CLUSTER_INTERVAL_HOURS ago AND
|
||||
* new unclustered sessions >= MIN_NEW_SESSIONS_FOR_CLUSTER.
|
||||
*
|
||||
* Stores last_cluster_time in session_clusters metadata.
|
||||
*/
|
||||
async shouldRunClustering(): Promise<boolean> {
|
||||
// Check last cluster time from cluster metadata
|
||||
const clusters = this.coreMemoryStore.listClusters('active');
|
||||
let lastClusterTime = 0;
|
||||
|
||||
for (const cluster of clusters) {
|
||||
const createdMs = new Date(cluster.created_at).getTime();
|
||||
if (createdMs > lastClusterTime) {
|
||||
lastClusterTime = createdMs;
|
||||
}
|
||||
const updatedMs = new Date(cluster.updated_at).getTime();
|
||||
if (updatedMs > lastClusterTime) {
|
||||
lastClusterTime = updatedMs;
|
||||
}
|
||||
}
|
||||
|
||||
// Check time interval
|
||||
const now = Date.now();
|
||||
const hoursSinceLastCluster = (now - lastClusterTime) / (1000 * 60 * 60);
|
||||
if (lastClusterTime > 0 && hoursSinceLastCluster < MIN_CLUSTER_INTERVAL_HOURS) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check number of unclustered sessions
|
||||
const allSessions = await this.collectSessions({ scope: 'recent' });
|
||||
const unclusteredCount = allSessions.filter(s => {
|
||||
const sessionClusters = this.coreMemoryStore.getSessionClusters(s.session_id);
|
||||
return sessionClusters.length === 0;
|
||||
}).length;
|
||||
|
||||
return unclusteredCount >= MIN_NEW_SESSIONS_FOR_CLUSTER;
|
||||
}
|
||||
|
||||
/**
|
||||
* Incremental clustering: process only a single new session.
|
||||
*
|
||||
* Computes the new session's similarity against existing cluster centroids
|
||||
* using HNSW search. If similarity >= CLUSTER_THRESHOLD, joins the best
|
||||
* matching cluster. Otherwise, remains unclustered until enough sessions
|
||||
* accumulate for a new cluster.
|
||||
*
|
||||
* @param sessionId - The session to incrementally cluster
|
||||
* @returns Result indicating what action was taken
|
||||
*/
|
||||
async incrementalCluster(sessionId: string): Promise<IncrementalClusterResult> {
|
||||
// Get or create session metadata
|
||||
let sessionMeta = this.coreMemoryStore.getSessionMetadata(sessionId);
|
||||
if (!sessionMeta) {
|
||||
// Try to build metadata from available sources
|
||||
const allSessions = await this.collectSessions({ scope: 'all' });
|
||||
sessionMeta = allSessions.find(s => s.session_id === sessionId) || null;
|
||||
|
||||
if (!sessionMeta) {
|
||||
return { sessionId, clusterId: null, action: 'skipped' };
|
||||
}
|
||||
this.coreMemoryStore.upsertSessionMetadata(sessionMeta);
|
||||
}
|
||||
|
||||
// Check if already clustered
|
||||
const existingClusters = this.coreMemoryStore.getSessionClusters(sessionId);
|
||||
if (existingClusters.length > 0) {
|
||||
return { sessionId, clusterId: existingClusters[0].id, action: 'skipped' };
|
||||
}
|
||||
|
||||
// Get all active clusters and their representative sessions
|
||||
const activeClusters = this.coreMemoryStore.listClusters('active');
|
||||
|
||||
if (activeClusters.length === 0) {
|
||||
return { sessionId, clusterId: null, action: 'skipped' };
|
||||
}
|
||||
|
||||
// Use HNSW to find nearest neighbors for the new session
|
||||
if (this.vectorIndex) {
|
||||
await this.preloadVectorNeighbors([sessionId]);
|
||||
}
|
||||
|
||||
// Calculate similarity against each cluster's member sessions
|
||||
let bestCluster: SessionCluster | null = null;
|
||||
let bestScore = 0;
|
||||
|
||||
for (const cluster of activeClusters) {
|
||||
const members = this.coreMemoryStore.getClusterMembers(cluster.id);
|
||||
if (members.length === 0) continue;
|
||||
|
||||
// Calculate average relevance against cluster members (sample up to 5)
|
||||
const sampleMembers = members.slice(0, 5);
|
||||
let totalScore = 0;
|
||||
let validCount = 0;
|
||||
|
||||
for (const member of sampleMembers) {
|
||||
const memberMeta = this.coreMemoryStore.getSessionMetadata(member.session_id);
|
||||
if (!memberMeta) continue;
|
||||
|
||||
const score = this.calculateRelevance(sessionMeta, memberMeta);
|
||||
totalScore += score;
|
||||
validCount++;
|
||||
}
|
||||
|
||||
if (validCount === 0) continue;
|
||||
|
||||
const avgScore = totalScore / validCount;
|
||||
if (avgScore > bestScore) {
|
||||
bestScore = avgScore;
|
||||
bestCluster = cluster;
|
||||
}
|
||||
}
|
||||
|
||||
// Join best cluster if above threshold
|
||||
if (bestCluster && bestScore >= CLUSTER_THRESHOLD) {
|
||||
const existingMembers = this.coreMemoryStore.getClusterMembers(bestCluster.id);
|
||||
|
||||
this.coreMemoryStore.addClusterMember({
|
||||
cluster_id: bestCluster.id,
|
||||
session_id: sessionId,
|
||||
session_type: sessionMeta.session_type as 'core_memory' | 'workflow' | 'cli_history' | 'native',
|
||||
sequence_order: existingMembers.length + 1,
|
||||
relevance_score: bestScore,
|
||||
});
|
||||
|
||||
// Update cluster description
|
||||
this.coreMemoryStore.updateCluster(bestCluster.id, {
|
||||
description: `Auto-generated cluster with ${existingMembers.length + 1} sessions`
|
||||
});
|
||||
|
||||
console.log(`[Clustering] Session ${sessionId} joined cluster '${bestCluster.name}' (score: ${bestScore.toFixed(3)})`);
|
||||
return { sessionId, clusterId: bestCluster.id, action: 'joined_existing' };
|
||||
}
|
||||
|
||||
// Not similar enough to any existing cluster
|
||||
return { sessionId, clusterId: null, action: 'skipped' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Agglomerative clustering algorithm
|
||||
* Returns array of clusters (each cluster is array of sessions)
|
||||
|
||||
410
ccw/src/core/unified-context-builder.ts
Normal file
410
ccw/src/core/unified-context-builder.ts
Normal file
@@ -0,0 +1,410 @@
|
||||
/**
|
||||
* UnifiedContextBuilder - Assembles context for Claude Code hooks
|
||||
*
|
||||
* Provides componentized context assembly for:
|
||||
* - session-start: MEMORY.md summary + cluster overview + hot entities + solidified patterns
|
||||
* - per-prompt: vector search + intent matching across all categories
|
||||
* - session-end: incremental embedding + clustering + heat score update tasks
|
||||
*
|
||||
* Character limits:
|
||||
* - session-start: <= 1000 chars
|
||||
* - per-prompt: <= 500 chars
|
||||
*/
|
||||
|
||||
import { existsSync, readdirSync } from 'fs';
|
||||
import { join, basename } from 'path';
|
||||
import { getProjectPaths } from '../config/storage-paths.js';
|
||||
import { getMemoryMdContent } from './memory-consolidation-pipeline.js';
|
||||
import { getMemoryStore } from './memory-store.js';
|
||||
import type { HotEntity } from './memory-store.js';
|
||||
import {
|
||||
UnifiedVectorIndex,
|
||||
isUnifiedEmbedderAvailable,
|
||||
} from './unified-vector-index.js';
|
||||
import type { VectorSearchMatch } from './unified-vector-index.js';
|
||||
import { SessionClusteringService } from './session-clustering-service.js';
|
||||
|
||||
// =============================================================================
|
||||
// Constants
|
||||
// =============================================================================
|
||||
|
||||
/** Maximum character count for session-start context */
|
||||
const SESSION_START_LIMIT = 1000;
|
||||
|
||||
/** Maximum character count for per-prompt context */
|
||||
const PER_PROMPT_LIMIT = 500;
|
||||
|
||||
/** Maximum characters for the MEMORY.md summary component */
|
||||
const MEMORY_SUMMARY_LIMIT = 500;
|
||||
|
||||
/** Number of top clusters to show in overview */
|
||||
const TOP_CLUSTERS = 3;
|
||||
|
||||
/** Number of top hot entities to show */
|
||||
const TOP_HOT_ENTITIES = 5;
|
||||
|
||||
/** Days to look back for hot entities */
|
||||
const HOT_ENTITY_DAYS = 7;
|
||||
|
||||
/** Number of vector search results for per-prompt */
|
||||
const VECTOR_TOP_K = 8;
|
||||
|
||||
/** Minimum vector similarity score */
|
||||
const VECTOR_MIN_SCORE = 0.3;
|
||||
|
||||
// =============================================================================
|
||||
// Types
|
||||
// =============================================================================
|
||||
|
||||
/** A task to be executed asynchronously at session-end */
|
||||
export interface SessionEndTask {
|
||||
/** Descriptive name of the task */
|
||||
name: string;
|
||||
/** Async function to execute */
|
||||
execute: () => Promise<void>;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// UnifiedContextBuilder
|
||||
// =============================================================================
|
||||
|
||||
export class UnifiedContextBuilder {
|
||||
private projectPath: string;
|
||||
private paths: ReturnType<typeof getProjectPaths>;
|
||||
|
||||
constructor(projectPath: string) {
|
||||
this.projectPath = projectPath;
|
||||
this.paths = getProjectPaths(projectPath);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public: session-start context
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Build context for session-start hook injection.
|
||||
*
|
||||
* Components (assembled in order, truncated to <= 1000 chars total):
|
||||
* 1. MEMORY.md summary (up to 500 chars)
|
||||
* 2. Cluster overview (top 3 active clusters)
|
||||
* 3. Hot entities (top 5 within last 7 days)
|
||||
* 4. Solidified patterns (skills/*.md file list)
|
||||
*/
|
||||
async buildSessionStartContext(): Promise<string> {
|
||||
const sections: string[] = [];
|
||||
|
||||
// Component 1: MEMORY.md summary
|
||||
const memorySummary = this.buildMemorySummary();
|
||||
if (memorySummary) {
|
||||
sections.push(memorySummary);
|
||||
}
|
||||
|
||||
// Component 2: Cluster overview
|
||||
const clusterOverview = await this.buildClusterOverview();
|
||||
if (clusterOverview) {
|
||||
sections.push(clusterOverview);
|
||||
}
|
||||
|
||||
// Component 3: Hot entities
|
||||
const hotEntities = this.buildHotEntities();
|
||||
if (hotEntities) {
|
||||
sections.push(hotEntities);
|
||||
}
|
||||
|
||||
// Component 4: Solidified patterns
|
||||
const patterns = this.buildSolidifiedPatterns();
|
||||
if (patterns) {
|
||||
sections.push(patterns);
|
||||
}
|
||||
|
||||
if (sections.length === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
// Assemble and truncate
|
||||
let content = '<ccw-memory-context>\n' + sections.join('\n') + '\n</ccw-memory-context>';
|
||||
|
||||
if (content.length > SESSION_START_LIMIT) {
|
||||
content = content.substring(0, SESSION_START_LIMIT - 20) + '\n</ccw-memory-context>';
|
||||
}
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public: per-prompt context
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Build context for per-prompt hook injection.
|
||||
*
|
||||
* Uses vector search across all categories to find relevant memories
|
||||
* matching the current prompt. Results are ranked by similarity score.
|
||||
*
|
||||
* @param prompt - Current user prompt text
|
||||
* @returns Context string (<= 500 chars) or empty string
|
||||
*/
|
||||
async buildPromptContext(prompt: string): Promise<string> {
|
||||
if (!prompt || !prompt.trim()) {
|
||||
return '';
|
||||
}
|
||||
|
||||
if (!isUnifiedEmbedderAvailable()) {
|
||||
return '';
|
||||
}
|
||||
|
||||
try {
|
||||
const vectorIndex = new UnifiedVectorIndex(this.projectPath);
|
||||
const result = await vectorIndex.search(prompt, {
|
||||
topK: VECTOR_TOP_K,
|
||||
minScore: VECTOR_MIN_SCORE,
|
||||
});
|
||||
|
||||
if (!result.success || result.matches.length === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
return this.formatPromptMatches(result.matches);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public: session-end tasks
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Build a list of async tasks to run at session-end.
|
||||
*
|
||||
* Tasks:
|
||||
* 1. Incremental vector embedding (index new/updated content)
|
||||
* 2. Incremental clustering (cluster unclustered sessions)
|
||||
* 3. Heat score updates (recalculate entity heat scores)
|
||||
*
|
||||
* @param sessionId - Current session ID for context
|
||||
* @returns Array of tasks with name and execute function
|
||||
*/
|
||||
buildSessionEndTasks(sessionId: string): SessionEndTask[] {
|
||||
const tasks: SessionEndTask[] = [];
|
||||
|
||||
// Task 1: Incremental vector embedding
|
||||
if (isUnifiedEmbedderAvailable()) {
|
||||
tasks.push({
|
||||
name: 'incremental-embedding',
|
||||
execute: async () => {
|
||||
try {
|
||||
const vectorIndex = new UnifiedVectorIndex(this.projectPath);
|
||||
// Re-index the MEMORY.md content if available
|
||||
const memoryContent = getMemoryMdContent(this.projectPath);
|
||||
if (memoryContent) {
|
||||
await vectorIndex.indexContent(memoryContent, {
|
||||
source_id: 'MEMORY_MD',
|
||||
source_type: 'core_memory',
|
||||
category: 'core_memory',
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
// Log but don't throw - session-end tasks are best-effort
|
||||
if (process.env.DEBUG) {
|
||||
console.error('[UnifiedContextBuilder] Embedding task failed:', (err as Error).message);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Task 2: Incremental clustering
|
||||
tasks.push({
|
||||
name: 'incremental-clustering',
|
||||
execute: async () => {
|
||||
try {
|
||||
const clusteringService = new SessionClusteringService(this.projectPath);
|
||||
await clusteringService.autocluster({ scope: 'unclustered' });
|
||||
} catch (err) {
|
||||
if (process.env.DEBUG) {
|
||||
console.error('[UnifiedContextBuilder] Clustering task failed:', (err as Error).message);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// Task 3: Heat score updates
|
||||
tasks.push({
|
||||
name: 'heat-score-update',
|
||||
execute: async () => {
|
||||
try {
|
||||
const memoryStore = getMemoryStore(this.projectPath);
|
||||
const hotEntities = memoryStore.getHotEntities(50);
|
||||
for (const entity of hotEntities) {
|
||||
if (entity.id != null) {
|
||||
memoryStore.calculateHeatScore(entity.id);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (process.env.DEBUG) {
|
||||
console.error('[UnifiedContextBuilder] Heat score update failed:', (err as Error).message);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
return tasks;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Private: Component builders
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Build MEMORY.md summary component.
|
||||
* Reads MEMORY.md and returns first MEMORY_SUMMARY_LIMIT characters.
|
||||
*/
|
||||
private buildMemorySummary(): string {
|
||||
const content = getMemoryMdContent(this.projectPath);
|
||||
if (!content) {
|
||||
return '';
|
||||
}
|
||||
|
||||
let summary = content.trim();
|
||||
if (summary.length > MEMORY_SUMMARY_LIMIT) {
|
||||
// Truncate at a newline boundary if possible
|
||||
const truncated = summary.substring(0, MEMORY_SUMMARY_LIMIT);
|
||||
const lastNewline = truncated.lastIndexOf('\n');
|
||||
summary = lastNewline > MEMORY_SUMMARY_LIMIT * 0.6
|
||||
? truncated.substring(0, lastNewline) + '...'
|
||||
: truncated + '...';
|
||||
}
|
||||
|
||||
return `## Memory Summary\n${summary}\n`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build cluster overview component.
|
||||
* Shows top N active clusters from the clustering service.
|
||||
*/
|
||||
private async buildClusterOverview(): Promise<string> {
|
||||
try {
|
||||
const { CoreMemoryStore } = await import('./core-memory-store.js');
|
||||
const store = new CoreMemoryStore(this.projectPath);
|
||||
const clusters = store.listClusters('active');
|
||||
|
||||
if (clusters.length === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
// Sort by most recent activity
|
||||
const sorted = clusters
|
||||
.map(c => {
|
||||
const members = store.getClusterMembers(c.id);
|
||||
return { cluster: c, memberCount: members.length };
|
||||
})
|
||||
.sort((a, b) => b.memberCount - a.memberCount)
|
||||
.slice(0, TOP_CLUSTERS);
|
||||
|
||||
let output = '## Active Clusters\n';
|
||||
for (const { cluster, memberCount } of sorted) {
|
||||
const intent = cluster.intent ? ` - ${cluster.intent}` : '';
|
||||
output += `- **${cluster.name}** (${memberCount})${intent}\n`;
|
||||
}
|
||||
|
||||
return output;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build hot entities component.
|
||||
* Shows top N entities by heat_score that were active within last 7 days.
|
||||
*/
|
||||
private buildHotEntities(): string {
|
||||
try {
|
||||
const memoryStore = getMemoryStore(this.projectPath);
|
||||
const allHot = memoryStore.getHotEntities(TOP_HOT_ENTITIES * 3);
|
||||
|
||||
if (allHot.length === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
// Filter to entities seen within the last HOT_ENTITY_DAYS days
|
||||
const cutoff = new Date();
|
||||
cutoff.setDate(cutoff.getDate() - HOT_ENTITY_DAYS);
|
||||
const cutoffStr = cutoff.toISOString();
|
||||
|
||||
const recentHot = allHot
|
||||
.filter(e => (e.last_seen_at || '') >= cutoffStr)
|
||||
.slice(0, TOP_HOT_ENTITIES);
|
||||
|
||||
if (recentHot.length === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
let output = '## Hot Entities (7d)\n';
|
||||
for (const entity of recentHot) {
|
||||
const heat = Math.round(entity.stats.heat_score);
|
||||
output += `- ${entity.type}:${entity.value} (heat:${heat})\n`;
|
||||
}
|
||||
|
||||
return output;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build solidified patterns component.
|
||||
* Scans skills/*.md files and lists their names.
|
||||
*/
|
||||
private buildSolidifiedPatterns(): string {
|
||||
try {
|
||||
const skillsDir = this.paths.memoryV2.skills;
|
||||
if (!existsSync(skillsDir)) {
|
||||
return '';
|
||||
}
|
||||
|
||||
const files = readdirSync(skillsDir).filter(f => f.endsWith('.md'));
|
||||
if (files.length === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
let output = '## Patterns\n';
|
||||
for (const file of files.slice(0, 5)) {
|
||||
const name = basename(file, '.md');
|
||||
output += `- ${name}\n`;
|
||||
}
|
||||
|
||||
return output;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Private: Formatting helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Format vector search matches for per-prompt context.
|
||||
* Builds a compact Markdown snippet within PER_PROMPT_LIMIT chars.
|
||||
*/
|
||||
private formatPromptMatches(matches: VectorSearchMatch[]): string {
|
||||
let output = '<ccw-related-memory>\n';
|
||||
|
||||
for (const match of matches) {
|
||||
const score = Math.round(match.score * 100);
|
||||
const snippet = match.content.substring(0, 80).replace(/\n/g, ' ').trim();
|
||||
const line = `- [${match.category}] ${snippet} (${score}%)\n`;
|
||||
|
||||
// Check if adding this line would exceed limit
|
||||
if (output.length + line.length + 25 > PER_PROMPT_LIMIT) {
|
||||
break;
|
||||
}
|
||||
output += line;
|
||||
}
|
||||
|
||||
output += '</ccw-related-memory>';
|
||||
|
||||
return output;
|
||||
}
|
||||
}
|
||||
488
ccw/src/core/unified-memory-service.ts
Normal file
488
ccw/src/core/unified-memory-service.ts
Normal file
@@ -0,0 +1,488 @@
|
||||
/**
|
||||
* Unified Memory Service - Cross-store search with RRF fusion
|
||||
*
|
||||
* Provides a single search() interface that combines:
|
||||
* - Vector search (HNSW via UnifiedVectorIndex)
|
||||
* - Full-text search (FTS5 via MemoryStore.searchPrompts)
|
||||
* - Heat-based scoring (entity heat from MemoryStore)
|
||||
*
|
||||
* Fusion: Reciprocal Rank Fusion (RRF)
|
||||
* score = sum(1 / (k + rank_i) * weight_i)
|
||||
* k = 60, weights = { vector: 0.6, fts: 0.3, heat: 0.1 }
|
||||
*/
|
||||
|
||||
import { UnifiedVectorIndex, isUnifiedEmbedderAvailable } from './unified-vector-index.js';
|
||||
import type {
|
||||
VectorCategory,
|
||||
VectorSearchMatch,
|
||||
VectorIndexStatus,
|
||||
} from './unified-vector-index.js';
|
||||
import { CoreMemoryStore, getCoreMemoryStore } from './core-memory-store.js';
|
||||
import type { CoreMemory } from './core-memory-store.js';
|
||||
import { MemoryStore, getMemoryStore } from './memory-store.js';
|
||||
import type { PromptHistory, HotEntity } from './memory-store.js';
|
||||
|
||||
// =============================================================================
|
||||
// Types
|
||||
// =============================================================================
|
||||
|
||||
/** Options for unified search */
|
||||
export interface UnifiedSearchOptions {
|
||||
/** Maximum number of results to return (default: 20) */
|
||||
limit?: number;
|
||||
/** Minimum relevance score threshold (default: 0.0) */
|
||||
minScore?: number;
|
||||
/** Filter by category */
|
||||
category?: VectorCategory;
|
||||
/** Vector search top-k (default: 30, fetched internally for fusion) */
|
||||
vectorTopK?: number;
|
||||
/** FTS search limit (default: 30, fetched internally for fusion) */
|
||||
ftsLimit?: number;
|
||||
}
|
||||
|
||||
/** A unified search result item */
|
||||
export interface UnifiedSearchResult {
|
||||
/** Unique identifier for the source item */
|
||||
source_id: string;
|
||||
/** Source type: core_memory, cli_history, workflow, entity, pattern */
|
||||
source_type: string;
|
||||
/** Fused relevance score (0..1 range, higher is better) */
|
||||
score: number;
|
||||
/** Text content (snippet or full) */
|
||||
content: string;
|
||||
/** Category of the result */
|
||||
category: string;
|
||||
/** Which ranking sources contributed to this result */
|
||||
rank_sources: {
|
||||
vector_rank?: number;
|
||||
vector_score?: number;
|
||||
fts_rank?: number;
|
||||
heat_score?: number;
|
||||
};
|
||||
}
|
||||
|
||||
/** Aggregated statistics from all stores + vector index */
|
||||
export interface UnifiedMemoryStats {
|
||||
core_memories: {
|
||||
total: number;
|
||||
archived: number;
|
||||
};
|
||||
stage1_outputs: number;
|
||||
entities: number;
|
||||
prompts: number;
|
||||
conversations: number;
|
||||
vector_index: {
|
||||
available: boolean;
|
||||
total_chunks: number;
|
||||
hnsw_available: boolean;
|
||||
hnsw_count: number;
|
||||
dimension: number;
|
||||
categories?: Record<string, number>;
|
||||
};
|
||||
}
|
||||
|
||||
/** KNN recommendation result */
|
||||
export interface RecommendationResult {
|
||||
source_id: string;
|
||||
source_type: string;
|
||||
score: number;
|
||||
content: string;
|
||||
category: string;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// RRF Constants
|
||||
// =============================================================================
|
||||
|
||||
/** RRF smoothing constant (standard value from the original RRF paper) */
|
||||
const RRF_K = 60;
|
||||
|
||||
/** Fusion weights */
|
||||
const WEIGHT_VECTOR = 0.6;
|
||||
const WEIGHT_FTS = 0.3;
|
||||
const WEIGHT_HEAT = 0.1;
|
||||
|
||||
// =============================================================================
|
||||
// UnifiedMemoryService
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Unified Memory Service providing cross-store search and recommendations.
|
||||
*
|
||||
* Combines vector similarity, full-text search, and entity heat scores
|
||||
* using Reciprocal Rank Fusion (RRF) for result ranking.
|
||||
*/
|
||||
export class UnifiedMemoryService {
|
||||
private projectPath: string;
|
||||
private vectorIndex: UnifiedVectorIndex | null = null;
|
||||
private coreMemoryStore: CoreMemoryStore;
|
||||
private memoryStore: MemoryStore;
|
||||
|
||||
constructor(projectPath: string) {
|
||||
this.projectPath = projectPath;
|
||||
this.coreMemoryStore = getCoreMemoryStore(projectPath);
|
||||
this.memoryStore = getMemoryStore(projectPath);
|
||||
|
||||
if (isUnifiedEmbedderAvailable()) {
|
||||
this.vectorIndex = new UnifiedVectorIndex(projectPath);
|
||||
}
|
||||
}
|
||||
|
||||
// ==========================================================================
|
||||
// Search
|
||||
// ==========================================================================
|
||||
|
||||
/**
|
||||
* Unified search across all memory stores.
|
||||
*
|
||||
* Pipeline:
|
||||
* 1. Vector search via UnifiedVectorIndex (semantic similarity)
|
||||
* 2. FTS5 search via MemoryStore.searchPrompts (keyword matching)
|
||||
* 3. Heat boost via entity heat scores
|
||||
* 4. RRF fusion to combine ranked lists
|
||||
*
|
||||
* @param query - Natural language search query
|
||||
* @param options - Search options
|
||||
* @returns Fused search results sorted by relevance
|
||||
*/
|
||||
async search(
|
||||
query: string,
|
||||
options: UnifiedSearchOptions = {}
|
||||
): Promise<UnifiedSearchResult[]> {
|
||||
const {
|
||||
limit = 20,
|
||||
minScore = 0.0,
|
||||
category,
|
||||
vectorTopK = 30,
|
||||
ftsLimit = 30,
|
||||
} = options;
|
||||
|
||||
// Run vector search and FTS search in parallel
|
||||
const [vectorResults, ftsResults, hotEntities] = await Promise.all([
|
||||
this.runVectorSearch(query, vectorTopK, category),
|
||||
this.runFtsSearch(query, ftsLimit),
|
||||
this.getHeatScores(),
|
||||
]);
|
||||
|
||||
// Build heat score lookup
|
||||
const heatMap = new Map<string, number>();
|
||||
for (const entity of hotEntities) {
|
||||
// Use normalized_value as key for heat lookup
|
||||
heatMap.set(entity.normalized_value, entity.stats.heat_score);
|
||||
}
|
||||
|
||||
// Collect all unique source_ids from both result sets
|
||||
const allSourceIds = new Set<string>();
|
||||
const vectorRankMap = new Map<string, { rank: number; score: number; match: VectorSearchMatch }>();
|
||||
const ftsRankMap = new Map<string, { rank: number; item: PromptHistory }>();
|
||||
|
||||
// Build vector rank map
|
||||
for (let i = 0; i < vectorResults.length; i++) {
|
||||
const match = vectorResults[i];
|
||||
const id = match.source_id;
|
||||
allSourceIds.add(id);
|
||||
vectorRankMap.set(id, { rank: i + 1, score: match.score, match });
|
||||
}
|
||||
|
||||
// Build FTS rank map
|
||||
for (let i = 0; i < ftsResults.length; i++) {
|
||||
const item = ftsResults[i];
|
||||
const id = item.session_id;
|
||||
allSourceIds.add(id);
|
||||
ftsRankMap.set(id, { rank: i + 1, item });
|
||||
}
|
||||
|
||||
// Calculate RRF score for each unique source_id
|
||||
const results: UnifiedSearchResult[] = [];
|
||||
|
||||
for (const sourceId of allSourceIds) {
|
||||
const vectorEntry = vectorRankMap.get(sourceId);
|
||||
const ftsEntry = ftsRankMap.get(sourceId);
|
||||
|
||||
// RRF: score = sum(weight_i / (k + rank_i))
|
||||
let rrfScore = 0;
|
||||
const rankSources: UnifiedSearchResult['rank_sources'] = {};
|
||||
|
||||
// Vector component
|
||||
if (vectorEntry) {
|
||||
rrfScore += WEIGHT_VECTOR / (RRF_K + vectorEntry.rank);
|
||||
rankSources.vector_rank = vectorEntry.rank;
|
||||
rankSources.vector_score = vectorEntry.score;
|
||||
}
|
||||
|
||||
// FTS component
|
||||
if (ftsEntry) {
|
||||
rrfScore += WEIGHT_FTS / (RRF_K + ftsEntry.rank);
|
||||
rankSources.fts_rank = ftsEntry.rank;
|
||||
}
|
||||
|
||||
// Heat component (boost based on entity heat)
|
||||
const heatScore = this.lookupHeatScore(sourceId, heatMap);
|
||||
if (heatScore > 0) {
|
||||
// Normalize heat score to a rank-like value (1 = hottest)
|
||||
// Use inverse: higher heat = lower rank number = higher contribution
|
||||
const heatRank = Math.max(1, Math.ceil(100 / (1 + heatScore)));
|
||||
rrfScore += WEIGHT_HEAT / (RRF_K + heatRank);
|
||||
rankSources.heat_score = heatScore;
|
||||
}
|
||||
|
||||
if (rrfScore < minScore) continue;
|
||||
|
||||
// Build result entry
|
||||
let content = '';
|
||||
let sourceType = '';
|
||||
let resultCategory = '';
|
||||
|
||||
if (vectorEntry) {
|
||||
content = vectorEntry.match.content;
|
||||
sourceType = vectorEntry.match.source_type;
|
||||
resultCategory = vectorEntry.match.category;
|
||||
} else if (ftsEntry) {
|
||||
content = ftsEntry.item.prompt_text || ftsEntry.item.context_summary || '';
|
||||
sourceType = 'cli_history';
|
||||
resultCategory = 'cli_history';
|
||||
}
|
||||
|
||||
results.push({
|
||||
source_id: sourceId,
|
||||
source_type: sourceType,
|
||||
score: rrfScore,
|
||||
content,
|
||||
category: resultCategory,
|
||||
rank_sources: rankSources,
|
||||
});
|
||||
}
|
||||
|
||||
// Sort by RRF score descending, take top `limit`
|
||||
results.sort((a, b) => b.score - a.score);
|
||||
return results.slice(0, limit);
|
||||
}
|
||||
|
||||
// ==========================================================================
|
||||
// Recommendations
|
||||
// ==========================================================================
|
||||
|
||||
/**
|
||||
* Get recommendations based on a memory's vector neighbors (KNN).
|
||||
*
|
||||
* Fetches the content of the given memory, then runs a vector search
|
||||
* to find similar content across all stores.
|
||||
*
|
||||
* @param memoryId - Core memory ID (CMEM-*)
|
||||
* @param limit - Number of recommendations (default: 5)
|
||||
* @returns Recommended items sorted by similarity
|
||||
*/
|
||||
async getRecommendations(
|
||||
memoryId: string,
|
||||
limit: number = 5
|
||||
): Promise<RecommendationResult[]> {
|
||||
// Get the memory content
|
||||
const memory = this.coreMemoryStore.getMemory(memoryId);
|
||||
if (!memory) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!this.vectorIndex) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Use memory content as query for KNN search
|
||||
// Request extra results so we can filter out self
|
||||
const searchResult = await this.vectorIndex.search(memory.content, {
|
||||
topK: limit + 5,
|
||||
minScore: 0.3,
|
||||
});
|
||||
|
||||
if (!searchResult.success) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Filter out self and map to recommendations
|
||||
const recommendations: RecommendationResult[] = [];
|
||||
for (const match of searchResult.matches) {
|
||||
// Skip the source memory itself
|
||||
if (match.source_id === memoryId) continue;
|
||||
|
||||
recommendations.push({
|
||||
source_id: match.source_id,
|
||||
source_type: match.source_type,
|
||||
score: match.score,
|
||||
content: match.content,
|
||||
category: match.category,
|
||||
});
|
||||
|
||||
if (recommendations.length >= limit) break;
|
||||
}
|
||||
|
||||
return recommendations;
|
||||
}
|
||||
|
||||
// ==========================================================================
|
||||
// Statistics
|
||||
// ==========================================================================
|
||||
|
||||
/**
|
||||
* Get aggregated statistics from all stores and the vector index.
|
||||
*
|
||||
* @returns Unified stats across core memories, V2 outputs, entities, prompts, and vectors
|
||||
*/
|
||||
async getStats(): Promise<UnifiedMemoryStats> {
|
||||
// Get core memory stats
|
||||
const allMemories = this.coreMemoryStore.getMemories({ limit: 100000 });
|
||||
const archivedMemories = allMemories.filter(m => m.archived);
|
||||
const stage1Count = this.coreMemoryStore.countStage1Outputs();
|
||||
|
||||
// Get memory store stats (entities, prompts, conversations)
|
||||
const db = (this.memoryStore as any).db;
|
||||
let entityCount = 0;
|
||||
let promptCount = 0;
|
||||
let conversationCount = 0;
|
||||
|
||||
try {
|
||||
entityCount = (db.prepare('SELECT COUNT(*) as count FROM entities').get() as { count: number }).count;
|
||||
} catch { /* table may not exist */ }
|
||||
|
||||
try {
|
||||
promptCount = (db.prepare('SELECT COUNT(*) as count FROM prompt_history').get() as { count: number }).count;
|
||||
} catch { /* table may not exist */ }
|
||||
|
||||
try {
|
||||
conversationCount = (db.prepare('SELECT COUNT(*) as count FROM conversations').get() as { count: number }).count;
|
||||
} catch { /* table may not exist */ }
|
||||
|
||||
// Get vector index status
|
||||
let vectorStatus: VectorIndexStatus = {
|
||||
success: false,
|
||||
total_chunks: 0,
|
||||
hnsw_available: false,
|
||||
hnsw_count: 0,
|
||||
dimension: 0,
|
||||
};
|
||||
|
||||
if (this.vectorIndex) {
|
||||
try {
|
||||
vectorStatus = await this.vectorIndex.getStatus();
|
||||
} catch {
|
||||
// Vector index not available
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
core_memories: {
|
||||
total: allMemories.length,
|
||||
archived: archivedMemories.length,
|
||||
},
|
||||
stage1_outputs: stage1Count,
|
||||
entities: entityCount,
|
||||
prompts: promptCount,
|
||||
conversations: conversationCount,
|
||||
vector_index: {
|
||||
available: vectorStatus.success,
|
||||
total_chunks: vectorStatus.total_chunks,
|
||||
hnsw_available: vectorStatus.hnsw_available,
|
||||
hnsw_count: vectorStatus.hnsw_count,
|
||||
dimension: vectorStatus.dimension,
|
||||
categories: vectorStatus.categories,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ==========================================================================
|
||||
// Internal helpers
|
||||
// ==========================================================================
|
||||
|
||||
/**
|
||||
* Run vector search via UnifiedVectorIndex.
|
||||
* Returns empty array if vector index is not available.
|
||||
*/
|
||||
private async runVectorSearch(
|
||||
query: string,
|
||||
topK: number,
|
||||
category?: VectorCategory
|
||||
): Promise<VectorSearchMatch[]> {
|
||||
if (!this.vectorIndex) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await this.vectorIndex.search(query, {
|
||||
topK,
|
||||
minScore: 0.1,
|
||||
category,
|
||||
});
|
||||
|
||||
if (!result.success) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return result.matches;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run FTS5 full-text search via MemoryStore.searchPrompts.
|
||||
* Returns empty array on error.
|
||||
*/
|
||||
private async runFtsSearch(
|
||||
query: string,
|
||||
limit: number
|
||||
): Promise<PromptHistory[]> {
|
||||
try {
|
||||
// FTS5 requires sanitized query (no special characters)
|
||||
const sanitized = this.sanitizeFtsQuery(query);
|
||||
if (!sanitized) return [];
|
||||
|
||||
return this.memoryStore.searchPrompts(sanitized, limit);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get hot entities for heat-based scoring.
|
||||
*/
|
||||
private async getHeatScores(): Promise<HotEntity[]> {
|
||||
try {
|
||||
return this.memoryStore.getHotEntities(50);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up heat score for a source ID.
|
||||
* Checks if any entity's normalized_value matches the source_id.
|
||||
*/
|
||||
private lookupHeatScore(
|
||||
sourceId: string,
|
||||
heatMap: Map<string, number>
|
||||
): number {
|
||||
// Direct match
|
||||
if (heatMap.has(sourceId)) {
|
||||
return heatMap.get(sourceId)!;
|
||||
}
|
||||
|
||||
// Check if source_id is a substring of any entity value (file paths)
|
||||
for (const [key, score] of heatMap) {
|
||||
if (sourceId.includes(key) || key.includes(sourceId)) {
|
||||
return score;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize a query string for FTS5 MATCH syntax.
|
||||
* Removes special characters that would cause FTS5 parse errors.
|
||||
*/
|
||||
private sanitizeFtsQuery(query: string): string {
|
||||
// Remove FTS5 special operators and punctuation
|
||||
return query
|
||||
.replace(/[*":(){}[\]^~\\/<>!@#$%&=+|;,.'`]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
}
|
||||
474
ccw/src/core/unified-vector-index.ts
Normal file
474
ccw/src/core/unified-vector-index.ts
Normal file
@@ -0,0 +1,474 @@
|
||||
/**
|
||||
* Unified Vector Index - TypeScript bridge to unified_memory_embedder.py
|
||||
*
|
||||
* Provides HNSW-backed vector indexing and search for all memory content
|
||||
* (core_memory, cli_history, workflow, entity, pattern) via CodexLens VectorStore.
|
||||
*
|
||||
* Features:
|
||||
* - JSON stdin/stdout protocol to Python embedder
|
||||
* - Content chunking (paragraph -> sentence splitting, CHUNK_SIZE=1500, OVERLAP=200)
|
||||
* - Batch embedding via CodexLens EmbedderFactory
|
||||
* - HNSW approximate nearest neighbor search (sub-10ms for 1000 chunks)
|
||||
* - Category-based filtering
|
||||
*/
|
||||
|
||||
import { spawn } from 'child_process';
|
||||
import { join, dirname } from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { getCodexLensPython } from '../utils/codexlens-path.js';
|
||||
import { StoragePaths, ensureStorageDir } from '../config/storage-paths.js';
|
||||
|
||||
// Get directory of this module
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// Venv python path (reuse CodexLens venv)
|
||||
const VENV_PYTHON = getCodexLensPython();
|
||||
|
||||
// Script path
|
||||
const EMBEDDER_SCRIPT = join(__dirname, '..', '..', 'scripts', 'unified_memory_embedder.py');
|
||||
|
||||
// Chunking constants (match existing core-memory-store.ts)
|
||||
const CHUNK_SIZE = 1500;
|
||||
const OVERLAP = 200;
|
||||
|
||||
// =============================================================================
|
||||
// Types
|
||||
// =============================================================================
|
||||
|
||||
/** Valid source types for vector content */
|
||||
export type SourceType = 'core_memory' | 'workflow' | 'cli_history';
|
||||
|
||||
/** Valid category values for vector filtering */
|
||||
export type VectorCategory = 'core_memory' | 'cli_history' | 'workflow' | 'entity' | 'pattern';
|
||||
|
||||
/** Metadata attached to each chunk in the vector store */
|
||||
export interface ChunkMetadata {
|
||||
/** Source identifier (e.g., memory ID, session ID) */
|
||||
source_id: string;
|
||||
/** Source type */
|
||||
source_type: SourceType;
|
||||
/** Category for filtering */
|
||||
category: VectorCategory;
|
||||
/** Chunk index within the source */
|
||||
chunk_index?: number;
|
||||
/** Additional metadata */
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/** A chunk to be embedded and indexed */
|
||||
export interface VectorChunk {
|
||||
/** Text content */
|
||||
content: string;
|
||||
/** Source identifier */
|
||||
source_id: string;
|
||||
/** Source type */
|
||||
source_type: SourceType;
|
||||
/** Category for filtering */
|
||||
category: VectorCategory;
|
||||
/** Chunk index */
|
||||
chunk_index: number;
|
||||
/** Additional metadata */
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** Result of an embed operation */
|
||||
export interface EmbedResult {
|
||||
success: boolean;
|
||||
chunks_processed: number;
|
||||
chunks_failed: number;
|
||||
elapsed_time: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/** A single search match */
|
||||
export interface VectorSearchMatch {
|
||||
content: string;
|
||||
score: number;
|
||||
source_id: string;
|
||||
source_type: string;
|
||||
chunk_index: number;
|
||||
category: string;
|
||||
metadata: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** Result of a search operation */
|
||||
export interface VectorSearchResult {
|
||||
success: boolean;
|
||||
matches: VectorSearchMatch[];
|
||||
elapsed_time?: number;
|
||||
total_searched?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/** Search options */
|
||||
export interface VectorSearchOptions {
|
||||
topK?: number;
|
||||
minScore?: number;
|
||||
category?: VectorCategory;
|
||||
}
|
||||
|
||||
/** Index status information */
|
||||
export interface VectorIndexStatus {
|
||||
success: boolean;
|
||||
total_chunks: number;
|
||||
hnsw_available: boolean;
|
||||
hnsw_count: number;
|
||||
dimension: number;
|
||||
categories?: Record<string, number>;
|
||||
model_config?: {
|
||||
backend: string;
|
||||
profile: string;
|
||||
dimension: number;
|
||||
max_tokens: number;
|
||||
};
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/** Reindex result */
|
||||
export interface ReindexResult {
|
||||
success: boolean;
|
||||
hnsw_count?: number;
|
||||
elapsed_time?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Python Bridge
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Check if the unified embedder is available (venv and script exist)
|
||||
*/
|
||||
export function isUnifiedEmbedderAvailable(): boolean {
|
||||
if (!existsSync(VENV_PYTHON)) {
|
||||
return false;
|
||||
}
|
||||
if (!existsSync(EMBEDDER_SCRIPT)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run Python script with JSON stdin/stdout protocol.
|
||||
*
|
||||
* @param request - JSON request object to send via stdin
|
||||
* @param timeout - Timeout in milliseconds (default: 5 minutes)
|
||||
* @returns Parsed JSON response
|
||||
*/
|
||||
function runPython<T>(request: Record<string, unknown>, timeout: number = 300000): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!isUnifiedEmbedderAvailable()) {
|
||||
reject(
|
||||
new Error(
|
||||
'Unified embedder not available. Ensure CodexLens venv exists at ~/.codexlens/venv'
|
||||
)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const child = spawn(VENV_PYTHON, [EMBEDDER_SCRIPT], {
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
timeout,
|
||||
});
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
|
||||
child.stdout.on('data', (data) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
|
||||
child.stderr.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
child.on('close', (code) => {
|
||||
if (code === 0 && stdout.trim()) {
|
||||
try {
|
||||
resolve(JSON.parse(stdout.trim()) as T);
|
||||
} catch {
|
||||
reject(new Error(`Failed to parse Python output: ${stdout.substring(0, 500)}`));
|
||||
}
|
||||
} else {
|
||||
reject(new Error(`Python script failed (exit code ${code}): ${stderr || stdout}`));
|
||||
}
|
||||
});
|
||||
|
||||
child.on('error', (err) => {
|
||||
if ((err as NodeJS.ErrnoException).code === 'ETIMEDOUT') {
|
||||
reject(new Error('Python script timed out'));
|
||||
} else {
|
||||
reject(new Error(`Failed to spawn Python: ${err.message}`));
|
||||
}
|
||||
});
|
||||
|
||||
// Write JSON request to stdin and close
|
||||
const jsonInput = JSON.stringify(request);
|
||||
child.stdin.write(jsonInput);
|
||||
child.stdin.end();
|
||||
});
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Content Chunking
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Chunk content into smaller pieces for embedding.
|
||||
* Uses paragraph-first, sentence-fallback strategy with overlap.
|
||||
*
|
||||
* Matches the chunking logic in core-memory-store.ts:
|
||||
* - CHUNK_SIZE = 1500 characters
|
||||
* - OVERLAP = 200 characters
|
||||
* - Split by paragraph boundaries (\n\n) first
|
||||
* - Fall back to sentence boundaries (. ) for oversized paragraphs
|
||||
*
|
||||
* @param content - Text content to chunk
|
||||
* @returns Array of chunk strings
|
||||
*/
|
||||
export function chunkContent(content: string): string[] {
|
||||
const chunks: string[] = [];
|
||||
|
||||
// Split by paragraph boundaries first
|
||||
const paragraphs = content.split(/\n\n+/);
|
||||
let currentChunk = '';
|
||||
|
||||
for (const paragraph of paragraphs) {
|
||||
// If adding this paragraph would exceed chunk size
|
||||
if (currentChunk.length + paragraph.length > CHUNK_SIZE && currentChunk.length > 0) {
|
||||
chunks.push(currentChunk.trim());
|
||||
|
||||
// Start new chunk with overlap
|
||||
const overlapText = currentChunk.slice(-OVERLAP);
|
||||
currentChunk = overlapText + '\n\n' + paragraph;
|
||||
} else {
|
||||
currentChunk += (currentChunk ? '\n\n' : '') + paragraph;
|
||||
}
|
||||
}
|
||||
|
||||
// Add remaining chunk
|
||||
if (currentChunk.trim()) {
|
||||
chunks.push(currentChunk.trim());
|
||||
}
|
||||
|
||||
// If chunks are still too large, split by sentences
|
||||
const finalChunks: string[] = [];
|
||||
for (const chunk of chunks) {
|
||||
if (chunk.length <= CHUNK_SIZE) {
|
||||
finalChunks.push(chunk);
|
||||
} else {
|
||||
// Split by sentence boundaries
|
||||
const sentences = chunk.split(/\. +/);
|
||||
let sentenceChunk = '';
|
||||
|
||||
for (const sentence of sentences) {
|
||||
const sentenceWithPeriod = sentence + '. ';
|
||||
if (
|
||||
sentenceChunk.length + sentenceWithPeriod.length > CHUNK_SIZE &&
|
||||
sentenceChunk.length > 0
|
||||
) {
|
||||
finalChunks.push(sentenceChunk.trim());
|
||||
const overlapText = sentenceChunk.slice(-OVERLAP);
|
||||
sentenceChunk = overlapText + sentenceWithPeriod;
|
||||
} else {
|
||||
sentenceChunk += sentenceWithPeriod;
|
||||
}
|
||||
}
|
||||
|
||||
if (sentenceChunk.trim()) {
|
||||
finalChunks.push(sentenceChunk.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return finalChunks.length > 0 ? finalChunks : [content];
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// UnifiedVectorIndex Class
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Unified vector index backed by CodexLens VectorStore (HNSW).
|
||||
*
|
||||
* Provides content chunking, embedding, storage, and search for all
|
||||
* memory content types through a single interface.
|
||||
*/
|
||||
export class UnifiedVectorIndex {
|
||||
private storePath: string;
|
||||
|
||||
/**
|
||||
* Create a UnifiedVectorIndex for a project.
|
||||
*
|
||||
* @param projectPath - Project root path (used to resolve storage location)
|
||||
*/
|
||||
constructor(projectPath: string) {
|
||||
const paths = StoragePaths.project(projectPath);
|
||||
this.storePath = paths.unifiedVectors.root;
|
||||
ensureStorageDir(this.storePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Index content by chunking, embedding, and storing in VectorStore.
|
||||
*
|
||||
* @param content - Text content to index
|
||||
* @param metadata - Metadata for all chunks (source_id, source_type, category)
|
||||
* @returns Embed result
|
||||
*/
|
||||
async indexContent(
|
||||
content: string,
|
||||
metadata: ChunkMetadata
|
||||
): Promise<EmbedResult> {
|
||||
if (!content.trim()) {
|
||||
return {
|
||||
success: true,
|
||||
chunks_processed: 0,
|
||||
chunks_failed: 0,
|
||||
elapsed_time: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Chunk content
|
||||
const textChunks = chunkContent(content);
|
||||
|
||||
// Build chunk objects for Python
|
||||
const chunks: VectorChunk[] = textChunks.map((text, index) => ({
|
||||
content: text,
|
||||
source_id: metadata.source_id,
|
||||
source_type: metadata.source_type,
|
||||
category: metadata.category,
|
||||
chunk_index: metadata.chunk_index != null ? metadata.chunk_index + index : index,
|
||||
metadata: { ...metadata },
|
||||
}));
|
||||
|
||||
try {
|
||||
const result = await runPython<EmbedResult>({
|
||||
operation: 'embed',
|
||||
store_path: this.storePath,
|
||||
chunks,
|
||||
batch_size: 8,
|
||||
});
|
||||
return result;
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
chunks_processed: 0,
|
||||
chunks_failed: textChunks.length,
|
||||
elapsed_time: 0,
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search the vector index using semantic similarity.
|
||||
*
|
||||
* @param query - Natural language search query
|
||||
* @param options - Search options (topK, minScore, category)
|
||||
* @returns Search results sorted by relevance
|
||||
*/
|
||||
async search(
|
||||
query: string,
|
||||
options: VectorSearchOptions = {}
|
||||
): Promise<VectorSearchResult> {
|
||||
const { topK = 10, minScore = 0.3, category } = options;
|
||||
|
||||
try {
|
||||
const result = await runPython<VectorSearchResult>({
|
||||
operation: 'search',
|
||||
store_path: this.storePath,
|
||||
query,
|
||||
top_k: topK,
|
||||
min_score: minScore,
|
||||
category: category || null,
|
||||
});
|
||||
return result;
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
matches: [],
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search the vector index using a pre-computed embedding vector.
|
||||
* Bypasses text embedding, directly querying HNSW with a raw vector.
|
||||
*
|
||||
* @param vector - Pre-computed embedding vector (array of floats)
|
||||
* @param options - Search options (topK, minScore, category)
|
||||
* @returns Search results sorted by relevance
|
||||
*/
|
||||
async searchByVector(
|
||||
vector: number[],
|
||||
options: VectorSearchOptions = {}
|
||||
): Promise<VectorSearchResult> {
|
||||
const { topK = 10, minScore = 0.3, category } = options;
|
||||
|
||||
try {
|
||||
const result = await runPython<VectorSearchResult>({
|
||||
operation: 'search_by_vector',
|
||||
store_path: this.storePath,
|
||||
vector,
|
||||
top_k: topK,
|
||||
min_score: minScore,
|
||||
category: category || null,
|
||||
});
|
||||
return result;
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
matches: [],
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild the HNSW index from scratch.
|
||||
*
|
||||
* @returns Reindex result
|
||||
*/
|
||||
async reindexAll(): Promise<ReindexResult> {
|
||||
try {
|
||||
const result = await runPython<ReindexResult>({
|
||||
operation: 'reindex',
|
||||
store_path: this.storePath,
|
||||
});
|
||||
return result;
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current status of the vector index.
|
||||
*
|
||||
* @returns Index status including chunk counts, HNSW availability, dimension
|
||||
*/
|
||||
async getStatus(): Promise<VectorIndexStatus> {
|
||||
try {
|
||||
const result = await runPython<VectorIndexStatus>({
|
||||
operation: 'status',
|
||||
store_path: this.storePath,
|
||||
});
|
||||
return result;
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
total_chunks: 0,
|
||||
hnsw_available: false,
|
||||
hnsw_count: 0,
|
||||
dimension: 0,
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import type {
|
||||
} from '../core/a2ui/A2UITypes.js';
|
||||
import http from 'http';
|
||||
import { a2uiWebSocketHandler } from '../core/a2ui/A2UIWebSocketHandler.js';
|
||||
import { remoteNotificationService } from '../core/services/remote-notification-service.js';
|
||||
|
||||
const DASHBOARD_PORT = Number(process.env.CCW_PORT || 3456);
|
||||
const POLL_INTERVAL_MS = 1000;
|
||||
@@ -466,6 +467,14 @@ export async function execute(params: AskQuestionParams): Promise<ToolResult<Ask
|
||||
const a2uiSurface = generateQuestionSurface(question, surfaceId);
|
||||
const sentCount = a2uiWebSocketHandler.sendSurface(a2uiSurface.surfaceUpdate);
|
||||
|
||||
// Trigger remote notification for ask-user-question event (if enabled)
|
||||
if (remoteNotificationService.shouldNotify('ask-user-question')) {
|
||||
remoteNotificationService.sendNotification('ask-user-question', {
|
||||
sessionId: surfaceId,
|
||||
questionText: question.title,
|
||||
});
|
||||
}
|
||||
|
||||
// If no local WS clients, start HTTP polling for answer from Dashboard
|
||||
if (sentCount === 0) {
|
||||
startAnswerPolling(question.id);
|
||||
@@ -1064,6 +1073,15 @@ async function executeSimpleFormat(
|
||||
// Send the surface
|
||||
const sentCount = a2uiWebSocketHandler.sendSurface(surfaceUpdate);
|
||||
|
||||
// Trigger remote notification for ask-user-question event (if enabled)
|
||||
if (remoteNotificationService.shouldNotify('ask-user-question')) {
|
||||
const questionTexts = questions.map(q => q.question).join('\n');
|
||||
remoteNotificationService.sendNotification('ask-user-question', {
|
||||
sessionId: compositeId,
|
||||
questionText: questionTexts,
|
||||
});
|
||||
}
|
||||
|
||||
// If no local WS clients, start HTTP polling for answer from Dashboard
|
||||
if (sentCount === 0) {
|
||||
startAnswerPolling(compositeId, true);
|
||||
|
||||
227
ccw/src/types/remote-notification.ts
Normal file
227
ccw/src/types/remote-notification.ts
Normal file
@@ -0,0 +1,227 @@
|
||||
// ========================================
|
||||
// Remote Notification Types
|
||||
// ========================================
|
||||
// Type definitions for remote notification system
|
||||
// Supports Discord, Telegram, and Generic Webhook platforms
|
||||
|
||||
/**
|
||||
* Supported notification platforms
|
||||
*/
|
||||
export type NotificationPlatform = 'discord' | 'telegram' | 'webhook';
|
||||
|
||||
/**
|
||||
* Event types that can trigger notifications
|
||||
*/
|
||||
export type NotificationEventType =
|
||||
| 'ask-user-question' // AskUserQuestion triggered
|
||||
| 'session-start' // CLI session started
|
||||
| 'session-end' // CLI session ended
|
||||
| 'task-completed' // Task completed successfully
|
||||
| 'task-failed'; // Task failed
|
||||
|
||||
/**
|
||||
* Discord platform configuration
|
||||
*/
|
||||
export interface DiscordConfig {
|
||||
/** Whether Discord notifications are enabled */
|
||||
enabled: boolean;
|
||||
/** Discord webhook URL */
|
||||
webhookUrl: string;
|
||||
/** Optional custom username for the webhook */
|
||||
username?: string;
|
||||
/** Optional avatar URL for the webhook */
|
||||
avatarUrl?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Telegram platform configuration
|
||||
*/
|
||||
export interface TelegramConfig {
|
||||
/** Whether Telegram notifications are enabled */
|
||||
enabled: boolean;
|
||||
/** Telegram bot token */
|
||||
botToken: string;
|
||||
/** Telegram chat ID (user or group) */
|
||||
chatId: string;
|
||||
/** Optional parse mode (HTML, Markdown, MarkdownV2) */
|
||||
parseMode?: 'HTML' | 'Markdown' | 'MarkdownV2';
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic Webhook platform configuration
|
||||
*/
|
||||
export interface WebhookConfig {
|
||||
/** Whether webhook notifications are enabled */
|
||||
enabled: boolean;
|
||||
/** Webhook URL */
|
||||
url: string;
|
||||
/** HTTP method (POST or PUT) */
|
||||
method: 'POST' | 'PUT';
|
||||
/** Custom headers */
|
||||
headers?: Record<string, string>;
|
||||
/** Request timeout in milliseconds */
|
||||
timeout?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Event configuration - maps events to platforms
|
||||
*/
|
||||
export interface EventConfig {
|
||||
/** Event type */
|
||||
event: NotificationEventType;
|
||||
/** Platforms to notify for this event */
|
||||
platforms: NotificationPlatform[];
|
||||
/** Whether this event's notifications are enabled */
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full remote notification configuration
|
||||
*/
|
||||
export interface RemoteNotificationConfig {
|
||||
/** Master switch for all remote notifications */
|
||||
enabled: boolean;
|
||||
/** Platform-specific configurations */
|
||||
platforms: {
|
||||
discord?: DiscordConfig;
|
||||
telegram?: TelegramConfig;
|
||||
webhook?: WebhookConfig;
|
||||
};
|
||||
/** Event-to-platform mappings */
|
||||
events: EventConfig[];
|
||||
/** Global timeout for all notification requests (ms) */
|
||||
timeout: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Context passed when sending a notification
|
||||
*/
|
||||
export interface NotificationContext {
|
||||
/** Event type that triggered the notification */
|
||||
eventType: NotificationEventType;
|
||||
/** Session ID if applicable */
|
||||
sessionId?: string;
|
||||
/** Question text for ask-user-question events */
|
||||
questionText?: string;
|
||||
/** Task description for task events */
|
||||
taskDescription?: string;
|
||||
/** Error message for task-failed events */
|
||||
errorMessage?: string;
|
||||
/** Timestamp of the event */
|
||||
timestamp: string;
|
||||
/** Additional metadata */
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of a single platform notification attempt
|
||||
*/
|
||||
export interface PlatformNotificationResult {
|
||||
/** Platform that was notified */
|
||||
platform: NotificationPlatform;
|
||||
/** Whether the notification succeeded */
|
||||
success: boolean;
|
||||
/** Error message if failed */
|
||||
error?: string;
|
||||
/** Response time in milliseconds */
|
||||
responseTime?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of sending notifications to all configured platforms
|
||||
*/
|
||||
export interface NotificationDispatchResult {
|
||||
/** Whether at least one notification succeeded */
|
||||
success: boolean;
|
||||
/** Results for each platform */
|
||||
results: PlatformNotificationResult[];
|
||||
/** Total dispatch time in milliseconds */
|
||||
totalTime: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test notification request
|
||||
*/
|
||||
export interface TestNotificationRequest {
|
||||
/** Platform to test */
|
||||
platform: NotificationPlatform;
|
||||
/** Platform configuration to test (temporary, not saved) */
|
||||
config: DiscordConfig | TelegramConfig | WebhookConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test notification result
|
||||
*/
|
||||
export interface TestNotificationResult {
|
||||
/** Whether the test succeeded */
|
||||
success: boolean;
|
||||
/** Error message if failed */
|
||||
error?: string;
|
||||
/** Response time in milliseconds */
|
||||
responseTime?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default configuration values
|
||||
*/
|
||||
export const DEFAULT_REMOTE_NOTIFICATION_CONFIG: RemoteNotificationConfig = {
|
||||
enabled: false,
|
||||
platforms: {},
|
||||
events: [
|
||||
{ event: 'ask-user-question', platforms: ['discord', 'telegram'], enabled: true },
|
||||
{ event: 'session-start', platforms: [], enabled: false },
|
||||
{ event: 'session-end', platforms: [], enabled: false },
|
||||
{ event: 'task-completed', platforms: [], enabled: false },
|
||||
{ event: 'task-failed', platforms: ['discord', 'telegram'], enabled: true },
|
||||
],
|
||||
timeout: 10000, // 10 seconds
|
||||
};
|
||||
|
||||
/**
|
||||
* Mask sensitive fields in config for API responses
|
||||
*/
|
||||
export function maskSensitiveConfig(config: RemoteNotificationConfig): RemoteNotificationConfig {
|
||||
return {
|
||||
...config,
|
||||
platforms: {
|
||||
discord: config.platforms.discord ? {
|
||||
...config.platforms.discord,
|
||||
webhookUrl: maskWebhookUrl(config.platforms.discord.webhookUrl),
|
||||
} : undefined,
|
||||
telegram: config.platforms.telegram ? {
|
||||
...config.platforms.telegram,
|
||||
botToken: maskToken(config.platforms.telegram.botToken),
|
||||
} : undefined,
|
||||
webhook: config.platforms.webhook ? {
|
||||
...config.platforms.webhook,
|
||||
// Don't mask webhook URL as it's needed for display
|
||||
} : undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Mask webhook URL for display (show only domain and last part)
|
||||
*/
|
||||
function maskWebhookUrl(url: string): string {
|
||||
if (!url) return '';
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
const pathParts = parsed.pathname.split('/');
|
||||
const lastPart = pathParts[pathParts.length - 1];
|
||||
if (lastPart && lastPart.length > 8) {
|
||||
return `${parsed.origin}/.../${lastPart.slice(0, 4)}****`;
|
||||
}
|
||||
return `${parsed.origin}/****`;
|
||||
} catch {
|
||||
return '****';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mask bot token for display
|
||||
*/
|
||||
function maskToken(token: string): string {
|
||||
if (!token || token.length < 10) return '****';
|
||||
return `${token.slice(0, 6)}****${token.slice(-4)}`;
|
||||
}
|
||||
75
ccw/src/types/util.ts
Normal file
75
ccw/src/types/util.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
// ========================================
|
||||
// Utility Types
|
||||
// ========================================
|
||||
// Common utility type definitions
|
||||
|
||||
/**
|
||||
* Deep partial type - makes all nested properties optional
|
||||
*/
|
||||
export type DeepPartial<T> = T extends object
|
||||
? {
|
||||
[P in keyof T]?: DeepPartial<T[P]>;
|
||||
}
|
||||
: T;
|
||||
|
||||
/**
|
||||
* Make specific keys optional
|
||||
*/
|
||||
export type PartialBy<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
|
||||
|
||||
/**
|
||||
* Make specific keys required
|
||||
*/
|
||||
export type RequiredBy<T, K extends keyof T> = Omit<T, K> & Required<Pick<T, K>>;
|
||||
|
||||
/**
|
||||
* Extract function parameter types
|
||||
*/
|
||||
export type Parameters<T> = T extends (...args: infer P) => unknown ? P : never;
|
||||
|
||||
/**
|
||||
* Extract function return type
|
||||
*/
|
||||
export type ReturnType<T> = T extends (...args: unknown[]) => infer R ? R : never;
|
||||
|
||||
// ========================================
|
||||
// Utility Functions
|
||||
// ========================================
|
||||
|
||||
/**
|
||||
* Deep merge utility for configuration updates
|
||||
* Recursively merges source into target, preserving nested objects
|
||||
*/
|
||||
export function deepMerge<T extends Record<string, unknown>>(
|
||||
target: T,
|
||||
source: DeepPartial<T>
|
||||
): T {
|
||||
const result = { ...target } as T;
|
||||
|
||||
for (const key in source) {
|
||||
if (Object.prototype.hasOwnProperty.call(source, key)) {
|
||||
const sourceValue = source[key];
|
||||
const targetValue = target[key];
|
||||
|
||||
if (
|
||||
sourceValue !== undefined &&
|
||||
sourceValue !== null &&
|
||||
typeof sourceValue === 'object' &&
|
||||
!Array.isArray(sourceValue) &&
|
||||
targetValue !== undefined &&
|
||||
targetValue !== null &&
|
||||
typeof targetValue === 'object' &&
|
||||
!Array.isArray(targetValue)
|
||||
) {
|
||||
(result as Record<string, unknown>)[key] = deepMerge(
|
||||
targetValue as Record<string, unknown>,
|
||||
sourceValue as DeepPartial<Record<string, unknown>>
|
||||
);
|
||||
} else if (sourceValue !== undefined) {
|
||||
(result as Record<string, unknown>)[key] = sourceValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
Reference in New Issue
Block a user