mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-02-14 02:42:04 +08:00
feat: unified task.json schema migration and multi-module updates
- Create task-schema.json (JSON Schema draft-07) with 10 field blocks fusing Unified JSONL, 6-field Task JSON, and Solution Schema advantages - Migrate unified-execute-with-file from JSONL to .task/*.json directory scanning - Migrate 3 producers (lite-plan, plan-converter, collaborative-plan) to .task/*.json multi-file output - Add review-cycle Phase 7.5 export-to-tasks (FIX-*.json) and issue-resolve --export-tasks option - Add schema compatibility annotations to action-planning-agent, workflow-plan, and tdd-plan - Add spec-generator skill phases and templates - Add memory v2 pipeline (consolidation, extraction, job scheduler, embedder) - Add secret-redactor utility and core-memory enhancements - Add codex-lens accuracy benchmarks and staged env config overrides
This commit is contained in:
catlog22
@@ -11,6 +11,7 @@ import {
|
||||
exportMemories,
|
||||
importMemories
|
||||
} from '../core/core-memory-store.js';
|
||||
import { MemoryJobScheduler } from '../core/memory-job-scheduler.js';
|
||||
import { notifyRefreshRequired } from '../tools/notifier.js';
|
||||
|
||||
interface CommandOptions {
|
||||
@@ -664,6 +665,185 @@ async function searchAction(keyword: string, options: CommandOptions): Promise<v
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Memory V2 CLI Subcommands
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Run batch extraction
|
||||
*/
|
||||
async function extractAction(options: CommandOptions): Promise<void> {
|
||||
try {
|
||||
const projectPath = getProjectPath();
|
||||
|
||||
console.log(chalk.cyan('\n Triggering memory extraction...\n'));
|
||||
|
||||
const { MemoryExtractionPipeline } = await import('../core/memory-extraction-pipeline.js');
|
||||
const pipeline = new MemoryExtractionPipeline(projectPath);
|
||||
|
||||
// Scan eligible sessions first
|
||||
const eligible = await pipeline.scanEligibleSessions();
|
||||
console.log(chalk.white(` Eligible sessions: ${eligible.length}`));
|
||||
|
||||
if (eligible.length === 0) {
|
||||
console.log(chalk.yellow(' No eligible sessions for extraction.\n'));
|
||||
return;
|
||||
}
|
||||
|
||||
// Run extraction (synchronous for CLI - shows progress)
|
||||
console.log(chalk.cyan(' Running batch extraction...'));
|
||||
await pipeline.runBatchExtraction();
|
||||
|
||||
const store = getCoreMemoryStore(projectPath);
|
||||
const stage1Count = store.countStage1Outputs();
|
||||
|
||||
console.log(chalk.green(`\n Extraction complete.`));
|
||||
console.log(chalk.white(` Total stage1 outputs: ${stage1Count}\n`));
|
||||
|
||||
notifyRefreshRequired('memory').catch(() => { /* ignore */ });
|
||||
|
||||
} catch (error) {
|
||||
console.error(chalk.red(`Error: ${(error as Error).message}`));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Show extraction status
|
||||
*/
|
||||
async function extractStatusAction(options: CommandOptions): Promise<void> {
|
||||
try {
|
||||
const projectPath = getProjectPath();
|
||||
const store = getCoreMemoryStore(projectPath);
|
||||
const scheduler = new MemoryJobScheduler(store.getDb());
|
||||
|
||||
const stage1Count = store.countStage1Outputs();
|
||||
const extractionJobs = scheduler.listJobs('extraction');
|
||||
|
||||
if (options.json) {
|
||||
console.log(JSON.stringify({ total_stage1: stage1Count, jobs: extractionJobs }, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(chalk.bold.cyan('\n Extraction Pipeline Status\n'));
|
||||
console.log(chalk.white(` Stage 1 outputs: ${stage1Count}`));
|
||||
console.log(chalk.white(` Extraction jobs: ${extractionJobs.length}`));
|
||||
|
||||
if (extractionJobs.length > 0) {
|
||||
console.log(chalk.gray('\n ─────────────────────────────────────────────────────────────────'));
|
||||
|
||||
for (const job of extractionJobs) {
|
||||
const statusColor = job.status === 'done' ? chalk.green
|
||||
: job.status === 'error' ? chalk.red
|
||||
: job.status === 'running' ? chalk.yellow
|
||||
: chalk.gray;
|
||||
|
||||
console.log(chalk.cyan(` ${job.job_key}`) + chalk.white(` [${statusColor(job.status)}]`));
|
||||
if (job.last_error) console.log(chalk.red(` Error: ${job.last_error}`));
|
||||
if (job.started_at) console.log(chalk.gray(` Started: ${new Date(job.started_at * 1000).toLocaleString()}`));
|
||||
if (job.finished_at) console.log(chalk.gray(` Finished: ${new Date(job.finished_at * 1000).toLocaleString()}`));
|
||||
console.log(chalk.gray(' ─────────────────────────────────────────────────────────────────'));
|
||||
}
|
||||
}
|
||||
|
||||
console.log();
|
||||
|
||||
} catch (error) {
|
||||
console.error(chalk.red(`Error: ${(error as Error).message}`));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run consolidation
|
||||
*/
|
||||
async function consolidateAction(options: CommandOptions): Promise<void> {
|
||||
try {
|
||||
const projectPath = getProjectPath();
|
||||
|
||||
console.log(chalk.cyan('\n Triggering memory consolidation...\n'));
|
||||
|
||||
const { MemoryConsolidationPipeline } = await import('../core/memory-consolidation-pipeline.js');
|
||||
const pipeline = new MemoryConsolidationPipeline(projectPath);
|
||||
|
||||
await pipeline.runConsolidation();
|
||||
|
||||
const memoryMd = pipeline.getMemoryMdContent();
|
||||
|
||||
console.log(chalk.green(' Consolidation complete.'));
|
||||
if (memoryMd) {
|
||||
console.log(chalk.white(` MEMORY.md generated (${memoryMd.length} chars)`));
|
||||
}
|
||||
console.log();
|
||||
|
||||
notifyRefreshRequired('memory').catch(() => { /* ignore */ });
|
||||
|
||||
} catch (error) {
|
||||
console.error(chalk.red(`Error: ${(error as Error).message}`));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List all V2 pipeline jobs
|
||||
*/
|
||||
async function jobsAction(options: CommandOptions): Promise<void> {
|
||||
try {
|
||||
const projectPath = getProjectPath();
|
||||
const store = getCoreMemoryStore(projectPath);
|
||||
const scheduler = new MemoryJobScheduler(store.getDb());
|
||||
|
||||
const kind = options.type || undefined;
|
||||
const jobs = scheduler.listJobs(kind);
|
||||
|
||||
if (options.json) {
|
||||
console.log(JSON.stringify({ jobs, total: jobs.length }, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(chalk.bold.cyan('\n Memory V2 Pipeline Jobs\n'));
|
||||
|
||||
if (jobs.length === 0) {
|
||||
console.log(chalk.yellow(' No jobs found.\n'));
|
||||
return;
|
||||
}
|
||||
|
||||
// Summary counts
|
||||
const byStatus: Record<string, number> = {};
|
||||
for (const job of jobs) {
|
||||
byStatus[job.status] = (byStatus[job.status] || 0) + 1;
|
||||
}
|
||||
|
||||
const statusParts = Object.entries(byStatus)
|
||||
.map(([s, c]) => `${s}: ${c}`)
|
||||
.join(' | ');
|
||||
console.log(chalk.white(` Summary: ${statusParts}`));
|
||||
console.log(chalk.gray(' ─────────────────────────────────────────────────────────────────'));
|
||||
|
||||
for (const job of jobs) {
|
||||
const statusColor = job.status === 'done' ? chalk.green
|
||||
: job.status === 'error' ? chalk.red
|
||||
: job.status === 'running' ? chalk.yellow
|
||||
: chalk.gray;
|
||||
|
||||
console.log(
|
||||
chalk.cyan(` [${job.kind}]`) +
|
||||
chalk.white(` ${job.job_key}`) +
|
||||
` [${statusColor(job.status)}]` +
|
||||
chalk.gray(` retries: ${job.retry_remaining}`)
|
||||
);
|
||||
if (job.last_error) console.log(chalk.red(` Error: ${job.last_error}`));
|
||||
console.log(chalk.gray(' ─────────────────────────────────────────────────────────────────'));
|
||||
}
|
||||
|
||||
console.log(chalk.gray(`\n Total: ${jobs.length}\n`));
|
||||
|
||||
} catch (error) {
|
||||
console.error(chalk.red(`Error: ${(error as Error).message}`));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Core Memory command entry point
|
||||
*/
|
||||
@@ -724,6 +904,23 @@ export async function coreMemoryCommand(
|
||||
await listFromAction(textArg, options);
|
||||
break;
|
||||
|
||||
// Memory V2 subcommands
|
||||
case 'extract':
|
||||
await extractAction(options);
|
||||
break;
|
||||
|
||||
case 'extract-status':
|
||||
await extractStatusAction(options);
|
||||
break;
|
||||
|
||||
case 'consolidate':
|
||||
await consolidateAction(options);
|
||||
break;
|
||||
|
||||
case 'jobs':
|
||||
await jobsAction(options);
|
||||
break;
|
||||
|
||||
default:
|
||||
console.log(chalk.bold.cyan('\n CCW Core Memory\n'));
|
||||
console.log(' Manage core memory entries and session clusters.\n');
|
||||
@@ -749,6 +946,12 @@ export async function coreMemoryCommand(
|
||||
console.log(chalk.white(' load-cluster <id> ') + chalk.gray('Load cluster context'));
|
||||
console.log(chalk.white(' search <keyword> ') + chalk.gray('Search sessions'));
|
||||
console.log();
|
||||
console.log(chalk.bold(' Memory V2 Pipeline:'));
|
||||
console.log(chalk.white(' extract ') + chalk.gray('Run batch memory extraction'));
|
||||
console.log(chalk.white(' extract-status ') + chalk.gray('Show extraction pipeline status'));
|
||||
console.log(chalk.white(' consolidate ') + chalk.gray('Run memory consolidation'));
|
||||
console.log(chalk.white(' jobs ') + chalk.gray('List all pipeline jobs'));
|
||||
console.log();
|
||||
console.log(chalk.bold(' Options:'));
|
||||
console.log(chalk.gray(' --id <id> Memory ID (for export/summary)'));
|
||||
console.log(chalk.gray(' --tool gemini|qwen AI tool for summary (default: gemini)'));
|
||||
@@ -765,6 +968,10 @@ export async function coreMemoryCommand(
|
||||
console.log(chalk.gray(' ccw core-memory list-from d--other-project'));
|
||||
console.log(chalk.gray(' ccw core-memory cluster --auto'));
|
||||
console.log(chalk.gray(' ccw core-memory cluster --dedup'));
|
||||
console.log(chalk.gray(' ccw core-memory extract # Run memory extraction'));
|
||||
console.log(chalk.gray(' ccw core-memory extract-status # Check extraction state'));
|
||||
console.log(chalk.gray(' ccw core-memory consolidate # Run consolidation'));
|
||||
console.log(chalk.gray(' ccw core-memory jobs # List pipeline jobs'));
|
||||
console.log();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -375,6 +375,19 @@ export interface ProjectPaths {
|
||||
config: string;
|
||||
/** CLI config file */
|
||||
cliConfig: string;
|
||||
/** Memory V2 paths */
|
||||
memoryV2: {
|
||||
/** Root: <projectRoot>/core-memory/v2/ */
|
||||
root: string;
|
||||
/** Rollout summaries directory */
|
||||
rolloutSummaries: string;
|
||||
/** Concatenated raw memories file */
|
||||
rawMemories: string;
|
||||
/** Final consolidated memory file */
|
||||
memoryMd: string;
|
||||
/** Skills directory */
|
||||
skills: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -434,6 +447,13 @@ export function getProjectPaths(projectPath: string): ProjectPaths {
|
||||
dashboardCache: join(projectDir, 'cache', 'dashboard-data.json'),
|
||||
config: join(projectDir, 'config'),
|
||||
cliConfig: join(projectDir, 'config', 'cli-config.json'),
|
||||
memoryV2: {
|
||||
root: join(projectDir, 'core-memory', 'v2'),
|
||||
rolloutSummaries: join(projectDir, 'core-memory', 'v2', 'rollout_summaries'),
|
||||
rawMemories: join(projectDir, 'core-memory', 'v2', 'raw_memories.md'),
|
||||
memoryMd: join(projectDir, 'core-memory', 'v2', 'MEMORY.md'),
|
||||
skills: join(projectDir, 'core-memory', 'v2', 'skills'),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -456,6 +476,13 @@ export function getProjectPathsById(projectId: string): ProjectPaths {
|
||||
dashboardCache: join(projectDir, 'cache', 'dashboard-data.json'),
|
||||
config: join(projectDir, 'config'),
|
||||
cliConfig: join(projectDir, 'config', 'cli-config.json'),
|
||||
memoryV2: {
|
||||
root: join(projectDir, 'core-memory', 'v2'),
|
||||
rolloutSummaries: join(projectDir, 'core-memory', 'v2', 'rollout_summaries'),
|
||||
rawMemories: join(projectDir, 'core-memory', 'v2', 'raw_memories.md'),
|
||||
memoryMd: join(projectDir, 'core-memory', 'v2', 'MEMORY.md'),
|
||||
skills: join(projectDir, 'core-memory', 'v2', 'skills'),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -682,4 +709,7 @@ export function initializeProjectStorage(projectPath: string): void {
|
||||
ensureStorageDir(paths.memory);
|
||||
ensureStorageDir(paths.cache);
|
||||
ensureStorageDir(paths.config);
|
||||
ensureStorageDir(paths.memoryV2.root);
|
||||
ensureStorageDir(paths.memoryV2.rolloutSummaries);
|
||||
ensureStorageDir(paths.memoryV2.skills);
|
||||
}
|
||||
|
||||
@@ -83,6 +83,17 @@ export interface ClaudeUpdateRecord {
|
||||
metadata?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Memory V2: Phase 1 extraction output row
|
||||
*/
|
||||
export interface Stage1Output {
|
||||
thread_id: string;
|
||||
source_updated_at: number;
|
||||
raw_memory: string;
|
||||
rollout_summary: string;
|
||||
generated_at: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Core Memory Store using SQLite
|
||||
*/
|
||||
@@ -215,6 +226,40 @@ export class CoreMemoryStore {
|
||||
CREATE INDEX IF NOT EXISTS idx_claude_history_path ON claude_update_history(file_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_claude_history_updated ON claude_update_history(updated_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_claude_history_module ON claude_update_history(module_path);
|
||||
|
||||
-- Memory V2: Phase 1 extraction outputs
|
||||
CREATE TABLE IF NOT EXISTS stage1_outputs (
|
||||
thread_id TEXT PRIMARY KEY,
|
||||
source_updated_at INTEGER NOT NULL,
|
||||
raw_memory TEXT NOT NULL,
|
||||
rollout_summary TEXT NOT NULL,
|
||||
generated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_stage1_generated ON stage1_outputs(generated_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_stage1_source_updated ON stage1_outputs(source_updated_at DESC);
|
||||
|
||||
-- Memory V2: Job scheduler
|
||||
CREATE TABLE IF NOT EXISTS jobs (
|
||||
kind TEXT NOT NULL,
|
||||
job_key TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'running', 'done', 'error')),
|
||||
worker_id TEXT,
|
||||
ownership_token TEXT,
|
||||
started_at INTEGER,
|
||||
finished_at INTEGER,
|
||||
lease_until INTEGER,
|
||||
retry_at INTEGER,
|
||||
retry_remaining INTEGER NOT NULL DEFAULT 3,
|
||||
last_error TEXT,
|
||||
input_watermark INTEGER DEFAULT 0,
|
||||
last_success_watermark INTEGER DEFAULT 0,
|
||||
PRIMARY KEY (kind, job_key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_jobs_kind_status ON jobs(kind, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_jobs_lease ON jobs(lease_until);
|
||||
`);
|
||||
}
|
||||
|
||||
@@ -275,6 +320,14 @@ export class CoreMemoryStore {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the underlying database instance.
|
||||
* Used by MemoryJobScheduler and other V2 components that share this DB.
|
||||
*/
|
||||
getDb(): Database.Database {
|
||||
return this.db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate timestamp-based ID for core memory
|
||||
*/
|
||||
@@ -1255,6 +1308,88 @@ ${memory.content}
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Memory V2: Stage 1 Output CRUD Operations
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Upsert a Phase 1 extraction output (idempotent by thread_id)
|
||||
*/
|
||||
upsertStage1Output(output: Stage1Output): void {
|
||||
const stmt = this.db.prepare(`
|
||||
INSERT INTO stage1_outputs (thread_id, source_updated_at, raw_memory, rollout_summary, generated_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(thread_id) DO UPDATE SET
|
||||
source_updated_at = excluded.source_updated_at,
|
||||
raw_memory = excluded.raw_memory,
|
||||
rollout_summary = excluded.rollout_summary,
|
||||
generated_at = excluded.generated_at
|
||||
`);
|
||||
|
||||
stmt.run(
|
||||
output.thread_id,
|
||||
output.source_updated_at,
|
||||
output.raw_memory,
|
||||
output.rollout_summary,
|
||||
output.generated_at
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a Phase 1 output by thread_id
|
||||
*/
|
||||
getStage1Output(threadId: string): Stage1Output | null {
|
||||
const stmt = this.db.prepare(`SELECT * FROM stage1_outputs WHERE thread_id = ?`);
|
||||
const row = stmt.get(threadId) as any;
|
||||
if (!row) return null;
|
||||
|
||||
return {
|
||||
thread_id: row.thread_id,
|
||||
source_updated_at: row.source_updated_at,
|
||||
raw_memory: row.raw_memory,
|
||||
rollout_summary: row.rollout_summary,
|
||||
generated_at: row.generated_at,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* List all Phase 1 outputs, ordered by generated_at descending
|
||||
*/
|
||||
listStage1Outputs(limit?: number): Stage1Output[] {
|
||||
const query = limit
|
||||
? `SELECT * FROM stage1_outputs ORDER BY generated_at DESC LIMIT ?`
|
||||
: `SELECT * FROM stage1_outputs ORDER BY generated_at DESC`;
|
||||
|
||||
const stmt = this.db.prepare(query);
|
||||
const rows = (limit ? stmt.all(limit) : stmt.all()) as any[];
|
||||
|
||||
return rows.map(row => ({
|
||||
thread_id: row.thread_id,
|
||||
source_updated_at: row.source_updated_at,
|
||||
raw_memory: row.raw_memory,
|
||||
rollout_summary: row.rollout_summary,
|
||||
generated_at: row.generated_at,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Count Phase 1 outputs
|
||||
*/
|
||||
countStage1Outputs(): number {
|
||||
const stmt = this.db.prepare(`SELECT COUNT(*) as count FROM stage1_outputs`);
|
||||
const row = stmt.get() as { count: number };
|
||||
return row.count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a Phase 1 output by thread_id
|
||||
*/
|
||||
deleteStage1Output(threadId: string): boolean {
|
||||
const stmt = this.db.prepare(`DELETE FROM stage1_outputs WHERE thread_id = ?`);
|
||||
const result = stmt.run(threadId);
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close database connection
|
||||
*/
|
||||
|
||||
474
ccw/src/core/memory-consolidation-pipeline.ts
Normal file
474
ccw/src/core/memory-consolidation-pipeline.ts
Normal file
@@ -0,0 +1,474 @@
|
||||
/**
|
||||
* Memory Consolidation Pipeline - Phase 2 Global Consolidation
|
||||
*
|
||||
* Orchestrates the global memory consolidation process:
|
||||
* Lock -> Materialize -> Agent -> Monitor -> Done
|
||||
*
|
||||
* Phase 1 outputs (per-session extractions stored in stage1_outputs DB table)
|
||||
* are materialized to disk as rollout_summaries/*.md + raw_memories.md,
|
||||
* then a CLI agent (--mode write) reads those files and produces MEMORY.md.
|
||||
*
|
||||
* The pipeline uses lease-based locking via MemoryJobScheduler to ensure
|
||||
* only one consolidation runs at a time, with heartbeat-based lease renewal.
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync, readdirSync, unlinkSync, writeFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { MemoryJobScheduler, type ClaimResult } from './memory-job-scheduler.js';
|
||||
import { getCoreMemoryStore, type Stage1Output } from './core-memory-store.js';
|
||||
import { getProjectPaths, ensureStorageDir } from '../config/storage-paths.js';
|
||||
import {
|
||||
HEARTBEAT_INTERVAL_SECONDS,
|
||||
MAX_RAW_MEMORIES_FOR_GLOBAL,
|
||||
} from './memory-v2-config.js';
|
||||
import {
|
||||
CONSOLIDATION_SYSTEM_PROMPT,
|
||||
buildConsolidationPrompt,
|
||||
} from './memory-consolidation-prompts.js';
|
||||
|
||||
// -- Types --
|
||||
|
||||
export interface ConsolidationStatus {
|
||||
status: 'idle' | 'running' | 'completed' | 'error';
|
||||
lastRun?: number;
|
||||
memoryMdExists: boolean;
|
||||
inputCount: number;
|
||||
lastError?: string;
|
||||
}
|
||||
|
||||
export interface MaterializationResult {
|
||||
summariesWritten: number;
|
||||
summariesPruned: number;
|
||||
rawMemoriesSize: number;
|
||||
}
|
||||
|
||||
// -- Constants --
|
||||
|
||||
const JOB_KIND = 'memory_consolidate_global';
|
||||
const JOB_KEY = 'global';
|
||||
const MAX_CONCURRENT = 1;
|
||||
const AGENT_TIMEOUT_MS = 300_000; // 5 minutes
|
||||
const DEFAULT_CLI_TOOL = 'gemini';
|
||||
|
||||
// -- Utility --
|
||||
|
||||
/**
|
||||
* Sanitize a thread ID for use as a filename.
|
||||
* Replaces filesystem-unsafe characters with underscores.
|
||||
*/
|
||||
function sanitizeFilename(name: string): string {
|
||||
return name.replace(/[<>:"/\\|?*\x00-\x1f]/g, '_');
|
||||
}
|
||||
|
||||
// -- Standalone Functions --
|
||||
|
||||
/**
|
||||
* Write one .md file per stage1_output to rollout_summaries/, prune orphans.
|
||||
*
|
||||
* Each file is named {sanitized_thread_id}.md and contains the rollout_summary.
|
||||
* Files in the directory that do not correspond to any DB row are deleted.
|
||||
*/
|
||||
export function syncRolloutSummaries(
|
||||
memoryHome: string,
|
||||
outputs: Stage1Output[]
|
||||
): MaterializationResult {
|
||||
const summariesDir = join(memoryHome, 'rollout_summaries');
|
||||
ensureStorageDir(summariesDir);
|
||||
|
||||
// Build set of expected filenames
|
||||
const expectedFiles = new Set<string>();
|
||||
let summariesWritten = 0;
|
||||
|
||||
for (const output of outputs) {
|
||||
const filename = `${sanitizeFilename(output.thread_id)}.md`;
|
||||
expectedFiles.add(filename);
|
||||
const filePath = join(summariesDir, filename);
|
||||
|
||||
// Write summary content with thread header
|
||||
const content = [
|
||||
`# Session: ${output.thread_id}`,
|
||||
`> Generated: ${new Date(output.generated_at * 1000).toISOString()}`,
|
||||
`> Source updated: ${new Date(output.source_updated_at * 1000).toISOString()}`,
|
||||
'',
|
||||
output.rollout_summary,
|
||||
].join('\n');
|
||||
|
||||
writeFileSync(filePath, content, 'utf-8');
|
||||
summariesWritten++;
|
||||
}
|
||||
|
||||
// Prune orphan files not in DB
|
||||
let summariesPruned = 0;
|
||||
const existingFiles = readdirSync(summariesDir);
|
||||
for (const file of existingFiles) {
|
||||
if (file.endsWith('.md') && !expectedFiles.has(file)) {
|
||||
unlinkSync(join(summariesDir, file));
|
||||
summariesPruned++;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
summariesWritten,
|
||||
summariesPruned,
|
||||
rawMemoriesSize: 0, // Not applicable for this function
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Concatenate the latest raw_memory entries into raw_memories.md with thread headers.
|
||||
*
|
||||
* Entries are sorted by generated_at descending, limited to maxCount.
|
||||
* Format per entry:
|
||||
* ## Thread: {thread_id}
|
||||
* {raw_memory content}
|
||||
* ---
|
||||
*
|
||||
* @returns The byte size of the written raw_memories.md file.
|
||||
*/
|
||||
export function rebuildRawMemories(
|
||||
memoryHome: string,
|
||||
outputs: Stage1Output[],
|
||||
maxCount: number
|
||||
): number {
|
||||
ensureStorageDir(memoryHome);
|
||||
|
||||
// Sort by generated_at descending, take up to maxCount
|
||||
const sorted = [...outputs]
|
||||
.sort((a, b) => b.generated_at - a.generated_at)
|
||||
.slice(0, maxCount);
|
||||
|
||||
const sections: string[] = [];
|
||||
for (const output of sorted) {
|
||||
sections.push(
|
||||
`## Thread: ${output.thread_id}`,
|
||||
`> Generated: ${new Date(output.generated_at * 1000).toISOString()}`,
|
||||
'',
|
||||
output.raw_memory,
|
||||
'',
|
||||
'---',
|
||||
'',
|
||||
);
|
||||
}
|
||||
|
||||
const content = sections.join('\n');
|
||||
const filePath = join(memoryHome, 'raw_memories.md');
|
||||
writeFileSync(filePath, content, 'utf-8');
|
||||
|
||||
return Buffer.byteLength(content, 'utf-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Read MEMORY.md content for session prompt injection.
|
||||
*
|
||||
* @param projectPath - Project root path (used to resolve storage paths)
|
||||
* @returns MEMORY.md content string, or null if the file does not exist
|
||||
*/
|
||||
export function getMemoryMdContent(projectPath: string): string | null {
|
||||
const paths = getProjectPaths(projectPath);
|
||||
const memoryMdPath = paths.memoryV2.memoryMd;
|
||||
|
||||
if (!existsSync(memoryMdPath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return readFileSync(memoryMdPath, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// -- Pipeline Class --
|
||||
|
||||
/**
|
||||
* MemoryConsolidationPipeline orchestrates global memory consolidation:
|
||||
* 1. Claim global lock via job scheduler
|
||||
* 2. Materialize Phase 1 outputs to disk (rollout_summaries/ + raw_memories.md)
|
||||
* 3. Invoke consolidation agent via executeCliTool --mode write
|
||||
* 4. Monitor with heartbeat lease renewal
|
||||
* 5. Mark job as succeeded or failed
|
||||
*/
|
||||
export class MemoryConsolidationPipeline {
|
||||
private projectPath: string;
|
||||
private store: ReturnType<typeof getCoreMemoryStore>;
|
||||
private scheduler: MemoryJobScheduler;
|
||||
private memoryHome: string;
|
||||
private cliTool: string;
|
||||
|
||||
constructor(projectPath: string, cliTool?: string) {
|
||||
this.projectPath = projectPath;
|
||||
this.store = getCoreMemoryStore(projectPath);
|
||||
this.scheduler = new MemoryJobScheduler(this.store.getDb());
|
||||
this.memoryHome = getProjectPaths(projectPath).memoryV2.root;
|
||||
this.cliTool = cliTool || DEFAULT_CLI_TOOL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to claim the global consolidation lock.
|
||||
*
|
||||
* Before claiming, ensures the job row exists and checks dirtiness.
|
||||
* The job scheduler handles the actual concurrency control.
|
||||
*/
|
||||
claimGlobalLock(): ClaimResult {
|
||||
return this.scheduler.claimJob(JOB_KIND, JOB_KEY, MAX_CONCURRENT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Materialize rollout summaries from DB to disk.
|
||||
* Writes one .md file per stage1_output row, prunes orphan files.
|
||||
*/
|
||||
materializeSummaries(): MaterializationResult {
|
||||
const outputs = this.store.listStage1Outputs();
|
||||
return syncRolloutSummaries(this.memoryHome, outputs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild raw_memories.md from DB entries.
|
||||
* Concatenates the latest entries up to MAX_RAW_MEMORIES_FOR_GLOBAL.
|
||||
*/
|
||||
materializeRawMemories(): number {
|
||||
const outputs = this.store.listStage1Outputs();
|
||||
return rebuildRawMemories(
|
||||
this.memoryHome,
|
||||
outputs,
|
||||
MAX_RAW_MEMORIES_FOR_GLOBAL
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the consolidation agent via executeCliTool with --mode write.
|
||||
* Starts a heartbeat timer to keep the lease alive during agent execution.
|
||||
*
|
||||
* @param token - Ownership token from claimGlobalLock
|
||||
* @returns true if agent completed successfully, false otherwise
|
||||
*/
|
||||
async runConsolidationAgent(token: string): Promise<boolean> {
|
||||
// Lazy import to avoid circular dependencies at module load time
|
||||
const { executeCliTool } = await import('../tools/cli-executor-core.js');
|
||||
|
||||
// Determine input state for prompt
|
||||
const summariesDir = join(this.memoryHome, 'rollout_summaries');
|
||||
let summaryCount = 0;
|
||||
if (existsSync(summariesDir)) {
|
||||
summaryCount = readdirSync(summariesDir).filter(f => f.endsWith('.md')).length;
|
||||
}
|
||||
const hasExistingMemoryMd = existsSync(join(this.memoryHome, 'MEMORY.md'));
|
||||
|
||||
// Ensure skills directory exists
|
||||
ensureStorageDir(join(this.memoryHome, 'skills'));
|
||||
|
||||
// Build the full prompt
|
||||
const userPrompt = buildConsolidationPrompt(summaryCount, hasExistingMemoryMd);
|
||||
const fullPrompt = `${CONSOLIDATION_SYSTEM_PROMPT}\n\n${userPrompt}`;
|
||||
|
||||
// Start heartbeat timer
|
||||
const heartbeatMs = HEARTBEAT_INTERVAL_SECONDS * 1000;
|
||||
const heartbeatTimer = setInterval(() => {
|
||||
const renewed = this.scheduler.heartbeat(JOB_KIND, JOB_KEY, token);
|
||||
if (!renewed) {
|
||||
// Heartbeat rejected - lease was lost
|
||||
clearInterval(heartbeatTimer);
|
||||
}
|
||||
}, heartbeatMs);
|
||||
|
||||
try {
|
||||
// Execute the consolidation agent
|
||||
const result = await Promise.race([
|
||||
executeCliTool({
|
||||
tool: this.cliTool,
|
||||
prompt: fullPrompt,
|
||||
mode: 'write',
|
||||
cd: this.memoryHome,
|
||||
category: 'internal',
|
||||
}),
|
||||
new Promise<never>((_, reject) =>
|
||||
setTimeout(
|
||||
() => reject(new Error('Consolidation agent timed out')),
|
||||
AGENT_TIMEOUT_MS
|
||||
)
|
||||
),
|
||||
]);
|
||||
|
||||
return result.success;
|
||||
} finally {
|
||||
clearInterval(heartbeatTimer);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify that Phase 1 artifacts were not modified by the agent.
|
||||
* Compares file count in rollout_summaries/ before and after agent run.
|
||||
*
|
||||
* @param expectedSummaryCount - Number of summaries before agent run
|
||||
* @returns true if artifacts are intact
|
||||
*/
|
||||
verifyPhase1Integrity(expectedSummaryCount: number): boolean {
|
||||
const summariesDir = join(this.memoryHome, 'rollout_summaries');
|
||||
if (!existsSync(summariesDir)) return expectedSummaryCount === 0;
|
||||
|
||||
const currentCount = readdirSync(summariesDir).filter(f => f.endsWith('.md')).length;
|
||||
return currentCount === expectedSummaryCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current consolidation status.
|
||||
*/
|
||||
getStatus(): ConsolidationStatus {
|
||||
const jobStatus = this.scheduler.getJobStatus(JOB_KIND, JOB_KEY);
|
||||
const inputCount = this.store.countStage1Outputs();
|
||||
const memoryMdExists = existsSync(join(this.memoryHome, 'MEMORY.md'));
|
||||
|
||||
if (!jobStatus) {
|
||||
return {
|
||||
status: 'idle',
|
||||
memoryMdExists,
|
||||
inputCount,
|
||||
};
|
||||
}
|
||||
|
||||
const statusMap: Record<string, ConsolidationStatus['status']> = {
|
||||
pending: 'idle',
|
||||
running: 'running',
|
||||
done: 'completed',
|
||||
error: 'error',
|
||||
};
|
||||
|
||||
return {
|
||||
status: statusMap[jobStatus.status] || 'idle',
|
||||
lastRun: jobStatus.finished_at,
|
||||
memoryMdExists,
|
||||
inputCount,
|
||||
lastError: jobStatus.last_error,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Read MEMORY.md content for session prompt injection.
|
||||
* Convenience method that delegates to the standalone function.
|
||||
*/
|
||||
getMemoryMdContent(): string | null {
|
||||
return getMemoryMdContent(this.projectPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the full consolidation pipeline.
|
||||
*
|
||||
* Pipeline flow:
|
||||
* 1. Check if there are inputs to process
|
||||
* 2. Claim global lock
|
||||
* 3. Materialize Phase 1 outputs to disk
|
||||
* 4. Run consolidation agent with heartbeat
|
||||
* 5. Verify Phase 1 integrity
|
||||
* 6. Mark job as succeeded or failed
|
||||
*
|
||||
* @returns Final consolidation status
|
||||
*/
|
||||
async runConsolidation(): Promise<ConsolidationStatus> {
|
||||
// Step 1: Check inputs
|
||||
const inputCount = this.store.countStage1Outputs();
|
||||
if (inputCount === 0) {
|
||||
return {
|
||||
status: 'idle',
|
||||
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
|
||||
inputCount: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Step 2: Claim global lock
|
||||
const claim = this.claimGlobalLock();
|
||||
if (!claim.claimed || !claim.ownership_token) {
|
||||
return {
|
||||
status: claim.reason === 'already_running' ? 'running' : 'idle',
|
||||
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
|
||||
inputCount,
|
||||
lastError: claim.reason
|
||||
? `Lock not acquired: ${claim.reason}`
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
const token = claim.ownership_token;
|
||||
|
||||
try {
|
||||
// Step 3: Materialize Phase 1 outputs to disk
|
||||
const matResult = this.materializeSummaries();
|
||||
const rawMemoriesSize = this.materializeRawMemories();
|
||||
|
||||
const expectedSummaryCount = matResult.summariesWritten;
|
||||
|
||||
// Step 4: Run consolidation agent with heartbeat
|
||||
const agentSuccess = await this.runConsolidationAgent(token);
|
||||
|
||||
if (!agentSuccess) {
|
||||
this.scheduler.markFailed(
|
||||
JOB_KIND,
|
||||
JOB_KEY,
|
||||
token,
|
||||
'Consolidation agent returned failure'
|
||||
);
|
||||
return {
|
||||
status: 'error',
|
||||
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
|
||||
inputCount,
|
||||
lastError: 'Consolidation agent returned failure',
|
||||
};
|
||||
}
|
||||
|
||||
// Step 5: Verify Phase 1 integrity
|
||||
if (!this.verifyPhase1Integrity(expectedSummaryCount)) {
|
||||
this.scheduler.markFailed(
|
||||
JOB_KIND,
|
||||
JOB_KEY,
|
||||
token,
|
||||
'Phase 1 artifacts were modified during consolidation'
|
||||
);
|
||||
return {
|
||||
status: 'error',
|
||||
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
|
||||
inputCount,
|
||||
lastError: 'Phase 1 artifacts were modified during consolidation',
|
||||
};
|
||||
}
|
||||
|
||||
// Step 6: Check that MEMORY.md was actually produced
|
||||
const memoryMdExists = existsSync(join(this.memoryHome, 'MEMORY.md'));
|
||||
if (!memoryMdExists) {
|
||||
this.scheduler.markFailed(
|
||||
JOB_KIND,
|
||||
JOB_KEY,
|
||||
token,
|
||||
'Agent completed but MEMORY.md was not produced'
|
||||
);
|
||||
return {
|
||||
status: 'error',
|
||||
memoryMdExists: false,
|
||||
inputCount,
|
||||
lastError: 'Agent completed but MEMORY.md was not produced',
|
||||
};
|
||||
}
|
||||
|
||||
// Step 7: Mark success with watermark
|
||||
// Use the current input count as the success watermark
|
||||
const watermark = inputCount;
|
||||
this.scheduler.markSucceeded(JOB_KIND, JOB_KEY, token, watermark);
|
||||
|
||||
return {
|
||||
status: 'completed',
|
||||
lastRun: Math.floor(Date.now() / 1000),
|
||||
memoryMdExists: true,
|
||||
inputCount,
|
||||
};
|
||||
} catch (err) {
|
||||
const errorMessage =
|
||||
err instanceof Error ? err.message : String(err);
|
||||
this.scheduler.markFailed(JOB_KIND, JOB_KEY, token, errorMessage);
|
||||
|
||||
return {
|
||||
status: 'error',
|
||||
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
|
||||
inputCount,
|
||||
lastError: errorMessage,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
108
ccw/src/core/memory-consolidation-prompts.ts
Normal file
108
ccw/src/core/memory-consolidation-prompts.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* Memory Consolidation Prompts - Phase 2 LLM Agent Prompt Templates
|
||||
*
|
||||
* System prompt and instruction templates for the consolidation agent that
|
||||
* reads Phase 1 outputs (rollout_summaries/ + raw_memories.md) and produces
|
||||
* MEMORY.md (and optional skills/ files).
|
||||
*
|
||||
* Design: The agent runs with --mode write in the MEMORY_HOME directory,
|
||||
* using standard file read/write tools to produce output.
|
||||
*/
|
||||
|
||||
/**
|
||||
* System-level instructions for the consolidation agent.
|
||||
* This is prepended to every consolidation prompt.
|
||||
*/
|
||||
export const CONSOLIDATION_SYSTEM_PROMPT = `You are a memory consolidation agent. Your job is to read phase-1 extraction artifacts and produce a consolidated MEMORY.md file that captures the most important, actionable knowledge from recent coding sessions.
|
||||
|
||||
## CRITICAL RULES
|
||||
|
||||
1. **Do NOT modify phase-1 artifacts.** The files in rollout_summaries/ and raw_memories.md are READ-ONLY inputs. Never edit, delete, or overwrite them.
|
||||
2. **Output only to MEMORY.md** (and optionally skills/ directory). Do not create files outside these paths.
|
||||
3. **Be concise and actionable.** Every line in MEMORY.md should help a future coding session be more productive.
|
||||
4. **Resolve conflicts.** When rollout summaries and raw memories disagree, prefer the more recent or more specific information.
|
||||
5. **Deduplicate.** Merge overlapping knowledge from different sessions into single authoritative entries.
|
||||
6. **Preserve attribution.** When a piece of knowledge comes from a specific session thread, note it briefly.`;
|
||||
|
||||
/**
|
||||
* Build the full consolidation prompt given the MEMORY_HOME directory path.
|
||||
*
|
||||
* The agent will be invoked with --cd pointing to memoryHome, so all file
|
||||
* references are relative to that directory.
|
||||
*
|
||||
* @param inputSummaryCount - Number of rollout summary files available
|
||||
* @param hasExistingMemoryMd - Whether a previous MEMORY.md exists to update
|
||||
* @returns Complete prompt string for executeCliTool
|
||||
*/
|
||||
export function buildConsolidationPrompt(
|
||||
inputSummaryCount: number,
|
||||
hasExistingMemoryMd: boolean
|
||||
): string {
|
||||
const action = hasExistingMemoryMd ? 'UPDATE' : 'CREATE';
|
||||
|
||||
return `## Task: ${action} MEMORY.md
|
||||
|
||||
You have access to the following phase-1 artifacts in the current directory:
|
||||
|
||||
### Input Files (READ-ONLY - do NOT modify these)
|
||||
- **rollout_summaries/*.md** - ${inputSummaryCount} per-session summary files. Each contains a concise summary of what was accomplished in that coding session. Use these for high-level routing and prioritization.
|
||||
- **raw_memories.md** - Concatenated detailed memories from recent sessions, organized by thread. Use this for detailed knowledge extraction and cross-referencing.
|
||||
${hasExistingMemoryMd ? '- **MEMORY.md** - The existing consolidated memory file. Update it with new knowledge while preserving still-relevant existing content.' : ''}
|
||||
|
||||
### Your Process
|
||||
1. Read all files in rollout_summaries/ to understand the scope and themes of recent sessions.
|
||||
2. Read raw_memories.md to extract detailed, actionable knowledge.
|
||||
3. Cross-reference summaries with raw memories to identify:
|
||||
- High-signal patterns and conventions discovered
|
||||
- Architecture decisions and their rationale
|
||||
- Common pitfalls and their solutions
|
||||
- Key APIs, interfaces, and integration points
|
||||
- Testing patterns and debugging approaches
|
||||
${hasExistingMemoryMd ? '4. Read the existing MEMORY.md and merge new knowledge, removing stale entries.' : '4. Organize extracted knowledge into the output structure below.'}
|
||||
5. Write the consolidated MEMORY.md file.
|
||||
6. Optionally, if there are reusable code patterns or workflows worth extracting, create files in the skills/ directory.
|
||||
|
||||
### Output: MEMORY.md Structure
|
||||
|
||||
Write MEMORY.md with the following sections. Omit any section that has no relevant content.
|
||||
|
||||
\`\`\`markdown
|
||||
# Project Memory
|
||||
|
||||
> Auto-generated by memory consolidation. Last updated: [current date]
|
||||
> Sources: [number] session summaries, [number] raw memory entries
|
||||
|
||||
## Architecture & Structure
|
||||
<!-- Key architectural decisions, module boundaries, data flow patterns -->
|
||||
|
||||
## Code Conventions
|
||||
<!-- Naming conventions, import patterns, error handling approaches, formatting rules -->
|
||||
|
||||
## Common Patterns
|
||||
<!-- Reusable patterns discovered across sessions: state management, API calls, testing approaches -->
|
||||
|
||||
## Key APIs & Interfaces
|
||||
<!-- Important interfaces, function signatures, configuration schemas that are frequently referenced -->
|
||||
|
||||
## Known Issues & Gotchas
|
||||
<!-- Pitfalls, edge cases, platform-specific behaviors, workarounds -->
|
||||
|
||||
## Recent Decisions
|
||||
<!-- Decisions made in recent sessions with brief rationale. Remove when no longer relevant. -->
|
||||
|
||||
## Session Insights
|
||||
<!-- High-value observations from individual sessions worth preserving -->
|
||||
\`\`\`
|
||||
|
||||
### Output: skills/ Directory (Optional)
|
||||
|
||||
If you identify reusable code snippets, shell commands, or workflow templates that appear across multiple sessions, create files in the skills/ directory:
|
||||
- skills/[name].md - Each file should be a self-contained, copy-paste ready reference
|
||||
|
||||
### Quality Criteria
|
||||
- Every entry should be actionable (helps write better code or avoid mistakes)
|
||||
- No vague platitudes - be specific with file paths, function names, config values
|
||||
- Prefer concrete examples over abstract descriptions
|
||||
- Keep total MEMORY.md under 5000 words - be ruthlessly concise
|
||||
- Remove outdated information that contradicts newer findings`;
|
||||
}
|
||||
@@ -9,6 +9,7 @@
|
||||
* - JSON protocol communication
|
||||
* - Three commands: embed, search, status
|
||||
* - Automatic availability checking
|
||||
* - Stage1 output embedding for V2 pipeline
|
||||
*/
|
||||
|
||||
import { spawn } from 'child_process';
|
||||
@@ -16,6 +17,9 @@ import { join, dirname } from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { getCodexLensPython } from '../utils/codexlens-path.js';
|
||||
import { getCoreMemoryStore } from './core-memory-store.js';
|
||||
import type { Stage1Output } from './core-memory-store.js';
|
||||
import { StoragePaths } from '../config/storage-paths.js';
|
||||
|
||||
// Get directory of this module
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
@@ -256,3 +260,111 @@ export async function getEmbeddingStatus(dbPath: string): Promise<EmbeddingStatu
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Memory V2: Stage1 Output Embedding
|
||||
// ============================================================================
|
||||
|
||||
/** Result of stage1 embedding operation */
|
||||
export interface Stage1EmbedResult {
|
||||
success: boolean;
|
||||
chunksCreated: number;
|
||||
chunksEmbedded: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunk and embed stage1_outputs (raw_memory + rollout_summary) for semantic search.
|
||||
*
|
||||
* Reads all stage1_outputs from the DB, chunks their raw_memory and rollout_summary
|
||||
* content, inserts chunks into memory_chunks with source_type='cli_history' and
|
||||
* metadata indicating the V2 origin, then triggers embedding generation.
|
||||
*
|
||||
* Uses source_id format: "s1:{thread_id}" to differentiate from regular cli_history chunks.
|
||||
*
|
||||
* @param projectPath - Project root path
|
||||
* @param force - Force re-chunking even if chunks exist
|
||||
* @returns Embedding result
|
||||
*/
|
||||
export async function embedStage1Outputs(
|
||||
projectPath: string,
|
||||
force: boolean = false
|
||||
): Promise<Stage1EmbedResult> {
|
||||
try {
|
||||
const store = getCoreMemoryStore(projectPath);
|
||||
const stage1Outputs = store.listStage1Outputs();
|
||||
|
||||
if (stage1Outputs.length === 0) {
|
||||
return { success: true, chunksCreated: 0, chunksEmbedded: 0 };
|
||||
}
|
||||
|
||||
let totalChunksCreated = 0;
|
||||
|
||||
for (const output of stage1Outputs) {
|
||||
const sourceId = `s1:${output.thread_id}`;
|
||||
|
||||
// Check if already chunked
|
||||
const existingChunks = store.getChunks(sourceId);
|
||||
if (existingChunks.length > 0 && !force) continue;
|
||||
|
||||
// Delete old chunks if force
|
||||
if (force && existingChunks.length > 0) {
|
||||
store.deleteChunks(sourceId);
|
||||
}
|
||||
|
||||
// Combine raw_memory and rollout_summary for richer semantic content
|
||||
const combinedContent = [
|
||||
output.rollout_summary ? `## Summary\n${output.rollout_summary}` : '',
|
||||
output.raw_memory ? `## Raw Memory\n${output.raw_memory}` : '',
|
||||
].filter(Boolean).join('\n\n');
|
||||
|
||||
if (!combinedContent.trim()) continue;
|
||||
|
||||
// Chunk using the store's built-in chunking
|
||||
const chunks = store.chunkContent(combinedContent, sourceId, 'cli_history');
|
||||
|
||||
// Insert chunks with V2 metadata
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
store.insertChunk({
|
||||
source_id: sourceId,
|
||||
source_type: 'cli_history',
|
||||
chunk_index: i,
|
||||
content: chunks[i],
|
||||
metadata: JSON.stringify({
|
||||
v2_source: 'stage1_output',
|
||||
thread_id: output.thread_id,
|
||||
generated_at: output.generated_at,
|
||||
}),
|
||||
created_at: new Date().toISOString(),
|
||||
});
|
||||
totalChunksCreated++;
|
||||
}
|
||||
}
|
||||
|
||||
// If we created chunks, generate embeddings
|
||||
let chunksEmbedded = 0;
|
||||
if (totalChunksCreated > 0) {
|
||||
const paths = StoragePaths.project(projectPath);
|
||||
const dbPath = join(paths.root, 'core-memory', 'core_memory.db');
|
||||
|
||||
const embedResult = await generateEmbeddings(dbPath, { force: false });
|
||||
if (embedResult.success) {
|
||||
chunksEmbedded = embedResult.chunks_processed;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
chunksCreated: totalChunksCreated,
|
||||
chunksEmbedded,
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
chunksCreated: 0,
|
||||
chunksEmbedded: 0,
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
466
ccw/src/core/memory-extraction-pipeline.ts
Normal file
466
ccw/src/core/memory-extraction-pipeline.ts
Normal file
@@ -0,0 +1,466 @@
|
||||
/**
|
||||
* Memory Extraction Pipeline - Phase 1 per-session extraction
|
||||
*
|
||||
* Orchestrates the full extraction flow for each CLI session:
|
||||
* Filter transcript -> Truncate -> LLM Extract -> SecretRedact -> PostProcess -> Store
|
||||
*
|
||||
* Uses CliHistoryStore for transcript access, executeCliTool for LLM invocation,
|
||||
* CoreMemoryStore for stage1_outputs storage, and MemoryJobScheduler for
|
||||
* concurrency control.
|
||||
*/
|
||||
|
||||
import type { ConversationRecord } from '../tools/cli-history-store.js';
|
||||
import { getHistoryStore } from '../tools/cli-history-store.js';
|
||||
import { getCoreMemoryStore, type Stage1Output } from './core-memory-store.js';
|
||||
import { MemoryJobScheduler } from './memory-job-scheduler.js';
|
||||
import {
|
||||
MAX_SESSION_AGE_DAYS,
|
||||
MIN_IDLE_HOURS,
|
||||
MAX_ROLLOUT_BYTES_FOR_PROMPT,
|
||||
MAX_RAW_MEMORY_CHARS,
|
||||
MAX_SUMMARY_CHARS,
|
||||
MAX_SESSIONS_PER_STARTUP,
|
||||
PHASE_ONE_CONCURRENCY,
|
||||
} from './memory-v2-config.js';
|
||||
import { EXTRACTION_SYSTEM_PROMPT, buildExtractionUserPrompt } from './memory-extraction-prompts.js';
|
||||
import { redactSecrets } from '../utils/secret-redactor.js';
|
||||
|
||||
// -- Types --
|
||||
|
||||
export interface ExtractionInput {
|
||||
sessionId: string;
|
||||
transcript: string;
|
||||
sourceUpdatedAt: number;
|
||||
}
|
||||
|
||||
export interface ExtractionOutput {
|
||||
raw_memory: string;
|
||||
rollout_summary: string;
|
||||
}
|
||||
|
||||
export interface TranscriptFilterOptions {
|
||||
/** Bitmask for turn type selection. Default ALL = 0x7FF */
|
||||
bitmask: number;
|
||||
/** Maximum bytes for the transcript sent to LLM */
|
||||
maxBytes: number;
|
||||
}
|
||||
|
||||
export interface BatchExtractionResult {
|
||||
processed: number;
|
||||
succeeded: number;
|
||||
failed: number;
|
||||
skipped: number;
|
||||
errors: Array<{ sessionId: string; error: string }>;
|
||||
}
|
||||
|
||||
// -- Turn type bitmask constants --
|
||||
|
||||
/** All turn types included */
|
||||
export const TURN_TYPE_ALL = 0x7FF;
|
||||
|
||||
// Individual turn type bits (for future filtering granularity)
|
||||
export const TURN_TYPE_USER_PROMPT = 0x001;
|
||||
export const TURN_TYPE_STDOUT = 0x002;
|
||||
export const TURN_TYPE_STDERR = 0x004;
|
||||
export const TURN_TYPE_PARSED = 0x008;
|
||||
|
||||
// -- Truncation marker --
|
||||
|
||||
const TRUNCATION_MARKER = '\n\n[... CONTENT TRUNCATED ...]\n\n';
|
||||
|
||||
// -- Job kind constant --
|
||||
|
||||
const JOB_KIND_EXTRACTION = 'phase1_extraction';
|
||||
|
||||
// -- Pipeline --
|
||||
|
||||
export class MemoryExtractionPipeline {
|
||||
private projectPath: string;
|
||||
/** Optional: override the LLM tool used for extraction. Defaults to 'gemini'. */
|
||||
private tool: string;
|
||||
/** Optional: current session ID to exclude from scanning */
|
||||
private currentSessionId?: string;
|
||||
|
||||
constructor(projectPath: string, options?: { tool?: string; currentSessionId?: string }) {
|
||||
this.projectPath = projectPath;
|
||||
this.tool = options?.tool || 'gemini';
|
||||
this.currentSessionId = options?.currentSessionId;
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Eligibility scanning
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Scan CLI history for sessions eligible for memory extraction.
|
||||
*
|
||||
* Eligibility criteria (from design spec section 4.1):
|
||||
* - Session age <= MAX_SESSION_AGE_DAYS (30 days)
|
||||
* - Session idle >= MIN_IDLE_HOURS (12 hours) since last update
|
||||
* - Not an ephemeral/internal session (category !== 'internal')
|
||||
* - Not the currently active session
|
||||
* - Has at least one turn with content
|
||||
*
|
||||
* @returns Array of eligible ConversationRecord objects, capped at MAX_SESSIONS_PER_STARTUP
|
||||
*/
|
||||
scanEligibleSessions(maxSessions?: number): ConversationRecord[] {
|
||||
const historyStore = getHistoryStore(this.projectPath);
|
||||
const now = Date.now();
|
||||
const maxAgeMs = MAX_SESSION_AGE_DAYS * 24 * 60 * 60 * 1000;
|
||||
const minIdleMs = MIN_IDLE_HOURS * 60 * 60 * 1000;
|
||||
|
||||
// Fetch recent conversations (generous limit to filter in-memory)
|
||||
const { executions } = historyStore.getHistory({ limit: 500 });
|
||||
const eligible: ConversationRecord[] = [];
|
||||
|
||||
for (const entry of executions) {
|
||||
// Skip current session
|
||||
if (this.currentSessionId && entry.id === this.currentSessionId) continue;
|
||||
|
||||
// Age check: created within MAX_SESSION_AGE_DAYS
|
||||
const createdAt = new Date(entry.timestamp).getTime();
|
||||
if (now - createdAt > maxAgeMs) continue;
|
||||
|
||||
// Idle check: last updated at least MIN_IDLE_HOURS ago
|
||||
const updatedAt = new Date(entry.updated_at || entry.timestamp).getTime();
|
||||
if (now - updatedAt < minIdleMs) continue;
|
||||
|
||||
// Skip internal/ephemeral sessions
|
||||
if (entry.category === 'internal') continue;
|
||||
|
||||
// Must have at least 1 turn
|
||||
if (!entry.turn_count || entry.turn_count < 1) continue;
|
||||
|
||||
// Load full conversation to include in result
|
||||
const conv = historyStore.getConversation(entry.id);
|
||||
if (!conv) continue;
|
||||
|
||||
eligible.push(conv);
|
||||
|
||||
if (eligible.length >= (maxSessions || MAX_SESSIONS_PER_STARTUP)) break;
|
||||
}
|
||||
|
||||
return eligible;
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Transcript filtering
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Extract transcript text from a ConversationRecord, keeping only turn types
|
||||
* that match the given bitmask.
|
||||
*
|
||||
* Default bitmask (ALL=0x7FF) includes all turn content: prompt, stdout, stderr, parsed.
|
||||
*
|
||||
* @param record - The conversation record to filter
|
||||
* @param bitmask - Bitmask for type selection (default: TURN_TYPE_ALL)
|
||||
* @returns Combined transcript text
|
||||
*/
|
||||
filterTranscript(record: ConversationRecord, bitmask: number = TURN_TYPE_ALL): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
for (const turn of record.turns) {
|
||||
const turnParts: string[] = [];
|
||||
|
||||
if (bitmask & TURN_TYPE_USER_PROMPT) {
|
||||
if (turn.prompt) {
|
||||
turnParts.push(`[USER] ${turn.prompt}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (bitmask & TURN_TYPE_STDOUT) {
|
||||
const stdout = turn.output?.parsed_output || turn.output?.stdout;
|
||||
if (stdout) {
|
||||
turnParts.push(`[ASSISTANT] ${stdout}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (bitmask & TURN_TYPE_STDERR) {
|
||||
if (turn.output?.stderr) {
|
||||
turnParts.push(`[STDERR] ${turn.output.stderr}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (bitmask & TURN_TYPE_PARSED) {
|
||||
// Use final_output if available and not already captured
|
||||
if (turn.output?.final_output && !(bitmask & TURN_TYPE_STDOUT)) {
|
||||
turnParts.push(`[FINAL] ${turn.output.final_output}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (turnParts.length > 0) {
|
||||
parts.push(`--- Turn ${turn.turn} ---\n${turnParts.join('\n')}`);
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('\n\n');
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Truncation
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Truncate transcript content to fit within LLM context limit.
|
||||
*
|
||||
* Strategy: Keep head 33% + truncation marker + tail 67%.
|
||||
* This preserves the session opening context and the most recent work.
|
||||
*
|
||||
* @param content - The full transcript text
|
||||
* @param maxBytes - Maximum allowed size in bytes (default: MAX_ROLLOUT_BYTES_FOR_PROMPT)
|
||||
* @returns Truncated content, or original if within limit
|
||||
*/
|
||||
truncateTranscript(content: string, maxBytes: number = MAX_ROLLOUT_BYTES_FOR_PROMPT): string {
|
||||
const contentBytes = Buffer.byteLength(content, 'utf-8');
|
||||
if (contentBytes <= maxBytes) {
|
||||
return content;
|
||||
}
|
||||
|
||||
// Calculate split sizes accounting for the marker
|
||||
const markerBytes = Buffer.byteLength(TRUNCATION_MARKER, 'utf-8');
|
||||
const availableBytes = maxBytes - markerBytes;
|
||||
const headBytes = Math.floor(availableBytes * 0.33);
|
||||
const tailBytes = availableBytes - headBytes;
|
||||
|
||||
// Convert to character-based approximation (safe for multi-byte)
|
||||
// Use Buffer slicing for byte-accurate truncation
|
||||
const buf = Buffer.from(content, 'utf-8');
|
||||
const headBuf = buf.subarray(0, headBytes);
|
||||
const tailBuf = buf.subarray(buf.length - tailBytes);
|
||||
|
||||
// Decode back to strings, trimming at character boundaries
|
||||
const head = headBuf.toString('utf-8').replace(/[\uFFFD]$/, '');
|
||||
const tail = tailBuf.toString('utf-8').replace(/^[\uFFFD]/, '');
|
||||
|
||||
return head + TRUNCATION_MARKER + tail;
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// LLM extraction
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Call the LLM to extract structured memory from a session transcript.
|
||||
*
|
||||
* Uses executeCliTool with the extraction prompts. The LLM is expected
|
||||
* to return a JSON object with raw_memory and rollout_summary fields.
|
||||
*
|
||||
* @param sessionId - Session ID for prompt context
|
||||
* @param transcript - The filtered and truncated transcript
|
||||
* @returns Raw LLM output string
|
||||
*/
|
||||
async extractMemory(sessionId: string, transcript: string): Promise<string> {
|
||||
const { executeCliTool } = await import('../tools/cli-executor-core.js');
|
||||
|
||||
const userPrompt = buildExtractionUserPrompt(sessionId, transcript);
|
||||
|
||||
const fullPrompt = `${EXTRACTION_SYSTEM_PROMPT}\n\n${userPrompt}`;
|
||||
|
||||
const result = await executeCliTool({
|
||||
tool: this.tool,
|
||||
prompt: fullPrompt,
|
||||
mode: 'analysis',
|
||||
cd: this.projectPath,
|
||||
category: 'internal',
|
||||
});
|
||||
|
||||
// Prefer parsedOutput (extracted text from stream JSON) over raw stdout
|
||||
const output = result.parsedOutput?.trim() || result.stdout?.trim() || '';
|
||||
return output;
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Post-processing
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Parse LLM output into structured ExtractionOutput.
|
||||
*
|
||||
* Supports 3 parsing modes:
|
||||
* 1. Pure JSON: Output is a valid JSON object
|
||||
* 2. Fenced JSON block: JSON wrapped in ```json ... ``` markers
|
||||
* 3. Text extraction: Non-conforming output wrapped in fallback structure
|
||||
*
|
||||
* Applies secret redaction and size limit enforcement.
|
||||
*
|
||||
* @param llmOutput - Raw text output from the LLM
|
||||
* @returns Validated ExtractionOutput with raw_memory and rollout_summary
|
||||
*/
|
||||
postProcess(llmOutput: string): ExtractionOutput {
|
||||
let parsed: { raw_memory?: string; rollout_summary?: string } | null = null;
|
||||
|
||||
// Mode 1: Pure JSON
|
||||
try {
|
||||
parsed = JSON.parse(llmOutput);
|
||||
} catch {
|
||||
// Not pure JSON, try next mode
|
||||
}
|
||||
|
||||
// Mode 2: Fenced JSON block
|
||||
if (!parsed) {
|
||||
const fencedMatch = llmOutput.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/);
|
||||
if (fencedMatch) {
|
||||
try {
|
||||
parsed = JSON.parse(fencedMatch[1]);
|
||||
} catch {
|
||||
// Fenced content is not valid JSON either
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Mode 3: Text extraction fallback
|
||||
if (!parsed || typeof parsed.raw_memory !== 'string') {
|
||||
parsed = {
|
||||
raw_memory: `# summary\n${llmOutput}\n\nMemory context:\n- Extracted from unstructured LLM output\n\nUser preferences:\n- (none detected)`,
|
||||
rollout_summary: llmOutput.substring(0, 200).replace(/\n/g, ' ').trim(),
|
||||
};
|
||||
}
|
||||
|
||||
// Apply secret redaction
|
||||
let rawMemory = redactSecrets(parsed.raw_memory || '');
|
||||
let rolloutSummary = redactSecrets(parsed.rollout_summary || '');
|
||||
|
||||
// Enforce size limits
|
||||
if (rawMemory.length > MAX_RAW_MEMORY_CHARS) {
|
||||
rawMemory = rawMemory.substring(0, MAX_RAW_MEMORY_CHARS);
|
||||
}
|
||||
if (rolloutSummary.length > MAX_SUMMARY_CHARS) {
|
||||
rolloutSummary = rolloutSummary.substring(0, MAX_SUMMARY_CHARS);
|
||||
}
|
||||
|
||||
return { raw_memory: rawMemory, rollout_summary: rolloutSummary };
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Single session extraction
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Run the full extraction pipeline for a single session.
|
||||
*
|
||||
* Pipeline stages: Filter -> Truncate -> LLM Extract -> PostProcess -> Store
|
||||
*
|
||||
* @param sessionId - The session to extract from
|
||||
* @returns The stored Stage1Output, or null if extraction failed
|
||||
*/
|
||||
async runExtractionJob(sessionId: string): Promise<Stage1Output | null> {
|
||||
const historyStore = getHistoryStore(this.projectPath);
|
||||
const record = historyStore.getConversation(sessionId);
|
||||
if (!record) {
|
||||
throw new Error(`Session not found: ${sessionId}`);
|
||||
}
|
||||
|
||||
// Stage 1: Filter transcript
|
||||
const transcript = this.filterTranscript(record);
|
||||
if (!transcript.trim()) {
|
||||
return null; // Empty transcript, nothing to extract
|
||||
}
|
||||
|
||||
// Stage 2: Truncate
|
||||
const truncated = this.truncateTranscript(transcript);
|
||||
|
||||
// Stage 3: LLM extraction
|
||||
const llmOutput = await this.extractMemory(sessionId, truncated);
|
||||
if (!llmOutput) {
|
||||
throw new Error(`LLM returned empty output for session: ${sessionId}`);
|
||||
}
|
||||
|
||||
// Stage 4: Post-process (parse + redact + validate)
|
||||
const extracted = this.postProcess(llmOutput);
|
||||
|
||||
// Stage 5: Store result
|
||||
const sourceUpdatedAt = Math.floor(new Date(record.updated_at).getTime() / 1000);
|
||||
const generatedAt = Math.floor(Date.now() / 1000);
|
||||
|
||||
const output: Stage1Output = {
|
||||
thread_id: sessionId,
|
||||
source_updated_at: sourceUpdatedAt,
|
||||
raw_memory: extracted.raw_memory,
|
||||
rollout_summary: extracted.rollout_summary,
|
||||
generated_at: generatedAt,
|
||||
};
|
||||
|
||||
const store = getCoreMemoryStore(this.projectPath);
|
||||
store.upsertStage1Output(output);
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Batch orchestration
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Run extraction for all eligible sessions with concurrency control.
|
||||
*
|
||||
* Uses MemoryJobScheduler to claim jobs and enforce PHASE_ONE_CONCURRENCY.
|
||||
* Failed extractions are recorded in the scheduler for retry.
|
||||
*
|
||||
* @returns BatchExtractionResult with counts and error details
|
||||
*/
|
||||
async runBatchExtraction(options?: { maxSessions?: number }): Promise<BatchExtractionResult> {
|
||||
const store = getCoreMemoryStore(this.projectPath);
|
||||
const scheduler = new MemoryJobScheduler(store.getDb());
|
||||
|
||||
// Scan eligible sessions
|
||||
const eligibleSessions = this.scanEligibleSessions(options?.maxSessions);
|
||||
|
||||
const result: BatchExtractionResult = {
|
||||
processed: 0,
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
if (eligibleSessions.length === 0) {
|
||||
return result;
|
||||
}
|
||||
|
||||
// Enqueue all eligible sessions
|
||||
for (const session of eligibleSessions) {
|
||||
const watermark = Math.floor(new Date(session.updated_at).getTime() / 1000);
|
||||
scheduler.enqueueJob(JOB_KIND_EXTRACTION, session.id, watermark);
|
||||
}
|
||||
|
||||
// Process with concurrency control using Promise.all with batching
|
||||
const batchSize = PHASE_ONE_CONCURRENCY;
|
||||
for (let i = 0; i < eligibleSessions.length; i += batchSize) {
|
||||
const batch = eligibleSessions.slice(i, i + batchSize);
|
||||
const promises = batch.map(async (session) => {
|
||||
// Try to claim the job
|
||||
const claim = scheduler.claimJob(JOB_KIND_EXTRACTION, session.id, batchSize);
|
||||
if (!claim.claimed) {
|
||||
result.skipped++;
|
||||
return;
|
||||
}
|
||||
|
||||
result.processed++;
|
||||
const token = claim.ownership_token!;
|
||||
|
||||
try {
|
||||
const output = await this.runExtractionJob(session.id);
|
||||
if (output) {
|
||||
const watermark = output.source_updated_at;
|
||||
scheduler.markSucceeded(JOB_KIND_EXTRACTION, session.id, token, watermark);
|
||||
result.succeeded++;
|
||||
} else {
|
||||
// Empty transcript - mark as done (nothing to extract)
|
||||
scheduler.markSucceeded(JOB_KIND_EXTRACTION, session.id, token, 0);
|
||||
result.skipped++;
|
||||
}
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : String(err);
|
||||
scheduler.markFailed(JOB_KIND_EXTRACTION, session.id, token, errorMsg);
|
||||
result.failed++;
|
||||
result.errors.push({ sessionId: session.id, error: errorMsg });
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(promises);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
91
ccw/src/core/memory-extraction-prompts.ts
Normal file
91
ccw/src/core/memory-extraction-prompts.ts
Normal file
@@ -0,0 +1,91 @@
|
||||
/**
|
||||
* Memory Extraction Prompts - LLM prompt templates for Phase 1 extraction
|
||||
*
|
||||
* Provides system and user prompt templates for extracting structured memory
|
||||
* from CLI session transcripts. The LLM output must conform to a JSON schema
|
||||
* with raw_memory and rollout_summary fields.
|
||||
*
|
||||
* Design spec section 4.4: Prompt structure with outcome triage rules.
|
||||
*/
|
||||
|
||||
/**
|
||||
* System prompt for the extraction LLM call.
|
||||
*
|
||||
* Instructs the model to:
|
||||
* - Produce a JSON object with raw_memory and rollout_summary
|
||||
* - Follow structure markers in raw_memory (# summary, Memory context, etc.)
|
||||
* - Apply outcome triage rules for categorizing task results
|
||||
* - Keep rollout_summary concise (1-2 sentences)
|
||||
*/
|
||||
export const EXTRACTION_SYSTEM_PROMPT = `You are a memory extraction agent. Your job is to read a CLI session transcript and produce structured memory output.
|
||||
|
||||
You MUST respond with a valid JSON object containing exactly two fields:
|
||||
|
||||
{
|
||||
"raw_memory": "<structured memory text>",
|
||||
"rollout_summary": "<1-2 sentence summary>"
|
||||
}
|
||||
|
||||
## raw_memory format
|
||||
|
||||
The raw_memory field must follow this structure:
|
||||
|
||||
# summary
|
||||
<One paragraph high-level summary of what was accomplished in this session>
|
||||
|
||||
Memory context:
|
||||
- Project: <project name or path if identifiable>
|
||||
- Tools used: <CLI tools, frameworks, languages mentioned>
|
||||
- Key files: <important files created or modified>
|
||||
|
||||
User preferences:
|
||||
- <Any coding style preferences, conventions, or patterns the user demonstrated>
|
||||
- <Tool preferences, workflow habits>
|
||||
|
||||
## Task: <task title or description>
|
||||
Outcome: <success | partial | failed | abandoned>
|
||||
<Detailed description of what was done, decisions made, and results>
|
||||
|
||||
### Key decisions
|
||||
- <Important architectural or design decisions>
|
||||
- <Trade-offs considered>
|
||||
|
||||
### Lessons learned
|
||||
- <What worked well>
|
||||
- <What did not work and why>
|
||||
- <Gotchas or pitfalls discovered>
|
||||
|
||||
## Outcome Triage Rules
|
||||
|
||||
- **success**: Task was completed as intended, tests pass, code works
|
||||
- **partial**: Some progress made but not fully complete; note what remains
|
||||
- **failed**: Attempted but could not achieve the goal; document root cause
|
||||
- **abandoned**: User switched direction or cancelled; note the reason
|
||||
|
||||
## rollout_summary format
|
||||
|
||||
A concise 1-2 sentence summary capturing:
|
||||
- What the session was about (the goal)
|
||||
- The outcome (success/partial/failed)
|
||||
- The most important takeaway
|
||||
|
||||
Do NOT include markdown code fences in your response. Return raw JSON only.`;
|
||||
|
||||
/**
|
||||
* Build the user prompt by injecting the session transcript.
|
||||
*
|
||||
* @param sessionId - The session/conversation ID for reference
|
||||
* @param transcript - The filtered and truncated transcript text
|
||||
* @returns The complete user prompt string
|
||||
*/
|
||||
export function buildExtractionUserPrompt(sessionId: string, transcript: string): string {
|
||||
return `Extract structured memory from the following CLI session transcript.
|
||||
|
||||
Session ID: ${sessionId}
|
||||
|
||||
--- BEGIN TRANSCRIPT ---
|
||||
${transcript}
|
||||
--- END TRANSCRIPT ---
|
||||
|
||||
Respond with a JSON object containing "raw_memory" and "rollout_summary" fields.`;
|
||||
}
|
||||
335
ccw/src/core/memory-job-scheduler.ts
Normal file
335
ccw/src/core/memory-job-scheduler.ts
Normal file
@@ -0,0 +1,335 @@
|
||||
/**
|
||||
* Memory Job Scheduler - Lease-based job scheduling backed by SQLite
|
||||
*
|
||||
* Provides atomic claim/release/heartbeat operations for coordinating
|
||||
* concurrent memory extraction and consolidation jobs.
|
||||
*
|
||||
* All state lives in the `jobs` table of the CoreMemoryStore database.
|
||||
* Concurrency control uses ownership_token + lease_until for distributed-safe
|
||||
* (but single-process) job dispatch.
|
||||
*/
|
||||
|
||||
import type Database from 'better-sqlite3';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { LEASE_SECONDS, MAX_RETRIES, RETRY_DELAY_SECONDS } from './memory-v2-config.js';
|
||||
|
||||
// -- Types --
|
||||
|
||||
export type JobStatus = 'pending' | 'running' | 'done' | 'error';
|
||||
|
||||
export interface JobRecord {
|
||||
kind: string;
|
||||
job_key: string;
|
||||
status: JobStatus;
|
||||
worker_id?: string;
|
||||
ownership_token?: string;
|
||||
started_at?: number;
|
||||
finished_at?: number;
|
||||
lease_until?: number;
|
||||
retry_at?: number;
|
||||
retry_remaining: number;
|
||||
last_error?: string;
|
||||
input_watermark: number;
|
||||
last_success_watermark: number;
|
||||
}
|
||||
|
||||
export interface ClaimResult {
|
||||
claimed: boolean;
|
||||
ownership_token?: string;
|
||||
reason?: 'already_running' | 'retry_exhausted' | 'retry_pending' | 'concurrency_limit';
|
||||
}
|
||||
|
||||
// -- Scheduler --
|
||||
|
||||
export class MemoryJobScheduler {
|
||||
private db: Database.Database;
|
||||
|
||||
constructor(db: Database.Database) {
|
||||
this.db = db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomically claim a job for processing.
|
||||
*
|
||||
* Logic:
|
||||
* 1. If job does not exist, insert it as 'running' with a fresh token.
|
||||
* 2. If job exists and is 'pending', transition to 'running'.
|
||||
* 3. If job exists and is 'running' but lease expired, reclaim it.
|
||||
* 4. If job exists and is 'error' with retry_remaining > 0 and retry_at <= now, reclaim it.
|
||||
* 5. Otherwise, return not claimed with reason.
|
||||
*
|
||||
* Respects maxConcurrent: total running jobs of this `kind` must not exceed limit.
|
||||
*/
|
||||
claimJob(kind: string, jobKey: string, maxConcurrent: number, workerId?: string): ClaimResult {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const token = randomUUID();
|
||||
const leaseUntil = now + LEASE_SECONDS;
|
||||
|
||||
// Use a transaction for atomicity
|
||||
const result = this.db.transaction(() => {
|
||||
// Check concurrency limit for this kind
|
||||
const runningCount = this.db.prepare(
|
||||
`SELECT COUNT(*) as cnt FROM jobs WHERE kind = ? AND status = 'running' AND lease_until > ?`
|
||||
).get(kind, now) as { cnt: number };
|
||||
|
||||
const existing = this.db.prepare(
|
||||
`SELECT * FROM jobs WHERE kind = ? AND job_key = ?`
|
||||
).get(kind, jobKey) as any | undefined;
|
||||
|
||||
if (!existing) {
|
||||
// No job row yet - check concurrency before inserting
|
||||
if (runningCount.cnt >= maxConcurrent) {
|
||||
return { claimed: false, reason: 'concurrency_limit' as const };
|
||||
}
|
||||
|
||||
this.db.prepare(`
|
||||
INSERT INTO jobs (kind, job_key, status, worker_id, ownership_token, started_at, lease_until, retry_remaining, input_watermark, last_success_watermark)
|
||||
VALUES (?, ?, 'running', ?, ?, ?, ?, ?, 0, 0)
|
||||
`).run(kind, jobKey, workerId || null, token, now, leaseUntil, MAX_RETRIES);
|
||||
|
||||
return { claimed: true, ownership_token: token };
|
||||
}
|
||||
|
||||
// Job exists - check status transitions
|
||||
if (existing.status === 'done') {
|
||||
// Already done - check dirty (input_watermark > last_success_watermark)
|
||||
if (existing.input_watermark <= existing.last_success_watermark) {
|
||||
return { claimed: false, reason: 'already_running' as const };
|
||||
}
|
||||
// Dirty - re-run
|
||||
if (runningCount.cnt >= maxConcurrent) {
|
||||
return { claimed: false, reason: 'concurrency_limit' as const };
|
||||
}
|
||||
this.db.prepare(`
|
||||
UPDATE jobs SET status = 'running', worker_id = ?, ownership_token = ?,
|
||||
started_at = ?, lease_until = ?, finished_at = NULL, last_error = NULL,
|
||||
retry_remaining = ?
|
||||
WHERE kind = ? AND job_key = ?
|
||||
`).run(workerId || null, token, now, leaseUntil, MAX_RETRIES, kind, jobKey);
|
||||
return { claimed: true, ownership_token: token };
|
||||
}
|
||||
|
||||
if (existing.status === 'running') {
|
||||
// Running - check lease expiry
|
||||
if (existing.lease_until > now) {
|
||||
return { claimed: false, reason: 'already_running' as const };
|
||||
}
|
||||
// Lease expired - reclaim if concurrency allows
|
||||
// The expired job doesn't count towards running total (lease_until <= now),
|
||||
// so runningCount already excludes it.
|
||||
if (runningCount.cnt >= maxConcurrent) {
|
||||
return { claimed: false, reason: 'concurrency_limit' as const };
|
||||
}
|
||||
this.db.prepare(`
|
||||
UPDATE jobs SET worker_id = ?, ownership_token = ?, started_at = ?,
|
||||
lease_until = ?, last_error = NULL
|
||||
WHERE kind = ? AND job_key = ?
|
||||
`).run(workerId || null, token, now, leaseUntil, kind, jobKey);
|
||||
return { claimed: true, ownership_token: token };
|
||||
}
|
||||
|
||||
if (existing.status === 'pending') {
|
||||
if (runningCount.cnt >= maxConcurrent) {
|
||||
return { claimed: false, reason: 'concurrency_limit' as const };
|
||||
}
|
||||
this.db.prepare(`
|
||||
UPDATE jobs SET status = 'running', worker_id = ?, ownership_token = ?,
|
||||
started_at = ?, lease_until = ?
|
||||
WHERE kind = ? AND job_key = ?
|
||||
`).run(workerId || null, token, now, leaseUntil, kind, jobKey);
|
||||
return { claimed: true, ownership_token: token };
|
||||
}
|
||||
|
||||
if (existing.status === 'error') {
|
||||
if (existing.retry_remaining <= 0) {
|
||||
return { claimed: false, reason: 'retry_exhausted' as const };
|
||||
}
|
||||
if (existing.retry_at && existing.retry_at > now) {
|
||||
return { claimed: false, reason: 'retry_pending' as const };
|
||||
}
|
||||
if (runningCount.cnt >= maxConcurrent) {
|
||||
return { claimed: false, reason: 'concurrency_limit' as const };
|
||||
}
|
||||
this.db.prepare(`
|
||||
UPDATE jobs SET status = 'running', worker_id = ?, ownership_token = ?,
|
||||
started_at = ?, lease_until = ?, last_error = NULL,
|
||||
retry_remaining = retry_remaining - 1
|
||||
WHERE kind = ? AND job_key = ?
|
||||
`).run(workerId || null, token, now, leaseUntil, kind, jobKey);
|
||||
return { claimed: true, ownership_token: token };
|
||||
}
|
||||
|
||||
return { claimed: false, reason: 'already_running' as const };
|
||||
})();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Release a job, marking it as done or error.
|
||||
* Only succeeds if the ownership_token matches.
|
||||
*/
|
||||
releaseJob(kind: string, jobKey: string, token: string, status: 'done' | 'error', error?: string): boolean {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
|
||||
const result = this.db.prepare(`
|
||||
UPDATE jobs SET
|
||||
status = ?,
|
||||
finished_at = ?,
|
||||
lease_until = NULL,
|
||||
ownership_token = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = ?
|
||||
WHERE kind = ? AND job_key = ? AND ownership_token = ?
|
||||
`).run(status, now, error || null, kind, jobKey, token);
|
||||
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Renew the lease for an active job.
|
||||
* Returns false if ownership_token does not match or job is not running.
|
||||
*/
|
||||
heartbeat(kind: string, jobKey: string, token: string, leaseSeconds: number = LEASE_SECONDS): boolean {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const newLeaseUntil = now + leaseSeconds;
|
||||
|
||||
const result = this.db.prepare(`
|
||||
UPDATE jobs SET lease_until = ?
|
||||
WHERE kind = ? AND job_key = ? AND ownership_token = ? AND status = 'running'
|
||||
`).run(newLeaseUntil, kind, jobKey, token);
|
||||
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueue a job or update its input_watermark.
|
||||
* Uses MAX(existing, new) for watermark to ensure monotonicity.
|
||||
* If job doesn't exist, creates it in 'pending' status.
|
||||
*/
|
||||
enqueueJob(kind: string, jobKey: string, inputWatermark: number): void {
|
||||
this.db.prepare(`
|
||||
INSERT INTO jobs (kind, job_key, status, retry_remaining, input_watermark, last_success_watermark)
|
||||
VALUES (?, ?, 'pending', ?, ?, 0)
|
||||
ON CONFLICT(kind, job_key) DO UPDATE SET
|
||||
input_watermark = MAX(jobs.input_watermark, excluded.input_watermark)
|
||||
`).run(kind, jobKey, MAX_RETRIES, inputWatermark);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a job as successfully completed and update success watermark.
|
||||
* Only succeeds if ownership_token matches.
|
||||
*/
|
||||
markSucceeded(kind: string, jobKey: string, token: string, watermark: number): boolean {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
|
||||
const result = this.db.prepare(`
|
||||
UPDATE jobs SET
|
||||
status = 'done',
|
||||
finished_at = ?,
|
||||
lease_until = NULL,
|
||||
ownership_token = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = NULL,
|
||||
last_success_watermark = ?
|
||||
WHERE kind = ? AND job_key = ? AND ownership_token = ?
|
||||
`).run(now, watermark, kind, jobKey, token);
|
||||
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a job as failed with error message and schedule retry.
|
||||
* Only succeeds if ownership_token matches.
|
||||
*/
|
||||
markFailed(kind: string, jobKey: string, token: string, error: string, retryDelay: number = RETRY_DELAY_SECONDS): boolean {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const retryAt = now + retryDelay;
|
||||
|
||||
const result = this.db.prepare(`
|
||||
UPDATE jobs SET
|
||||
status = 'error',
|
||||
finished_at = ?,
|
||||
lease_until = NULL,
|
||||
ownership_token = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = ?,
|
||||
retry_at = ?
|
||||
WHERE kind = ? AND job_key = ? AND ownership_token = ?
|
||||
`).run(now, error, retryAt, kind, jobKey, token);
|
||||
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current status of a specific job.
|
||||
*/
|
||||
getJobStatus(kind: string, jobKey: string): JobRecord | null {
|
||||
const row = this.db.prepare(
|
||||
`SELECT * FROM jobs WHERE kind = ? AND job_key = ?`
|
||||
).get(kind, jobKey) as any;
|
||||
|
||||
if (!row) return null;
|
||||
return this.rowToJobRecord(row);
|
||||
}
|
||||
|
||||
/**
|
||||
* List jobs, optionally filtered by kind and/or status.
|
||||
*/
|
||||
listJobs(kind?: string, status?: JobStatus): JobRecord[] {
|
||||
let query = 'SELECT * FROM jobs';
|
||||
const params: any[] = [];
|
||||
const conditions: string[] = [];
|
||||
|
||||
if (kind) {
|
||||
conditions.push('kind = ?');
|
||||
params.push(kind);
|
||||
}
|
||||
if (status) {
|
||||
conditions.push('status = ?');
|
||||
params.push(status);
|
||||
}
|
||||
|
||||
if (conditions.length > 0) {
|
||||
query += ' WHERE ' + conditions.join(' AND ');
|
||||
}
|
||||
|
||||
query += ' ORDER BY kind, job_key';
|
||||
|
||||
const rows = this.db.prepare(query).all(...params) as any[];
|
||||
return rows.map(row => this.rowToJobRecord(row));
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a job is dirty (input_watermark > last_success_watermark).
|
||||
*/
|
||||
isDirty(kind: string, jobKey: string): boolean {
|
||||
const row = this.db.prepare(
|
||||
`SELECT input_watermark, last_success_watermark FROM jobs WHERE kind = ? AND job_key = ?`
|
||||
).get(kind, jobKey) as any;
|
||||
|
||||
if (!row) return false;
|
||||
return row.input_watermark > row.last_success_watermark;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a database row to a typed JobRecord.
|
||||
*/
|
||||
private rowToJobRecord(row: any): JobRecord {
|
||||
return {
|
||||
kind: row.kind,
|
||||
job_key: row.job_key,
|
||||
status: row.status,
|
||||
worker_id: row.worker_id || undefined,
|
||||
ownership_token: row.ownership_token || undefined,
|
||||
started_at: row.started_at || undefined,
|
||||
finished_at: row.finished_at || undefined,
|
||||
lease_until: row.lease_until || undefined,
|
||||
retry_at: row.retry_at || undefined,
|
||||
retry_remaining: row.retry_remaining,
|
||||
last_error: row.last_error || undefined,
|
||||
input_watermark: row.input_watermark ?? 0,
|
||||
last_success_watermark: row.last_success_watermark ?? 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
84
ccw/src/core/memory-v2-config.ts
Normal file
84
ccw/src/core/memory-v2-config.ts
Normal file
@@ -0,0 +1,84 @@
|
||||
/**
|
||||
* Memory V2 Configuration Constants
|
||||
*
|
||||
* All tuning parameters for the two-phase memory extraction and consolidation pipeline.
|
||||
* Phase 1: Per-session extraction (transcript -> structured memory)
|
||||
* Phase 2: Global consolidation (structured memories -> MEMORY.md)
|
||||
*/
|
||||
|
||||
// -- Batch orchestration --
|
||||
|
||||
/** Maximum sessions to process per startup/trigger */
|
||||
export const MAX_SESSIONS_PER_STARTUP = 64;
|
||||
|
||||
/** Maximum concurrent Phase 1 extraction jobs */
|
||||
export const PHASE_ONE_CONCURRENCY = 64;
|
||||
|
||||
// -- Session eligibility --
|
||||
|
||||
/** Maximum session age in days to consider for extraction */
|
||||
export const MAX_SESSION_AGE_DAYS = 30;
|
||||
|
||||
/** Minimum idle hours before a session becomes eligible */
|
||||
export const MIN_IDLE_HOURS = 12;
|
||||
|
||||
// -- Job scheduler --
|
||||
|
||||
/** Default lease duration in seconds (1 hour) */
|
||||
export const LEASE_SECONDS = 3600;
|
||||
|
||||
/** Delay in seconds before retrying a failed job */
|
||||
export const RETRY_DELAY_SECONDS = 3600;
|
||||
|
||||
/** Maximum retry attempts for a failed job */
|
||||
export const MAX_RETRIES = 3;
|
||||
|
||||
/** Interval in seconds between heartbeat renewals */
|
||||
export const HEARTBEAT_INTERVAL_SECONDS = 30;
|
||||
|
||||
// -- Content size limits --
|
||||
|
||||
/** Maximum characters for raw_memory field in stage1_outputs */
|
||||
export const MAX_RAW_MEMORY_CHARS = 300_000;
|
||||
|
||||
/** Maximum characters for rollout_summary field in stage1_outputs */
|
||||
export const MAX_SUMMARY_CHARS = 1200;
|
||||
|
||||
/** Maximum bytes of transcript to send to LLM for extraction */
|
||||
export const MAX_ROLLOUT_BYTES_FOR_PROMPT = 1_000_000;
|
||||
|
||||
/** Maximum number of raw memories included in global consolidation input */
|
||||
export const MAX_RAW_MEMORIES_FOR_GLOBAL = 64;
|
||||
|
||||
// -- Typed configuration object --
|
||||
|
||||
export interface MemoryV2Config {
|
||||
MAX_SESSIONS_PER_STARTUP: number;
|
||||
PHASE_ONE_CONCURRENCY: number;
|
||||
MAX_SESSION_AGE_DAYS: number;
|
||||
MIN_IDLE_HOURS: number;
|
||||
LEASE_SECONDS: number;
|
||||
RETRY_DELAY_SECONDS: number;
|
||||
MAX_RETRIES: number;
|
||||
HEARTBEAT_INTERVAL_SECONDS: number;
|
||||
MAX_RAW_MEMORY_CHARS: number;
|
||||
MAX_SUMMARY_CHARS: number;
|
||||
MAX_ROLLOUT_BYTES_FOR_PROMPT: number;
|
||||
MAX_RAW_MEMORIES_FOR_GLOBAL: number;
|
||||
}
|
||||
|
||||
/** Default configuration object - use individual exports for direct access */
|
||||
export const MEMORY_V2_DEFAULTS: Readonly<MemoryV2Config> = {
|
||||
MAX_SESSIONS_PER_STARTUP,
|
||||
PHASE_ONE_CONCURRENCY,
|
||||
MAX_SESSION_AGE_DAYS,
|
||||
MIN_IDLE_HOURS,
|
||||
LEASE_SECONDS,
|
||||
RETRY_DELAY_SECONDS,
|
||||
MAX_RETRIES,
|
||||
MAX_RAW_MEMORY_CHARS,
|
||||
MAX_SUMMARY_CHARS,
|
||||
MAX_ROLLOUT_BYTES_FOR_PROMPT,
|
||||
MAX_RAW_MEMORIES_FOR_GLOBAL,
|
||||
HEARTBEAT_INTERVAL_SECONDS,
|
||||
} as const;
|
||||
@@ -4,6 +4,8 @@ import { getCoreMemoryStore } from '../core-memory-store.js';
|
||||
import type { CoreMemory, SessionCluster, ClusterMember, ClusterRelation } from '../core-memory-store.js';
|
||||
import { getEmbeddingStatus, generateEmbeddings } from '../memory-embedder-bridge.js';
|
||||
import { checkSemanticStatus } from '../../tools/codex-lens.js';
|
||||
import { MemoryJobScheduler } from '../memory-job-scheduler.js';
|
||||
import type { JobStatus } from '../memory-job-scheduler.js';
|
||||
import { StoragePaths } from '../../config/storage-paths.js';
|
||||
import { join } from 'path';
|
||||
import { getDefaultTool } from '../../tools/claude-cli-tools.js';
|
||||
@@ -233,6 +235,199 @@ export async function handleCoreMemoryRoutes(ctx: RouteContext): Promise<boolean
|
||||
return true;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Memory V2 Pipeline API Endpoints
|
||||
// ============================================================
|
||||
|
||||
// API: Trigger batch extraction (fire-and-forget)
|
||||
if (pathname === '/api/core-memory/extract' && req.method === 'POST') {
|
||||
handlePostRequest(req, res, async (body) => {
|
||||
const { maxSessions, path: projectPath } = body;
|
||||
const basePath = projectPath || initialPath;
|
||||
|
||||
try {
|
||||
const { MemoryExtractionPipeline } = await import('../memory-extraction-pipeline.js');
|
||||
const pipeline = new MemoryExtractionPipeline(basePath);
|
||||
|
||||
// Broadcast start event
|
||||
broadcastToClients({
|
||||
type: 'MEMORY_EXTRACTION_STARTED',
|
||||
payload: {
|
||||
timestamp: new Date().toISOString(),
|
||||
maxSessions: maxSessions || 'default',
|
||||
}
|
||||
});
|
||||
|
||||
// Fire-and-forget: trigger async, notify on completion
|
||||
const batchPromise = pipeline.runBatchExtraction();
|
||||
batchPromise.then(() => {
|
||||
broadcastToClients({
|
||||
type: 'MEMORY_EXTRACTION_COMPLETED',
|
||||
payload: { timestamp: new Date().toISOString() }
|
||||
});
|
||||
}).catch((err: Error) => {
|
||||
broadcastToClients({
|
||||
type: 'MEMORY_EXTRACTION_FAILED',
|
||||
payload: {
|
||||
timestamp: new Date().toISOString(),
|
||||
error: err.message,
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Scan eligible sessions to report count
|
||||
const eligible = pipeline.scanEligibleSessions();
|
||||
|
||||
return {
|
||||
success: true,
|
||||
triggered: true,
|
||||
eligibleCount: eligible.length,
|
||||
message: `Extraction triggered for ${eligible.length} eligible sessions`,
|
||||
};
|
||||
} catch (error: unknown) {
|
||||
return { error: (error as Error).message, status: 500 };
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
// API: Get extraction pipeline status
|
||||
if (pathname === '/api/core-memory/extract/status' && req.method === 'GET') {
|
||||
const projectPath = url.searchParams.get('path') || initialPath;
|
||||
|
||||
try {
|
||||
const store = getCoreMemoryStore(projectPath);
|
||||
const scheduler = new MemoryJobScheduler(store.getDb());
|
||||
|
||||
const stage1Count = store.countStage1Outputs();
|
||||
const extractionJobs = scheduler.listJobs('phase1_extraction');
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
success: true,
|
||||
total_stage1: stage1Count,
|
||||
jobs: extractionJobs.map(j => ({
|
||||
job_key: j.job_key,
|
||||
status: j.status,
|
||||
started_at: j.started_at,
|
||||
finished_at: j.finished_at,
|
||||
last_error: j.last_error,
|
||||
retry_remaining: j.retry_remaining,
|
||||
})),
|
||||
}));
|
||||
} catch (error: unknown) {
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: (error as Error).message }));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// API: Trigger consolidation (fire-and-forget)
|
||||
if (pathname === '/api/core-memory/consolidate' && req.method === 'POST') {
|
||||
handlePostRequest(req, res, async (body) => {
|
||||
const { path: projectPath } = body;
|
||||
const basePath = projectPath || initialPath;
|
||||
|
||||
try {
|
||||
const { MemoryConsolidationPipeline } = await import('../memory-consolidation-pipeline.js');
|
||||
const pipeline = new MemoryConsolidationPipeline(basePath);
|
||||
|
||||
// Broadcast start event
|
||||
broadcastToClients({
|
||||
type: 'MEMORY_CONSOLIDATION_STARTED',
|
||||
payload: { timestamp: new Date().toISOString() }
|
||||
});
|
||||
|
||||
// Fire-and-forget
|
||||
const consolidatePromise = pipeline.runConsolidation();
|
||||
consolidatePromise.then(() => {
|
||||
broadcastToClients({
|
||||
type: 'MEMORY_CONSOLIDATION_COMPLETED',
|
||||
payload: { timestamp: new Date().toISOString() }
|
||||
});
|
||||
}).catch((err: Error) => {
|
||||
broadcastToClients({
|
||||
type: 'MEMORY_CONSOLIDATION_FAILED',
|
||||
payload: {
|
||||
timestamp: new Date().toISOString(),
|
||||
error: err.message,
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
triggered: true,
|
||||
message: 'Consolidation triggered',
|
||||
};
|
||||
} catch (error: unknown) {
|
||||
return { error: (error as Error).message, status: 500 };
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
// API: Get consolidation status
|
||||
if (pathname === '/api/core-memory/consolidate/status' && req.method === 'GET') {
|
||||
const projectPath = url.searchParams.get('path') || initialPath;
|
||||
|
||||
try {
|
||||
const { MemoryConsolidationPipeline } = await import('../memory-consolidation-pipeline.js');
|
||||
const pipeline = new MemoryConsolidationPipeline(projectPath);
|
||||
const status = pipeline.getStatus();
|
||||
const memoryMd = pipeline.getMemoryMdContent();
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
success: true,
|
||||
status: status?.status || 'unknown',
|
||||
memoryMdAvailable: !!memoryMd,
|
||||
memoryMdPreview: memoryMd ? memoryMd.substring(0, 500) : undefined,
|
||||
}));
|
||||
} catch (error: unknown) {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
success: true,
|
||||
status: 'unavailable',
|
||||
memoryMdAvailable: false,
|
||||
error: (error as Error).message,
|
||||
}));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// API: List all V2 pipeline jobs
|
||||
if (pathname === '/api/core-memory/jobs' && req.method === 'GET') {
|
||||
const projectPath = url.searchParams.get('path') || initialPath;
|
||||
const kind = url.searchParams.get('kind') || undefined;
|
||||
const statusFilter = url.searchParams.get('status') as JobStatus | undefined;
|
||||
|
||||
try {
|
||||
const store = getCoreMemoryStore(projectPath);
|
||||
const scheduler = new MemoryJobScheduler(store.getDb());
|
||||
|
||||
const jobs = scheduler.listJobs(kind, statusFilter);
|
||||
|
||||
// Compute byStatus counts
|
||||
const byStatus: Record<string, number> = {};
|
||||
for (const job of jobs) {
|
||||
byStatus[job.status] = (byStatus[job.status] || 0) + 1;
|
||||
}
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
success: true,
|
||||
jobs,
|
||||
total: jobs.length,
|
||||
byStatus,
|
||||
}));
|
||||
} catch (error: unknown) {
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: (error as Error).message }));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Session Clustering API Endpoints
|
||||
// ============================================================
|
||||
|
||||
@@ -7,12 +7,17 @@ import { z } from 'zod';
|
||||
import type { ToolSchema, ToolResult } from '../types/tool.js';
|
||||
import { getCoreMemoryStore, findMemoryAcrossProjects } from '../core/core-memory-store.js';
|
||||
import * as MemoryEmbedder from '../core/memory-embedder-bridge.js';
|
||||
import { MemoryJobScheduler } from '../core/memory-job-scheduler.js';
|
||||
import type { JobRecord, JobStatus } from '../core/memory-job-scheduler.js';
|
||||
import { StoragePaths } from '../config/storage-paths.js';
|
||||
import { join } from 'path';
|
||||
import { getProjectRoot } from '../utils/path-validator.js';
|
||||
|
||||
// Zod schemas
|
||||
const OperationEnum = z.enum(['list', 'import', 'export', 'summary', 'embed', 'search', 'embed_status']);
|
||||
const OperationEnum = z.enum([
|
||||
'list', 'import', 'export', 'summary', 'embed', 'search', 'embed_status',
|
||||
'extract', 'extract_status', 'consolidate', 'consolidate_status', 'jobs',
|
||||
]);
|
||||
|
||||
const ParamsSchema = z.object({
|
||||
operation: OperationEnum,
|
||||
@@ -31,6 +36,11 @@ const ParamsSchema = z.object({
|
||||
source_id: z.string().optional(),
|
||||
batch_size: z.number().optional().default(8),
|
||||
force: z.boolean().optional().default(false),
|
||||
// V2 extract parameters
|
||||
max_sessions: z.number().optional(),
|
||||
// V2 jobs parameters
|
||||
kind: z.string().optional(),
|
||||
status_filter: z.enum(['pending', 'running', 'done', 'error']).optional(),
|
||||
});
|
||||
|
||||
type Params = z.infer<typeof ParamsSchema>;
|
||||
@@ -105,7 +115,44 @@ interface EmbedStatusResult {
|
||||
by_type: Record<string, { total: number; embedded: number }>;
|
||||
}
|
||||
|
||||
type OperationResult = ListResult | ImportResult | ExportResult | SummaryResult | EmbedResult | SearchResult | EmbedStatusResult;
|
||||
// -- Memory V2 operation result types --
|
||||
|
||||
interface ExtractResult {
|
||||
operation: 'extract';
|
||||
triggered: boolean;
|
||||
jobIds: string[];
|
||||
message: string;
|
||||
}
|
||||
|
||||
interface ExtractStatusResult {
|
||||
operation: 'extract_status';
|
||||
total_stage1: number;
|
||||
jobs: Array<{ job_key: string; status: string; last_error?: string }>;
|
||||
}
|
||||
|
||||
interface ConsolidateResult {
|
||||
operation: 'consolidate';
|
||||
triggered: boolean;
|
||||
message: string;
|
||||
}
|
||||
|
||||
interface ConsolidateStatusResult {
|
||||
operation: 'consolidate_status';
|
||||
status: string;
|
||||
memoryMdAvailable: boolean;
|
||||
memoryMdPreview?: string;
|
||||
}
|
||||
|
||||
interface JobsResult {
|
||||
operation: 'jobs';
|
||||
jobs: JobRecord[];
|
||||
total: number;
|
||||
byStatus: Record<string, number>;
|
||||
}
|
||||
|
||||
type OperationResult = ListResult | ImportResult | ExportResult | SummaryResult
|
||||
| EmbedResult | SearchResult | EmbedStatusResult
|
||||
| ExtractResult | ExtractStatusResult | ConsolidateResult | ConsolidateStatusResult | JobsResult;
|
||||
|
||||
/**
|
||||
* Get project path - uses explicit path if provided, otherwise falls back to current working directory
|
||||
@@ -333,6 +380,165 @@ async function executeEmbedStatus(params: Params): Promise<EmbedStatusResult> {
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Memory V2 Operation Handlers
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Operation: extract
|
||||
* Trigger batch extraction (fire-and-forget). Returns job IDs immediately.
|
||||
*/
|
||||
async function executeExtract(params: Params): Promise<ExtractResult> {
|
||||
const { max_sessions, path } = params;
|
||||
const projectPath = getProjectPath(path);
|
||||
|
||||
try {
|
||||
const { MemoryExtractionPipeline } = await import('../core/memory-extraction-pipeline.js');
|
||||
const pipeline = new MemoryExtractionPipeline(projectPath);
|
||||
|
||||
// Fire-and-forget: trigger batch extraction asynchronously
|
||||
const batchPromise = pipeline.runBatchExtraction({ maxSessions: max_sessions });
|
||||
|
||||
// Don't await - let it run in background
|
||||
batchPromise.catch((err: Error) => {
|
||||
// Log errors but don't throw - fire-and-forget
|
||||
console.error(`[memory-v2] Batch extraction error: ${err.message}`);
|
||||
});
|
||||
|
||||
// Scan eligible sessions to report count
|
||||
const eligible = pipeline.scanEligibleSessions(max_sessions);
|
||||
const sessionIds = eligible.map(s => s.id);
|
||||
|
||||
return {
|
||||
operation: 'extract',
|
||||
triggered: true,
|
||||
jobIds: sessionIds,
|
||||
message: `Extraction triggered for ${eligible.length} eligible sessions (max: ${max_sessions || 'default'})`,
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
operation: 'extract',
|
||||
triggered: false,
|
||||
jobIds: [],
|
||||
message: `Failed to trigger extraction: ${(err as Error).message}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Operation: extract_status
|
||||
* Get extraction pipeline state.
|
||||
*/
|
||||
async function executeExtractStatus(params: Params): Promise<ExtractStatusResult> {
|
||||
const { path } = params;
|
||||
const projectPath = getProjectPath(path);
|
||||
|
||||
const store = getCoreMemoryStore(projectPath);
|
||||
const scheduler = new MemoryJobScheduler(store.getDb());
|
||||
|
||||
const stage1Count = store.countStage1Outputs();
|
||||
const extractionJobs = scheduler.listJobs('extraction');
|
||||
|
||||
return {
|
||||
operation: 'extract_status',
|
||||
total_stage1: stage1Count,
|
||||
jobs: extractionJobs.map(j => ({
|
||||
job_key: j.job_key,
|
||||
status: j.status,
|
||||
last_error: j.last_error,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Operation: consolidate
|
||||
* Trigger consolidation (fire-and-forget).
|
||||
*/
|
||||
async function executeConsolidate(params: Params): Promise<ConsolidateResult> {
|
||||
const { path } = params;
|
||||
const projectPath = getProjectPath(path);
|
||||
|
||||
try {
|
||||
const { MemoryConsolidationPipeline } = await import('../core/memory-consolidation-pipeline.js');
|
||||
const pipeline = new MemoryConsolidationPipeline(projectPath);
|
||||
|
||||
// Fire-and-forget: trigger consolidation asynchronously
|
||||
const consolidatePromise = pipeline.runConsolidation();
|
||||
|
||||
consolidatePromise.catch((err: Error) => {
|
||||
console.error(`[memory-v2] Consolidation error: ${err.message}`);
|
||||
});
|
||||
|
||||
return {
|
||||
operation: 'consolidate',
|
||||
triggered: true,
|
||||
message: 'Consolidation triggered',
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
operation: 'consolidate',
|
||||
triggered: false,
|
||||
message: `Failed to trigger consolidation: ${(err as Error).message}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Operation: consolidate_status
|
||||
* Get consolidation pipeline state.
|
||||
*/
|
||||
async function executeConsolidateStatus(params: Params): Promise<ConsolidateStatusResult> {
|
||||
const { path } = params;
|
||||
const projectPath = getProjectPath(path);
|
||||
|
||||
try {
|
||||
const { MemoryConsolidationPipeline } = await import('../core/memory-consolidation-pipeline.js');
|
||||
const pipeline = new MemoryConsolidationPipeline(projectPath);
|
||||
const status = pipeline.getStatus();
|
||||
const memoryMd = pipeline.getMemoryMdContent();
|
||||
|
||||
return {
|
||||
operation: 'consolidate_status',
|
||||
status: status?.status || 'unknown',
|
||||
memoryMdAvailable: !!memoryMd,
|
||||
memoryMdPreview: memoryMd ? memoryMd.substring(0, 500) : undefined,
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
operation: 'consolidate_status',
|
||||
status: 'unavailable',
|
||||
memoryMdAvailable: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Operation: jobs
|
||||
* List all V2 jobs with optional kind filter.
|
||||
*/
|
||||
function executeJobs(params: Params): JobsResult {
|
||||
const { kind, status_filter, path } = params;
|
||||
const projectPath = getProjectPath(path);
|
||||
|
||||
const store = getCoreMemoryStore(projectPath);
|
||||
const scheduler = new MemoryJobScheduler(store.getDb());
|
||||
|
||||
const jobs = scheduler.listJobs(kind, status_filter as JobStatus | undefined);
|
||||
|
||||
// Compute byStatus counts
|
||||
const byStatus: Record<string, number> = {};
|
||||
for (const job of jobs) {
|
||||
byStatus[job.status] = (byStatus[job.status] || 0) + 1;
|
||||
}
|
||||
|
||||
return {
|
||||
operation: 'jobs',
|
||||
jobs,
|
||||
total: jobs.length,
|
||||
byStatus,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Route to appropriate operation handler
|
||||
*/
|
||||
@@ -354,9 +560,19 @@ async function execute(params: Params): Promise<OperationResult> {
|
||||
return executeSearch(params);
|
||||
case 'embed_status':
|
||||
return executeEmbedStatus(params);
|
||||
case 'extract':
|
||||
return executeExtract(params);
|
||||
case 'extract_status':
|
||||
return executeExtractStatus(params);
|
||||
case 'consolidate':
|
||||
return executeConsolidate(params);
|
||||
case 'consolidate_status':
|
||||
return executeConsolidateStatus(params);
|
||||
case 'jobs':
|
||||
return executeJobs(params);
|
||||
default:
|
||||
throw new Error(
|
||||
`Unknown operation: ${operation}. Valid operations: list, import, export, summary, embed, search, embed_status`
|
||||
`Unknown operation: ${operation}. Valid operations: list, import, export, summary, embed, search, embed_status, extract, extract_status, consolidate, consolidate_status, jobs`
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -374,6 +590,11 @@ Usage:
|
||||
core_memory(operation="embed", source_id="CMEM-xxx") # Generate embeddings for memory
|
||||
core_memory(operation="search", query="authentication") # Search memories semantically
|
||||
core_memory(operation="embed_status") # Check embedding status
|
||||
core_memory(operation="extract") # Trigger batch memory extraction (V2)
|
||||
core_memory(operation="extract_status") # Check extraction pipeline status
|
||||
core_memory(operation="consolidate") # Trigger memory consolidation (V2)
|
||||
core_memory(operation="consolidate_status") # Check consolidation status
|
||||
core_memory(operation="jobs") # List all V2 pipeline jobs
|
||||
|
||||
Path parameter (highest priority):
|
||||
core_memory(operation="list", path="/path/to/project") # Use specific project path
|
||||
@@ -384,7 +605,10 @@ Memory IDs use format: CMEM-YYYYMMDD-HHMMSS`,
|
||||
properties: {
|
||||
operation: {
|
||||
type: 'string',
|
||||
enum: ['list', 'import', 'export', 'summary', 'embed', 'search', 'embed_status'],
|
||||
enum: [
|
||||
'list', 'import', 'export', 'summary', 'embed', 'search', 'embed_status',
|
||||
'extract', 'extract_status', 'consolidate', 'consolidate_status', 'jobs',
|
||||
],
|
||||
description: 'Operation to perform',
|
||||
},
|
||||
path: {
|
||||
@@ -437,6 +661,19 @@ Memory IDs use format: CMEM-YYYYMMDD-HHMMSS`,
|
||||
type: 'boolean',
|
||||
description: 'Force re-embedding even if embeddings exist (default: false)',
|
||||
},
|
||||
max_sessions: {
|
||||
type: 'number',
|
||||
description: 'Max sessions to extract in one batch (for extract operation)',
|
||||
},
|
||||
kind: {
|
||||
type: 'string',
|
||||
description: 'Filter jobs by kind (for jobs operation, e.g. "extraction" or "consolidation")',
|
||||
},
|
||||
status_filter: {
|
||||
type: 'string',
|
||||
enum: ['pending', 'running', 'done', 'error'],
|
||||
description: 'Filter jobs by status (for jobs operation)',
|
||||
},
|
||||
},
|
||||
required: ['operation'],
|
||||
},
|
||||
|
||||
50
ccw/src/utils/secret-redactor.ts
Normal file
50
ccw/src/utils/secret-redactor.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* Secret Redactor - Regex-based secret pattern detection and replacement
|
||||
*
|
||||
* Scans text for common secret patterns (API keys, tokens, credentials)
|
||||
* and replaces them with [REDACTED_SECRET] to prevent leakage into
|
||||
* memory extraction outputs.
|
||||
*
|
||||
* Patterns are intentionally specific (prefix-based) to minimize false positives.
|
||||
*/
|
||||
|
||||
const REDACTED = '[REDACTED_SECRET]';
|
||||
|
||||
/**
|
||||
* Secret patterns with named regex for each category.
|
||||
* Each pattern targets a specific, well-known secret format.
|
||||
*/
|
||||
const SECRET_PATTERNS: ReadonlyArray<{ name: string; regex: RegExp }> = [
|
||||
// OpenAI API keys: sk-<20+ alphanumeric chars>
|
||||
{ name: 'openai_key', regex: /sk-[A-Za-z0-9]{20,}/g },
|
||||
// AWS Access Key IDs: AKIA<16 uppercase alphanumeric chars>
|
||||
{ name: 'aws_key', regex: /AKIA[0-9A-Z]{16}/g },
|
||||
// Bearer tokens: Bearer <16+ token chars>
|
||||
{ name: 'bearer_token', regex: /Bearer\s+[A-Za-z0-9._\-]{16,}/g },
|
||||
// Secret assignments: key=value or key:value patterns for known secret variable names
|
||||
{ name: 'secret_assignment', regex: /(?:api_key|token|secret|password)[:=]\S+/gi },
|
||||
];
|
||||
|
||||
/**
|
||||
* Apply regex-based secret pattern matching and replacement.
|
||||
*
|
||||
* Scans the input text for 4 pattern categories:
|
||||
* 1. OpenAI API keys (sk-...)
|
||||
* 2. AWS Access Key IDs (AKIA...)
|
||||
* 3. Bearer tokens (Bearer ...)
|
||||
* 4. Secret variable assignments (api_key=..., token:..., etc.)
|
||||
*
|
||||
* @param text - Input text to scan for secrets
|
||||
* @returns Text with all matched secrets replaced by [REDACTED_SECRET]
|
||||
*/
|
||||
export function redactSecrets(text: string): string {
|
||||
if (!text) return text;
|
||||
|
||||
let result = text;
|
||||
for (const { regex } of SECRET_PATTERNS) {
|
||||
// Reset lastIndex for global regexes to ensure fresh match on each call
|
||||
regex.lastIndex = 0;
|
||||
result = result.replace(regex, REDACTED);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
Reference in New Issue
Block a user