mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-02-12 02:37:45 +08:00
feat: unified task.json schema migration and multi-module updates
- Create task-schema.json (JSON Schema draft-07) with 10 field blocks fusing Unified JSONL, 6-field Task JSON, and Solution Schema advantages - Migrate unified-execute-with-file from JSONL to .task/*.json directory scanning - Migrate 3 producers (lite-plan, plan-converter, collaborative-plan) to .task/*.json multi-file output - Add review-cycle Phase 7.5 export-to-tasks (FIX-*.json) and issue-resolve --export-tasks option - Add schema compatibility annotations to action-planning-agent, workflow-plan, and tdd-plan - Add spec-generator skill phases and templates - Add memory v2 pipeline (consolidation, extraction, job scheduler, embedder) - Add secret-redactor utility and core-memory enhancements - Add codex-lens accuracy benchmarks and staged env config overrides
This commit is contained in:
catlog22
379
.ccw/workflows/cli-templates/schemas/task-schema.json
Normal file
379
.ccw/workflows/cli-templates/schemas/task-schema.json
Normal file
@@ -0,0 +1,379 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"$id": "task-schema.json",
|
||||||
|
"title": "Unified Task JSON Schema",
|
||||||
|
"description": "统一任务定义 schema v1.0 — 每个任务一个独立 JSON 文件，自包含所有执行所需信息。融合 Unified JSONL (convergence)、6-field Task JSON (execution_config)、Solution Schema (modification_points) 三者优势。",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["id", "title", "description", "depends_on", "convergence"],
|
||||||
|
|
||||||
|
"properties": {
|
||||||
|
|
||||||
|
"_comment_IDENTITY": "IDENTITY 区块 (必填) — 任务基本信息",
|
||||||
|
"id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "任务ID，前缀由生产者决定 (TASK-001 / IMPL-001 / L0 / T1 / FIX-001 等)"
|
||||||
|
},
|
||||||
|
"title": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "任务标题 (动词+目标，如 'Create unified task schema')"
|
||||||
|
},
|
||||||
|
"description": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "目标+原因 (1-3 句，描述做什么和为什么)"
|
||||||
|
},
|
||||||
|
|
||||||
|
"_comment_CLASSIFICATION": "CLASSIFICATION 区块 (可选) — 任务分类",
|
||||||
|
"type": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["infrastructure", "feature", "enhancement", "fix", "refactor", "testing", "docs", "chore"],
|
||||||
|
"description": "任务类型"
|
||||||
|
},
|
||||||
|
"priority": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["critical", "high", "medium", "low"],
|
||||||
|
"description": "优先级"
|
||||||
|
},
|
||||||
|
"effort": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["small", "medium", "large"],
|
||||||
|
"description": "工作量估算"
|
||||||
|
},
|
||||||
|
"action": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["Create", "Update", "Implement", "Refactor", "Add", "Delete", "Configure", "Test", "Fix"],
|
||||||
|
"description": "操作动作 (便于 issue 系统分类)"
|
||||||
|
},
|
||||||
|
|
||||||
|
"_comment_SCOPE": "SCOPE 区块 (可选) — 覆盖范围",
|
||||||
|
"scope": {
|
||||||
|
"oneOf": [
|
||||||
|
{ "type": "string" },
|
||||||
|
{ "type": "array", "items": { "type": "string" } }
|
||||||
|
],
|
||||||
|
"description": "覆盖范围 (模块路径或功能区域)"
|
||||||
|
},
|
||||||
|
"excludes": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"description": "明确排除的范围"
|
||||||
|
},
|
||||||
|
"focus_paths": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"description": "重点文件/目录路径"
|
||||||
|
},
|
||||||
|
|
||||||
|
"_comment_DEPENDENCIES": "DEPENDENCIES 区块 (必填) — 依赖关系",
|
||||||
|
"depends_on": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"default": [],
|
||||||
|
"description": "依赖任务 ID 列表 (无依赖则 [])"
|
||||||
|
},
|
||||||
|
"parallel_group": {
|
||||||
|
"type": "number",
|
||||||
|
"description": "并行分组编号 (同组可并行执行)"
|
||||||
|
},
|
||||||
|
|
||||||
|
"_comment_CONVERGENCE": "CONVERGENCE 区块 (必填) — 完成标准",
|
||||||
|
"convergence": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["criteria"],
|
||||||
|
"properties": {
|
||||||
|
"criteria": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"minItems": 1,
|
||||||
|
"description": "可测试的完成条件 (可写为断言或手动步骤)"
|
||||||
|
},
|
||||||
|
"verification": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "可执行的验证步骤 (命令或明确步骤)"
|
||||||
|
},
|
||||||
|
"definition_of_done": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "业务语言完成定义 (非技术人员可判断)"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
|
||||||
|
"_comment_FILES": "FILES 区块 (可选) — 文件级修改点",
|
||||||
|
"files": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["path"],
|
||||||
|
"properties": {
|
||||||
|
"path": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "文件路径"
|
||||||
|
},
|
||||||
|
"action": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["modify", "create", "delete"],
|
||||||
|
"description": "文件操作类型"
|
||||||
|
},
|
||||||
|
"target": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "修改目标 (函数名/类名，来自 Solution Schema)"
|
||||||
|
},
|
||||||
|
"changes": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"description": "修改描述列表"
|
||||||
|
},
|
||||||
|
"conflict_risk": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["low", "medium", "high"],
|
||||||
|
"description": "冲突风险等级"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
"description": "涉及文件列表及修改详情"
|
||||||
|
},
|
||||||
|
|
||||||
|
"_comment_IMPLEMENTATION": "IMPLEMENTATION 区块 (可选) — 实施指南",
|
||||||
|
"implementation": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"description": "步骤化实施指南 (来自 Solution Schema)"
|
||||||
|
},
|
||||||
|
"test": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"commands": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"description": "测试命令"
|
||||||
|
},
|
||||||
|
"unit": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"description": "单元测试要求"
|
||||||
|
},
|
||||||
|
"integration": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"description": "集成测试要求"
|
||||||
|
},
|
||||||
|
"coverage_target": {
|
||||||
|
"type": "number",
|
||||||
|
"minimum": 0,
|
||||||
|
"maximum": 100,
|
||||||
|
"description": "覆盖率目标 (%)"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false,
|
||||||
|
"description": "测试要求"
|
||||||
|
},
|
||||||
|
"regression": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"description": "回归检查点"
|
||||||
|
},
|
||||||
|
|
||||||
|
"_comment_EXECUTION": "EXECUTION 区块 (可选) — 执行策略",
|
||||||
|
"meta": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"agent": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "分配的 agent (@code-developer / @test-fix-agent 等)"
|
||||||
|
},
|
||||||
|
"module": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "所属模块 (frontend/backend/shared)"
|
||||||
|
},
|
||||||
|
"execution_config": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"method": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["agent", "cli"],
|
||||||
|
"description": "执行方式"
|
||||||
|
},
|
||||||
|
"cli_tool": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["codex", "gemini", "qwen", "auto"],
|
||||||
|
"description": "CLI 工具选择"
|
||||||
|
},
|
||||||
|
"enable_resume": {
|
||||||
|
"type": "boolean",
|
||||||
|
"description": "是否启用会话恢复"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": true,
|
||||||
|
"description": "执行元信息"
|
||||||
|
},
|
||||||
|
"cli_execution": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "CLI 会话 ID (WFS-{session}-TASK-{id})"
|
||||||
|
},
|
||||||
|
"strategy": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["new", "resume", "fork", "merge_fork"],
|
||||||
|
"description": "CLI 执行策略"
|
||||||
|
},
|
||||||
|
"resume_from": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "父任务 CLI ID (用于 resume/fork)"
|
||||||
|
},
|
||||||
|
"merge_from": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"description": "合并来源 CLI ID 列表 (用于 merge_fork)"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false,
|
||||||
|
"description": "CLI 执行配置"
|
||||||
|
},
|
||||||
|
|
||||||
|
"_comment_CONTEXT": "CONTEXT 区块 (可选) — 来源与上下文",
|
||||||
|
"source": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"tool": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "产出工具名 (workflow-plan / lite-plan / issue-resolve / review-cycle 等)"
|
||||||
|
},
|
||||||
|
"session_id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "来源 session ID"
|
||||||
|
},
|
||||||
|
"original_id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "转换前原始 ID"
|
||||||
|
},
|
||||||
|
"issue_id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "关联的 Issue ID"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false,
|
||||||
|
"description": "任务来源信息"
|
||||||
|
},
|
||||||
|
"context_package_path": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "上下文包路径"
|
||||||
|
},
|
||||||
|
"evidence": {
|
||||||
|
"type": "array",
|
||||||
|
"description": "支撑证据"
|
||||||
|
},
|
||||||
|
"risk_items": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"description": "风险项"
|
||||||
|
},
|
||||||
|
"inputs": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"description": "消费的产物 (文件/资源)"
|
||||||
|
},
|
||||||
|
"outputs": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"description": "产出的产物 (文件/资源)"
|
||||||
|
},
|
||||||
|
"commit": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"type": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["feat", "fix", "refactor", "test", "docs", "chore"],
|
||||||
|
"description": "提交类型"
|
||||||
|
},
|
||||||
|
"scope": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "提交范围"
|
||||||
|
},
|
||||||
|
"message_template": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "提交消息模板"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false,
|
||||||
|
"description": "提交信息模板"
|
||||||
|
},
|
||||||
|
|
||||||
|
"_comment_RUNTIME": "RUNTIME 区块 (执行时填充) — 运行时状态",
|
||||||
|
"status": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["pending", "in_progress", "completed", "failed", "skipped", "blocked"],
|
||||||
|
"default": "pending",
|
||||||
|
"description": "执行状态 (执行引擎填充)"
|
||||||
|
},
|
||||||
|
"executed_at": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "date-time",
|
||||||
|
"description": "执行时间戳 (ISO 8601)"
|
||||||
|
},
|
||||||
|
"result": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"success": {
|
||||||
|
"type": "boolean",
|
||||||
|
"description": "是否成功"
|
||||||
|
},
|
||||||
|
"files_modified": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"description": "修改的文件列表"
|
||||||
|
},
|
||||||
|
"summary": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "执行摘要"
|
||||||
|
},
|
||||||
|
"error": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "错误信息 (失败时)"
|
||||||
|
},
|
||||||
|
"convergence_verified": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "boolean" },
|
||||||
|
"description": "收敛标准验证结果 (对应 convergence.criteria 每项)"
|
||||||
|
},
|
||||||
|
"commit_hash": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "提交哈希"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false,
|
||||||
|
"description": "执行结果 (执行引擎填充)"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
"additionalProperties": true,
|
||||||
|
|
||||||
|
"_field_usage_by_producer": {
|
||||||
|
"workflow-plan": "IDENTITY + CLASSIFICATION + SCOPE + DEPENDENCIES + CONVERGENCE + FILES + EXECUTION(meta+cli_execution) + CONTEXT(context_package_path)",
|
||||||
|
"lite-plan": "IDENTITY + CLASSIFICATION + DEPENDENCIES + CONVERGENCE + FILES",
|
||||||
|
"req-plan": "IDENTITY + CLASSIFICATION + SCOPE + DEPENDENCIES + CONVERGENCE + CONTEXT(inputs/outputs/risk_items)",
|
||||||
|
"collaborative-plan": "IDENTITY + CLASSIFICATION + SCOPE + DEPENDENCIES + CONVERGENCE + FILES + CONTEXT(source)",
|
||||||
|
"issue-resolve": "IDENTITY + CLASSIFICATION + SCOPE + DEPENDENCIES + CONVERGENCE + FILES(with target) + IMPLEMENTATION + CONTEXT(commit/source)",
|
||||||
|
"review-cycle": "IDENTITY + CLASSIFICATION + FILES + CONVERGENCE + IMPLEMENTATION + CONTEXT(source/evidence)",
|
||||||
|
"tdd-plan": "IDENTITY + CLASSIFICATION + DEPENDENCIES + CONVERGENCE + EXECUTION + IMPLEMENTATION(test focused)",
|
||||||
|
"analyze-brainstorm": "IDENTITY + CLASSIFICATION + DEPENDENCIES + CONVERGENCE + CONTEXT(evidence/source)"
|
||||||
|
},
|
||||||
|
|
||||||
|
"_directory_convention": {
|
||||||
|
"standard_path": "{session_folder}/.task/",
|
||||||
|
"file_naming": "TASK-{id}.json (或保留原有 ID 前缀: IMPL-, L0-, T1-, FIX-)",
|
||||||
|
"examples": {
|
||||||
|
"workflow-plan": ".workflow/active/WFS-xxx/.task/IMPL-001.json",
|
||||||
|
"lite-plan": ".workflow/.lite-plan/{id}/.task/TASK-001.json",
|
||||||
|
"req-plan": ".workflow/.req-plan/RPLAN-{id}/.task/TASK-001.json",
|
||||||
|
"collab-plan": ".workflow/.planning/CPLAN-{id}/.task/TASK-001.json",
|
||||||
|
"review-cycle": ".workflow/active/WFS-{id}/.review/.task/FIX-001.json",
|
||||||
|
"issue-resolve": ".workflow/issues/{issue-id}/.task/TASK-001.json"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -283,6 +283,34 @@ Generate individual `.task/IMPL-*.json` files with the following structure:
|
|||||||
- `resume_from`: Parent task's cli_execution_id (for resume/fork)
|
- `resume_from`: Parent task's cli_execution_id (for resume/fork)
|
||||||
- `merge_from`: Array of parent cli_execution_ids (for merge_fork)
|
- `merge_from`: Array of parent cli_execution_ids (for merge_fork)
|
||||||
|
|
||||||
|
#### Schema Compatibility
|
||||||
|
|
||||||
|
The 6-field task JSON is a **superset** of `task-schema.json` (the unified task schema at `.ccw/workflows/cli-templates/schemas/task-schema.json`). All generated `.task/IMPL-*.json` files are compatible with the unified schema via the following field mapping:
|
||||||
|
|
||||||
|
| 6-Field Task JSON (this schema) | task-schema.json (unified) | Notes |
|
||||||
|
|--------------------------------|---------------------------|-------|
|
||||||
|
| `id` | `id` | Direct mapping |
|
||||||
|
| `title` | `title` | Direct mapping |
|
||||||
|
| `status` | `status` | Direct mapping |
|
||||||
|
| `meta.type` | `type` | Flattened in unified schema |
|
||||||
|
| `meta.agent` | `meta.agent` | Same path, preserved |
|
||||||
|
| `meta.execution_config` | `meta.execution_config` | Same path, preserved |
|
||||||
|
| `context.requirements` | `description` + `implementation` | Unified schema splits into goal description and step-by-step guide |
|
||||||
|
| `context.acceptance` | `convergence.criteria` | **Key mapping**: acceptance criteria become convergence criteria |
|
||||||
|
| `context.focus_paths` | `focus_paths` | Moved to top-level in unified schema |
|
||||||
|
| `context.depends_on` | `depends_on` | Moved to top-level in unified schema |
|
||||||
|
| `context.shared_context` | _(no direct equivalent)_ | 6-field extension for tech stack and conventions |
|
||||||
|
| `context.artifacts` | `evidence` + `inputs` | Unified schema uses generic evidence/inputs arrays |
|
||||||
|
| `flow_control.target_files` | `files[].path` | Unified schema uses structured file objects |
|
||||||
|
| `flow_control.implementation_approach` | `implementation` | Unified schema uses flat string array |
|
||||||
|
| `flow_control.pre_analysis` | _(no direct equivalent)_ | 6-field extension for pre-execution analysis |
|
||||||
|
| `context_package_path` | `context_package_path` | Direct mapping |
|
||||||
|
| `cli_execution_id` | `cli_execution.id` | Nested in unified schema |
|
||||||
|
| `cli_execution` | `cli_execution` | Direct mapping |
|
||||||
|
|
||||||
|
**Backward Compatibility**: The 6-field schema retains all existing fields. The unified schema fields (`convergence`, `depends_on` at top-level, `files`, `implementation`) are accepted as **optional aliases** when present. Consumers SHOULD check both locations (e.g., `convergence.criteria` OR `context.acceptance`).
|
||||||
|
|
||||||
|
|
||||||
**CLI Execution Strategy Rules** (MANDATORY - apply to all tasks):
|
**CLI Execution Strategy Rules** (MANDATORY - apply to all tasks):
|
||||||
|
|
||||||
| Dependency Pattern | Strategy | CLI Command Pattern |
|
| Dependency Pattern | Strategy | CLI Command Pattern |
|
||||||
@@ -418,6 +446,14 @@ userConfig.executionMethod → meta.execution_config
|
|||||||
"auth_strategy": "JWT with refresh tokens",
|
"auth_strategy": "JWT with refresh tokens",
|
||||||
"conventions": ["Follow existing auth patterns in src/auth/legacy/"]
|
"conventions": ["Follow existing auth patterns in src/auth/legacy/"]
|
||||||
},
|
},
|
||||||
|
"convergence": {
|
||||||
|
"criteria": [
|
||||||
|
"3 features implemented: verify by npm test -- auth (exit code 0)",
|
||||||
|
"5 files created: verify by ls src/auth/*.ts | wc -l = 5"
|
||||||
|
],
|
||||||
|
"verification": "npm test -- auth && ls src/auth/*.ts | wc -l",
|
||||||
|
"definition_of_done": "Authentication module fully functional with all endpoints and tests passing"
|
||||||
|
},
|
||||||
"artifacts": [
|
"artifacts": [
|
||||||
{
|
{
|
||||||
"type": "feature_spec|cross_cutting_spec|synthesis_specification|topic_framework|individual_role_analysis",
|
"type": "feature_spec|cross_cutting_spec|synthesis_specification|topic_framework|individual_role_analysis",
|
||||||
@@ -437,6 +473,10 @@ userConfig.executionMethod → meta.execution_config
|
|||||||
- `requirements`: **QUANTIFIED** implementation requirements (MUST include explicit counts and enumerated lists, e.g., "5 files: [list]")
|
- `requirements`: **QUANTIFIED** implementation requirements (MUST include explicit counts and enumerated lists, e.g., "5 files: [list]")
|
||||||
- `focus_paths`: Target directories/files (concrete paths without wildcards)
|
- `focus_paths`: Target directories/files (concrete paths without wildcards)
|
||||||
- `acceptance`: **MEASURABLE** acceptance criteria (MUST include verification commands, e.g., "verify by ls ... | wc -l = N")
|
- `acceptance`: **MEASURABLE** acceptance criteria (MUST include verification commands, e.g., "verify by ls ... | wc -l = N")
|
||||||
|
- `convergence`: _(Optional, unified schema alias)_ Structured completion criteria object following `task-schema.json` format. When present, `convergence.criteria` maps to `acceptance`. Use **either** `acceptance` (6-field native) **or** `convergence` (unified schema native), not both. See [Schema Compatibility](#schema-compatibility) for full mapping.
|
||||||
|
- `criteria`: Array of testable completion conditions (equivalent to `acceptance`)
|
||||||
|
- `verification`: Executable verification command or steps
|
||||||
|
- `definition_of_done`: Business-language completion definition (non-technical)
|
||||||
- `depends_on`: Prerequisite task IDs that must complete before this task starts
|
- `depends_on`: Prerequisite task IDs that must complete before this task starts
|
||||||
- `inherited`: Context, patterns, and dependencies passed from parent task
|
- `inherited`: Context, patterns, and dependencies passed from parent task
|
||||||
- `shared_context`: Tech stack, conventions, and architectural strategies for the task
|
- `shared_context`: Tech stack, conventions, and architectural strategies for the task
|
||||||
|
|||||||
81
.claude/skills/spec-generator/README.md
Normal file
81
.claude/skills/spec-generator/README.md
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
# Spec Generator
|
||||||
|
|
||||||
|
Structured specification document generator producing a complete document chain (Product Brief -> PRD -> Architecture -> Epics).
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Via workflow command
|
||||||
|
/workflow:spec "Build a task management system"
|
||||||
|
/workflow:spec -y "User auth with OAuth2" # Auto mode
|
||||||
|
/workflow:spec -c "task management" # Resume session
|
||||||
|
```
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
spec-generator/
|
||||||
|
|- SKILL.md # Entry point: metadata + architecture + flow
|
||||||
|
|- phases/
|
||||||
|
| |- 01-discovery.md # Seed analysis + codebase exploration
|
||||||
|
| |- 02-product-brief.md # Multi-CLI product brief generation
|
||||||
|
| |- 03-requirements.md # PRD with MoSCoW priorities
|
||||||
|
| |- 04-architecture.md # Architecture decisions + review
|
||||||
|
| |- 05-epics-stories.md # Epic/Story decomposition
|
||||||
|
| |- 06-readiness-check.md # Quality validation + handoff
|
||||||
|
|- specs/
|
||||||
|
| |- document-standards.md # Format, frontmatter, naming rules
|
||||||
|
| |- quality-gates.md # Per-phase quality criteria
|
||||||
|
|- templates/
|
||||||
|
| |- product-brief.md # Product brief template
|
||||||
|
| |- requirements-prd.md # PRD template
|
||||||
|
| |- architecture-doc.md # Architecture document template
|
||||||
|
| |- epics-template.md # Epic/Story template
|
||||||
|
|- README.md # This file
|
||||||
|
```
|
||||||
|
|
||||||
|
## 6-Phase Pipeline
|
||||||
|
|
||||||
|
| Phase | Name | Output | CLI Tools |
|
||||||
|
|-------|------|--------|-----------|
|
||||||
|
| 1 | Discovery | spec-config.json | Gemini (analysis) |
|
||||||
|
| 2 | Product Brief | product-brief.md | Gemini + Codex + Claude (parallel) |
|
||||||
|
| 3 | Requirements | requirements.md | Gemini (analysis) |
|
||||||
|
| 4 | Architecture | architecture.md | Gemini + Codex (sequential) |
|
||||||
|
| 5 | Epics & Stories | epics.md | Gemini (analysis) |
|
||||||
|
| 6 | Readiness Check | readiness-report.md, spec-summary.md | Gemini (validation) |
|
||||||
|
|
||||||
|
## Runtime Output
|
||||||
|
|
||||||
|
```
|
||||||
|
.workflow/.spec/SPEC-{slug}-{YYYY-MM-DD}/
|
||||||
|
|- spec-config.json # Session state
|
||||||
|
|- discovery-context.json # Codebase context (optional)
|
||||||
|
|- product-brief.md # Phase 2
|
||||||
|
|- requirements.md # Phase 3
|
||||||
|
|- architecture.md # Phase 4
|
||||||
|
|- epics.md # Phase 5
|
||||||
|
|- readiness-report.md # Phase 6
|
||||||
|
|- spec-summary.md # Phase 6
|
||||||
|
```
|
||||||
|
|
||||||
|
## Flags
|
||||||
|
|
||||||
|
- `-y|--yes`: Auto mode - skip all interactive confirmations
|
||||||
|
- `-c|--continue`: Resume from last completed phase
|
||||||
|
|
||||||
|
## Handoff
|
||||||
|
|
||||||
|
After Phase 6, choose execution path:
|
||||||
|
- `workflow:lite-plan` - Execute per Epic
|
||||||
|
- `workflow:req-plan-with-file` - Roadmap decomposition
|
||||||
|
- `workflow:plan` - Full planning
|
||||||
|
- `issue:new` - Create issues per Epic
|
||||||
|
|
||||||
|
## Design Principles
|
||||||
|
|
||||||
|
- **Document chain**: Each phase builds on previous outputs
|
||||||
|
- **Multi-perspective**: Gemini/Codex/Claude provide different viewpoints
|
||||||
|
- **Template-driven**: Consistent format via templates + frontmatter
|
||||||
|
- **Resumable**: spec-config.json tracks completed phases
|
||||||
|
- **Pure documentation**: No code generation - clean handoff to execution workflows
|
||||||
242
.claude/skills/spec-generator/phases/01-discovery.md
Normal file
242
.claude/skills/spec-generator/phases/01-discovery.md
Normal file
@@ -0,0 +1,242 @@
|
|||||||
|
# Phase 1: Discovery
|
||||||
|
|
||||||
|
Parse input, analyze the seed idea, optionally explore codebase, establish session configuration.
|
||||||
|
|
||||||
|
## Objective
|
||||||
|
|
||||||
|
- Generate session ID and create output directory
|
||||||
|
- Parse user input (text description or file reference)
|
||||||
|
- Analyze seed via Gemini CLI to extract problem space dimensions
|
||||||
|
- Conditionally explore codebase for existing patterns and constraints
|
||||||
|
- Gather user preferences (depth, focus areas) via interactive confirmation
|
||||||
|
- Write `spec-config.json` as the session state file
|
||||||
|
|
||||||
|
## Input
|
||||||
|
|
||||||
|
- Dependency: `$ARGUMENTS` (user input from command)
|
||||||
|
- Flags: `-y` (auto mode), `-c` (continue mode)
|
||||||
|
|
||||||
|
## Execution Steps
|
||||||
|
|
||||||
|
### Step 1: Session Initialization
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// Parse arguments
|
||||||
|
const args = $ARGUMENTS;
|
||||||
|
const autoMode = args.includes('-y') || args.includes('--yes');
|
||||||
|
const continueMode = args.includes('-c') || args.includes('--continue');
|
||||||
|
|
||||||
|
// Extract the idea/topic (remove flags)
|
||||||
|
const idea = args.replace(/(-y|--yes|-c|--continue)\s*/g, '').trim();
|
||||||
|
|
||||||
|
// Generate session ID
|
||||||
|
const slug = idea.toLowerCase()
|
||||||
|
.replace(/[^a-z0-9\u4e00-\u9fff]+/g, '-')
|
||||||
|
.replace(/^-|-$/g, '')
|
||||||
|
.slice(0, 40);
|
||||||
|
const date = new Date().toISOString().slice(0, 10);
|
||||||
|
const sessionId = `SPEC-${slug}-${date}`;
|
||||||
|
const workDir = `.workflow/.spec/${sessionId}`;
|
||||||
|
|
||||||
|
// Check for continue mode
|
||||||
|
if (continueMode) {
|
||||||
|
// Find existing session
|
||||||
|
const existingSessions = Glob('.workflow/.spec/SPEC-*/spec-config.json');
|
||||||
|
// If slug matches an existing session, load it and resume
|
||||||
|
// Read spec-config.json, find first incomplete phase, jump to that phase
|
||||||
|
return; // Resume logic handled by orchestrator
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create output directory
|
||||||
|
Bash(`mkdir -p "${workDir}"`);
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 2: Input Parsing
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// Determine input type
|
||||||
|
if (idea.startsWith('@') || idea.endsWith('.md') || idea.endsWith('.txt')) {
|
||||||
|
// File reference - read and extract content
|
||||||
|
const filePath = idea.replace(/^@/, '');
|
||||||
|
const fileContent = Read(filePath);
|
||||||
|
// Use file content as the seed
|
||||||
|
inputType = 'file';
|
||||||
|
seedInput = fileContent;
|
||||||
|
} else {
|
||||||
|
// Direct text description
|
||||||
|
inputType = 'text';
|
||||||
|
seedInput = idea;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 3: Seed Analysis via Gemini CLI
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
Bash({
|
||||||
|
command: `ccw cli -p "PURPOSE: Analyze this seed idea/requirement to extract structured problem space dimensions.
|
||||||
|
Success: Clear problem statement, target users, domain identification, 3-5 exploration dimensions.
|
||||||
|
|
||||||
|
SEED INPUT:
|
||||||
|
${seedInput}
|
||||||
|
|
||||||
|
TASK:
|
||||||
|
- Extract a clear problem statement (what problem does this solve?)
|
||||||
|
- Identify target users (who benefits?)
|
||||||
|
- Determine the domain (technical, business, consumer, etc.)
|
||||||
|
- List constraints (budget, time, technical, regulatory)
|
||||||
|
- Generate 3-5 exploration dimensions (key areas to investigate)
|
||||||
|
- Assess complexity: simple (1-2 components), moderate (3-5 components), complex (6+ components)
|
||||||
|
|
||||||
|
MODE: analysis
|
||||||
|
EXPECTED: JSON output with fields: problem_statement, target_users[], domain, constraints[], dimensions[], complexity
|
||||||
|
CONSTRAINTS: Be specific and actionable, not vague
|
||||||
|
" --tool gemini --mode analysis`,
|
||||||
|
run_in_background: true
|
||||||
|
});
|
||||||
|
// Wait for CLI result before continuing
|
||||||
|
```
|
||||||
|
|
||||||
|
Parse the CLI output into structured `seedAnalysis`:
|
||||||
|
```javascript
|
||||||
|
const seedAnalysis = {
|
||||||
|
problem_statement: "...",
|
||||||
|
target_users: ["..."],
|
||||||
|
domain: "...",
|
||||||
|
constraints: ["..."],
|
||||||
|
dimensions: ["..."]
|
||||||
|
};
|
||||||
|
const complexity = "moderate"; // from CLI output
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 4: Codebase Exploration (Conditional)
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// Detect if running inside a project with code
|
||||||
|
const hasCodebase = Glob('**/*.{ts,js,py,java,go,rs}').length > 0
|
||||||
|
|| Glob('package.json').length > 0
|
||||||
|
|| Glob('Cargo.toml').length > 0;
|
||||||
|
|
||||||
|
if (hasCodebase) {
|
||||||
|
Task({
|
||||||
|
subagent_type: "cli-explore-agent",
|
||||||
|
run_in_background: false,
|
||||||
|
description: `Explore codebase for spec: ${slug}`,
|
||||||
|
prompt: `
|
||||||
|
## Spec Generator Context
|
||||||
|
Topic: ${seedInput}
|
||||||
|
Dimensions: ${seedAnalysis.dimensions.join(', ')}
|
||||||
|
Session: ${workDir}
|
||||||
|
|
||||||
|
## MANDATORY FIRST STEPS
|
||||||
|
1. Search for code related to topic keywords
|
||||||
|
2. Read project config files (package.json, pyproject.toml, etc.) if they exist
|
||||||
|
|
||||||
|
## Exploration Focus
|
||||||
|
- Identify existing implementations related to the topic
|
||||||
|
- Find patterns that could inform architecture decisions
|
||||||
|
- Map current architecture constraints
|
||||||
|
- Locate integration points and dependencies
|
||||||
|
|
||||||
|
## Output
|
||||||
|
Write findings to: ${workDir}/discovery-context.json
|
||||||
|
|
||||||
|
Schema:
|
||||||
|
{
|
||||||
|
"relevant_files": [{"path": "...", "relevance": "high|medium|low", "rationale": "..."}],
|
||||||
|
"existing_patterns": ["pattern descriptions"],
|
||||||
|
"architecture_constraints": ["constraint descriptions"],
|
||||||
|
"integration_points": ["integration point descriptions"],
|
||||||
|
"tech_stack": {"languages": [], "frameworks": [], "databases": []},
|
||||||
|
"_metadata": { "exploration_type": "spec-discovery", "timestamp": "ISO8601" }
|
||||||
|
}
|
||||||
|
`
|
||||||
|
});
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 5: User Confirmation (Interactive)
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
if (!autoMode) {
|
||||||
|
// Confirm problem statement and select depth
|
||||||
|
AskUserQuestion({
|
||||||
|
questions: [
|
||||||
|
{
|
||||||
|
question: `Problem statement: "${seedAnalysis.problem_statement}" - Is this accurate?`,
|
||||||
|
header: "Problem",
|
||||||
|
multiSelect: false,
|
||||||
|
options: [
|
||||||
|
{ label: "Accurate", description: "Proceed with this problem statement" },
|
||||||
|
{ label: "Needs adjustment", description: "I'll refine the problem statement" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
question: "What specification depth do you need?",
|
||||||
|
header: "Depth",
|
||||||
|
multiSelect: false,
|
||||||
|
options: [
|
||||||
|
{ label: "Light", description: "Quick overview - key decisions only" },
|
||||||
|
{ label: "Standard (Recommended)", description: "Balanced detail for most projects" },
|
||||||
|
{ label: "Comprehensive", description: "Maximum detail for complex/critical projects" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
question: "Which areas should we focus on?",
|
||||||
|
header: "Focus",
|
||||||
|
multiSelect: true,
|
||||||
|
options: seedAnalysis.dimensions.map(d => ({ label: d, description: `Explore ${d} in depth` }))
|
||||||
|
}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
// Auto mode defaults
|
||||||
|
depth = "standard";
|
||||||
|
focusAreas = seedAnalysis.dimensions;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 6: Write spec-config.json
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
const specConfig = {
|
||||||
|
session_id: sessionId,
|
||||||
|
seed_input: seedInput,
|
||||||
|
input_type: inputType,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
mode: autoMode ? "auto" : "interactive",
|
||||||
|
complexity: complexity,
|
||||||
|
depth: depth,
|
||||||
|
focus_areas: focusAreas,
|
||||||
|
seed_analysis: seedAnalysis,
|
||||||
|
has_codebase: hasCodebase,
|
||||||
|
phasesCompleted: [
|
||||||
|
{
|
||||||
|
phase: 1,
|
||||||
|
name: "discovery",
|
||||||
|
output_file: "spec-config.json",
|
||||||
|
completed_at: new Date().toISOString()
|
||||||
|
}
|
||||||
|
]
|
||||||
|
};
|
||||||
|
|
||||||
|
Write(`${workDir}/spec-config.json`, JSON.stringify(specConfig, null, 2));
|
||||||
|
```
|
||||||
|
|
||||||
|
## Output
|
||||||
|
|
||||||
|
- **File**: `spec-config.json`
|
||||||
|
- **File**: `discovery-context.json` (optional, if codebase detected)
|
||||||
|
- **Format**: JSON
|
||||||
|
|
||||||
|
## Quality Checklist
|
||||||
|
|
||||||
|
- [ ] Session ID matches `SPEC-{slug}-{date}` format
|
||||||
|
- [ ] Problem statement exists and is >= 20 characters
|
||||||
|
- [ ] Target users identified (>= 1)
|
||||||
|
- [ ] 3-5 exploration dimensions generated
|
||||||
|
- [ ] spec-config.json written with all required fields
|
||||||
|
- [ ] Output directory created
|
||||||
|
|
||||||
|
## Next Phase
|
||||||
|
|
||||||
|
Proceed to [Phase 2: Product Brief](02-product-brief.md) with the generated spec-config.json.
|
||||||
226
.claude/skills/spec-generator/phases/02-product-brief.md
Normal file
226
.claude/skills/spec-generator/phases/02-product-brief.md
Normal file
@@ -0,0 +1,226 @@
|
|||||||
|
# Phase 2: Product Brief
|
||||||
|
|
||||||
|
Generate a product brief through multi-perspective CLI analysis, establishing "what" and "why".
|
||||||
|
|
||||||
|
## Objective
|
||||||
|
|
||||||
|
- Read Phase 1 outputs (spec-config.json, discovery-context.json)
|
||||||
|
- Launch 3 parallel CLI analyses from product, technical, and user perspectives
|
||||||
|
- Synthesize convergent themes and conflicting views
|
||||||
|
- Optionally refine with user input
|
||||||
|
- Generate product-brief.md using template
|
||||||
|
|
||||||
|
## Input
|
||||||
|
|
||||||
|
- Dependency: `{workDir}/spec-config.json`
|
||||||
|
- Optional: `{workDir}/discovery-context.json`
|
||||||
|
- Config: `{workDir}/spec-config.json`
|
||||||
|
- Template: `templates/product-brief.md`
|
||||||
|
|
||||||
|
## Execution Steps
|
||||||
|
|
||||||
|
### Step 1: Load Phase 1 Context
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
const specConfig = JSON.parse(Read(`${workDir}/spec-config.json`));
|
||||||
|
const { seed_analysis, seed_input, has_codebase, depth, focus_areas } = specConfig;
|
||||||
|
|
||||||
|
let discoveryContext = null;
|
||||||
|
if (has_codebase) {
|
||||||
|
try {
|
||||||
|
discoveryContext = JSON.parse(Read(`${workDir}/discovery-context.json`));
|
||||||
|
} catch (e) {
|
||||||
|
// No discovery context available, proceed without
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build shared context string for CLI prompts
|
||||||
|
const sharedContext = `
|
||||||
|
SEED: ${seed_input}
|
||||||
|
PROBLEM: ${seed_analysis.problem_statement}
|
||||||
|
TARGET USERS: ${seed_analysis.target_users.join(', ')}
|
||||||
|
DOMAIN: ${seed_analysis.domain}
|
||||||
|
CONSTRAINTS: ${seed_analysis.constraints.join(', ')}
|
||||||
|
FOCUS AREAS: ${focus_areas.join(', ')}
|
||||||
|
${discoveryContext ? `
|
||||||
|
CODEBASE CONTEXT:
|
||||||
|
- Existing patterns: ${discoveryContext.existing_patterns?.slice(0,5).join(', ') || 'none'}
|
||||||
|
- Architecture constraints: ${discoveryContext.architecture_constraints?.slice(0,3).join(', ') || 'none'}
|
||||||
|
- Tech stack: ${JSON.stringify(discoveryContext.tech_stack || {})}
|
||||||
|
` : ''}`;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 2: Multi-CLI Parallel Analysis (3 perspectives)
|
||||||
|
|
||||||
|
Launch 3 CLI calls in parallel:
|
||||||
|
|
||||||
|
**Product Perspective (Gemini)**:
|
||||||
|
```javascript
|
||||||
|
Bash({
|
||||||
|
command: `ccw cli -p "PURPOSE: Product analysis for specification - identify market fit, user value, and success criteria.
|
||||||
|
Success: Clear vision, measurable goals, competitive positioning.
|
||||||
|
|
||||||
|
${sharedContext}
|
||||||
|
|
||||||
|
TASK:
|
||||||
|
- Define product vision (1-3 sentences, aspirational)
|
||||||
|
- Analyze market/competitive landscape
|
||||||
|
- Define 3-5 measurable success metrics
|
||||||
|
- Identify scope boundaries (in-scope vs out-of-scope)
|
||||||
|
- Assess user value proposition
|
||||||
|
- List assumptions that need validation
|
||||||
|
|
||||||
|
MODE: analysis
|
||||||
|
EXPECTED: Structured product analysis with: vision, goals with metrics, scope, competitive positioning, assumptions
|
||||||
|
CONSTRAINTS: Focus on 'what' and 'why', not 'how'
|
||||||
|
" --tool gemini --mode analysis`,
|
||||||
|
run_in_background: true
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
**Technical Perspective (Codex)**:
|
||||||
|
```javascript
|
||||||
|
Bash({
|
||||||
|
command: `ccw cli -p "PURPOSE: Technical feasibility analysis for specification - assess implementation viability and constraints.
|
||||||
|
Success: Clear technical constraints, integration complexity, technology recommendations.
|
||||||
|
|
||||||
|
${sharedContext}
|
||||||
|
|
||||||
|
TASK:
|
||||||
|
- Assess technical feasibility of the core concept
|
||||||
|
- Identify technical constraints and blockers
|
||||||
|
- Evaluate integration complexity with existing systems
|
||||||
|
- Recommend technology approach (high-level)
|
||||||
|
- Identify technical risks and dependencies
|
||||||
|
- Estimate complexity: simple/moderate/complex
|
||||||
|
|
||||||
|
MODE: analysis
|
||||||
|
EXPECTED: Technical analysis with: feasibility assessment, constraints, integration complexity, tech recommendations, risks
|
||||||
|
CONSTRAINTS: Focus on feasibility and constraints, not detailed architecture
|
||||||
|
" --tool codex --mode analysis`,
|
||||||
|
run_in_background: true
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
**User Perspective (Claude)**:
|
||||||
|
```javascript
|
||||||
|
Bash({
|
||||||
|
command: `ccw cli -p "PURPOSE: User experience analysis for specification - understand user journeys, pain points, and UX considerations.
|
||||||
|
Success: Clear user personas, journey maps, UX requirements.
|
||||||
|
|
||||||
|
${sharedContext}
|
||||||
|
|
||||||
|
TASK:
|
||||||
|
- Elaborate user personas with goals and frustrations
|
||||||
|
- Map primary user journey (happy path)
|
||||||
|
- Identify key pain points in current experience
|
||||||
|
- Define UX success criteria
|
||||||
|
- List accessibility and usability considerations
|
||||||
|
- Suggest interaction patterns
|
||||||
|
|
||||||
|
MODE: analysis
|
||||||
|
EXPECTED: User analysis with: personas, journey map, pain points, UX criteria, interaction recommendations
|
||||||
|
CONSTRAINTS: Focus on user needs and experience, not implementation
|
||||||
|
" --tool claude --mode analysis`,
|
||||||
|
run_in_background: true
|
||||||
|
});
|
||||||
|
|
||||||
|
// STOP: Wait for all 3 CLI results before continuing
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 3: Synthesize Perspectives
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// After receiving all 3 CLI results:
|
||||||
|
// Extract convergent themes (all agree)
|
||||||
|
// Identify conflicting views (need resolution)
|
||||||
|
// Note unique contributions from each perspective
|
||||||
|
|
||||||
|
const synthesis = {
|
||||||
|
convergent_themes: [], // themes all 3 perspectives agree on
|
||||||
|
conflicts: [], // areas where perspectives differ
|
||||||
|
product_insights: [], // unique from product perspective
|
||||||
|
technical_insights: [], // unique from technical perspective
|
||||||
|
user_insights: [] // unique from user perspective
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 4: Interactive Refinement (Optional)
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
if (!autoMode) {
|
||||||
|
// Present synthesis summary to user
|
||||||
|
// AskUserQuestion with:
|
||||||
|
// - Confirm vision statement
|
||||||
|
// - Resolve any conflicts between perspectives
|
||||||
|
// - Adjust scope if needed
|
||||||
|
AskUserQuestion({
|
||||||
|
questions: [
|
||||||
|
{
|
||||||
|
question: "Review the synthesized product brief. Any adjustments needed?",
|
||||||
|
header: "Review",
|
||||||
|
multiSelect: false,
|
||||||
|
options: [
|
||||||
|
{ label: "Looks good", description: "Proceed to PRD generation" },
|
||||||
|
{ label: "Adjust scope", description: "Narrow or expand the scope" },
|
||||||
|
{ label: "Revise vision", description: "Refine the vision statement" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 5: Generate product-brief.md
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// Read template
|
||||||
|
const template = Read('templates/product-brief.md');
|
||||||
|
|
||||||
|
// Fill template with synthesized content
|
||||||
|
// Apply document-standards.md formatting rules
|
||||||
|
// Write with YAML frontmatter
|
||||||
|
|
||||||
|
const frontmatter = `---
|
||||||
|
session_id: ${specConfig.session_id}
|
||||||
|
phase: 2
|
||||||
|
document_type: product-brief
|
||||||
|
status: ${autoMode ? 'complete' : 'draft'}
|
||||||
|
generated_at: ${new Date().toISOString()}
|
||||||
|
stepsCompleted: ["load-context", "multi-cli-analysis", "synthesis", "generation"]
|
||||||
|
version: 1
|
||||||
|
dependencies:
|
||||||
|
- spec-config.json
|
||||||
|
---`;
|
||||||
|
|
||||||
|
// Combine frontmatter + filled template content
|
||||||
|
Write(`${workDir}/product-brief.md`, `${frontmatter}\n\n${filledContent}`);
|
||||||
|
|
||||||
|
// Update spec-config.json
|
||||||
|
specConfig.phasesCompleted.push({
|
||||||
|
phase: 2,
|
||||||
|
name: "product-brief",
|
||||||
|
output_file: "product-brief.md",
|
||||||
|
completed_at: new Date().toISOString()
|
||||||
|
});
|
||||||
|
Write(`${workDir}/spec-config.json`, JSON.stringify(specConfig, null, 2));
|
||||||
|
```
|
||||||
|
|
||||||
|
## Output
|
||||||
|
|
||||||
|
- **File**: `product-brief.md`
|
||||||
|
- **Format**: Markdown with YAML frontmatter
|
||||||
|
|
||||||
|
## Quality Checklist
|
||||||
|
|
||||||
|
- [ ] Vision statement: clear, 1-3 sentences
|
||||||
|
- [ ] Problem statement: specific and measurable
|
||||||
|
- [ ] Target users: >= 1 persona with needs
|
||||||
|
- [ ] Goals: >= 2 with measurable metrics
|
||||||
|
- [ ] Scope: in-scope and out-of-scope defined
|
||||||
|
- [ ] Multi-perspective synthesis included
|
||||||
|
- [ ] YAML frontmatter valid
|
||||||
|
|
||||||
|
## Next Phase
|
||||||
|
|
||||||
|
Proceed to [Phase 3: Requirements](03-requirements.md) with the generated product-brief.md.
|
||||||
192
.claude/skills/spec-generator/specs/document-standards.md
Normal file
192
.claude/skills/spec-generator/specs/document-standards.md
Normal file
@@ -0,0 +1,192 @@
|
|||||||
|
# Document Standards
|
||||||
|
|
||||||
|
Defines format conventions, YAML frontmatter schema, naming rules, and content structure for all spec-generator outputs.
|
||||||
|
|
||||||
|
## When to Use
|
||||||
|
|
||||||
|
| Phase | Usage | Section |
|
||||||
|
|-------|-------|---------|
|
||||||
|
| All Phases | Frontmatter format | YAML Frontmatter Schema |
|
||||||
|
| All Phases | File naming | Naming Conventions |
|
||||||
|
| Phase 2-5 | Document structure | Content Structure |
|
||||||
|
| Phase 6 | Validation reference | All sections |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## YAML Frontmatter Schema
|
||||||
|
|
||||||
|
Every generated document MUST begin with YAML frontmatter:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
---
|
||||||
|
session_id: SPEC-{slug}-{YYYY-MM-DD}
|
||||||
|
phase: {1-6}
|
||||||
|
document_type: {product-brief|requirements|architecture|epics|readiness-report|spec-summary}
|
||||||
|
status: draft|review|complete
|
||||||
|
generated_at: {ISO8601 timestamp}
|
||||||
|
stepsCompleted: []
|
||||||
|
version: 1
|
||||||
|
dependencies:
|
||||||
|
- {list of input documents used}
|
||||||
|
---
|
||||||
|
```
|
||||||
|
|
||||||
|
### Field Definitions
|
||||||
|
|
||||||
|
| Field | Type | Required | Description |
|
||||||
|
|-------|------|----------|-------------|
|
||||||
|
| `session_id` | string | Yes | Session identifier matching spec-config.json |
|
||||||
|
| `phase` | number | Yes | Phase number that generated this document (1-6) |
|
||||||
|
| `document_type` | string | Yes | One of: product-brief, requirements, architecture, epics, readiness-report, spec-summary |
|
||||||
|
| `status` | enum | Yes | draft (initial), review (user reviewed), complete (finalized) |
|
||||||
|
| `generated_at` | string | Yes | ISO8601 timestamp of generation |
|
||||||
|
| `stepsCompleted` | array | Yes | List of step IDs completed during generation |
|
||||||
|
| `version` | number | Yes | Document version, incremented on re-generation |
|
||||||
|
| `dependencies` | array | No | List of input files this document depends on |
|
||||||
|
|
||||||
|
### Status Transitions
|
||||||
|
|
||||||
|
```
|
||||||
|
draft -> review -> complete
|
||||||
|
| ^
|
||||||
|
+-------------------+ (direct promotion in auto mode)
|
||||||
|
```
|
||||||
|
|
||||||
|
- **draft**: Initial generation, not yet user-reviewed
|
||||||
|
- **review**: User has reviewed and provided feedback
|
||||||
|
- **complete**: Finalized, ready for downstream consumption
|
||||||
|
|
||||||
|
In auto mode (`-y`), documents are promoted directly from `draft` to `complete`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Naming Conventions
|
||||||
|
|
||||||
|
### Session ID Format
|
||||||
|
|
||||||
|
```
|
||||||
|
SPEC-{slug}-{YYYY-MM-DD}
|
||||||
|
```
|
||||||
|
|
||||||
|
- **slug**: Lowercase, alphanumeric + Chinese characters, hyphens as separators, max 40 chars
|
||||||
|
- **date**: UTC+8 date in YYYY-MM-DD format
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
- `SPEC-task-management-system-2026-02-11`
|
||||||
|
- `SPEC-user-auth-oauth-2026-02-11`
|
||||||
|
|
||||||
|
### Output Files
|
||||||
|
|
||||||
|
| File | Phase | Description |
|
||||||
|
|------|-------|-------------|
|
||||||
|
| `spec-config.json` | 1 | Session configuration and state |
|
||||||
|
| `discovery-context.json` | 1 | Codebase exploration results (optional) |
|
||||||
|
| `product-brief.md` | 2 | Product brief document |
|
||||||
|
| `requirements.md` | 3 | PRD document |
|
||||||
|
| `architecture.md` | 4 | Architecture decisions document |
|
||||||
|
| `epics.md` | 5 | Epic/Story breakdown document |
|
||||||
|
| `readiness-report.md` | 6 | Quality validation report |
|
||||||
|
| `spec-summary.md` | 6 | One-page executive summary |
|
||||||
|
|
||||||
|
### Output Directory
|
||||||
|
|
||||||
|
```
|
||||||
|
.workflow/.spec/{session-id}/
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Content Structure
|
||||||
|
|
||||||
|
### Heading Hierarchy
|
||||||
|
|
||||||
|
- `#` (H1): Document title only (one per document)
|
||||||
|
- `##` (H2): Major sections
|
||||||
|
- `###` (H3): Subsections
|
||||||
|
- `####` (H4): Detail items (use sparingly)
|
||||||
|
|
||||||
|
Maximum depth: 4 levels. Prefer flat structures.
|
||||||
|
|
||||||
|
### Section Ordering
|
||||||
|
|
||||||
|
Every document follows this general pattern:
|
||||||
|
|
||||||
|
1. **YAML Frontmatter** (mandatory)
|
||||||
|
2. **Title** (H1)
|
||||||
|
3. **Executive Summary** (2-3 sentences)
|
||||||
|
4. **Core Content Sections** (H2, document-specific)
|
||||||
|
5. **Open Questions / Risks** (if applicable)
|
||||||
|
6. **References / Traceability** (links to upstream/downstream docs)
|
||||||
|
|
||||||
|
### Formatting Rules
|
||||||
|
|
||||||
|
| Element | Format | Example |
|
||||||
|
|---------|--------|---------|
|
||||||
|
| Requirements | `REQ-{NNN}` prefix | REQ-001: User login |
|
||||||
|
| Acceptance criteria | Checkbox list | `- [ ] User can log in with email` |
|
||||||
|
| Architecture decisions | `ADR-{NNN}` prefix | ADR-001: Use PostgreSQL |
|
||||||
|
| Epics | `EPIC-{NNN}` prefix | EPIC-001: Authentication |
|
||||||
|
| Stories | `STORY-{EPIC}-{NNN}` prefix | STORY-001-001: Login form |
|
||||||
|
| Priority tags | MoSCoW labels | `[Must]`, `[Should]`, `[Could]`, `[Won't]` |
|
||||||
|
| Mermaid diagrams | Fenced code blocks | ````mermaid ... ``` `` |
|
||||||
|
| Code examples | Language-tagged blocks | ````typescript ... ``` `` |
|
||||||
|
|
||||||
|
### Cross-Reference Format
|
||||||
|
|
||||||
|
Use relative references between documents:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
See [Product Brief](product-brief.md#section-name) for details.
|
||||||
|
Derived from [REQ-001](requirements.md#req-001).
|
||||||
|
```
|
||||||
|
|
||||||
|
### Language
|
||||||
|
|
||||||
|
- Document body: Follow user's input language (Chinese or English)
|
||||||
|
- Technical identifiers: Always English (REQ-001, ADR-001, EPIC-001)
|
||||||
|
- YAML frontmatter keys: Always English
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## spec-config.json Schema
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"session_id": "string (required)",
|
||||||
|
"seed_input": "string (required) - original user input",
|
||||||
|
"input_type": "text|file (required)",
|
||||||
|
"timestamp": "ISO8601 (required)",
|
||||||
|
"mode": "interactive|auto (required)",
|
||||||
|
"complexity": "simple|moderate|complex (required)",
|
||||||
|
"depth": "light|standard|comprehensive (required)",
|
||||||
|
"focus_areas": ["string array"],
|
||||||
|
"seed_analysis": {
|
||||||
|
"problem_statement": "string",
|
||||||
|
"target_users": ["string array"],
|
||||||
|
"domain": "string",
|
||||||
|
"constraints": ["string array"],
|
||||||
|
"dimensions": ["string array - 3-5 exploration dimensions"]
|
||||||
|
},
|
||||||
|
"has_codebase": "boolean",
|
||||||
|
"phasesCompleted": [
|
||||||
|
{
|
||||||
|
"phase": "number (1-6)",
|
||||||
|
"name": "string (phase name)",
|
||||||
|
"output_file": "string (primary output file)",
|
||||||
|
"completed_at": "ISO8601"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Validation Checklist
|
||||||
|
|
||||||
|
- [ ] Every document starts with valid YAML frontmatter
|
||||||
|
- [ ] `session_id` matches across all documents in a session
|
||||||
|
- [ ] `status` field reflects current document state
|
||||||
|
- [ ] All cross-references resolve to valid targets
|
||||||
|
- [ ] Heading hierarchy is correct (no skipped levels)
|
||||||
|
- [ ] Technical identifiers use correct prefixes
|
||||||
|
- [ ] Output files are in the correct directory
|
||||||
207
.claude/skills/spec-generator/specs/quality-gates.md
Normal file
207
.claude/skills/spec-generator/specs/quality-gates.md
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
# Quality Gates
|
||||||
|
|
||||||
|
Per-phase quality gate criteria and scoring dimensions for spec-generator outputs.
|
||||||
|
|
||||||
|
## When to Use
|
||||||
|
|
||||||
|
| Phase | Usage | Section |
|
||||||
|
|-------|-------|---------|
|
||||||
|
| Phase 2-5 | Post-generation self-check | Per-Phase Gates |
|
||||||
|
| Phase 6 | Cross-document validation | Cross-Document Validation |
|
||||||
|
| Phase 6 | Final scoring | Scoring Dimensions |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Quality Thresholds
|
||||||
|
|
||||||
|
| Gate | Score | Action |
|
||||||
|
|------|-------|--------|
|
||||||
|
| **Pass** | >= 80% | Continue to next phase |
|
||||||
|
| **Review** | 60-79% | Log warnings, continue with caveats |
|
||||||
|
| **Fail** | < 60% | Must address issues before continuing |
|
||||||
|
|
||||||
|
In auto mode (`-y`), Review-level issues are logged but do not block progress.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Scoring Dimensions
|
||||||
|
|
||||||
|
### 1. Completeness (25%)
|
||||||
|
|
||||||
|
All required sections present with substantive content.
|
||||||
|
|
||||||
|
| Score | Criteria |
|
||||||
|
|-------|----------|
|
||||||
|
| 100% | All template sections filled with detailed content |
|
||||||
|
| 75% | All sections present, some lack detail |
|
||||||
|
| 50% | Major sections present but minor sections missing |
|
||||||
|
| 25% | Multiple major sections missing or empty |
|
||||||
|
| 0% | Document is a skeleton only |
|
||||||
|
|
||||||
|
### 2. Consistency (25%)
|
||||||
|
|
||||||
|
Terminology, formatting, and references are uniform across documents.
|
||||||
|
|
||||||
|
| Score | Criteria |
|
||||||
|
|-------|----------|
|
||||||
|
| 100% | All terms consistent, all references valid, formatting uniform |
|
||||||
|
| 75% | Minor terminology variations, all references valid |
|
||||||
|
| 50% | Some inconsistent terms, 1-2 broken references |
|
||||||
|
| 25% | Frequent inconsistencies, multiple broken references |
|
||||||
|
| 0% | Documents contradict each other |
|
||||||
|
|
||||||
|
### 3. Traceability (25%)
|
||||||
|
|
||||||
|
Requirements, architecture decisions, and stories trace back to goals.
|
||||||
|
|
||||||
|
| Score | Criteria |
|
||||||
|
|-------|----------|
|
||||||
|
| 100% | Every story traces to a requirement, every requirement traces to a goal |
|
||||||
|
| 75% | Most items traceable, few orphans |
|
||||||
|
| 50% | Partial traceability, some disconnected items |
|
||||||
|
| 25% | Weak traceability, many orphan items |
|
||||||
|
| 0% | No traceability between documents |
|
||||||
|
|
||||||
|
### 4. Depth (25%)
|
||||||
|
|
||||||
|
Content provides sufficient detail for execution teams.
|
||||||
|
|
||||||
|
| Score | Criteria |
|
||||||
|
|-------|----------|
|
||||||
|
| 100% | Acceptance criteria specific and testable, architecture decisions justified, stories estimable |
|
||||||
|
| 75% | Most items detailed enough, few vague areas |
|
||||||
|
| 50% | Mix of detailed and vague content |
|
||||||
|
| 25% | Mostly high-level, lacking actionable detail |
|
||||||
|
| 0% | Too abstract for execution |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Per-Phase Quality Gates
|
||||||
|
|
||||||
|
### Phase 1: Discovery
|
||||||
|
|
||||||
|
| Check | Criteria | Severity |
|
||||||
|
|-------|----------|----------|
|
||||||
|
| Session ID valid | Matches `SPEC-{slug}-{date}` format | Error |
|
||||||
|
| Problem statement exists | Non-empty, >= 20 characters | Error |
|
||||||
|
| Target users identified | >= 1 user group | Error |
|
||||||
|
| Dimensions generated | 3-5 exploration dimensions | Warning |
|
||||||
|
| Constraints listed | >= 0 (can be empty with justification) | Info |
|
||||||
|
|
||||||
|
### Phase 2: Product Brief
|
||||||
|
|
||||||
|
| Check | Criteria | Severity |
|
||||||
|
|-------|----------|----------|
|
||||||
|
| Vision statement | Clear, 1-3 sentences | Error |
|
||||||
|
| Problem statement | Specific and measurable | Error |
|
||||||
|
| Target users | >= 1 persona with needs described | Error |
|
||||||
|
| Goals defined | >= 2 measurable goals | Error |
|
||||||
|
| Success metrics | >= 2 quantifiable metrics | Warning |
|
||||||
|
| Scope boundaries | In-scope and out-of-scope listed | Warning |
|
||||||
|
| Multi-perspective | >= 2 CLI perspectives synthesized | Info |
|
||||||
|
|
||||||
|
### Phase 3: Requirements (PRD)
|
||||||
|
|
||||||
|
| Check | Criteria | Severity |
|
||||||
|
|-------|----------|----------|
|
||||||
|
| Functional requirements | >= 3 with REQ-NNN IDs | Error |
|
||||||
|
| Acceptance criteria | Every requirement has >= 1 criterion | Error |
|
||||||
|
| MoSCoW priority | Every requirement tagged | Error |
|
||||||
|
| Non-functional requirements | >= 1 (performance, security, etc.) | Warning |
|
||||||
|
| User stories | >= 1 per Must-have requirement | Warning |
|
||||||
|
| Traceability | Requirements trace to product brief goals | Warning |
|
||||||
|
|
||||||
|
### Phase 4: Architecture
|
||||||
|
|
||||||
|
| Check | Criteria | Severity |
|
||||||
|
|-------|----------|----------|
|
||||||
|
| Component diagram | Present (Mermaid or ASCII) | Error |
|
||||||
|
| Tech stack specified | Languages, frameworks, key libraries | Error |
|
||||||
|
| ADR present | >= 1 Architecture Decision Record | Error |
|
||||||
|
| ADR has alternatives | Each ADR lists >= 2 options considered | Warning |
|
||||||
|
| Integration points | External systems/APIs identified | Warning |
|
||||||
|
| Data model | Key entities and relationships described | Warning |
|
||||||
|
| Codebase mapping | Mapped to existing code (if has_codebase) | Info |
|
||||||
|
|
||||||
|
### Phase 5: Epics & Stories
|
||||||
|
|
||||||
|
| Check | Criteria | Severity |
|
||||||
|
|-------|----------|----------|
|
||||||
|
| Epics defined | 3-7 epics with EPIC-NNN IDs | Error |
|
||||||
|
| MVP subset | >= 1 epic tagged as MVP | Error |
|
||||||
|
| Stories per epic | 2-5 stories per epic | Error |
|
||||||
|
| Story format | "As a...I want...So that..." pattern | Warning |
|
||||||
|
| Dependency map | Cross-epic dependencies documented | Warning |
|
||||||
|
| Estimation hints | Relative sizing (S/M/L/XL) per story | Info |
|
||||||
|
| Traceability | Stories trace to requirements | Warning |
|
||||||
|
|
||||||
|
### Phase 6: Readiness Check
|
||||||
|
|
||||||
|
| Check | Criteria | Severity |
|
||||||
|
|-------|----------|----------|
|
||||||
|
| All documents exist | product-brief, requirements, architecture, epics | Error |
|
||||||
|
| Frontmatter valid | All YAML frontmatter parseable and correct | Error |
|
||||||
|
| Cross-references valid | All document links resolve | Error |
|
||||||
|
| Overall score >= 60% | Weighted average across 4 dimensions | Error |
|
||||||
|
| No unresolved Errors | All Error-severity issues addressed | Error |
|
||||||
|
| Summary generated | spec-summary.md created | Warning |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Cross-Document Validation
|
||||||
|
|
||||||
|
Checks performed during Phase 6 across all documents:
|
||||||
|
|
||||||
|
### Completeness Matrix
|
||||||
|
|
||||||
|
```
|
||||||
|
Product Brief goals -> Requirements (each goal has >= 1 requirement)
|
||||||
|
Requirements -> Architecture (each Must requirement has design coverage)
|
||||||
|
Requirements -> Epics (each Must requirement appears in >= 1 story)
|
||||||
|
Architecture ADRs -> Epics (tech choices reflected in implementation stories)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Consistency Checks
|
||||||
|
|
||||||
|
| Check | Documents | Rule |
|
||||||
|
|-------|-----------|------|
|
||||||
|
| Terminology | All | Same term used consistently (no synonyms for same concept) |
|
||||||
|
| User personas | Brief + PRD + Epics | Same user names/roles throughout |
|
||||||
|
| Scope | Brief + PRD | PRD scope does not exceed brief scope |
|
||||||
|
| Tech stack | Architecture + Epics | Stories reference correct technologies |
|
||||||
|
|
||||||
|
### Traceability Matrix Format
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
| Goal | Requirements | Architecture | Epics |
|
||||||
|
|------|-------------|--------------|-------|
|
||||||
|
| G-001: ... | REQ-001, REQ-002 | ADR-001 | EPIC-001 |
|
||||||
|
| G-002: ... | REQ-003 | ADR-002 | EPIC-002, EPIC-003 |
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Issue Classification
|
||||||
|
|
||||||
|
### Error (Must Fix)
|
||||||
|
|
||||||
|
- Missing required document or section
|
||||||
|
- Broken cross-references
|
||||||
|
- Contradictory information between documents
|
||||||
|
- Empty acceptance criteria on Must-have requirements
|
||||||
|
- No MVP subset defined in epics
|
||||||
|
|
||||||
|
### Warning (Should Fix)
|
||||||
|
|
||||||
|
- Vague acceptance criteria
|
||||||
|
- Missing non-functional requirements
|
||||||
|
- No success metrics defined
|
||||||
|
- Incomplete traceability
|
||||||
|
- Missing architecture review notes
|
||||||
|
|
||||||
|
### Info (Nice to Have)
|
||||||
|
|
||||||
|
- Could add more detailed personas
|
||||||
|
- Consider additional ADR alternatives
|
||||||
|
- Story estimation hints missing
|
||||||
|
- Mermaid diagrams could be more detailed
|
||||||
133
.claude/skills/spec-generator/templates/product-brief.md
Normal file
133
.claude/skills/spec-generator/templates/product-brief.md
Normal file
@@ -0,0 +1,133 @@
|
|||||||
|
# Product Brief Template
|
||||||
|
|
||||||
|
Template for generating product brief documents in Phase 2.
|
||||||
|
|
||||||
|
## Usage Context
|
||||||
|
|
||||||
|
| Phase | Usage |
|
||||||
|
|-------|-------|
|
||||||
|
| Phase 2 (Product Brief) | Generate product-brief.md from multi-CLI analysis |
|
||||||
|
| Output Location | `{workDir}/product-brief.md` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Template
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
---
|
||||||
|
session_id: {session_id}
|
||||||
|
phase: 2
|
||||||
|
document_type: product-brief
|
||||||
|
status: draft
|
||||||
|
generated_at: {timestamp}
|
||||||
|
stepsCompleted: []
|
||||||
|
version: 1
|
||||||
|
dependencies:
|
||||||
|
- spec-config.json
|
||||||
|
---
|
||||||
|
|
||||||
|
# Product Brief: {product_name}
|
||||||
|
|
||||||
|
{executive_summary - 2-3 sentences capturing the essence of the product/feature}
|
||||||
|
|
||||||
|
## Vision
|
||||||
|
|
||||||
|
{vision_statement - clear, aspirational 1-3 sentence statement of what success looks like}
|
||||||
|
|
||||||
|
## Problem Statement
|
||||||
|
|
||||||
|
### Current Situation
|
||||||
|
{description of the current state and pain points}
|
||||||
|
|
||||||
|
### Impact
|
||||||
|
{quantified impact of the problem - who is affected, how much, how often}
|
||||||
|
|
||||||
|
## Target Users
|
||||||
|
|
||||||
|
{for each user persona:}
|
||||||
|
|
||||||
|
### {Persona Name}
|
||||||
|
- **Role**: {user's role/context}
|
||||||
|
- **Needs**: {primary needs related to this product}
|
||||||
|
- **Pain Points**: {current frustrations}
|
||||||
|
- **Success Criteria**: {what success looks like for this user}
|
||||||
|
|
||||||
|
## Goals & Success Metrics
|
||||||
|
|
||||||
|
| Goal ID | Goal | Success Metric | Target |
|
||||||
|
|---------|------|----------------|--------|
|
||||||
|
| G-001 | {goal description} | {measurable metric} | {specific target} |
|
||||||
|
| G-002 | {goal description} | {measurable metric} | {specific target} |
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
### In Scope
|
||||||
|
- {feature/capability 1}
|
||||||
|
- {feature/capability 2}
|
||||||
|
- {feature/capability 3}
|
||||||
|
|
||||||
|
### Out of Scope
|
||||||
|
- {explicitly excluded item 1}
|
||||||
|
- {explicitly excluded item 2}
|
||||||
|
|
||||||
|
### Assumptions
|
||||||
|
- {key assumption 1}
|
||||||
|
- {key assumption 2}
|
||||||
|
|
||||||
|
## Competitive Landscape
|
||||||
|
|
||||||
|
| Aspect | Current State | Proposed Solution | Advantage |
|
||||||
|
|--------|--------------|-------------------|-----------|
|
||||||
|
| {aspect} | {how it's done now} | {our approach} | {differentiator} |
|
||||||
|
|
||||||
|
## Constraints & Dependencies
|
||||||
|
|
||||||
|
### Technical Constraints
|
||||||
|
- {constraint 1}
|
||||||
|
- {constraint 2}
|
||||||
|
|
||||||
|
### Business Constraints
|
||||||
|
- {constraint 1}
|
||||||
|
|
||||||
|
### Dependencies
|
||||||
|
- {external dependency 1}
|
||||||
|
- {external dependency 2}
|
||||||
|
|
||||||
|
## Multi-Perspective Synthesis
|
||||||
|
|
||||||
|
### Product Perspective
|
||||||
|
{summary of product/market analysis findings}
|
||||||
|
|
||||||
|
### Technical Perspective
|
||||||
|
{summary of technical feasibility and constraints}
|
||||||
|
|
||||||
|
### User Perspective
|
||||||
|
{summary of user journey and UX considerations}
|
||||||
|
|
||||||
|
### Convergent Themes
|
||||||
|
{themes where all perspectives agree}
|
||||||
|
|
||||||
|
### Conflicting Views
|
||||||
|
{areas where perspectives differ, with notes on resolution approach}
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
- [ ] {unresolved question 1}
|
||||||
|
- [ ] {unresolved question 2}
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- Derived from: [spec-config.json](spec-config.json)
|
||||||
|
- Next: [Requirements PRD](requirements.md)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Variable Descriptions
|
||||||
|
|
||||||
|
| Variable | Source | Description |
|
||||||
|
|----------|--------|-------------|
|
||||||
|
| `{session_id}` | spec-config.json | Session identifier |
|
||||||
|
| `{timestamp}` | Runtime | ISO8601 generation timestamp |
|
||||||
|
| `{product_name}` | Seed analysis | Product/feature name |
|
||||||
|
| `{executive_summary}` | CLI synthesis | 2-3 sentence summary |
|
||||||
|
| `{vision_statement}` | CLI product perspective | Aspirational vision |
|
||||||
|
| All `{...}` fields | CLI analysis outputs | Filled from multi-perspective analysis |
|
||||||
@@ -55,11 +55,11 @@ The key innovation is the **Plan Note** architecture — a shared collaborative
|
|||||||
│ │
|
│ │
|
||||||
│ Phase 2: Serial Domain Planning │
|
│ Phase 2: Serial Domain Planning │
|
||||||
│ ┌──────────────┐ │
|
│ ┌──────────────┐ │
|
||||||
│ │ Domain 1 │→ Explore codebase → Generate tasks.jsonl │
|
│ │ Domain 1 │→ Explore codebase → Generate .task/TASK-*.json │
|
||||||
│ │ Section 1 │→ Fill task pool + evidence in plan-note.md │
|
│ │ Section 1 │→ Fill task pool + evidence in plan-note.md │
|
||||||
│ └──────┬───────┘ │
|
│ └──────┬───────┘ │
|
||||||
│ ┌──────▼───────┐ │
|
│ ┌──────▼───────┐ │
|
||||||
│ │ Domain 2 │→ Explore codebase → Generate tasks.jsonl │
|
│ │ Domain 2 │→ Explore codebase → Generate .task/TASK-*.json │
|
||||||
│ │ Section 2 │→ Fill task pool + evidence in plan-note.md │
|
│ │ Section 2 │→ Fill task pool + evidence in plan-note.md │
|
||||||
│ └──────┬───────┘ │
|
│ └──────┬───────┘ │
|
||||||
│ ┌──────▼───────┐ │
|
│ ┌──────▼───────┐ │
|
||||||
@@ -72,7 +72,7 @@ The key innovation is the **Plan Note** architecture — a shared collaborative
|
|||||||
│ └─ Update plan-note.md conflict section │
|
│ └─ Update plan-note.md conflict section │
|
||||||
│ │
|
│ │
|
||||||
│ Phase 4: Completion (No Merge) │
|
│ Phase 4: Completion (No Merge) │
|
||||||
│ ├─ Merge domain tasks.jsonl → session tasks.jsonl │
|
│ ├─ Collect domain .task/*.json → session .task/*.json │
|
||||||
│ ├─ Generate plan.md (human-readable) │
|
│ ├─ Generate plan.md (human-readable) │
|
||||||
│ └─ Ready for execution │
|
│ └─ Ready for execution │
|
||||||
│ │
|
│ │
|
||||||
@@ -81,17 +81,26 @@ The key innovation is the **Plan Note** architecture — a shared collaborative
|
|||||||
|
|
||||||
## Output Structure
|
## Output Structure
|
||||||
|
|
||||||
|
> **Schema**: `cat ~/.ccw/workflows/cli-templates/schemas/task-schema.json`
|
||||||
|
|
||||||
```
|
```
|
||||||
{projectRoot}/.workflow/.planning/CPLAN-{slug}-{date}/
|
{projectRoot}/.workflow/.planning/CPLAN-{slug}-{date}/
|
||||||
├── plan-note.md # ⭐ Core: Requirements + Tasks + Conflicts
|
├── plan-note.md # ⭐ Core: Requirements + Tasks + Conflicts
|
||||||
├── requirement-analysis.json # Phase 1: Sub-domain assignments
|
├── requirement-analysis.json # Phase 1: Sub-domain assignments
|
||||||
├── domains/ # Phase 2: Per-domain plans
|
├── domains/ # Phase 2: Per-domain plans
|
||||||
│ ├── {domain-1}/
|
│ ├── {domain-1}/
|
||||||
│ │ └── tasks.jsonl # Unified JSONL (one task per line)
|
│ │ └── .task/ # Per-domain task JSON files
|
||||||
|
│ │ ├── TASK-001.json
|
||||||
|
│ │ └── ...
|
||||||
│ ├── {domain-2}/
|
│ ├── {domain-2}/
|
||||||
│ │ └── tasks.jsonl
|
│ │ └── .task/
|
||||||
|
│ │ ├── TASK-101.json
|
||||||
|
│ │ └── ...
|
||||||
|
│ └── ...
|
||||||
|
├── .task/ # ⭐ Merged task JSON files (all domains)
|
||||||
|
│ ├── TASK-001.json
|
||||||
|
│ ├── TASK-101.json
|
||||||
│ └── ...
|
│ └── ...
|
||||||
├── tasks.jsonl # ⭐ Merged unified JSONL (all domains)
|
|
||||||
├── conflicts.json # Phase 3: Conflict report
|
├── conflicts.json # Phase 3: Conflict report
|
||||||
└── plan.md # Phase 4: Human-readable summary
|
└── plan.md # Phase 4: Human-readable summary
|
||||||
```
|
```
|
||||||
@@ -109,7 +118,7 @@ The key innovation is the **Plan Note** architecture — a shared collaborative
|
|||||||
|
|
||||||
| Artifact | Purpose |
|
| Artifact | Purpose |
|
||||||
|----------|---------|
|
|----------|---------|
|
||||||
| `domains/{domain}/tasks.jsonl` | Unified JSONL per domain (one task per line with convergence) |
|
| `domains/{domain}/.task/TASK-*.json` | Task JSON files per domain (one file per task with convergence) |
|
||||||
| Updated `plan-note.md` | Task pool and evidence sections filled for each domain |
|
| Updated `plan-note.md` | Task pool and evidence sections filled for each domain |
|
||||||
|
|
||||||
### Phase 3: Conflict Detection
|
### Phase 3: Conflict Detection
|
||||||
@@ -123,7 +132,7 @@ The key innovation is the **Plan Note** architecture — a shared collaborative
|
|||||||
|
|
||||||
| Artifact | Purpose |
|
| Artifact | Purpose |
|
||||||
|----------|---------|
|
|----------|---------|
|
||||||
| `tasks.jsonl` | Merged unified JSONL from all domains (consumable by unified-execute) |
|
| `.task/TASK-*.json` | Merged task JSON files from all domains (consumable by unified-execute) |
|
||||||
| `plan.md` | Human-readable summary with requirements, tasks, and conflicts |
|
| `plan.md` | Human-readable summary with requirements, tasks, and conflicts |
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -294,8 +303,8 @@ For each sub-domain, execute the full planning cycle inline:
|
|||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
for (const sub of subDomains) {
|
for (const sub of subDomains) {
|
||||||
// 1. Create domain directory
|
// 1. Create domain directory with .task/ subfolder
|
||||||
Bash(`mkdir -p ${sessionFolder}/domains/${sub.focus_area}`)
|
Bash(`mkdir -p ${sessionFolder}/domains/${sub.focus_area}/.task`)
|
||||||
|
|
||||||
// 2. Explore codebase for domain-relevant context
|
// 2. Explore codebase for domain-relevant context
|
||||||
// Use: mcp__ace-tool__search_context, Grep, Glob, Read
|
// Use: mcp__ace-tool__search_context, Grep, Glob, Read
|
||||||
@@ -305,7 +314,7 @@ for (const sub of subDomains) {
|
|||||||
// - Integration points with other domains
|
// - Integration points with other domains
|
||||||
// - Architecture constraints
|
// - Architecture constraints
|
||||||
|
|
||||||
// 3. Generate unified JSONL tasks (one task per line)
|
// 3. Generate task JSON records (following task-schema.json)
|
||||||
const domainTasks = [
|
const domainTasks = [
|
||||||
// For each task within the assigned ID range:
|
// For each task within the assigned ID range:
|
||||||
{
|
{
|
||||||
@@ -339,9 +348,11 @@ for (const sub of subDomains) {
|
|||||||
// ... more tasks
|
// ... more tasks
|
||||||
]
|
]
|
||||||
|
|
||||||
// 4. Write domain tasks.jsonl (one task per line)
|
// 4. Write individual task JSON files (one per task)
|
||||||
const jsonlContent = domainTasks.map(t => JSON.stringify(t)).join('\n')
|
domainTasks.forEach(task => {
|
||||||
Write(`${sessionFolder}/domains/${sub.focus_area}/tasks.jsonl`, jsonlContent)
|
Write(`${sessionFolder}/domains/${sub.focus_area}/.task/${task.id}.json`,
|
||||||
|
JSON.stringify(task, null, 2))
|
||||||
|
})
|
||||||
|
|
||||||
// 5. Sync summary to plan-note.md
|
// 5. Sync summary to plan-note.md
|
||||||
// Read current plan-note.md
|
// Read current plan-note.md
|
||||||
@@ -399,7 +410,7 @@ After all domains are planned, verify the shared document.
|
|||||||
5. Check for any section format inconsistencies
|
5. Check for any section format inconsistencies
|
||||||
|
|
||||||
**Success Criteria**:
|
**Success Criteria**:
|
||||||
- `domains/{domain}/tasks.jsonl` created for each domain (unified JSONL format)
|
- `domains/{domain}/.task/TASK-*.json` created for each domain (one file per task)
|
||||||
- Each task has convergence (criteria + verification + definition_of_done)
|
- Each task has convergence (criteria + verification + definition_of_done)
|
||||||
- `plan-note.md` updated with all task pools and evidence sections
|
- `plan-note.md` updated with all task pools and evidence sections
|
||||||
- Task summaries follow consistent format
|
- Task summaries follow consistent format
|
||||||
@@ -413,7 +424,7 @@ After all domains are planned, verify the shared document.
|
|||||||
|
|
||||||
### Step 3.1: Parse plan-note.md
|
### Step 3.1: Parse plan-note.md
|
||||||
|
|
||||||
Extract all tasks from all "任务池" sections and domain tasks.jsonl files.
|
Extract all tasks from all "任务池" sections and domain .task/*.json files.
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// parsePlanNote(markdown)
|
// parsePlanNote(markdown)
|
||||||
@@ -422,13 +433,14 @@ Extract all tasks from all "任务池" sections and domain tasks.jsonl files.
|
|||||||
// - Build sections array: { level, heading, start, content }
|
// - Build sections array: { level, heading, start, content }
|
||||||
// - Return: { frontmatter, sections }
|
// - Return: { frontmatter, sections }
|
||||||
|
|
||||||
// Also load all domain tasks.jsonl for detailed data
|
// Also load all domain .task/*.json for detailed data
|
||||||
// loadDomainTasks(sessionFolder, subDomains):
|
// loadDomainTasks(sessionFolder, subDomains):
|
||||||
// const allTasks = []
|
// const allTasks = []
|
||||||
// for (const sub of subDomains) {
|
// for (const sub of subDomains) {
|
||||||
// const jsonl = Read(`${sessionFolder}/domains/${sub.focus_area}/tasks.jsonl`)
|
// const taskDir = `${sessionFolder}/domains/${sub.focus_area}/.task`
|
||||||
// jsonl.split('\n').filter(l => l.trim()).forEach(line => {
|
// const taskFiles = Glob(`${taskDir}/TASK-*.json`)
|
||||||
// allTasks.push(JSON.parse(line))
|
// taskFiles.forEach(file => {
|
||||||
|
// allTasks.push(JSON.parse(Read(file)))
|
||||||
// })
|
// })
|
||||||
// }
|
// }
|
||||||
// return allTasks
|
// return allTasks
|
||||||
@@ -543,23 +555,24 @@ Write(`${sessionFolder}/conflicts.json`, JSON.stringify({
|
|||||||
|
|
||||||
**Objective**: Generate human-readable plan summary and finalize workflow.
|
**Objective**: Generate human-readable plan summary and finalize workflow.
|
||||||
|
|
||||||
### Step 4.1: Merge Domain tasks.jsonl
|
### Step 4.1: Collect Domain .task/*.json to Session .task/
|
||||||
|
|
||||||
Merge all per-domain JSONL files into a single session-level `tasks.jsonl`.
|
Copy all per-domain task JSON files into a single session-level `.task/` directory.
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// Collect all domain tasks
|
// Create session-level .task/ directory
|
||||||
const allDomainTasks = []
|
Bash(`mkdir -p ${sessionFolder}/.task`)
|
||||||
|
|
||||||
|
// Collect all domain task files
|
||||||
for (const sub of subDomains) {
|
for (const sub of subDomains) {
|
||||||
const jsonl = Read(`${sessionFolder}/domains/${sub.focus_area}/tasks.jsonl`)
|
const taskDir = `${sessionFolder}/domains/${sub.focus_area}/.task`
|
||||||
jsonl.split('\n').filter(l => l.trim()).forEach(line => {
|
const taskFiles = Glob(`${taskDir}/TASK-*.json`)
|
||||||
allDomainTasks.push(JSON.parse(line))
|
taskFiles.forEach(file => {
|
||||||
|
const filename = path.basename(file)
|
||||||
|
// Copy domain task file to session .task/ directory
|
||||||
|
Bash(`cp ${file} ${sessionFolder}/.task/${filename}`)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write merged tasks.jsonl at session root
|
|
||||||
const mergedJsonl = allDomainTasks.map(t => JSON.stringify(t)).join('\n')
|
|
||||||
Write(`${sessionFolder}/tasks.jsonl`, mergedJsonl)
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Step 4.2: Generate plan.md
|
### Step 4.2: Generate plan.md
|
||||||
@@ -613,7 +626,7 @@ ${allConflicts.length === 0
|
|||||||
## 执行
|
## 执行
|
||||||
|
|
||||||
\`\`\`bash
|
\`\`\`bash
|
||||||
/workflow:unified-execute-with-file PLAN="${sessionFolder}/tasks.jsonl"
|
/workflow:unified-execute-with-file PLAN="${sessionFolder}/.task/"
|
||||||
\`\`\`
|
\`\`\`
|
||||||
|
|
||||||
**Session artifacts**: \`${sessionFolder}/\`
|
**Session artifacts**: \`${sessionFolder}/\`
|
||||||
@@ -652,14 +665,14 @@ if (!autoMode) {
|
|||||||
|
|
||||||
| Selection | Action |
|
| Selection | Action |
|
||||||
|-----------|--------|
|
|-----------|--------|
|
||||||
| Execute Plan | `Skill(skill="workflow:unified-execute-with-file", args="PLAN=\"${sessionFolder}/tasks.jsonl\"")` |
|
| Execute Plan | `Skill(skill="workflow:unified-execute-with-file", args="PLAN=\"${sessionFolder}/.task/\"")` |
|
||||||
| Review Conflicts | Display conflicts.json content for manual resolution |
|
| Review Conflicts | Display conflicts.json content for manual resolution |
|
||||||
| Export | Copy plan.md + plan-note.md to user-specified location |
|
| Export | Copy plan.md + plan-note.md to user-specified location |
|
||||||
| Done | Display artifact paths, end workflow |
|
| Done | Display artifact paths, end workflow |
|
||||||
|
|
||||||
**Success Criteria**:
|
**Success Criteria**:
|
||||||
- `plan.md` generated with complete summary
|
- `plan.md` generated with complete summary
|
||||||
- `tasks.jsonl` merged at session root (consumable by unified-execute)
|
- `.task/TASK-*.json` collected at session root (consumable by unified-execute)
|
||||||
- All artifacts present in session directory
|
- All artifacts present in session directory
|
||||||
- User informed of completion and next steps
|
- User informed of completion and next steps
|
||||||
|
|
||||||
@@ -685,11 +698,11 @@ User initiates: TASK="task description"
|
|||||||
├─ Generate requirement-analysis.json
|
├─ Generate requirement-analysis.json
|
||||||
│
|
│
|
||||||
├─ Serial domain planning:
|
├─ Serial domain planning:
|
||||||
│ ├─ Domain 1: explore → tasks.jsonl → fill plan-note.md
|
│ ├─ Domain 1: explore → .task/TASK-*.json → fill plan-note.md
|
||||||
│ ├─ Domain 2: explore → tasks.jsonl → fill plan-note.md
|
│ ├─ Domain 2: explore → .task/TASK-*.json → fill plan-note.md
|
||||||
│ └─ Domain N: ...
|
│ └─ Domain N: ...
|
||||||
│
|
│
|
||||||
├─ Merge domain tasks.jsonl → session tasks.jsonl
|
├─ Collect domain .task/*.json → session .task/
|
||||||
│
|
│
|
||||||
├─ Verify plan-note.md consistency
|
├─ Verify plan-note.md consistency
|
||||||
├─ Detect conflicts
|
├─ Detect conflicts
|
||||||
@@ -738,7 +751,7 @@ User resumes: TASK="same task"
|
|||||||
1. **Review Plan Note**: Check plan-note.md between domains to verify progress
|
1. **Review Plan Note**: Check plan-note.md between domains to verify progress
|
||||||
2. **Verify Independence**: Ensure sub-domains are truly independent and have minimal overlap
|
2. **Verify Independence**: Ensure sub-domains are truly independent and have minimal overlap
|
||||||
3. **Check Dependencies**: Cross-domain dependencies should be documented explicitly
|
3. **Check Dependencies**: Cross-domain dependencies should be documented explicitly
|
||||||
4. **Inspect Details**: Review `domains/{domain}/tasks.jsonl` for specifics when needed
|
4. **Inspect Details**: Review `domains/{domain}/.task/TASK-*.json` for specifics when needed
|
||||||
5. **Consistent Format**: Follow task summary format strictly across all domains
|
5. **Consistent Format**: Follow task summary format strictly across all domains
|
||||||
6. **TASK ID Isolation**: Use pre-assigned non-overlapping ranges to prevent ID conflicts
|
6. **TASK ID Isolation**: Use pre-assigned non-overlapping ranges to prevent ID conflicts
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
---
|
---
|
||||||
name: issue-resolve
|
name: issue-resolve
|
||||||
description: Unified issue resolution pipeline with source selection. Plan issues via AI exploration, convert from artifacts, import from brainstorm sessions, or form execution queues. Triggers on "issue:plan", "issue:queue", "issue:convert-to-plan", "issue:from-brainstorm", "resolve issue", "plan issue", "queue issues", "convert plan to issue".
|
description: Unified issue resolution pipeline with source selection. Plan issues via AI exploration, convert from artifacts, import from brainstorm sessions, form execution queues, or export solutions to task JSON. Triggers on "issue:plan", "issue:queue", "issue:convert-to-plan", "issue:from-brainstorm", "export-to-tasks", "resolve issue", "plan issue", "queue issues", "convert plan to issue".
|
||||||
allowed-tools: spawn_agent, wait, send_input, close_agent, AskUserQuestion, Read, Write, Edit, Bash, Glob, Grep
|
allowed-tools: spawn_agent, wait, send_input, close_agent, AskUserQuestion, Read, Write, Edit, Bash, Glob, Grep
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -28,6 +28,10 @@ Unified issue resolution pipeline that orchestrates solution creation from multi
|
|||||||
↓ ↓ ↓ ↓ │
|
↓ ↓ ↓ ↓ │
|
||||||
Solutions Solutions Issue+Sol Exec Queue │
|
Solutions Solutions Issue+Sol Exec Queue │
|
||||||
(bound) (bound) (bound) (ordered) │
|
(bound) (bound) (bound) (ordered) │
|
||||||
|
│ │ │ │
|
||||||
|
└─────┬─────┘───────────┘ │
|
||||||
|
↓ (optional --export-tasks) │
|
||||||
|
.task/TASK-*.json │
|
||||||
│
|
│
|
||||||
┌────────────────────────────────┘
|
┌────────────────────────────────┘
|
||||||
↓
|
↓
|
||||||
@@ -119,6 +123,7 @@ codex -p "@.codex/prompts/issue-resolve.md [FLAGS] \"<input>\""
|
|||||||
--issue <id> Bind to existing issue (convert mode)
|
--issue <id> Bind to existing issue (convert mode)
|
||||||
--supplement Add tasks to existing solution (convert mode)
|
--supplement Add tasks to existing solution (convert mode)
|
||||||
--queues <n> Number of parallel queues (queue mode, default: 1)
|
--queues <n> Number of parallel queues (queue mode, default: 1)
|
||||||
|
--export-tasks Export solution tasks to .task/TASK-*.json (task-schema.json format)
|
||||||
|
|
||||||
# Examples
|
# Examples
|
||||||
codex -p "@.codex/prompts/issue-resolve.md GH-123,GH-124" # Explore & plan issues
|
codex -p "@.codex/prompts/issue-resolve.md GH-123,GH-124" # Explore & plan issues
|
||||||
@@ -151,6 +156,8 @@ Phase Execution (load one phase):
|
|||||||
└─ Phase 4: Form Queue → phases/04-issue-queue.md
|
└─ Phase 4: Form Queue → phases/04-issue-queue.md
|
||||||
|
|
||||||
Post-Phase:
|
Post-Phase:
|
||||||
|
├─ Export to Task JSON (optional, with --export-tasks flag)
|
||||||
|
│ └─ For each solution.tasks[] → write .task/TASK-{T-id}.json
|
||||||
└─ Summary + Next steps recommendation
|
└─ Summary + Next steps recommendation
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -263,6 +270,29 @@ User Input (issue IDs / artifact path / session ID / flags)
|
|||||||
[Summary + Next Steps]
|
[Summary + Next Steps]
|
||||||
├─ After Plan/Convert/Brainstorm → Suggest /issue:queue or /issue:execute
|
├─ After Plan/Convert/Brainstorm → Suggest /issue:queue or /issue:execute
|
||||||
└─ After Queue → Suggest /issue:execute
|
└─ After Queue → Suggest /issue:execute
|
||||||
|
|
||||||
|
(Optional) Export to Task JSON (when --export-tasks flag is set):
|
||||||
|
├─ For each solution.tasks[] entry:
|
||||||
|
│ ├─ solution.task.id → id (prefixed as TASK-{T-id})
|
||||||
|
│ ├─ solution.task.title → title
|
||||||
|
│ ├─ solution.task.description → description
|
||||||
|
│ ├─ solution.task.action → action
|
||||||
|
│ ├─ solution.task.scope → scope
|
||||||
|
│ ├─ solution.task.modification_points[] → files[]
|
||||||
|
│ │ ├─ mp.file → files[].path
|
||||||
|
│ │ ├─ mp.target → files[].target
|
||||||
|
│ │ └─ mp.change → files[].changes[]
|
||||||
|
│ ├─ solution.task.acceptance → convergence
|
||||||
|
│ │ ├─ acceptance.criteria[] → convergence.criteria[]
|
||||||
|
│ │ └─ acceptance.verification[]→ convergence.verification (joined)
|
||||||
|
│ ├─ solution.task.implementation → implementation[]
|
||||||
|
│ ├─ solution.task.test → test
|
||||||
|
│ ├─ solution.task.depends_on → depends_on
|
||||||
|
│ ├─ solution.task.commit → commit
|
||||||
|
│ └─ solution.task.priority → priority (1→critical, 2→high, 3→medium, 4-5→low)
|
||||||
|
├─ Output path: .workflow/issues/{issue-id}/.task/TASK-{T-id}.json
|
||||||
|
├─ Each file follows task-schema.json (IDENTITY + CONVERGENCE + FILES required)
|
||||||
|
└─ source.tool = "issue-resolve", source.issue_id = {issue-id}
|
||||||
```
|
```
|
||||||
|
|
||||||
## Task Tracking Pattern
|
## Task Tracking Pattern
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
---
|
---
|
||||||
name: plan-converter
|
name: plan-converter
|
||||||
description: Convert any planning/analysis/brainstorm output to unified JSONL task format. Supports roadmap.jsonl, plan.json, plan-note.md, conclusions.json, synthesis.json.
|
description: Convert any planning/analysis/brainstorm output to .task/*.json multi-file format. Supports roadmap.jsonl, plan.json, plan-note.md, conclusions.json, synthesis.json.
|
||||||
argument-hint: "<input-file> [-o <output-file>]"
|
argument-hint: "<input-file> [-o <output-file>]"
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -8,70 +8,46 @@ argument-hint: "<input-file> [-o <output-file>]"
|
|||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
Converts any planning artifact to **unified JSONL task format** — the single standard consumed by `unified-execute-with-file`.
|
Converts any planning artifact to **`.task/*.json` multi-file format** — the single standard consumed by `unified-execute-with-file`.
|
||||||
|
|
||||||
|
> **Schema**: `cat ~/.ccw/workflows/cli-templates/schemas/task-schema.json`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Auto-detect format, output to same directory
|
# Auto-detect format, output to same directory .task/
|
||||||
/codex:plan-converter ".workflow/.req-plan/RPLAN-auth-2025-01-21/roadmap.jsonl"
|
/codex:plan-converter ".workflow/.req-plan/RPLAN-auth-2025-01-21/roadmap.jsonl"
|
||||||
|
|
||||||
# Specify output path
|
# Specify output directory
|
||||||
/codex:plan-converter ".workflow/.planning/CPLAN-xxx/plan-note.md" -o tasks.jsonl
|
/codex:plan-converter ".workflow/.planning/CPLAN-xxx/plan-note.md" -o .task/
|
||||||
|
|
||||||
# Convert brainstorm synthesis
|
# Convert brainstorm synthesis
|
||||||
/codex:plan-converter ".workflow/.brainstorm/BS-xxx/synthesis.json"
|
/codex:plan-converter ".workflow/.brainstorm/BS-xxx/synthesis.json"
|
||||||
```
|
```
|
||||||
|
|
||||||
**Supported inputs**: roadmap.jsonl, tasks.jsonl (per-domain), plan-note.md, conclusions.json, synthesis.json
|
**Supported inputs**: roadmap.jsonl, .task/*.json (per-domain), plan-note.md, conclusions.json, synthesis.json
|
||||||
|
|
||||||
**Output**: Unified JSONL (`tasks.jsonl` in same directory, or specified `-o` path)
|
**Output**: `.task/*.json` (one file per task, in same directory's `.task/` subfolder, or specified `-o` path)
|
||||||
|
|
||||||
## Unified JSONL Schema
|
## Task JSON Schema
|
||||||
|
|
||||||
每行一条自包含任务记录：
|
每个任务一个独立 JSON 文件 (`.task/TASK-{id}.json`)，遵循统一 schema：
|
||||||
|
|
||||||
|
> **Schema 定义**: `cat ~/.ccw/workflows/cli-templates/schemas/task-schema.json`
|
||||||
|
|
||||||
|
**Producer 使用的字段集** (plan-converter 输出):
|
||||||
|
|
||||||
```
|
```
|
||||||
┌─ IDENTITY (必填) ──────────────────────────────────────────┐
|
IDENTITY (必填): id, title, description
|
||||||
│ id string 任务 ID (L0/T1/TASK-001) │
|
CLASSIFICATION (可选): type, priority, effort
|
||||||
│ title string 任务标题 │
|
SCOPE (可选): scope, excludes
|
||||||
│ description string 目标 + 原因 │
|
DEPENDENCIES (必填): depends_on, parallel_group, inputs, outputs
|
||||||
├─ CLASSIFICATION (可选) ────────────────────────────────────┤
|
CONVERGENCE (必填): convergence.criteria, convergence.verification, convergence.definition_of_done
|
||||||
│ type enum infrastructure|feature|enhancement│
|
FILES (可选): files[].path, files[].action, files[].changes, files[].conflict_risk
|
||||||
│ enum |fix|refactor|testing │
|
CONTEXT (可选): source.tool, source.session_id, source.original_id, evidence, risk_items
|
||||||
│ priority enum high|medium|low │
|
RUNTIME (执行时填充): status, executed_at, result
|
||||||
│ effort enum small|medium|large │
|
|
||||||
├─ SCOPE (可选) ─────────────────────────────────────────────┤
|
|
||||||
│ scope string|[] 覆盖范围 │
|
|
||||||
│ excludes string[] 明确排除 │
|
|
||||||
├─ DEPENDENCIES (depends_on 必填) ───────────────────────────┤
|
|
||||||
│ depends_on string[] 依赖任务 ID（无依赖则 []） │
|
|
||||||
│ parallel_group number 并行分组（同组可并行） │
|
|
||||||
│ inputs string[] 消费的产物 │
|
|
||||||
│ outputs string[] 产出的产物 │
|
|
||||||
├─ CONVERGENCE (必填) ───────────────────────────────────────┤
|
|
||||||
│ convergence.criteria string[] 可测试的完成条件 │
|
|
||||||
│ convergence.verification string 可执行的验证步骤 │
|
|
||||||
│ convergence.definition_of_done string 业务语言完成定义 │
|
|
||||||
├─ FILES (可选，渐进详细) ───────────────────────────────────┤
|
|
||||||
│ files[].path string 文件路径 (必填*) │
|
|
||||||
│ files[].action enum modify|create|delete │
|
|
||||||
│ files[].changes string[] 修改描述 │
|
|
||||||
│ files[].conflict_risk enum low|medium|high │
|
|
||||||
├─ CONTEXT (可选) ───────────────────────────────────────────┤
|
|
||||||
│ source.tool string 产出工具名 │
|
|
||||||
│ source.session_id string 来源 session │
|
|
||||||
│ source.original_id string 转换前原始 ID │
|
|
||||||
│ evidence any[] 支撑证据 │
|
|
||||||
│ risk_items string[] 风险项 │
|
|
||||||
├─ EXECUTION (执行时填充，规划时不存在) ─────────────────────┤
|
|
||||||
│ _execution.status enum pending|in_progress| │
|
|
||||||
│ completed|failed|skipped │
|
|
||||||
│ _execution.executed_at string ISO 时间戳 │
|
|
||||||
│ _execution.result object { success, files_modified, │
|
|
||||||
│ summary, error, │
|
|
||||||
│ convergence_verified[] } │
|
|
||||||
└────────────────────────────────────────────────────────────┘
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**文件命名**: `TASK-{id}.json` (保留原有 ID 前缀: L0-, T1-, IDEA- 等)
|
||||||
|
|
||||||
## Target Input
|
## Target Input
|
||||||
|
|
||||||
**$ARGUMENTS**
|
**$ARGUMENTS**
|
||||||
@@ -82,9 +58,9 @@ Converts any planning artifact to **unified JSONL task format** — the single s
|
|||||||
Step 0: Parse arguments, resolve input path
|
Step 0: Parse arguments, resolve input path
|
||||||
Step 1: Detect input format
|
Step 1: Detect input format
|
||||||
Step 2: Parse input → extract raw records
|
Step 2: Parse input → extract raw records
|
||||||
Step 3: Transform → unified JSONL records
|
Step 3: Transform → unified task records
|
||||||
Step 4: Validate convergence quality
|
Step 4: Validate convergence quality
|
||||||
Step 5: Write output + display summary
|
Step 5: Write .task/*.json output + display summary
|
||||||
```
|
```
|
||||||
|
|
||||||
## Implementation
|
## Implementation
|
||||||
@@ -110,7 +86,7 @@ const content = Read(resolvedInput)
|
|||||||
|
|
||||||
function detectFormat(filename, content) {
|
function detectFormat(filename, content) {
|
||||||
if (filename === 'roadmap.jsonl') return 'roadmap-jsonl'
|
if (filename === 'roadmap.jsonl') return 'roadmap-jsonl'
|
||||||
if (filename === 'tasks.jsonl') return 'tasks-jsonl' // already unified or per-domain
|
if (filename === 'tasks.jsonl') return 'tasks-jsonl' // legacy JSONL or per-domain
|
||||||
if (filename === 'plan-note.md') return 'plan-note-md'
|
if (filename === 'plan-note.md') return 'plan-note-md'
|
||||||
if (filename === 'conclusions.json') return 'conclusions-json'
|
if (filename === 'conclusions.json') return 'conclusions-json'
|
||||||
if (filename === 'synthesis.json') return 'synthesis-json'
|
if (filename === 'synthesis.json') return 'synthesis-json'
|
||||||
@@ -132,7 +108,7 @@ function detectFormat(filename, content) {
|
|||||||
| Filename | Format ID | Source Tool |
|
| Filename | Format ID | Source Tool |
|
||||||
|----------|-----------|------------|
|
|----------|-----------|------------|
|
||||||
| `roadmap.jsonl` | roadmap-jsonl | req-plan-with-file |
|
| `roadmap.jsonl` | roadmap-jsonl | req-plan-with-file |
|
||||||
| `tasks.jsonl` (per-domain) | tasks-jsonl | collaborative-plan-with-file |
|
| `tasks.jsonl` (legacy) / `.task/*.json` | tasks-jsonl / task-json | collaborative-plan-with-file |
|
||||||
| `plan-note.md` | plan-note-md | collaborative-plan-with-file |
|
| `plan-note.md` | plan-note-md | collaborative-plan-with-file |
|
||||||
| `conclusions.json` | conclusions-json | analyze-with-file |
|
| `conclusions.json` | conclusions-json | analyze-with-file |
|
||||||
| `synthesis.json` | synthesis-json | brainstorm-with-file |
|
| `synthesis.json` | synthesis-json | brainstorm-with-file |
|
||||||
@@ -394,12 +370,15 @@ function validateConvergence(records) {
|
|||||||
// | Technical DoD | Rewrite in business language |
|
// | Technical DoD | Rewrite in business language |
|
||||||
```
|
```
|
||||||
|
|
||||||
### Step 5: Write Output & Summary
|
### Step 5: Write .task/*.json Output & Summary
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// Determine output path
|
// Determine output directory
|
||||||
const outputFile = outputPath
|
const outputDir = outputPath
|
||||||
|| `${path.dirname(resolvedInput)}/tasks.jsonl`
|
|| `${path.dirname(resolvedInput)}/.task`
|
||||||
|
|
||||||
|
// Create output directory
|
||||||
|
Bash(`mkdir -p ${outputDir}`)
|
||||||
|
|
||||||
// Clean records: remove undefined/null optional fields
|
// Clean records: remove undefined/null optional fields
|
||||||
const cleanedRecords = records.map(rec => {
|
const cleanedRecords = records.map(rec => {
|
||||||
@@ -411,16 +390,18 @@ const cleanedRecords = records.map(rec => {
|
|||||||
return clean
|
return clean
|
||||||
})
|
})
|
||||||
|
|
||||||
// Write JSONL
|
// Write individual task JSON files
|
||||||
const jsonlContent = cleanedRecords.map(r => JSON.stringify(r)).join('\n')
|
cleanedRecords.forEach(record => {
|
||||||
Write(outputFile, jsonlContent)
|
const filename = `${record.id}.json`
|
||||||
|
Write(`${outputDir}/${filename}`, JSON.stringify(record, null, 2))
|
||||||
|
})
|
||||||
|
|
||||||
// Display summary
|
// Display summary
|
||||||
// | Source | Format | Records | Issues |
|
// | Source | Format | Records | Issues |
|
||||||
// |-----------------|-------------------|---------|--------|
|
// |-----------------|-------------------|---------|--------|
|
||||||
// | roadmap.jsonl | roadmap-jsonl | 4 | 0 |
|
// | roadmap.jsonl | roadmap-jsonl | 4 | 0 |
|
||||||
//
|
//
|
||||||
// Output: .workflow/.req-plan/RPLAN-xxx/tasks.jsonl
|
// Output: .workflow/.req-plan/RPLAN-xxx/.task/ (4 files)
|
||||||
// Records: 4 tasks with convergence criteria
|
// Records: 4 tasks with convergence criteria
|
||||||
// Quality: All convergence checks passed
|
// Quality: All convergence checks passed
|
||||||
```
|
```
|
||||||
@@ -433,7 +414,7 @@ Write(outputFile, jsonlContent)
|
|||||||
|--------|-----------|------------|-----------------|-----------|--------------|-----------|
|
|--------|-----------|------------|-----------------|-----------|--------------|-----------|
|
||||||
| roadmap.jsonl (progressive) | req-plan | L0-L3 | **Yes** | No | No | **Yes** |
|
| roadmap.jsonl (progressive) | req-plan | L0-L3 | **Yes** | No | No | **Yes** |
|
||||||
| roadmap.jsonl (direct) | req-plan | T1-TN | **Yes** | No | No | **Yes** |
|
| roadmap.jsonl (direct) | req-plan | T1-TN | **Yes** | No | No | **Yes** |
|
||||||
| tasks.jsonl (per-domain) | collaborative-plan | TASK-NNN | **Yes** | **Yes** (detailed) | Optional | **Yes** |
|
| .task/TASK-*.json (per-domain) | collaborative-plan | TASK-NNN | **Yes** | **Yes** (detailed) | Optional | **Yes** |
|
||||||
| plan-note.md | collaborative-plan | TASK-NNN | **Generate** | **Yes** (from 修改文件) | From effort | No |
|
| plan-note.md | collaborative-plan | TASK-NNN | **Generate** | **Yes** (from 修改文件) | From effort | No |
|
||||||
| conclusions.json | analyze | TASK-NNN | **Generate** | No | **Yes** | No |
|
| conclusions.json | analyze | TASK-NNN | **Generate** | No | **Yes** | No |
|
||||||
| synthesis.json | brainstorm | IDEA-NNN | **Generate** | No | From score | No |
|
| synthesis.json | brainstorm | IDEA-NNN | **Generate** | No | From score | No |
|
||||||
|
|||||||
@@ -34,14 +34,14 @@ Unified multi-dimensional code review orchestrator with dual-mode (session/modul
|
|||||||
┌─────────────────────────────┼─────────────────────────────────┐
|
┌─────────────────────────────┼─────────────────────────────────┐
|
||||||
│ Fix Pipeline (Phase 6-9) │
|
│ Fix Pipeline (Phase 6-9) │
|
||||||
│ │
|
│ │
|
||||||
│ ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌─────────┐
|
│ ┌─────────┐ ┌─────────┐ ┌──────────┐ ┌─────────┐ ┌─────────┐
|
||||||
│ │ Phase 6 │→ │ Phase 7 │→ │ Phase 8 │→ │ Phase 9 │
|
│ │ Phase 6 │→ │ Phase 7 │→ │Phase 7.5 │→ │ Phase 8 │→ │ Phase 9 │
|
||||||
│ │Discovery│ │Parallel │ │Execution│ │Complete │
|
│ │Discovery│ │Parallel │ │Export to │ │Execution│ │Complete │
|
||||||
│ │Batching │ │Planning │ │Orchestr.│ │ │
|
│ │Batching │ │Planning │ │Task JSON │ │Orchestr.│ │ │
|
||||||
│ └─────────┘ └─────────┘ └─────────┘ └─────────┘
|
│ └─────────┘ └─────────┘ └──────────┘ └─────────┘ └─────────┘
|
||||||
│ grouping N agents M agents aggregate
|
│ grouping N agents fix-plan → M agents aggregate
|
||||||
│ + batch ×cli-plan ×cli-exec + summary
|
│ + batch ×cli-plan .task/FIX-* ×cli-exec + summary
|
||||||
└────────────────────────────────────────────────────────────────┘
|
└────────────────────────────────────────────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
## Key Design Principles
|
## Key Design Principles
|
||||||
@@ -73,6 +73,7 @@ review-cycle --fix <review-dir> [FLAGS] # Fix with flags
|
|||||||
--fix Enter fix pipeline after review or standalone
|
--fix Enter fix pipeline after review or standalone
|
||||||
--resume Resume interrupted fix session
|
--resume Resume interrupted fix session
|
||||||
--batch-size=N Findings per planning batch (default: 5, fix mode only)
|
--batch-size=N Findings per planning batch (default: 5, fix mode only)
|
||||||
|
--export-tasks Export fix-plan findings to .task/FIX-*.json (auto-enabled with --fix)
|
||||||
|
|
||||||
# Examples
|
# Examples
|
||||||
review-cycle src/auth/** # Module: review auth
|
review-cycle src/auth/** # Module: review auth
|
||||||
@@ -159,6 +160,20 @@ Phase 7: Fix Parallel Planning
|
|||||||
├─ Lifecycle: spawn_agent → batch wait → close_agent
|
├─ Lifecycle: spawn_agent → batch wait → close_agent
|
||||||
└─ Orchestrator aggregates → fix-plan.json
|
└─ Orchestrator aggregates → fix-plan.json
|
||||||
|
|
||||||
|
Phase 7.5: Export to Task JSON (auto with --fix, or explicit --export-tasks)
|
||||||
|
└─ Convert fix-plan.json findings → .task/FIX-{seq}.json
|
||||||
|
├─ For each finding in fix-plan.json:
|
||||||
|
│ ├─ finding.file → files[].path (action: "modify")
|
||||||
|
│ ├─ finding.severity → priority (critical|high|medium|low)
|
||||||
|
│ ├─ finding.fix_description → description
|
||||||
|
│ ├─ finding.dimension → scope
|
||||||
|
│ ├─ finding.verification → convergence.verification
|
||||||
|
│ ├─ finding.changes[] → convergence.criteria[]
|
||||||
|
│ └─ finding.fix_steps[] → implementation[]
|
||||||
|
├─ Output path: {projectRoot}/.workflow/active/WFS-{id}/.review/.task/FIX-{seq}.json
|
||||||
|
├─ Each file follows task-schema.json (IDENTITY + CONVERGENCE + FILES required)
|
||||||
|
└─ source.tool = "review-cycle", source.session_id = WFS-{id}
|
||||||
|
|
||||||
Phase 8: Fix Execution
|
Phase 8: Fix Execution
|
||||||
└─ Ref: phases/08-fix-execution.md
|
└─ Ref: phases/08-fix-execution.md
|
||||||
├─ Stage-based execution per aggregated timeline
|
├─ Stage-based execution per aggregated timeline
|
||||||
@@ -185,7 +200,8 @@ Complete: Review reports + optional fix results
|
|||||||
| 5 | [phases/05-review-completion.md](phases/05-review-completion.md) | No more iterations needed | Shared from both review commands Phase 5 |
|
| 5 | [phases/05-review-completion.md](phases/05-review-completion.md) | No more iterations needed | Shared from both review commands Phase 5 |
|
||||||
| 6 | [phases/06-fix-discovery-batching.md](phases/06-fix-discovery-batching.md) | Fix mode entry | review-cycle-fix Phase 1 + 1.5 |
|
| 6 | [phases/06-fix-discovery-batching.md](phases/06-fix-discovery-batching.md) | Fix mode entry | review-cycle-fix Phase 1 + 1.5 |
|
||||||
| 7 | [phases/07-fix-parallel-planning.md](phases/07-fix-parallel-planning.md) | Phase 6 complete | review-cycle-fix Phase 2 |
|
| 7 | [phases/07-fix-parallel-planning.md](phases/07-fix-parallel-planning.md) | Phase 6 complete | review-cycle-fix Phase 2 |
|
||||||
| 8 | [phases/08-fix-execution.md](phases/08-fix-execution.md) | Phase 7 complete | review-cycle-fix Phase 3 |
|
| 7.5 | _(inline in SKILL.md)_ | Phase 7 complete | Export fix-plan findings to .task/FIX-*.json |
|
||||||
|
| 8 | [phases/08-fix-execution.md](phases/08-fix-execution.md) | Phase 7.5 complete | review-cycle-fix Phase 3 |
|
||||||
| 9 | [phases/09-fix-completion.md](phases/09-fix-completion.md) | Phase 8 complete | review-cycle-fix Phase 4 + 5 |
|
| 9 | [phases/09-fix-completion.md](phases/09-fix-completion.md) | Phase 8 complete | review-cycle-fix Phase 4 + 5 |
|
||||||
|
|
||||||
## Core Rules
|
## Core Rules
|
||||||
@@ -225,6 +241,8 @@ Phase 6: Fix Discovery & Batching
|
|||||||
↓ Output: finding batches (in-memory)
|
↓ Output: finding batches (in-memory)
|
||||||
Phase 7: Fix Parallel Planning
|
Phase 7: Fix Parallel Planning
|
||||||
↓ Output: partial-plan-*.json → fix-plan.json (aggregated)
|
↓ Output: partial-plan-*.json → fix-plan.json (aggregated)
|
||||||
|
Phase 7.5: Export to Task JSON
|
||||||
|
↓ Output: .task/FIX-{seq}.json (per finding, follows task-schema.json)
|
||||||
Phase 8: Fix Execution
|
Phase 8: Fix Execution
|
||||||
↓ Output: fix-progress-*.json, git commits
|
↓ Output: fix-progress-*.json, git commits
|
||||||
Phase 9: Fix Completion
|
Phase 9: Fix Completion
|
||||||
@@ -324,10 +342,11 @@ Phase 5: Review Completion → pending
|
|||||||
|
|
||||||
**Fix Pipeline (added after Phase 5 if triggered)**:
|
**Fix Pipeline (added after Phase 5 if triggered)**:
|
||||||
```
|
```
|
||||||
Phase 6: Fix Discovery & Batching → pending
|
Phase 6: Fix Discovery & Batching → pending
|
||||||
Phase 7: Parallel Planning → pending
|
Phase 7: Parallel Planning → pending
|
||||||
Phase 8: Execution → pending
|
Phase 7.5: Export to Task JSON → pending
|
||||||
Phase 9: Fix Completion → pending
|
Phase 8: Execution → pending
|
||||||
|
Phase 9: Fix Completion → pending
|
||||||
```
|
```
|
||||||
|
|
||||||
## Error Handling
|
## Error Handling
|
||||||
@@ -386,6 +405,10 @@ Gemini → Qwen → Codex → degraded mode
|
|||||||
│ ├── security-cli-output.txt
|
│ ├── security-cli-output.txt
|
||||||
│ ├── deep-dive-1-{uuid}.md
|
│ ├── deep-dive-1-{uuid}.md
|
||||||
│ └── ...
|
│ └── ...
|
||||||
|
├── .task/ # Task JSON exports (Phase 7.5)
|
||||||
|
│ ├── FIX-001.json # Per-finding task (task-schema.json)
|
||||||
|
│ ├── FIX-002.json
|
||||||
|
│ └── ...
|
||||||
└── fixes/{fix-session-id}/ # Fix results (Phase 6-9)
|
└── fixes/{fix-session-id}/ # Fix results (Phase 6-9)
|
||||||
├── partial-plan-*.json
|
├── partial-plan-*.json
|
||||||
├── fix-plan.json
|
├── fix-plan.json
|
||||||
|
|||||||
@@ -1,40 +1,41 @@
|
|||||||
---
|
---
|
||||||
name: unified-execute-with-file
|
name: unified-execute-with-file
|
||||||
description: Universal execution engine consuming unified JSONL task format. Serial task execution with convergence verification, progress tracking via execution.md + execution-events.md.
|
description: Universal execution engine consuming .task/*.json directory format. Serial task execution with convergence verification, progress tracking via execution.md + execution-events.md.
|
||||||
argument-hint: "PLAN=\"<path/to/tasks.jsonl>\" [--auto-commit] [--dry-run]"
|
argument-hint: "PLAN=\"<path/to/.task/>\" [--auto-commit] [--dry-run]"
|
||||||
---
|
---
|
||||||
|
|
||||||
# Unified-Execute-With-File Workflow
|
# Unified-Execute-With-File Workflow
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
Universal execution engine consuming **unified JSONL** (`tasks.jsonl`) and executing tasks serially with convergence verification and progress tracking.
|
Universal execution engine consuming **`.task/*.json`** directory and executing tasks serially with convergence verification and progress tracking.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Execute from req-plan output
|
# Execute from lite-plan output
|
||||||
/codex:unified-execute-with-file PLAN=".workflow/.req-plan/RPLAN-auth-2025-01-21/tasks.jsonl"
|
/codex:unified-execute-with-file PLAN=".workflow/.lite-plan/LPLAN-auth-2025-01-21/.task/"
|
||||||
|
|
||||||
# Execute from collaborative-plan output
|
# Execute from workflow session output
|
||||||
/codex:unified-execute-with-file PLAN=".workflow/.planning/CPLAN-xxx/tasks.jsonl" --auto-commit
|
/codex:unified-execute-with-file PLAN=".workflow/active/WFS-xxx/.task/" --auto-commit
|
||||||
|
|
||||||
# Dry-run mode
|
# Execute a single task JSON file
|
||||||
/codex:unified-execute-with-file PLAN="tasks.jsonl" --dry-run
|
/codex:unified-execute-with-file PLAN=".workflow/active/WFS-xxx/.task/IMPL-001.json" --dry-run
|
||||||
|
|
||||||
# Auto-detect from .workflow/ directories
|
# Auto-detect from .workflow/ directories
|
||||||
/codex:unified-execute-with-file
|
/codex:unified-execute-with-file
|
||||||
```
|
```
|
||||||
|
|
||||||
**Core workflow**: Load JSONL → Validate → Pre-Execution Analysis → Execute → Verify Convergence → Track Progress
|
**Core workflow**: Scan .task/*.json → Validate → Pre-Execution Analysis → Execute → Verify Convergence → Track Progress
|
||||||
|
|
||||||
**Key features**:
|
**Key features**:
|
||||||
- **Single format**: Only consumes unified JSONL (`tasks.jsonl`)
|
- **Directory-based**: Consumes `.task/` directory containing individual task JSON files
|
||||||
- **Convergence-driven**: Verifies each task's convergence criteria after execution
|
- **Convergence-driven**: Verifies each task's convergence criteria after execution
|
||||||
- **Serial execution**: Process tasks in topological order with dependency tracking
|
- **Serial execution**: Process tasks in topological order with dependency tracking
|
||||||
- **Dual progress tracking**: `execution.md` (overview) + `execution-events.md` (event stream)
|
- **Dual progress tracking**: `execution.md` (overview) + `execution-events.md` (event stream)
|
||||||
- **Auto-commit**: Optional conventional commits per task
|
- **Auto-commit**: Optional conventional commits per task
|
||||||
- **Dry-run mode**: Simulate execution without changes
|
- **Dry-run mode**: Simulate execution without changes
|
||||||
|
- **Flexible input**: Accepts `.task/` directory path or a single `.json` file path
|
||||||
|
|
||||||
**Input format**: Use `plan-converter` to convert other formats (roadmap.jsonl, plan-note.md, conclusions.json, synthesis.json) to unified JSONL first.
|
**Input format**: Each task is a standalone JSON file in `.task/` directory (e.g., `IMPL-001.json`). Use `plan-converter` to convert other formats to `.task/*.json` first.
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
@@ -44,7 +45,7 @@ Universal execution engine consuming **unified JSONL** (`tasks.jsonl`) and execu
|
|||||||
├─────────────────────────────────────────────────────────────┤
|
├─────────────────────────────────────────────────────────────┤
|
||||||
│ │
|
│ │
|
||||||
│ Phase 1: Load & Validate │
|
│ Phase 1: Load & Validate │
|
||||||
│ ├─ Parse tasks.jsonl (one task per line) │
|
│ ├─ Scan .task/*.json (one task per file) │
|
||||||
│ ├─ Validate schema (id, title, depends_on, convergence) │
|
│ ├─ Validate schema (id, title, depends_on, convergence) │
|
||||||
│ ├─ Detect cycles, build topological order │
|
│ ├─ Detect cycles, build topological order │
|
||||||
│ └─ Initialize execution.md + execution-events.md │
|
│ └─ Initialize execution.md + execution-events.md │
|
||||||
@@ -63,13 +64,13 @@ Universal execution engine consuming **unified JSONL** (`tasks.jsonl`) and execu
|
|||||||
│ ├─ Verify convergence.criteria[] │
|
│ ├─ Verify convergence.criteria[] │
|
||||||
│ ├─ Run convergence.verification command │
|
│ ├─ Run convergence.verification command │
|
||||||
│ ├─ Record COMPLETE/FAIL event with verification results │
|
│ ├─ Record COMPLETE/FAIL event with verification results │
|
||||||
│ ├─ Update _execution state in JSONL │
|
│ ├─ Update _execution state in task JSON file │
|
||||||
│ └─ Auto-commit if enabled │
|
│ └─ Auto-commit if enabled │
|
||||||
│ │
|
│ │
|
||||||
│ Phase 4: Completion │
|
│ Phase 4: Completion │
|
||||||
│ ├─ Finalize execution.md with summary statistics │
|
│ ├─ Finalize execution.md with summary statistics │
|
||||||
│ ├─ Finalize execution-events.md with session footer │
|
│ ├─ Finalize execution-events.md with session footer │
|
||||||
│ ├─ Write back tasks.jsonl with _execution states │
|
│ ├─ Write back .task/*.json with _execution states │
|
||||||
│ └─ Offer follow-up actions │
|
│ └─ Offer follow-up actions │
|
||||||
│ │
|
│ │
|
||||||
└─────────────────────────────────────────────────────────────┘
|
└─────────────────────────────────────────────────────────────┘
|
||||||
@@ -83,7 +84,7 @@ ${projectRoot}/.workflow/.execution/EXEC-{slug}-{date}-{random}/
|
|||||||
└── execution-events.md # ⭐ Unified event log (single source of truth)
|
└── execution-events.md # ⭐ Unified event log (single source of truth)
|
||||||
```
|
```
|
||||||
|
|
||||||
Additionally, the source `tasks.jsonl` is updated in-place with `_execution` states.
|
Additionally, each source `.task/*.json` file is updated in-place with `_execution` states.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -105,12 +106,12 @@ let planPath = planMatch ? planMatch[1] : null
|
|||||||
|
|
||||||
// Auto-detect if no PLAN specified
|
// Auto-detect if no PLAN specified
|
||||||
if (!planPath) {
|
if (!planPath) {
|
||||||
// Search in order:
|
// Search in order (most recent first):
|
||||||
// .workflow/.req-plan/*/tasks.jsonl
|
// .workflow/active/*/.task/
|
||||||
// .workflow/.planning/*/tasks.jsonl
|
// .workflow/.lite-plan/*/.task/
|
||||||
// .workflow/.analysis/*/tasks.jsonl
|
// .workflow/.req-plan/*/.task/
|
||||||
// .workflow/.brainstorm/*/tasks.jsonl
|
// .workflow/.planning/*/.task/
|
||||||
// Use most recently modified
|
// Use most recently modified directory containing *.json files
|
||||||
}
|
}
|
||||||
|
|
||||||
// Resolve path
|
// Resolve path
|
||||||
@@ -130,41 +131,85 @@ Bash(`mkdir -p ${sessionFolder}`)
|
|||||||
|
|
||||||
## Phase 1: Load & Validate
|
## Phase 1: Load & Validate
|
||||||
|
|
||||||
**Objective**: Parse unified JSONL, validate schema and dependencies, build execution order.
|
**Objective**: Scan `.task/` directory, parse individual task JSON files, validate schema and dependencies, build execution order.
|
||||||
|
|
||||||
### Step 1.1: Parse Unified JSONL
|
### Step 1.1: Scan .task/ Directory and Parse Task Files
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
const content = Read(planPath)
|
// Determine if planPath is a directory or single file
|
||||||
const tasks = content.split('\n')
|
const isDirectory = planPath.endsWith('/') || Bash(`test -d "${planPath}" && echo dir || echo file`).trim() === 'dir'
|
||||||
.filter(line => line.trim())
|
|
||||||
.map((line, i) => {
|
|
||||||
try { return JSON.parse(line) }
|
|
||||||
catch (e) { throw new Error(`Line ${i + 1}: Invalid JSON — ${e.message}`) }
|
|
||||||
})
|
|
||||||
|
|
||||||
if (tasks.length === 0) throw new Error('No tasks found in JSONL file')
|
let taskFiles, tasks
|
||||||
|
|
||||||
|
if (isDirectory) {
|
||||||
|
// Directory mode: scan for all *.json files
|
||||||
|
taskFiles = Glob('*.json', planPath)
|
||||||
|
if (taskFiles.length === 0) throw new Error(`No .json files found in ${planPath}`)
|
||||||
|
|
||||||
|
tasks = taskFiles.map(filePath => {
|
||||||
|
try {
|
||||||
|
const content = Read(filePath)
|
||||||
|
const task = JSON.parse(content)
|
||||||
|
task._source_file = filePath // Track source file for write-back
|
||||||
|
return task
|
||||||
|
} catch (e) {
|
||||||
|
throw new Error(`${path.basename(filePath)}: Invalid JSON - ${e.message}`)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
// Single file mode: parse one task JSON
|
||||||
|
try {
|
||||||
|
const content = Read(planPath)
|
||||||
|
const task = JSON.parse(content)
|
||||||
|
task._source_file = planPath
|
||||||
|
tasks = [task]
|
||||||
|
} catch (e) {
|
||||||
|
throw new Error(`${path.basename(planPath)}: Invalid JSON - ${e.message}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tasks.length === 0) throw new Error('No tasks found')
|
||||||
```
|
```
|
||||||
|
|
||||||
### Step 1.2: Validate Schema
|
### Step 1.2: Validate Schema
|
||||||
|
|
||||||
|
Validate against unified task schema: `~/.ccw/workflows/cli-templates/schemas/task-schema.json`
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
const errors = []
|
const errors = []
|
||||||
tasks.forEach((task, i) => {
|
tasks.forEach((task, i) => {
|
||||||
// Required fields
|
const src = task._source_file ? path.basename(task._source_file) : `Task ${i + 1}`
|
||||||
if (!task.id) errors.push(`Task ${i + 1}: missing 'id'`)
|
|
||||||
if (!task.title) errors.push(`Task ${i + 1}: missing 'title'`)
|
|
||||||
if (!task.description) errors.push(`Task ${i + 1}: missing 'description'`)
|
|
||||||
if (!Array.isArray(task.depends_on)) errors.push(`${task.id}: missing 'depends_on' array`)
|
|
||||||
|
|
||||||
// Convergence required
|
// Required fields (per task-schema.json)
|
||||||
|
if (!task.id) errors.push(`${src}: missing 'id'`)
|
||||||
|
if (!task.title) errors.push(`${src}: missing 'title'`)
|
||||||
|
if (!task.description) errors.push(`${src}: missing 'description'`)
|
||||||
|
if (!Array.isArray(task.depends_on)) errors.push(`${task.id || src}: missing 'depends_on' array`)
|
||||||
|
|
||||||
|
// Context block (optional but validated if present)
|
||||||
|
if (task.context) {
|
||||||
|
if (task.context.requirements && !Array.isArray(task.context.requirements))
|
||||||
|
errors.push(`${task.id}: context.requirements must be array`)
|
||||||
|
if (task.context.acceptance && !Array.isArray(task.context.acceptance))
|
||||||
|
errors.push(`${task.id}: context.acceptance must be array`)
|
||||||
|
if (task.context.focus_paths && !Array.isArray(task.context.focus_paths))
|
||||||
|
errors.push(`${task.id}: context.focus_paths must be array`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convergence (required for execution verification)
|
||||||
if (!task.convergence) {
|
if (!task.convergence) {
|
||||||
errors.push(`${task.id}: missing 'convergence'`)
|
errors.push(`${task.id || src}: missing 'convergence'`)
|
||||||
} else {
|
} else {
|
||||||
if (!task.convergence.criteria?.length) errors.push(`${task.id}: empty convergence.criteria`)
|
if (!task.convergence.criteria?.length) errors.push(`${task.id}: empty convergence.criteria`)
|
||||||
if (!task.convergence.verification) errors.push(`${task.id}: missing convergence.verification`)
|
if (!task.convergence.verification) errors.push(`${task.id}: missing convergence.verification`)
|
||||||
if (!task.convergence.definition_of_done) errors.push(`${task.id}: missing convergence.definition_of_done`)
|
if (!task.convergence.definition_of_done) errors.push(`${task.id}: missing convergence.definition_of_done`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Flow control (optional but validated if present)
|
||||||
|
if (task.flow_control) {
|
||||||
|
if (task.flow_control.target_files && !Array.isArray(task.flow_control.target_files))
|
||||||
|
errors.push(`${task.id}: flow_control.target_files must be array`)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
if (errors.length) {
|
if (errors.length) {
|
||||||
@@ -610,14 +655,29 @@ appendToEvents(`
|
|||||||
`)
|
`)
|
||||||
```
|
```
|
||||||
|
|
||||||
### Step 4.3: Write Back tasks.jsonl with _execution
|
### Step 4.3: Write Back .task/*.json with _execution
|
||||||
|
|
||||||
Update the source JSONL file with execution states:
|
Update each source task JSON file with execution states:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
const updatedJsonl = tasks.map(task => JSON.stringify(task)).join('\n')
|
tasks.forEach(task => {
|
||||||
Write(planPath, updatedJsonl)
|
const filePath = task._source_file
|
||||||
// Each task now has _execution: { status, executed_at, result }
|
if (!filePath) return
|
||||||
|
|
||||||
|
// Read current file to preserve formatting and non-execution fields
|
||||||
|
const current = JSON.parse(Read(filePath))
|
||||||
|
|
||||||
|
// Update _execution status and result
|
||||||
|
current._execution = {
|
||||||
|
status: task._execution?.status || 'pending',
|
||||||
|
executed_at: task._execution?.executed_at || null,
|
||||||
|
result: task._execution?.result || null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write back individual task file
|
||||||
|
Write(filePath, JSON.stringify(current, null, 2))
|
||||||
|
})
|
||||||
|
// Each task JSON file now has _execution: { status, executed_at, result }
|
||||||
```
|
```
|
||||||
|
|
||||||
### Step 4.4: Post-Completion Options
|
### Step 4.4: Post-Completion Options
|
||||||
@@ -651,20 +711,20 @@ AskUserQuestion({
|
|||||||
|
|
||||||
| Flag | Default | Description |
|
| Flag | Default | Description |
|
||||||
|------|---------|-------------|
|
|------|---------|-------------|
|
||||||
| `PLAN="..."` | auto-detect | Path to unified JSONL file (`tasks.jsonl`) |
|
| `PLAN="..."` | auto-detect | Path to `.task/` directory or single task `.json` file |
|
||||||
| `--auto-commit` | false | Commit changes after each successful task |
|
| `--auto-commit` | false | Commit changes after each successful task |
|
||||||
| `--dry-run` | false | Simulate execution without making changes |
|
| `--dry-run` | false | Simulate execution without making changes |
|
||||||
|
|
||||||
### Plan Auto-Detection Order
|
### Plan Auto-Detection Order
|
||||||
|
|
||||||
When no `PLAN` specified, search in order (most recent first):
|
When no `PLAN` specified, search for `.task/` directories in order (most recent first):
|
||||||
|
|
||||||
1. `.workflow/.req-plan/*/tasks.jsonl`
|
1. `.workflow/active/*/.task/`
|
||||||
2. `.workflow/.planning/*/tasks.jsonl`
|
2. `.workflow/.lite-plan/*/.task/`
|
||||||
3. `.workflow/.analysis/*/tasks.jsonl`
|
3. `.workflow/.req-plan/*/.task/`
|
||||||
4. `.workflow/.brainstorm/*/tasks.jsonl`
|
4. `.workflow/.planning/*/.task/`
|
||||||
|
|
||||||
**If source is not unified JSONL**: Run `plan-converter` first.
|
**If source is not `.task/*.json`**: Run `plan-converter` first to generate `.task/` directory.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -672,10 +732,10 @@ When no `PLAN` specified, search in order (most recent first):
|
|||||||
|
|
||||||
| Situation | Action | Recovery |
|
| Situation | Action | Recovery |
|
||||||
|-----------|--------|----------|
|
|-----------|--------|----------|
|
||||||
| JSONL file not found | Report error with path | Check path, run plan-converter |
|
| .task/ directory not found | Report error with path | Check path, run plan-converter |
|
||||||
| Invalid JSON line | Report line number and error | Fix JSONL file manually |
|
| Invalid JSON in task file | Report filename and error | Fix task JSON file manually |
|
||||||
| Missing convergence | Report validation error | Run plan-converter to add convergence |
|
| Missing convergence | Report validation error | Run plan-converter to add convergence |
|
||||||
| Circular dependency | Stop, report cycle path | Fix dependencies in JSONL |
|
| Circular dependency | Stop, report cycle path | Fix dependencies in task JSON |
|
||||||
| Task execution fails | Record in events, ask user | Retry, skip, accept, or abort |
|
| Task execution fails | Record in events, ask user | Retry, skip, accept, or abort |
|
||||||
| Convergence verification fails | Mark task failed, ask user | Fix code and retry, or accept |
|
| Convergence verification fails | Mark task failed, ask user | Fix code and retry, or accept |
|
||||||
| Verification command timeout | Mark as unverified | Manual verification needed |
|
| Verification command timeout | Mark as unverified | Manual verification needed |
|
||||||
@@ -692,7 +752,7 @@ When no `PLAN` specified, search in order (most recent first):
|
|||||||
2. **Check Convergence**: Ensure all tasks have meaningful convergence criteria
|
2. **Check Convergence**: Ensure all tasks have meaningful convergence criteria
|
||||||
3. **Review Dependencies**: Verify execution order makes sense
|
3. **Review Dependencies**: Verify execution order makes sense
|
||||||
4. **Backup**: Commit pending changes before starting
|
4. **Backup**: Commit pending changes before starting
|
||||||
5. **Convert First**: Use `plan-converter` for non-JSONL sources
|
5. **Convert First**: Use `plan-converter` for non-.task/ sources
|
||||||
|
|
||||||
### During Execution
|
### During Execution
|
||||||
|
|
||||||
@@ -704,7 +764,7 @@ When no `PLAN` specified, search in order (most recent first):
|
|||||||
|
|
||||||
1. **Review Summary**: Check execution.md statistics and failed tasks
|
1. **Review Summary**: Check execution.md statistics and failed tasks
|
||||||
2. **Verify Changes**: Inspect modified files match expectations
|
2. **Verify Changes**: Inspect modified files match expectations
|
||||||
3. **Check JSONL**: Review `_execution` states in tasks.jsonl
|
3. **Check Task Files**: Review `_execution` states in `.task/*.json` files
|
||||||
4. **Next Steps**: Use completion options for follow-up
|
4. **Next Steps**: Use completion options for follow-up
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
@@ -1,19 +1,21 @@
|
|||||||
---
|
---
|
||||||
name: workflow-lite-plan-execute
|
name: workflow-lite-plan-execute
|
||||||
description: Lightweight planning + execution workflow. Serial CLI exploration → Search verification → Clarification → Planning → Unified JSONL output → Execution via unified-execute.
|
description: Lightweight planning + execution workflow. Serial CLI exploration → Search verification → Clarification → Planning → .task/*.json multi-file output → Execution via unified-execute.
|
||||||
allowed-tools: AskUserQuestion, Read, Write, Edit, Bash, Glob, Grep, mcp__ace-tool__search_context
|
allowed-tools: AskUserQuestion, Read, Write, Edit, Bash, Glob, Grep, mcp__ace-tool__search_context
|
||||||
---
|
---
|
||||||
|
|
||||||
# Planning Workflow
|
# Planning Workflow
|
||||||
|
|
||||||
Lite Plan produces a unified JSONL (`tasks.jsonl`) implementation plan via serial CLI exploration and direct planning, then hands off to unified-execute-with-file for task execution.
|
Lite Plan produces `.task/TASK-*.json` (one file per task) implementation plan via serial CLI exploration and direct planning, then hands off to unified-execute-with-file for task execution.
|
||||||
|
|
||||||
|
> **Schema**: `cat ~/.ccw/workflows/cli-templates/schemas/task-schema.json`
|
||||||
|
|
||||||
## Key Design Principles
|
## Key Design Principles
|
||||||
|
|
||||||
1. **Serial Execution**: All phases execute serially inline, no agent delegation
|
1. **Serial Execution**: All phases execute serially inline, no agent delegation
|
||||||
2. **CLI Exploration**: Multi-angle codebase exploration via `ccw cli` calls (default gemini, fallback claude)
|
2. **CLI Exploration**: Multi-angle codebase exploration via `ccw cli` calls (default gemini, fallback claude)
|
||||||
3. **Search Verification**: Verify CLI findings with ACE search / Grep / Glob before incorporating
|
3. **Search Verification**: Verify CLI findings with ACE search / Grep / Glob before incorporating
|
||||||
4. **Unified JSONL Output**: Produces `tasks.jsonl` compatible with `collaborative-plan-with-file` and `unified-execute-with-file`
|
4. **Multi-File Task Output**: Produces `.task/TASK-*.json` (one file per task) compatible with `collaborative-plan-with-file` and `unified-execute-with-file`
|
||||||
5. **Progressive Phase Loading**: Only load phase docs when about to execute
|
5. **Progressive Phase Loading**: Only load phase docs when about to execute
|
||||||
|
|
||||||
## Auto Mode
|
## Auto Mode
|
||||||
@@ -49,7 +51,7 @@ $workflow-lite-plan-execute "docs/todo.md"
|
|||||||
|
|
||||||
| Phase | Document | Purpose |
|
| Phase | Document | Purpose |
|
||||||
|-------|----------|---------|
|
|-------|----------|---------|
|
||||||
| 1 | `phases/01-lite-plan.md` | Serial CLI exploration, clarification, plan generation → tasks.jsonl |
|
| 1 | `phases/01-lite-plan.md` | Serial CLI exploration, clarification, plan generation → .task/TASK-*.json |
|
||||||
| 2 | `phases/02-lite-execute.md` | Handoff to unified-execute-with-file for task execution |
|
| 2 | `phases/02-lite-execute.md` | Handoff to unified-execute-with-file for task execution |
|
||||||
|
|
||||||
## Orchestrator Logic
|
## Orchestrator Logic
|
||||||
@@ -72,7 +74,7 @@ function extractTaskDescription(args) {
|
|||||||
|
|
||||||
const taskDescription = extractTaskDescription($ARGUMENTS)
|
const taskDescription = extractTaskDescription($ARGUMENTS)
|
||||||
|
|
||||||
// Phase 1: Lite Plan → tasks.jsonl
|
// Phase 1: Lite Plan → .task/TASK-*.json
|
||||||
Read('phases/01-lite-plan.md')
|
Read('phases/01-lite-plan.md')
|
||||||
// Execute planning phase...
|
// Execute planning phase...
|
||||||
|
|
||||||
@@ -84,12 +86,14 @@ if (planResult?.userSelection?.confirmation !== 'Allow' && !autoYes) {
|
|||||||
|
|
||||||
// Phase 2: Handoff to unified-execute-with-file
|
// Phase 2: Handoff to unified-execute-with-file
|
||||||
Read('phases/02-lite-execute.md')
|
Read('phases/02-lite-execute.md')
|
||||||
// Invoke unified-execute-with-file with tasks.jsonl path
|
// Invoke unified-execute-with-file with .task/ directory path
|
||||||
```
|
```
|
||||||
|
|
||||||
## Output Contract
|
## Output Contract
|
||||||
|
|
||||||
Phase 1 produces `tasks.jsonl` (unified JSONL format) — compatible with `collaborative-plan-with-file` and consumable by `unified-execute-with-file`.
|
Phase 1 produces `.task/TASK-*.json` (one file per task) — compatible with `collaborative-plan-with-file` and consumable by `unified-execute-with-file`.
|
||||||
|
|
||||||
|
> **Schema**: `cat ~/.ccw/workflows/cli-templates/schemas/task-schema.json`
|
||||||
|
|
||||||
**Output Directory**: `{projectRoot}/.workflow/.lite-plan/{session-id}/`
|
**Output Directory**: `{projectRoot}/.workflow/.lite-plan/{session-id}/`
|
||||||
|
|
||||||
@@ -100,37 +104,41 @@ Phase 1 produces `tasks.jsonl` (unified JSONL format) — compatible with `colla
|
|||||||
├── explorations-manifest.json # Exploration index
|
├── explorations-manifest.json # Exploration index
|
||||||
├── exploration-notes.md # Synthesized exploration notes
|
├── exploration-notes.md # Synthesized exploration notes
|
||||||
├── requirement-analysis.json # Complexity assessment
|
├── requirement-analysis.json # Complexity assessment
|
||||||
├── tasks.jsonl # ⭐ Unified JSONL (collaborative-plan-with-file compatible)
|
├── .task/ # ⭐ Task JSON files (one per task)
|
||||||
|
│ ├── TASK-001.json # Individual task definition
|
||||||
|
│ ├── TASK-002.json
|
||||||
|
│ └── ...
|
||||||
└── plan.md # Human-readable summary
|
└── plan.md # Human-readable summary
|
||||||
```
|
```
|
||||||
|
|
||||||
**Unified JSONL Task Format** (one task per line):
|
**Task JSON Format** (one file per task, following task-schema.json):
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
|
// File: .task/TASK-001.json
|
||||||
{
|
{
|
||||||
id: "TASK-001",
|
"id": "TASK-001",
|
||||||
title: string,
|
"title": "string",
|
||||||
description: string,
|
"description": "string",
|
||||||
type: "feature|fix|refactor|enhancement|testing|infrastructure",
|
"type": "feature|fix|refactor|enhancement|testing|infrastructure",
|
||||||
priority: "high|medium|low",
|
"priority": "high|medium|low",
|
||||||
effort: "small|medium|large",
|
"effort": "small|medium|large",
|
||||||
scope: string,
|
"scope": "string",
|
||||||
depends_on: ["TASK-xxx"],
|
"depends_on": ["TASK-xxx"],
|
||||||
convergence: {
|
"convergence": {
|
||||||
criteria: string[], // Testable conditions
|
"criteria": ["string"], // Testable conditions
|
||||||
verification: string, // Executable command or manual steps
|
"verification": "string", // Executable command or manual steps
|
||||||
definition_of_done: string // Business language
|
"definition_of_done": "string" // Business language
|
||||||
},
|
},
|
||||||
files: [{
|
"files": [{
|
||||||
path: string,
|
"path": "string",
|
||||||
action: "modify|create|delete",
|
"action": "modify|create|delete",
|
||||||
changes: string[],
|
"changes": ["string"],
|
||||||
conflict_risk: "low|medium|high"
|
"conflict_risk": "low|medium|high"
|
||||||
}],
|
}],
|
||||||
source: {
|
"source": {
|
||||||
tool: "workflow-lite-plan-execute",
|
"tool": "workflow-lite-plan-execute",
|
||||||
session_id: string,
|
"session_id": "string",
|
||||||
original_id: string
|
"original_id": "string"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
@@ -158,7 +166,7 @@ After planning completes:
|
|||||||
1. **Planning phase NEVER modifies project code** — it may write planning artifacts, but all implementation is delegated to unified-execute
|
1. **Planning phase NEVER modifies project code** — it may write planning artifacts, but all implementation is delegated to unified-execute
|
||||||
2. **All phases serial, no agent delegation** — everything runs inline, no spawn_agent
|
2. **All phases serial, no agent delegation** — everything runs inline, no spawn_agent
|
||||||
3. **CLI exploration with search verification** — CLI calls produce findings, ACE/Grep/Glob verify them
|
3. **CLI exploration with search verification** — CLI calls produce findings, ACE/Grep/Glob verify them
|
||||||
4. **tasks.jsonl is the output contract** — unified JSONL format passed to unified-execute-with-file
|
4. **`.task/*.json` is the output contract** — individual task JSON files passed to unified-execute-with-file
|
||||||
5. **Progressive loading**: Read phase doc only when about to execute
|
5. **Progressive loading**: Read phase doc only when about to execute
|
||||||
6. **File-path detection**: Treat input as a file path only if the path exists; do not infer from file extensions
|
6. **File-path detection**: Treat input as a file path only if the path exists; do not infer from file extensions
|
||||||
|
|
||||||
@@ -168,7 +176,7 @@ After planning completes:
|
|||||||
|-------|------------|
|
|-------|------------|
|
||||||
| CLI exploration failure | Skip angle, continue with remaining; fallback gemini → claude |
|
| CLI exploration failure | Skip angle, continue with remaining; fallback gemini → claude |
|
||||||
| Planning phase failure | Display error, offer retry |
|
| Planning phase failure | Display error, offer retry |
|
||||||
| tasks.jsonl missing | Error: planning phase did not produce output |
|
| .task/ directory empty | Error: planning phase did not produce output |
|
||||||
| Phase file not found | Error with file path for debugging |
|
| Phase file not found | Error with file path for debugging |
|
||||||
|
|
||||||
## Related Skills
|
## Related Skills
|
||||||
|
|||||||
@@ -77,6 +77,7 @@ Phase 2: Context Gathering & Conflict Resolution
|
|||||||
Phase 3: Task Generation
|
Phase 3: Task Generation
|
||||||
└─ Ref: phases/03-task-generation.md
|
└─ Ref: phases/03-task-generation.md
|
||||||
└─ Output: IMPL_PLAN.md, task JSONs, TODO_LIST.md
|
└─ Output: IMPL_PLAN.md, task JSONs, TODO_LIST.md
|
||||||
|
└─ Schema: .task/IMPL-*.json follows 6-field superset of task-schema.json
|
||||||
|
|
||||||
User Decision (or --yes auto):
|
User Decision (or --yes auto):
|
||||||
└─ "Start Execution" → Phase 4
|
└─ "Start Execution" → Phase 4
|
||||||
@@ -426,6 +427,20 @@ if (autoYes) {
|
|||||||
- After each phase, automatically continue to next phase based on TodoList status
|
- After each phase, automatically continue to next phase based on TodoList status
|
||||||
- **Always close_agent after wait completes**
|
- **Always close_agent after wait completes**
|
||||||
|
|
||||||
|
## Task JSON Schema Compatibility
|
||||||
|
|
||||||
|
Phase 3 generates `.task/IMPL-*.json` files using the **6-field schema** defined in `action-planning-agent.md`. These task JSONs are a **superset** of the unified `task-schema.json` (located at `.ccw/workflows/cli-templates/schemas/task-schema.json`).
|
||||||
|
|
||||||
|
**Key field mappings** (6-field → unified schema):
|
||||||
|
- `context.acceptance` → `convergence.criteria`
|
||||||
|
- `context.requirements` → `description` + `implementation`
|
||||||
|
- `context.depends_on` → `depends_on` (top-level)
|
||||||
|
- `context.focus_paths` → `focus_paths` (top-level)
|
||||||
|
- `meta.type` → `type` (top-level)
|
||||||
|
- `flow_control.target_files` → `files[].path`
|
||||||
|
|
||||||
|
All existing 6-field schema fields are preserved. The unified schema fields are accepted as optional aliases for cross-tool interoperability. See `action-planning-agent.md` Section 2.1 "Schema Compatibility" for the full mapping table.
|
||||||
|
|
||||||
## Related Commands
|
## Related Commands
|
||||||
|
|
||||||
**Prerequisite Commands**:
|
**Prerequisite Commands**:
|
||||||
|
|||||||
@@ -233,6 +233,7 @@ Phase 5: TDD Task Generation ← ATTACHED (3 tasks)
|
|||||||
├─ Phase 5.2: Planning - design Red-Green-Refactor cycles
|
├─ Phase 5.2: Planning - design Red-Green-Refactor cycles
|
||||||
└─ Phase 5.3: Output - generate IMPL tasks with internal TDD phases
|
└─ Phase 5.3: Output - generate IMPL tasks with internal TDD phases
|
||||||
└─ Output: IMPL-*.json, IMPL_PLAN.md ← COLLAPSED
|
└─ Output: IMPL-*.json, IMPL_PLAN.md ← COLLAPSED
|
||||||
|
└─ Schema: .task/IMPL-*.json follows 6-field superset of task-schema.json
|
||||||
|
|
||||||
Phase 6: TDD Structure Validation (inline)
|
Phase 6: TDD Structure Validation (inline)
|
||||||
└─ Internal validation + summary returned
|
└─ Internal validation + summary returned
|
||||||
@@ -766,6 +767,20 @@ Read and execute: `phases/03-tdd-verify.md` with `--session [sessionId]`
|
|||||||
|
|
||||||
This generates a comprehensive TDD_COMPLIANCE_REPORT.md with quality gate recommendation.
|
This generates a comprehensive TDD_COMPLIANCE_REPORT.md with quality gate recommendation.
|
||||||
|
|
||||||
|
## Task JSON Schema Compatibility
|
||||||
|
|
||||||
|
Phase 5 generates `.task/IMPL-*.json` files using the **6-field schema** defined in `action-planning-agent.md`. These task JSONs are a **superset** of the unified `task-schema.json` (located at `.ccw/workflows/cli-templates/schemas/task-schema.json`).
|
||||||
|
|
||||||
|
**Key field mappings** (6-field → unified schema):
|
||||||
|
- `context.acceptance` → `convergence.criteria`
|
||||||
|
- `context.requirements` → `description` + `implementation`
|
||||||
|
- `context.depends_on` → `depends_on` (top-level)
|
||||||
|
- `context.focus_paths` → `focus_paths` (top-level)
|
||||||
|
- `meta.type` → `type` (top-level)
|
||||||
|
- `flow_control.target_files` → `files[].path`
|
||||||
|
|
||||||
|
All existing 6-field schema fields are preserved. TDD-specific extensions (`meta.tdd_workflow`, `tdd_phase` in implementation_approach steps) are additive and do not conflict with the unified schema. See `action-planning-agent.md` Section 2.1 "Schema Compatibility" for the full mapping table.
|
||||||
|
|
||||||
## Related Skills
|
## Related Skills
|
||||||
|
|
||||||
**Prerequisite**:
|
**Prerequisite**:
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import {
|
|||||||
exportMemories,
|
exportMemories,
|
||||||
importMemories
|
importMemories
|
||||||
} from '../core/core-memory-store.js';
|
} from '../core/core-memory-store.js';
|
||||||
|
import { MemoryJobScheduler } from '../core/memory-job-scheduler.js';
|
||||||
import { notifyRefreshRequired } from '../tools/notifier.js';
|
import { notifyRefreshRequired } from '../tools/notifier.js';
|
||||||
|
|
||||||
interface CommandOptions {
|
interface CommandOptions {
|
||||||
@@ -664,6 +665,185 @@ async function searchAction(keyword: string, options: CommandOptions): Promise<v
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Memory V2 CLI Subcommands
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run batch extraction
|
||||||
|
*/
|
||||||
|
async function extractAction(options: CommandOptions): Promise<void> {
|
||||||
|
try {
|
||||||
|
const projectPath = getProjectPath();
|
||||||
|
|
||||||
|
console.log(chalk.cyan('\n Triggering memory extraction...\n'));
|
||||||
|
|
||||||
|
const { MemoryExtractionPipeline } = await import('../core/memory-extraction-pipeline.js');
|
||||||
|
const pipeline = new MemoryExtractionPipeline(projectPath);
|
||||||
|
|
||||||
|
// Scan eligible sessions first
|
||||||
|
const eligible = await pipeline.scanEligibleSessions();
|
||||||
|
console.log(chalk.white(` Eligible sessions: ${eligible.length}`));
|
||||||
|
|
||||||
|
if (eligible.length === 0) {
|
||||||
|
console.log(chalk.yellow(' No eligible sessions for extraction.\n'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run extraction (synchronous for CLI - shows progress)
|
||||||
|
console.log(chalk.cyan(' Running batch extraction...'));
|
||||||
|
await pipeline.runBatchExtraction();
|
||||||
|
|
||||||
|
const store = getCoreMemoryStore(projectPath);
|
||||||
|
const stage1Count = store.countStage1Outputs();
|
||||||
|
|
||||||
|
console.log(chalk.green(`\n Extraction complete.`));
|
||||||
|
console.log(chalk.white(` Total stage1 outputs: ${stage1Count}\n`));
|
||||||
|
|
||||||
|
notifyRefreshRequired('memory').catch(() => { /* ignore */ });
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error(chalk.red(`Error: ${(error as Error).message}`));
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Show extraction status
|
||||||
|
*/
|
||||||
|
async function extractStatusAction(options: CommandOptions): Promise<void> {
|
||||||
|
try {
|
||||||
|
const projectPath = getProjectPath();
|
||||||
|
const store = getCoreMemoryStore(projectPath);
|
||||||
|
const scheduler = new MemoryJobScheduler(store.getDb());
|
||||||
|
|
||||||
|
const stage1Count = store.countStage1Outputs();
|
||||||
|
const extractionJobs = scheduler.listJobs('extraction');
|
||||||
|
|
||||||
|
if (options.json) {
|
||||||
|
console.log(JSON.stringify({ total_stage1: stage1Count, jobs: extractionJobs }, null, 2));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(chalk.bold.cyan('\n Extraction Pipeline Status\n'));
|
||||||
|
console.log(chalk.white(` Stage 1 outputs: ${stage1Count}`));
|
||||||
|
console.log(chalk.white(` Extraction jobs: ${extractionJobs.length}`));
|
||||||
|
|
||||||
|
if (extractionJobs.length > 0) {
|
||||||
|
console.log(chalk.gray('\n ─────────────────────────────────────────────────────────────────'));
|
||||||
|
|
||||||
|
for (const job of extractionJobs) {
|
||||||
|
const statusColor = job.status === 'done' ? chalk.green
|
||||||
|
: job.status === 'error' ? chalk.red
|
||||||
|
: job.status === 'running' ? chalk.yellow
|
||||||
|
: chalk.gray;
|
||||||
|
|
||||||
|
console.log(chalk.cyan(` ${job.job_key}`) + chalk.white(` [${statusColor(job.status)}]`));
|
||||||
|
if (job.last_error) console.log(chalk.red(` Error: ${job.last_error}`));
|
||||||
|
if (job.started_at) console.log(chalk.gray(` Started: ${new Date(job.started_at * 1000).toLocaleString()}`));
|
||||||
|
if (job.finished_at) console.log(chalk.gray(` Finished: ${new Date(job.finished_at * 1000).toLocaleString()}`));
|
||||||
|
console.log(chalk.gray(' ─────────────────────────────────────────────────────────────────'));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log();
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error(chalk.red(`Error: ${(error as Error).message}`));
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run consolidation
|
||||||
|
*/
|
||||||
|
async function consolidateAction(options: CommandOptions): Promise<void> {
|
||||||
|
try {
|
||||||
|
const projectPath = getProjectPath();
|
||||||
|
|
||||||
|
console.log(chalk.cyan('\n Triggering memory consolidation...\n'));
|
||||||
|
|
||||||
|
const { MemoryConsolidationPipeline } = await import('../core/memory-consolidation-pipeline.js');
|
||||||
|
const pipeline = new MemoryConsolidationPipeline(projectPath);
|
||||||
|
|
||||||
|
await pipeline.runConsolidation();
|
||||||
|
|
||||||
|
const memoryMd = pipeline.getMemoryMdContent();
|
||||||
|
|
||||||
|
console.log(chalk.green(' Consolidation complete.'));
|
||||||
|
if (memoryMd) {
|
||||||
|
console.log(chalk.white(` MEMORY.md generated (${memoryMd.length} chars)`));
|
||||||
|
}
|
||||||
|
console.log();
|
||||||
|
|
||||||
|
notifyRefreshRequired('memory').catch(() => { /* ignore */ });
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error(chalk.red(`Error: ${(error as Error).message}`));
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* List all V2 pipeline jobs
|
||||||
|
*/
|
||||||
|
async function jobsAction(options: CommandOptions): Promise<void> {
|
||||||
|
try {
|
||||||
|
const projectPath = getProjectPath();
|
||||||
|
const store = getCoreMemoryStore(projectPath);
|
||||||
|
const scheduler = new MemoryJobScheduler(store.getDb());
|
||||||
|
|
||||||
|
const kind = options.type || undefined;
|
||||||
|
const jobs = scheduler.listJobs(kind);
|
||||||
|
|
||||||
|
if (options.json) {
|
||||||
|
console.log(JSON.stringify({ jobs, total: jobs.length }, null, 2));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(chalk.bold.cyan('\n Memory V2 Pipeline Jobs\n'));
|
||||||
|
|
||||||
|
if (jobs.length === 0) {
|
||||||
|
console.log(chalk.yellow(' No jobs found.\n'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Summary counts
|
||||||
|
const byStatus: Record<string, number> = {};
|
||||||
|
for (const job of jobs) {
|
||||||
|
byStatus[job.status] = (byStatus[job.status] || 0) + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const statusParts = Object.entries(byStatus)
|
||||||
|
.map(([s, c]) => `${s}: ${c}`)
|
||||||
|
.join(' | ');
|
||||||
|
console.log(chalk.white(` Summary: ${statusParts}`));
|
||||||
|
console.log(chalk.gray(' ─────────────────────────────────────────────────────────────────'));
|
||||||
|
|
||||||
|
for (const job of jobs) {
|
||||||
|
const statusColor = job.status === 'done' ? chalk.green
|
||||||
|
: job.status === 'error' ? chalk.red
|
||||||
|
: job.status === 'running' ? chalk.yellow
|
||||||
|
: chalk.gray;
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
chalk.cyan(` [${job.kind}]`) +
|
||||||
|
chalk.white(` ${job.job_key}`) +
|
||||||
|
` [${statusColor(job.status)}]` +
|
||||||
|
chalk.gray(` retries: ${job.retry_remaining}`)
|
||||||
|
);
|
||||||
|
if (job.last_error) console.log(chalk.red(` Error: ${job.last_error}`));
|
||||||
|
console.log(chalk.gray(' ─────────────────────────────────────────────────────────────────'));
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(chalk.gray(`\n Total: ${jobs.length}\n`));
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error(chalk.red(`Error: ${(error as Error).message}`));
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Core Memory command entry point
|
* Core Memory command entry point
|
||||||
*/
|
*/
|
||||||
@@ -724,6 +904,23 @@ export async function coreMemoryCommand(
|
|||||||
await listFromAction(textArg, options);
|
await listFromAction(textArg, options);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
// Memory V2 subcommands
|
||||||
|
case 'extract':
|
||||||
|
await extractAction(options);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'extract-status':
|
||||||
|
await extractStatusAction(options);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'consolidate':
|
||||||
|
await consolidateAction(options);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'jobs':
|
||||||
|
await jobsAction(options);
|
||||||
|
break;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
console.log(chalk.bold.cyan('\n CCW Core Memory\n'));
|
console.log(chalk.bold.cyan('\n CCW Core Memory\n'));
|
||||||
console.log(' Manage core memory entries and session clusters.\n');
|
console.log(' Manage core memory entries and session clusters.\n');
|
||||||
@@ -749,6 +946,12 @@ export async function coreMemoryCommand(
|
|||||||
console.log(chalk.white(' load-cluster <id> ') + chalk.gray('Load cluster context'));
|
console.log(chalk.white(' load-cluster <id> ') + chalk.gray('Load cluster context'));
|
||||||
console.log(chalk.white(' search <keyword> ') + chalk.gray('Search sessions'));
|
console.log(chalk.white(' search <keyword> ') + chalk.gray('Search sessions'));
|
||||||
console.log();
|
console.log();
|
||||||
|
console.log(chalk.bold(' Memory V2 Pipeline:'));
|
||||||
|
console.log(chalk.white(' extract ') + chalk.gray('Run batch memory extraction'));
|
||||||
|
console.log(chalk.white(' extract-status ') + chalk.gray('Show extraction pipeline status'));
|
||||||
|
console.log(chalk.white(' consolidate ') + chalk.gray('Run memory consolidation'));
|
||||||
|
console.log(chalk.white(' jobs ') + chalk.gray('List all pipeline jobs'));
|
||||||
|
console.log();
|
||||||
console.log(chalk.bold(' Options:'));
|
console.log(chalk.bold(' Options:'));
|
||||||
console.log(chalk.gray(' --id <id> Memory ID (for export/summary)'));
|
console.log(chalk.gray(' --id <id> Memory ID (for export/summary)'));
|
||||||
console.log(chalk.gray(' --tool gemini|qwen AI tool for summary (default: gemini)'));
|
console.log(chalk.gray(' --tool gemini|qwen AI tool for summary (default: gemini)'));
|
||||||
@@ -765,6 +968,10 @@ export async function coreMemoryCommand(
|
|||||||
console.log(chalk.gray(' ccw core-memory list-from d--other-project'));
|
console.log(chalk.gray(' ccw core-memory list-from d--other-project'));
|
||||||
console.log(chalk.gray(' ccw core-memory cluster --auto'));
|
console.log(chalk.gray(' ccw core-memory cluster --auto'));
|
||||||
console.log(chalk.gray(' ccw core-memory cluster --dedup'));
|
console.log(chalk.gray(' ccw core-memory cluster --dedup'));
|
||||||
|
console.log(chalk.gray(' ccw core-memory extract # Run memory extraction'));
|
||||||
|
console.log(chalk.gray(' ccw core-memory extract-status # Check extraction state'));
|
||||||
|
console.log(chalk.gray(' ccw core-memory consolidate # Run consolidation'));
|
||||||
|
console.log(chalk.gray(' ccw core-memory jobs # List pipeline jobs'));
|
||||||
console.log();
|
console.log();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -375,6 +375,19 @@ export interface ProjectPaths {
|
|||||||
config: string;
|
config: string;
|
||||||
/** CLI config file */
|
/** CLI config file */
|
||||||
cliConfig: string;
|
cliConfig: string;
|
||||||
|
/** Memory V2 paths */
|
||||||
|
memoryV2: {
|
||||||
|
/** Root: <projectRoot>/core-memory/v2/ */
|
||||||
|
root: string;
|
||||||
|
/** Rollout summaries directory */
|
||||||
|
rolloutSummaries: string;
|
||||||
|
/** Concatenated raw memories file */
|
||||||
|
rawMemories: string;
|
||||||
|
/** Final consolidated memory file */
|
||||||
|
memoryMd: string;
|
||||||
|
/** Skills directory */
|
||||||
|
skills: string;
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -434,6 +447,13 @@ export function getProjectPaths(projectPath: string): ProjectPaths {
|
|||||||
dashboardCache: join(projectDir, 'cache', 'dashboard-data.json'),
|
dashboardCache: join(projectDir, 'cache', 'dashboard-data.json'),
|
||||||
config: join(projectDir, 'config'),
|
config: join(projectDir, 'config'),
|
||||||
cliConfig: join(projectDir, 'config', 'cli-config.json'),
|
cliConfig: join(projectDir, 'config', 'cli-config.json'),
|
||||||
|
memoryV2: {
|
||||||
|
root: join(projectDir, 'core-memory', 'v2'),
|
||||||
|
rolloutSummaries: join(projectDir, 'core-memory', 'v2', 'rollout_summaries'),
|
||||||
|
rawMemories: join(projectDir, 'core-memory', 'v2', 'raw_memories.md'),
|
||||||
|
memoryMd: join(projectDir, 'core-memory', 'v2', 'MEMORY.md'),
|
||||||
|
skills: join(projectDir, 'core-memory', 'v2', 'skills'),
|
||||||
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -456,6 +476,13 @@ export function getProjectPathsById(projectId: string): ProjectPaths {
|
|||||||
dashboardCache: join(projectDir, 'cache', 'dashboard-data.json'),
|
dashboardCache: join(projectDir, 'cache', 'dashboard-data.json'),
|
||||||
config: join(projectDir, 'config'),
|
config: join(projectDir, 'config'),
|
||||||
cliConfig: join(projectDir, 'config', 'cli-config.json'),
|
cliConfig: join(projectDir, 'config', 'cli-config.json'),
|
||||||
|
memoryV2: {
|
||||||
|
root: join(projectDir, 'core-memory', 'v2'),
|
||||||
|
rolloutSummaries: join(projectDir, 'core-memory', 'v2', 'rollout_summaries'),
|
||||||
|
rawMemories: join(projectDir, 'core-memory', 'v2', 'raw_memories.md'),
|
||||||
|
memoryMd: join(projectDir, 'core-memory', 'v2', 'MEMORY.md'),
|
||||||
|
skills: join(projectDir, 'core-memory', 'v2', 'skills'),
|
||||||
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -682,4 +709,7 @@ export function initializeProjectStorage(projectPath: string): void {
|
|||||||
ensureStorageDir(paths.memory);
|
ensureStorageDir(paths.memory);
|
||||||
ensureStorageDir(paths.cache);
|
ensureStorageDir(paths.cache);
|
||||||
ensureStorageDir(paths.config);
|
ensureStorageDir(paths.config);
|
||||||
|
ensureStorageDir(paths.memoryV2.root);
|
||||||
|
ensureStorageDir(paths.memoryV2.rolloutSummaries);
|
||||||
|
ensureStorageDir(paths.memoryV2.skills);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -83,6 +83,17 @@ export interface ClaudeUpdateRecord {
|
|||||||
metadata?: string;
|
metadata?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Memory V2: Phase 1 extraction output row
|
||||||
|
*/
|
||||||
|
export interface Stage1Output {
|
||||||
|
thread_id: string;
|
||||||
|
source_updated_at: number;
|
||||||
|
raw_memory: string;
|
||||||
|
rollout_summary: string;
|
||||||
|
generated_at: number;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Core Memory Store using SQLite
|
* Core Memory Store using SQLite
|
||||||
*/
|
*/
|
||||||
@@ -215,6 +226,40 @@ export class CoreMemoryStore {
|
|||||||
CREATE INDEX IF NOT EXISTS idx_claude_history_path ON claude_update_history(file_path);
|
CREATE INDEX IF NOT EXISTS idx_claude_history_path ON claude_update_history(file_path);
|
||||||
CREATE INDEX IF NOT EXISTS idx_claude_history_updated ON claude_update_history(updated_at DESC);
|
CREATE INDEX IF NOT EXISTS idx_claude_history_updated ON claude_update_history(updated_at DESC);
|
||||||
CREATE INDEX IF NOT EXISTS idx_claude_history_module ON claude_update_history(module_path);
|
CREATE INDEX IF NOT EXISTS idx_claude_history_module ON claude_update_history(module_path);
|
||||||
|
|
||||||
|
-- Memory V2: Phase 1 extraction outputs
|
||||||
|
CREATE TABLE IF NOT EXISTS stage1_outputs (
|
||||||
|
thread_id TEXT PRIMARY KEY,
|
||||||
|
source_updated_at INTEGER NOT NULL,
|
||||||
|
raw_memory TEXT NOT NULL,
|
||||||
|
rollout_summary TEXT NOT NULL,
|
||||||
|
generated_at INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_stage1_generated ON stage1_outputs(generated_at DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_stage1_source_updated ON stage1_outputs(source_updated_at DESC);
|
||||||
|
|
||||||
|
-- Memory V2: Job scheduler
|
||||||
|
CREATE TABLE IF NOT EXISTS jobs (
|
||||||
|
kind TEXT NOT NULL,
|
||||||
|
job_key TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'running', 'done', 'error')),
|
||||||
|
worker_id TEXT,
|
||||||
|
ownership_token TEXT,
|
||||||
|
started_at INTEGER,
|
||||||
|
finished_at INTEGER,
|
||||||
|
lease_until INTEGER,
|
||||||
|
retry_at INTEGER,
|
||||||
|
retry_remaining INTEGER NOT NULL DEFAULT 3,
|
||||||
|
last_error TEXT,
|
||||||
|
input_watermark INTEGER DEFAULT 0,
|
||||||
|
last_success_watermark INTEGER DEFAULT 0,
|
||||||
|
PRIMARY KEY (kind, job_key)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_jobs_kind_status ON jobs(kind, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_jobs_lease ON jobs(lease_until);
|
||||||
`);
|
`);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -275,6 +320,14 @@ export class CoreMemoryStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the underlying database instance.
|
||||||
|
* Used by MemoryJobScheduler and other V2 components that share this DB.
|
||||||
|
*/
|
||||||
|
getDb(): Database.Database {
|
||||||
|
return this.db;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generate timestamp-based ID for core memory
|
* Generate timestamp-based ID for core memory
|
||||||
*/
|
*/
|
||||||
@@ -1255,6 +1308,88 @@ ${memory.content}
|
|||||||
return result.changes;
|
return result.changes;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Memory V2: Stage 1 Output CRUD Operations
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Upsert a Phase 1 extraction output (idempotent by thread_id)
|
||||||
|
*/
|
||||||
|
upsertStage1Output(output: Stage1Output): void {
|
||||||
|
const stmt = this.db.prepare(`
|
||||||
|
INSERT INTO stage1_outputs (thread_id, source_updated_at, raw_memory, rollout_summary, generated_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(thread_id) DO UPDATE SET
|
||||||
|
source_updated_at = excluded.source_updated_at,
|
||||||
|
raw_memory = excluded.raw_memory,
|
||||||
|
rollout_summary = excluded.rollout_summary,
|
||||||
|
generated_at = excluded.generated_at
|
||||||
|
`);
|
||||||
|
|
||||||
|
stmt.run(
|
||||||
|
output.thread_id,
|
||||||
|
output.source_updated_at,
|
||||||
|
output.raw_memory,
|
||||||
|
output.rollout_summary,
|
||||||
|
output.generated_at
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get a Phase 1 output by thread_id
|
||||||
|
*/
|
||||||
|
getStage1Output(threadId: string): Stage1Output | null {
|
||||||
|
const stmt = this.db.prepare(`SELECT * FROM stage1_outputs WHERE thread_id = ?`);
|
||||||
|
const row = stmt.get(threadId) as any;
|
||||||
|
if (!row) return null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
thread_id: row.thread_id,
|
||||||
|
source_updated_at: row.source_updated_at,
|
||||||
|
raw_memory: row.raw_memory,
|
||||||
|
rollout_summary: row.rollout_summary,
|
||||||
|
generated_at: row.generated_at,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* List all Phase 1 outputs, ordered by generated_at descending
|
||||||
|
*/
|
||||||
|
listStage1Outputs(limit?: number): Stage1Output[] {
|
||||||
|
const query = limit
|
||||||
|
? `SELECT * FROM stage1_outputs ORDER BY generated_at DESC LIMIT ?`
|
||||||
|
: `SELECT * FROM stage1_outputs ORDER BY generated_at DESC`;
|
||||||
|
|
||||||
|
const stmt = this.db.prepare(query);
|
||||||
|
const rows = (limit ? stmt.all(limit) : stmt.all()) as any[];
|
||||||
|
|
||||||
|
return rows.map(row => ({
|
||||||
|
thread_id: row.thread_id,
|
||||||
|
source_updated_at: row.source_updated_at,
|
||||||
|
raw_memory: row.raw_memory,
|
||||||
|
rollout_summary: row.rollout_summary,
|
||||||
|
generated_at: row.generated_at,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Count Phase 1 outputs
|
||||||
|
*/
|
||||||
|
countStage1Outputs(): number {
|
||||||
|
const stmt = this.db.prepare(`SELECT COUNT(*) as count FROM stage1_outputs`);
|
||||||
|
const row = stmt.get() as { count: number };
|
||||||
|
return row.count;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete a Phase 1 output by thread_id
|
||||||
|
*/
|
||||||
|
deleteStage1Output(threadId: string): boolean {
|
||||||
|
const stmt = this.db.prepare(`DELETE FROM stage1_outputs WHERE thread_id = ?`);
|
||||||
|
const result = stmt.run(threadId);
|
||||||
|
return result.changes > 0;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Close database connection
|
* Close database connection
|
||||||
*/
|
*/
|
||||||
|
|||||||
474
ccw/src/core/memory-consolidation-pipeline.ts
Normal file
474
ccw/src/core/memory-consolidation-pipeline.ts
Normal file
@@ -0,0 +1,474 @@
|
|||||||
|
/**
|
||||||
|
* Memory Consolidation Pipeline - Phase 2 Global Consolidation
|
||||||
|
*
|
||||||
|
* Orchestrates the global memory consolidation process:
|
||||||
|
* Lock -> Materialize -> Agent -> Monitor -> Done
|
||||||
|
*
|
||||||
|
* Phase 1 outputs (per-session extractions stored in stage1_outputs DB table)
|
||||||
|
* are materialized to disk as rollout_summaries/*.md + raw_memories.md,
|
||||||
|
* then a CLI agent (--mode write) reads those files and produces MEMORY.md.
|
||||||
|
*
|
||||||
|
* The pipeline uses lease-based locking via MemoryJobScheduler to ensure
|
||||||
|
* only one consolidation runs at a time, with heartbeat-based lease renewal.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { existsSync, readFileSync, readdirSync, unlinkSync, writeFileSync } from 'fs';
|
||||||
|
import { join } from 'path';
|
||||||
|
import { MemoryJobScheduler, type ClaimResult } from './memory-job-scheduler.js';
|
||||||
|
import { getCoreMemoryStore, type Stage1Output } from './core-memory-store.js';
|
||||||
|
import { getProjectPaths, ensureStorageDir } from '../config/storage-paths.js';
|
||||||
|
import {
|
||||||
|
HEARTBEAT_INTERVAL_SECONDS,
|
||||||
|
MAX_RAW_MEMORIES_FOR_GLOBAL,
|
||||||
|
} from './memory-v2-config.js';
|
||||||
|
import {
|
||||||
|
CONSOLIDATION_SYSTEM_PROMPT,
|
||||||
|
buildConsolidationPrompt,
|
||||||
|
} from './memory-consolidation-prompts.js';
|
||||||
|
|
||||||
|
// -- Types --
|
||||||
|
|
||||||
|
export interface ConsolidationStatus {
|
||||||
|
status: 'idle' | 'running' | 'completed' | 'error';
|
||||||
|
lastRun?: number;
|
||||||
|
memoryMdExists: boolean;
|
||||||
|
inputCount: number;
|
||||||
|
lastError?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MaterializationResult {
|
||||||
|
summariesWritten: number;
|
||||||
|
summariesPruned: number;
|
||||||
|
rawMemoriesSize: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- Constants --
|
||||||
|
|
||||||
|
const JOB_KIND = 'memory_consolidate_global';
|
||||||
|
const JOB_KEY = 'global';
|
||||||
|
const MAX_CONCURRENT = 1;
|
||||||
|
const AGENT_TIMEOUT_MS = 300_000; // 5 minutes
|
||||||
|
const DEFAULT_CLI_TOOL = 'gemini';
|
||||||
|
|
||||||
|
// -- Utility --
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sanitize a thread ID for use as a filename.
|
||||||
|
* Replaces filesystem-unsafe characters with underscores.
|
||||||
|
*/
|
||||||
|
function sanitizeFilename(name: string): string {
|
||||||
|
return name.replace(/[<>:"/\\|?*\x00-\x1f]/g, '_');
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- Standalone Functions --
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write one .md file per stage1_output to rollout_summaries/, prune orphans.
|
||||||
|
*
|
||||||
|
* Each file is named {sanitized_thread_id}.md and contains the rollout_summary.
|
||||||
|
* Files in the directory that do not correspond to any DB row are deleted.
|
||||||
|
*/
|
||||||
|
export function syncRolloutSummaries(
|
||||||
|
memoryHome: string,
|
||||||
|
outputs: Stage1Output[]
|
||||||
|
): MaterializationResult {
|
||||||
|
const summariesDir = join(memoryHome, 'rollout_summaries');
|
||||||
|
ensureStorageDir(summariesDir);
|
||||||
|
|
||||||
|
// Build set of expected filenames
|
||||||
|
const expectedFiles = new Set<string>();
|
||||||
|
let summariesWritten = 0;
|
||||||
|
|
||||||
|
for (const output of outputs) {
|
||||||
|
const filename = `${sanitizeFilename(output.thread_id)}.md`;
|
||||||
|
expectedFiles.add(filename);
|
||||||
|
const filePath = join(summariesDir, filename);
|
||||||
|
|
||||||
|
// Write summary content with thread header
|
||||||
|
const content = [
|
||||||
|
`# Session: ${output.thread_id}`,
|
||||||
|
`> Generated: ${new Date(output.generated_at * 1000).toISOString()}`,
|
||||||
|
`> Source updated: ${new Date(output.source_updated_at * 1000).toISOString()}`,
|
||||||
|
'',
|
||||||
|
output.rollout_summary,
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
writeFileSync(filePath, content, 'utf-8');
|
||||||
|
summariesWritten++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prune orphan files not in DB
|
||||||
|
let summariesPruned = 0;
|
||||||
|
const existingFiles = readdirSync(summariesDir);
|
||||||
|
for (const file of existingFiles) {
|
||||||
|
if (file.endsWith('.md') && !expectedFiles.has(file)) {
|
||||||
|
unlinkSync(join(summariesDir, file));
|
||||||
|
summariesPruned++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
summariesWritten,
|
||||||
|
summariesPruned,
|
||||||
|
rawMemoriesSize: 0, // Not applicable for this function
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Concatenate the latest raw_memory entries into raw_memories.md with thread headers.
|
||||||
|
*
|
||||||
|
* Entries are sorted by generated_at descending, limited to maxCount.
|
||||||
|
* Format per entry:
|
||||||
|
* ## Thread: {thread_id}
|
||||||
|
* {raw_memory content}
|
||||||
|
* ---
|
||||||
|
*
|
||||||
|
* @returns The byte size of the written raw_memories.md file.
|
||||||
|
*/
|
||||||
|
export function rebuildRawMemories(
|
||||||
|
memoryHome: string,
|
||||||
|
outputs: Stage1Output[],
|
||||||
|
maxCount: number
|
||||||
|
): number {
|
||||||
|
ensureStorageDir(memoryHome);
|
||||||
|
|
||||||
|
// Sort by generated_at descending, take up to maxCount
|
||||||
|
const sorted = [...outputs]
|
||||||
|
.sort((a, b) => b.generated_at - a.generated_at)
|
||||||
|
.slice(0, maxCount);
|
||||||
|
|
||||||
|
const sections: string[] = [];
|
||||||
|
for (const output of sorted) {
|
||||||
|
sections.push(
|
||||||
|
`## Thread: ${output.thread_id}`,
|
||||||
|
`> Generated: ${new Date(output.generated_at * 1000).toISOString()}`,
|
||||||
|
'',
|
||||||
|
output.raw_memory,
|
||||||
|
'',
|
||||||
|
'---',
|
||||||
|
'',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const content = sections.join('\n');
|
||||||
|
const filePath = join(memoryHome, 'raw_memories.md');
|
||||||
|
writeFileSync(filePath, content, 'utf-8');
|
||||||
|
|
||||||
|
return Buffer.byteLength(content, 'utf-8');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read MEMORY.md content for session prompt injection.
|
||||||
|
*
|
||||||
|
* @param projectPath - Project root path (used to resolve storage paths)
|
||||||
|
* @returns MEMORY.md content string, or null if the file does not exist
|
||||||
|
*/
|
||||||
|
export function getMemoryMdContent(projectPath: string): string | null {
|
||||||
|
const paths = getProjectPaths(projectPath);
|
||||||
|
const memoryMdPath = paths.memoryV2.memoryMd;
|
||||||
|
|
||||||
|
if (!existsSync(memoryMdPath)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return readFileSync(memoryMdPath, 'utf-8');
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- Pipeline Class --
|
||||||
|
|
||||||
|
/**
|
||||||
|
* MemoryConsolidationPipeline orchestrates global memory consolidation:
|
||||||
|
* 1. Claim global lock via job scheduler
|
||||||
|
* 2. Materialize Phase 1 outputs to disk (rollout_summaries/ + raw_memories.md)
|
||||||
|
* 3. Invoke consolidation agent via executeCliTool --mode write
|
||||||
|
* 4. Monitor with heartbeat lease renewal
|
||||||
|
* 5. Mark job as succeeded or failed
|
||||||
|
*/
|
||||||
|
export class MemoryConsolidationPipeline {
|
||||||
|
private projectPath: string;
|
||||||
|
private store: ReturnType<typeof getCoreMemoryStore>;
|
||||||
|
private scheduler: MemoryJobScheduler;
|
||||||
|
private memoryHome: string;
|
||||||
|
private cliTool: string;
|
||||||
|
|
||||||
|
constructor(projectPath: string, cliTool?: string) {
|
||||||
|
this.projectPath = projectPath;
|
||||||
|
this.store = getCoreMemoryStore(projectPath);
|
||||||
|
this.scheduler = new MemoryJobScheduler(this.store.getDb());
|
||||||
|
this.memoryHome = getProjectPaths(projectPath).memoryV2.root;
|
||||||
|
this.cliTool = cliTool || DEFAULT_CLI_TOOL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attempt to claim the global consolidation lock.
|
||||||
|
*
|
||||||
|
* Before claiming, ensures the job row exists and checks dirtiness.
|
||||||
|
* The job scheduler handles the actual concurrency control.
|
||||||
|
*/
|
||||||
|
claimGlobalLock(): ClaimResult {
|
||||||
|
return this.scheduler.claimJob(JOB_KIND, JOB_KEY, MAX_CONCURRENT);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Materialize rollout summaries from DB to disk.
|
||||||
|
* Writes one .md file per stage1_output row, prunes orphan files.
|
||||||
|
*/
|
||||||
|
materializeSummaries(): MaterializationResult {
|
||||||
|
const outputs = this.store.listStage1Outputs();
|
||||||
|
return syncRolloutSummaries(this.memoryHome, outputs);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Rebuild raw_memories.md from DB entries.
|
||||||
|
* Concatenates the latest entries up to MAX_RAW_MEMORIES_FOR_GLOBAL.
|
||||||
|
*/
|
||||||
|
materializeRawMemories(): number {
|
||||||
|
const outputs = this.store.listStage1Outputs();
|
||||||
|
return rebuildRawMemories(
|
||||||
|
this.memoryHome,
|
||||||
|
outputs,
|
||||||
|
MAX_RAW_MEMORIES_FOR_GLOBAL
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run the consolidation agent via executeCliTool with --mode write.
|
||||||
|
* Starts a heartbeat timer to keep the lease alive during agent execution.
|
||||||
|
*
|
||||||
|
* @param token - Ownership token from claimGlobalLock
|
||||||
|
* @returns true if agent completed successfully, false otherwise
|
||||||
|
*/
|
||||||
|
async runConsolidationAgent(token: string): Promise<boolean> {
|
||||||
|
// Lazy import to avoid circular dependencies at module load time
|
||||||
|
const { executeCliTool } = await import('../tools/cli-executor-core.js');
|
||||||
|
|
||||||
|
// Determine input state for prompt
|
||||||
|
const summariesDir = join(this.memoryHome, 'rollout_summaries');
|
||||||
|
let summaryCount = 0;
|
||||||
|
if (existsSync(summariesDir)) {
|
||||||
|
summaryCount = readdirSync(summariesDir).filter(f => f.endsWith('.md')).length;
|
||||||
|
}
|
||||||
|
const hasExistingMemoryMd = existsSync(join(this.memoryHome, 'MEMORY.md'));
|
||||||
|
|
||||||
|
// Ensure skills directory exists
|
||||||
|
ensureStorageDir(join(this.memoryHome, 'skills'));
|
||||||
|
|
||||||
|
// Build the full prompt
|
||||||
|
const userPrompt = buildConsolidationPrompt(summaryCount, hasExistingMemoryMd);
|
||||||
|
const fullPrompt = `${CONSOLIDATION_SYSTEM_PROMPT}\n\n${userPrompt}`;
|
||||||
|
|
||||||
|
// Start heartbeat timer
|
||||||
|
const heartbeatMs = HEARTBEAT_INTERVAL_SECONDS * 1000;
|
||||||
|
const heartbeatTimer = setInterval(() => {
|
||||||
|
const renewed = this.scheduler.heartbeat(JOB_KIND, JOB_KEY, token);
|
||||||
|
if (!renewed) {
|
||||||
|
// Heartbeat rejected - lease was lost
|
||||||
|
clearInterval(heartbeatTimer);
|
||||||
|
}
|
||||||
|
}, heartbeatMs);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Execute the consolidation agent
|
||||||
|
const result = await Promise.race([
|
||||||
|
executeCliTool({
|
||||||
|
tool: this.cliTool,
|
||||||
|
prompt: fullPrompt,
|
||||||
|
mode: 'write',
|
||||||
|
cd: this.memoryHome,
|
||||||
|
category: 'internal',
|
||||||
|
}),
|
||||||
|
new Promise<never>((_, reject) =>
|
||||||
|
setTimeout(
|
||||||
|
() => reject(new Error('Consolidation agent timed out')),
|
||||||
|
AGENT_TIMEOUT_MS
|
||||||
|
)
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
|
return result.success;
|
||||||
|
} finally {
|
||||||
|
clearInterval(heartbeatTimer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verify that Phase 1 artifacts were not modified by the agent.
|
||||||
|
* Compares file count in rollout_summaries/ before and after agent run.
|
||||||
|
*
|
||||||
|
* @param expectedSummaryCount - Number of summaries before agent run
|
||||||
|
* @returns true if artifacts are intact
|
||||||
|
*/
|
||||||
|
verifyPhase1Integrity(expectedSummaryCount: number): boolean {
|
||||||
|
const summariesDir = join(this.memoryHome, 'rollout_summaries');
|
||||||
|
if (!existsSync(summariesDir)) return expectedSummaryCount === 0;
|
||||||
|
|
||||||
|
const currentCount = readdirSync(summariesDir).filter(f => f.endsWith('.md')).length;
|
||||||
|
return currentCount === expectedSummaryCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the current consolidation status.
|
||||||
|
*/
|
||||||
|
getStatus(): ConsolidationStatus {
|
||||||
|
const jobStatus = this.scheduler.getJobStatus(JOB_KIND, JOB_KEY);
|
||||||
|
const inputCount = this.store.countStage1Outputs();
|
||||||
|
const memoryMdExists = existsSync(join(this.memoryHome, 'MEMORY.md'));
|
||||||
|
|
||||||
|
if (!jobStatus) {
|
||||||
|
return {
|
||||||
|
status: 'idle',
|
||||||
|
memoryMdExists,
|
||||||
|
inputCount,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const statusMap: Record<string, ConsolidationStatus['status']> = {
|
||||||
|
pending: 'idle',
|
||||||
|
running: 'running',
|
||||||
|
done: 'completed',
|
||||||
|
error: 'error',
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: statusMap[jobStatus.status] || 'idle',
|
||||||
|
lastRun: jobStatus.finished_at,
|
||||||
|
memoryMdExists,
|
||||||
|
inputCount,
|
||||||
|
lastError: jobStatus.last_error,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read MEMORY.md content for session prompt injection.
|
||||||
|
* Convenience method that delegates to the standalone function.
|
||||||
|
*/
|
||||||
|
getMemoryMdContent(): string | null {
|
||||||
|
return getMemoryMdContent(this.projectPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run the full consolidation pipeline.
|
||||||
|
*
|
||||||
|
* Pipeline flow:
|
||||||
|
* 1. Check if there are inputs to process
|
||||||
|
* 2. Claim global lock
|
||||||
|
* 3. Materialize Phase 1 outputs to disk
|
||||||
|
* 4. Run consolidation agent with heartbeat
|
||||||
|
* 5. Verify Phase 1 integrity
|
||||||
|
* 6. Mark job as succeeded or failed
|
||||||
|
*
|
||||||
|
* @returns Final consolidation status
|
||||||
|
*/
|
||||||
|
async runConsolidation(): Promise<ConsolidationStatus> {
|
||||||
|
// Step 1: Check inputs
|
||||||
|
const inputCount = this.store.countStage1Outputs();
|
||||||
|
if (inputCount === 0) {
|
||||||
|
return {
|
||||||
|
status: 'idle',
|
||||||
|
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
|
||||||
|
inputCount: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 2: Claim global lock
|
||||||
|
const claim = this.claimGlobalLock();
|
||||||
|
if (!claim.claimed || !claim.ownership_token) {
|
||||||
|
return {
|
||||||
|
status: claim.reason === 'already_running' ? 'running' : 'idle',
|
||||||
|
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
|
||||||
|
inputCount,
|
||||||
|
lastError: claim.reason
|
||||||
|
? `Lock not acquired: ${claim.reason}`
|
||||||
|
: undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const token = claim.ownership_token;
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Step 3: Materialize Phase 1 outputs to disk
|
||||||
|
const matResult = this.materializeSummaries();
|
||||||
|
const rawMemoriesSize = this.materializeRawMemories();
|
||||||
|
|
||||||
|
const expectedSummaryCount = matResult.summariesWritten;
|
||||||
|
|
||||||
|
// Step 4: Run consolidation agent with heartbeat
|
||||||
|
const agentSuccess = await this.runConsolidationAgent(token);
|
||||||
|
|
||||||
|
if (!agentSuccess) {
|
||||||
|
this.scheduler.markFailed(
|
||||||
|
JOB_KIND,
|
||||||
|
JOB_KEY,
|
||||||
|
token,
|
||||||
|
'Consolidation agent returned failure'
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
status: 'error',
|
||||||
|
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
|
||||||
|
inputCount,
|
||||||
|
lastError: 'Consolidation agent returned failure',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 5: Verify Phase 1 integrity
|
||||||
|
if (!this.verifyPhase1Integrity(expectedSummaryCount)) {
|
||||||
|
this.scheduler.markFailed(
|
||||||
|
JOB_KIND,
|
||||||
|
JOB_KEY,
|
||||||
|
token,
|
||||||
|
'Phase 1 artifacts were modified during consolidation'
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
status: 'error',
|
||||||
|
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
|
||||||
|
inputCount,
|
||||||
|
lastError: 'Phase 1 artifacts were modified during consolidation',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 6: Check that MEMORY.md was actually produced
|
||||||
|
const memoryMdExists = existsSync(join(this.memoryHome, 'MEMORY.md'));
|
||||||
|
if (!memoryMdExists) {
|
||||||
|
this.scheduler.markFailed(
|
||||||
|
JOB_KIND,
|
||||||
|
JOB_KEY,
|
||||||
|
token,
|
||||||
|
'Agent completed but MEMORY.md was not produced'
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
status: 'error',
|
||||||
|
memoryMdExists: false,
|
||||||
|
inputCount,
|
||||||
|
lastError: 'Agent completed but MEMORY.md was not produced',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 7: Mark success with watermark
|
||||||
|
// Use the current input count as the success watermark
|
||||||
|
const watermark = inputCount;
|
||||||
|
this.scheduler.markSucceeded(JOB_KIND, JOB_KEY, token, watermark);
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: 'completed',
|
||||||
|
lastRun: Math.floor(Date.now() / 1000),
|
||||||
|
memoryMdExists: true,
|
||||||
|
inputCount,
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
const errorMessage =
|
||||||
|
err instanceof Error ? err.message : String(err);
|
||||||
|
this.scheduler.markFailed(JOB_KIND, JOB_KEY, token, errorMessage);
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: 'error',
|
||||||
|
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
|
||||||
|
inputCount,
|
||||||
|
lastError: errorMessage,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
108
ccw/src/core/memory-consolidation-prompts.ts
Normal file
108
ccw/src/core/memory-consolidation-prompts.ts
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
/**
|
||||||
|
* Memory Consolidation Prompts - Phase 2 LLM Agent Prompt Templates
|
||||||
|
*
|
||||||
|
* System prompt and instruction templates for the consolidation agent that
|
||||||
|
* reads Phase 1 outputs (rollout_summaries/ + raw_memories.md) and produces
|
||||||
|
* MEMORY.md (and optional skills/ files).
|
||||||
|
*
|
||||||
|
* Design: The agent runs with --mode write in the MEMORY_HOME directory,
|
||||||
|
* using standard file read/write tools to produce output.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* System-level instructions for the consolidation agent.
|
||||||
|
* This is prepended to every consolidation prompt.
|
||||||
|
*/
|
||||||
|
export const CONSOLIDATION_SYSTEM_PROMPT = `You are a memory consolidation agent. Your job is to read phase-1 extraction artifacts and produce a consolidated MEMORY.md file that captures the most important, actionable knowledge from recent coding sessions.
|
||||||
|
|
||||||
|
## CRITICAL RULES
|
||||||
|
|
||||||
|
1. **Do NOT modify phase-1 artifacts.** The files in rollout_summaries/ and raw_memories.md are READ-ONLY inputs. Never edit, delete, or overwrite them.
|
||||||
|
2. **Output only to MEMORY.md** (and optionally skills/ directory). Do not create files outside these paths.
|
||||||
|
3. **Be concise and actionable.** Every line in MEMORY.md should help a future coding session be more productive.
|
||||||
|
4. **Resolve conflicts.** When rollout summaries and raw memories disagree, prefer the more recent or more specific information.
|
||||||
|
5. **Deduplicate.** Merge overlapping knowledge from different sessions into single authoritative entries.
|
||||||
|
6. **Preserve attribution.** When a piece of knowledge comes from a specific session thread, note it briefly.`;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the full consolidation prompt given the MEMORY_HOME directory path.
|
||||||
|
*
|
||||||
|
* The agent will be invoked with --cd pointing to memoryHome, so all file
|
||||||
|
* references are relative to that directory.
|
||||||
|
*
|
||||||
|
* @param inputSummaryCount - Number of rollout summary files available
|
||||||
|
* @param hasExistingMemoryMd - Whether a previous MEMORY.md exists to update
|
||||||
|
* @returns Complete prompt string for executeCliTool
|
||||||
|
*/
|
||||||
|
export function buildConsolidationPrompt(
|
||||||
|
inputSummaryCount: number,
|
||||||
|
hasExistingMemoryMd: boolean
|
||||||
|
): string {
|
||||||
|
const action = hasExistingMemoryMd ? 'UPDATE' : 'CREATE';
|
||||||
|
|
||||||
|
return `## Task: ${action} MEMORY.md
|
||||||
|
|
||||||
|
You have access to the following phase-1 artifacts in the current directory:
|
||||||
|
|
||||||
|
### Input Files (READ-ONLY - do NOT modify these)
|
||||||
|
- **rollout_summaries/*.md** - ${inputSummaryCount} per-session summary files. Each contains a concise summary of what was accomplished in that coding session. Use these for high-level routing and prioritization.
|
||||||
|
- **raw_memories.md** - Concatenated detailed memories from recent sessions, organized by thread. Use this for detailed knowledge extraction and cross-referencing.
|
||||||
|
${hasExistingMemoryMd ? '- **MEMORY.md** - The existing consolidated memory file. Update it with new knowledge while preserving still-relevant existing content.' : ''}
|
||||||
|
|
||||||
|
### Your Process
|
||||||
|
1. Read all files in rollout_summaries/ to understand the scope and themes of recent sessions.
|
||||||
|
2. Read raw_memories.md to extract detailed, actionable knowledge.
|
||||||
|
3. Cross-reference summaries with raw memories to identify:
|
||||||
|
- High-signal patterns and conventions discovered
|
||||||
|
- Architecture decisions and their rationale
|
||||||
|
- Common pitfalls and their solutions
|
||||||
|
- Key APIs, interfaces, and integration points
|
||||||
|
- Testing patterns and debugging approaches
|
||||||
|
${hasExistingMemoryMd ? '4. Read the existing MEMORY.md and merge new knowledge, removing stale entries.' : '4. Organize extracted knowledge into the output structure below.'}
|
||||||
|
5. Write the consolidated MEMORY.md file.
|
||||||
|
6. Optionally, if there are reusable code patterns or workflows worth extracting, create files in the skills/ directory.
|
||||||
|
|
||||||
|
### Output: MEMORY.md Structure
|
||||||
|
|
||||||
|
Write MEMORY.md with the following sections. Omit any section that has no relevant content.
|
||||||
|
|
||||||
|
\`\`\`markdown
|
||||||
|
# Project Memory
|
||||||
|
|
||||||
|
> Auto-generated by memory consolidation. Last updated: [current date]
|
||||||
|
> Sources: [number] session summaries, [number] raw memory entries
|
||||||
|
|
||||||
|
## Architecture & Structure
|
||||||
|
<!-- Key architectural decisions, module boundaries, data flow patterns -->
|
||||||
|
|
||||||
|
## Code Conventions
|
||||||
|
<!-- Naming conventions, import patterns, error handling approaches, formatting rules -->
|
||||||
|
|
||||||
|
## Common Patterns
|
||||||
|
<!-- Reusable patterns discovered across sessions: state management, API calls, testing approaches -->
|
||||||
|
|
||||||
|
## Key APIs & Interfaces
|
||||||
|
<!-- Important interfaces, function signatures, configuration schemas that are frequently referenced -->
|
||||||
|
|
||||||
|
## Known Issues & Gotchas
|
||||||
|
<!-- Pitfalls, edge cases, platform-specific behaviors, workarounds -->
|
||||||
|
|
||||||
|
## Recent Decisions
|
||||||
|
<!-- Decisions made in recent sessions with brief rationale. Remove when no longer relevant. -->
|
||||||
|
|
||||||
|
## Session Insights
|
||||||
|
<!-- High-value observations from individual sessions worth preserving -->
|
||||||
|
\`\`\`
|
||||||
|
|
||||||
|
### Output: skills/ Directory (Optional)
|
||||||
|
|
||||||
|
If you identify reusable code snippets, shell commands, or workflow templates that appear across multiple sessions, create files in the skills/ directory:
|
||||||
|
- skills/[name].md - Each file should be a self-contained, copy-paste ready reference
|
||||||
|
|
||||||
|
### Quality Criteria
|
||||||
|
- Every entry should be actionable (helps write better code or avoid mistakes)
|
||||||
|
- No vague platitudes - be specific with file paths, function names, config values
|
||||||
|
- Prefer concrete examples over abstract descriptions
|
||||||
|
- Keep total MEMORY.md under 5000 words - be ruthlessly concise
|
||||||
|
- Remove outdated information that contradicts newer findings`;
|
||||||
|
}
|
||||||
@@ -9,6 +9,7 @@
|
|||||||
* - JSON protocol communication
|
* - JSON protocol communication
|
||||||
* - Three commands: embed, search, status
|
* - Three commands: embed, search, status
|
||||||
* - Automatic availability checking
|
* - Automatic availability checking
|
||||||
|
* - Stage1 output embedding for V2 pipeline
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { spawn } from 'child_process';
|
import { spawn } from 'child_process';
|
||||||
@@ -16,6 +17,9 @@ import { join, dirname } from 'path';
|
|||||||
import { existsSync } from 'fs';
|
import { existsSync } from 'fs';
|
||||||
import { fileURLToPath } from 'url';
|
import { fileURLToPath } from 'url';
|
||||||
import { getCodexLensPython } from '../utils/codexlens-path.js';
|
import { getCodexLensPython } from '../utils/codexlens-path.js';
|
||||||
|
import { getCoreMemoryStore } from './core-memory-store.js';
|
||||||
|
import type { Stage1Output } from './core-memory-store.js';
|
||||||
|
import { StoragePaths } from '../config/storage-paths.js';
|
||||||
|
|
||||||
// Get directory of this module
|
// Get directory of this module
|
||||||
const __filename = fileURLToPath(import.meta.url);
|
const __filename = fileURLToPath(import.meta.url);
|
||||||
@@ -256,3 +260,111 @@ export async function getEmbeddingStatus(dbPath: string): Promise<EmbeddingStatu
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Memory V2: Stage1 Output Embedding
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/** Result of stage1 embedding operation */
|
||||||
|
export interface Stage1EmbedResult {
|
||||||
|
success: boolean;
|
||||||
|
chunksCreated: number;
|
||||||
|
chunksEmbedded: number;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Chunk and embed stage1_outputs (raw_memory + rollout_summary) for semantic search.
|
||||||
|
*
|
||||||
|
* Reads all stage1_outputs from the DB, chunks their raw_memory and rollout_summary
|
||||||
|
* content, inserts chunks into memory_chunks with source_type='cli_history' and
|
||||||
|
* metadata indicating the V2 origin, then triggers embedding generation.
|
||||||
|
*
|
||||||
|
* Uses source_id format: "s1:{thread_id}" to differentiate from regular cli_history chunks.
|
||||||
|
*
|
||||||
|
* @param projectPath - Project root path
|
||||||
|
* @param force - Force re-chunking even if chunks exist
|
||||||
|
* @returns Embedding result
|
||||||
|
*/
|
||||||
|
export async function embedStage1Outputs(
|
||||||
|
projectPath: string,
|
||||||
|
force: boolean = false
|
||||||
|
): Promise<Stage1EmbedResult> {
|
||||||
|
try {
|
||||||
|
const store = getCoreMemoryStore(projectPath);
|
||||||
|
const stage1Outputs = store.listStage1Outputs();
|
||||||
|
|
||||||
|
if (stage1Outputs.length === 0) {
|
||||||
|
return { success: true, chunksCreated: 0, chunksEmbedded: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
let totalChunksCreated = 0;
|
||||||
|
|
||||||
|
for (const output of stage1Outputs) {
|
||||||
|
const sourceId = `s1:${output.thread_id}`;
|
||||||
|
|
||||||
|
// Check if already chunked
|
||||||
|
const existingChunks = store.getChunks(sourceId);
|
||||||
|
if (existingChunks.length > 0 && !force) continue;
|
||||||
|
|
||||||
|
// Delete old chunks if force
|
||||||
|
if (force && existingChunks.length > 0) {
|
||||||
|
store.deleteChunks(sourceId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Combine raw_memory and rollout_summary for richer semantic content
|
||||||
|
const combinedContent = [
|
||||||
|
output.rollout_summary ? `## Summary\n${output.rollout_summary}` : '',
|
||||||
|
output.raw_memory ? `## Raw Memory\n${output.raw_memory}` : '',
|
||||||
|
].filter(Boolean).join('\n\n');
|
||||||
|
|
||||||
|
if (!combinedContent.trim()) continue;
|
||||||
|
|
||||||
|
// Chunk using the store's built-in chunking
|
||||||
|
const chunks = store.chunkContent(combinedContent, sourceId, 'cli_history');
|
||||||
|
|
||||||
|
// Insert chunks with V2 metadata
|
||||||
|
for (let i = 0; i < chunks.length; i++) {
|
||||||
|
store.insertChunk({
|
||||||
|
source_id: sourceId,
|
||||||
|
source_type: 'cli_history',
|
||||||
|
chunk_index: i,
|
||||||
|
content: chunks[i],
|
||||||
|
metadata: JSON.stringify({
|
||||||
|
v2_source: 'stage1_output',
|
||||||
|
thread_id: output.thread_id,
|
||||||
|
generated_at: output.generated_at,
|
||||||
|
}),
|
||||||
|
created_at: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
totalChunksCreated++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we created chunks, generate embeddings
|
||||||
|
let chunksEmbedded = 0;
|
||||||
|
if (totalChunksCreated > 0) {
|
||||||
|
const paths = StoragePaths.project(projectPath);
|
||||||
|
const dbPath = join(paths.root, 'core-memory', 'core_memory.db');
|
||||||
|
|
||||||
|
const embedResult = await generateEmbeddings(dbPath, { force: false });
|
||||||
|
if (embedResult.success) {
|
||||||
|
chunksEmbedded = embedResult.chunks_processed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
chunksCreated: totalChunksCreated,
|
||||||
|
chunksEmbedded,
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
chunksCreated: 0,
|
||||||
|
chunksEmbedded: 0,
|
||||||
|
error: (err as Error).message,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|||||||
466
ccw/src/core/memory-extraction-pipeline.ts
Normal file
466
ccw/src/core/memory-extraction-pipeline.ts
Normal file
@@ -0,0 +1,466 @@
|
|||||||
|
/**
|
||||||
|
* Memory Extraction Pipeline - Phase 1 per-session extraction
|
||||||
|
*
|
||||||
|
* Orchestrates the full extraction flow for each CLI session:
|
||||||
|
* Filter transcript -> Truncate -> LLM Extract -> SecretRedact -> PostProcess -> Store
|
||||||
|
*
|
||||||
|
* Uses CliHistoryStore for transcript access, executeCliTool for LLM invocation,
|
||||||
|
* CoreMemoryStore for stage1_outputs storage, and MemoryJobScheduler for
|
||||||
|
* concurrency control.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { ConversationRecord } from '../tools/cli-history-store.js';
|
||||||
|
import { getHistoryStore } from '../tools/cli-history-store.js';
|
||||||
|
import { getCoreMemoryStore, type Stage1Output } from './core-memory-store.js';
|
||||||
|
import { MemoryJobScheduler } from './memory-job-scheduler.js';
|
||||||
|
import {
|
||||||
|
MAX_SESSION_AGE_DAYS,
|
||||||
|
MIN_IDLE_HOURS,
|
||||||
|
MAX_ROLLOUT_BYTES_FOR_PROMPT,
|
||||||
|
MAX_RAW_MEMORY_CHARS,
|
||||||
|
MAX_SUMMARY_CHARS,
|
||||||
|
MAX_SESSIONS_PER_STARTUP,
|
||||||
|
PHASE_ONE_CONCURRENCY,
|
||||||
|
} from './memory-v2-config.js';
|
||||||
|
import { EXTRACTION_SYSTEM_PROMPT, buildExtractionUserPrompt } from './memory-extraction-prompts.js';
|
||||||
|
import { redactSecrets } from '../utils/secret-redactor.js';
|
||||||
|
|
||||||
|
// -- Types --
|
||||||
|
|
||||||
|
export interface ExtractionInput {
|
||||||
|
sessionId: string;
|
||||||
|
transcript: string;
|
||||||
|
sourceUpdatedAt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ExtractionOutput {
|
||||||
|
raw_memory: string;
|
||||||
|
rollout_summary: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TranscriptFilterOptions {
|
||||||
|
/** Bitmask for turn type selection. Default ALL = 0x7FF */
|
||||||
|
bitmask: number;
|
||||||
|
/** Maximum bytes for the transcript sent to LLM */
|
||||||
|
maxBytes: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface BatchExtractionResult {
|
||||||
|
processed: number;
|
||||||
|
succeeded: number;
|
||||||
|
failed: number;
|
||||||
|
skipped: number;
|
||||||
|
errors: Array<{ sessionId: string; error: string }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- Turn type bitmask constants --
|
||||||
|
|
||||||
|
/** All turn types included */
|
||||||
|
export const TURN_TYPE_ALL = 0x7FF;
|
||||||
|
|
||||||
|
// Individual turn type bits (for future filtering granularity)
|
||||||
|
export const TURN_TYPE_USER_PROMPT = 0x001;
|
||||||
|
export const TURN_TYPE_STDOUT = 0x002;
|
||||||
|
export const TURN_TYPE_STDERR = 0x004;
|
||||||
|
export const TURN_TYPE_PARSED = 0x008;
|
||||||
|
|
||||||
|
// -- Truncation marker --
|
||||||
|
|
||||||
|
const TRUNCATION_MARKER = '\n\n[... CONTENT TRUNCATED ...]\n\n';
|
||||||
|
|
||||||
|
// -- Job kind constant --
|
||||||
|
|
||||||
|
const JOB_KIND_EXTRACTION = 'phase1_extraction';
|
||||||
|
|
||||||
|
// -- Pipeline --
|
||||||
|
|
||||||
|
export class MemoryExtractionPipeline {
|
||||||
|
private projectPath: string;
|
||||||
|
/** Optional: override the LLM tool used for extraction. Defaults to 'gemini'. */
|
||||||
|
private tool: string;
|
||||||
|
/** Optional: current session ID to exclude from scanning */
|
||||||
|
private currentSessionId?: string;
|
||||||
|
|
||||||
|
constructor(projectPath: string, options?: { tool?: string; currentSessionId?: string }) {
|
||||||
|
this.projectPath = projectPath;
|
||||||
|
this.tool = options?.tool || 'gemini';
|
||||||
|
this.currentSessionId = options?.currentSessionId;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ========================================================================
|
||||||
|
// Eligibility scanning
|
||||||
|
// ========================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Scan CLI history for sessions eligible for memory extraction.
|
||||||
|
*
|
||||||
|
* Eligibility criteria (from design spec section 4.1):
|
||||||
|
* - Session age <= MAX_SESSION_AGE_DAYS (30 days)
|
||||||
|
* - Session idle >= MIN_IDLE_HOURS (12 hours) since last update
|
||||||
|
* - Not an ephemeral/internal session (category !== 'internal')
|
||||||
|
* - Not the currently active session
|
||||||
|
* - Has at least one turn with content
|
||||||
|
*
|
||||||
|
* @returns Array of eligible ConversationRecord objects, capped at MAX_SESSIONS_PER_STARTUP
|
||||||
|
*/
|
||||||
|
scanEligibleSessions(maxSessions?: number): ConversationRecord[] {
|
||||||
|
const historyStore = getHistoryStore(this.projectPath);
|
||||||
|
const now = Date.now();
|
||||||
|
const maxAgeMs = MAX_SESSION_AGE_DAYS * 24 * 60 * 60 * 1000;
|
||||||
|
const minIdleMs = MIN_IDLE_HOURS * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
// Fetch recent conversations (generous limit to filter in-memory)
|
||||||
|
const { executions } = historyStore.getHistory({ limit: 500 });
|
||||||
|
const eligible: ConversationRecord[] = [];
|
||||||
|
|
||||||
|
for (const entry of executions) {
|
||||||
|
// Skip current session
|
||||||
|
if (this.currentSessionId && entry.id === this.currentSessionId) continue;
|
||||||
|
|
||||||
|
// Age check: created within MAX_SESSION_AGE_DAYS
|
||||||
|
const createdAt = new Date(entry.timestamp).getTime();
|
||||||
|
if (now - createdAt > maxAgeMs) continue;
|
||||||
|
|
||||||
|
// Idle check: last updated at least MIN_IDLE_HOURS ago
|
||||||
|
const updatedAt = new Date(entry.updated_at || entry.timestamp).getTime();
|
||||||
|
if (now - updatedAt < minIdleMs) continue;
|
||||||
|
|
||||||
|
// Skip internal/ephemeral sessions
|
||||||
|
if (entry.category === 'internal') continue;
|
||||||
|
|
||||||
|
// Must have at least 1 turn
|
||||||
|
if (!entry.turn_count || entry.turn_count < 1) continue;
|
||||||
|
|
||||||
|
// Load full conversation to include in result
|
||||||
|
const conv = historyStore.getConversation(entry.id);
|
||||||
|
if (!conv) continue;
|
||||||
|
|
||||||
|
eligible.push(conv);
|
||||||
|
|
||||||
|
if (eligible.length >= (maxSessions || MAX_SESSIONS_PER_STARTUP)) break;
|
||||||
|
}
|
||||||
|
|
||||||
|
return eligible;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ========================================================================
|
||||||
|
// Transcript filtering
|
||||||
|
// ========================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract transcript text from a ConversationRecord, keeping only turn types
|
||||||
|
* that match the given bitmask.
|
||||||
|
*
|
||||||
|
* Default bitmask (ALL=0x7FF) includes all turn content: prompt, stdout, stderr, parsed.
|
||||||
|
*
|
||||||
|
* @param record - The conversation record to filter
|
||||||
|
* @param bitmask - Bitmask for type selection (default: TURN_TYPE_ALL)
|
||||||
|
* @returns Combined transcript text
|
||||||
|
*/
|
||||||
|
filterTranscript(record: ConversationRecord, bitmask: number = TURN_TYPE_ALL): string {
|
||||||
|
const parts: string[] = [];
|
||||||
|
|
||||||
|
for (const turn of record.turns) {
|
||||||
|
const turnParts: string[] = [];
|
||||||
|
|
||||||
|
if (bitmask & TURN_TYPE_USER_PROMPT) {
|
||||||
|
if (turn.prompt) {
|
||||||
|
turnParts.push(`[USER] ${turn.prompt}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bitmask & TURN_TYPE_STDOUT) {
|
||||||
|
const stdout = turn.output?.parsed_output || turn.output?.stdout;
|
||||||
|
if (stdout) {
|
||||||
|
turnParts.push(`[ASSISTANT] ${stdout}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bitmask & TURN_TYPE_STDERR) {
|
||||||
|
if (turn.output?.stderr) {
|
||||||
|
turnParts.push(`[STDERR] ${turn.output.stderr}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bitmask & TURN_TYPE_PARSED) {
|
||||||
|
// Use final_output if available and not already captured
|
||||||
|
if (turn.output?.final_output && !(bitmask & TURN_TYPE_STDOUT)) {
|
||||||
|
turnParts.push(`[FINAL] ${turn.output.final_output}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (turnParts.length > 0) {
|
||||||
|
parts.push(`--- Turn ${turn.turn} ---\n${turnParts.join('\n')}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return parts.join('\n\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
// ========================================================================
|
||||||
|
// Truncation
|
||||||
|
// ========================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Truncate transcript content to fit within LLM context limit.
|
||||||
|
*
|
||||||
|
* Strategy: Keep head 33% + truncation marker + tail 67%.
|
||||||
|
* This preserves the session opening context and the most recent work.
|
||||||
|
*
|
||||||
|
* @param content - The full transcript text
|
||||||
|
* @param maxBytes - Maximum allowed size in bytes (default: MAX_ROLLOUT_BYTES_FOR_PROMPT)
|
||||||
|
* @returns Truncated content, or original if within limit
|
||||||
|
*/
|
||||||
|
truncateTranscript(content: string, maxBytes: number = MAX_ROLLOUT_BYTES_FOR_PROMPT): string {
|
||||||
|
const contentBytes = Buffer.byteLength(content, 'utf-8');
|
||||||
|
if (contentBytes <= maxBytes) {
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate split sizes accounting for the marker
|
||||||
|
const markerBytes = Buffer.byteLength(TRUNCATION_MARKER, 'utf-8');
|
||||||
|
const availableBytes = maxBytes - markerBytes;
|
||||||
|
const headBytes = Math.floor(availableBytes * 0.33);
|
||||||
|
const tailBytes = availableBytes - headBytes;
|
||||||
|
|
||||||
|
// Convert to character-based approximation (safe for multi-byte)
|
||||||
|
// Use Buffer slicing for byte-accurate truncation
|
||||||
|
const buf = Buffer.from(content, 'utf-8');
|
||||||
|
const headBuf = buf.subarray(0, headBytes);
|
||||||
|
const tailBuf = buf.subarray(buf.length - tailBytes);
|
||||||
|
|
||||||
|
// Decode back to strings, trimming at character boundaries
|
||||||
|
const head = headBuf.toString('utf-8').replace(/[\uFFFD]$/, '');
|
||||||
|
const tail = tailBuf.toString('utf-8').replace(/^[\uFFFD]/, '');
|
||||||
|
|
||||||
|
return head + TRUNCATION_MARKER + tail;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ========================================================================
|
||||||
|
// LLM extraction
|
||||||
|
// ========================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Call the LLM to extract structured memory from a session transcript.
|
||||||
|
*
|
||||||
|
* Uses executeCliTool with the extraction prompts. The LLM is expected
|
||||||
|
* to return a JSON object with raw_memory and rollout_summary fields.
|
||||||
|
*
|
||||||
|
* @param sessionId - Session ID for prompt context
|
||||||
|
* @param transcript - The filtered and truncated transcript
|
||||||
|
* @returns Raw LLM output string
|
||||||
|
*/
|
||||||
|
async extractMemory(sessionId: string, transcript: string): Promise<string> {
|
||||||
|
const { executeCliTool } = await import('../tools/cli-executor-core.js');
|
||||||
|
|
||||||
|
const userPrompt = buildExtractionUserPrompt(sessionId, transcript);
|
||||||
|
|
||||||
|
const fullPrompt = `${EXTRACTION_SYSTEM_PROMPT}\n\n${userPrompt}`;
|
||||||
|
|
||||||
|
const result = await executeCliTool({
|
||||||
|
tool: this.tool,
|
||||||
|
prompt: fullPrompt,
|
||||||
|
mode: 'analysis',
|
||||||
|
cd: this.projectPath,
|
||||||
|
category: 'internal',
|
||||||
|
});
|
||||||
|
|
||||||
|
// Prefer parsedOutput (extracted text from stream JSON) over raw stdout
|
||||||
|
const output = result.parsedOutput?.trim() || result.stdout?.trim() || '';
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ========================================================================
|
||||||
|
// Post-processing
|
||||||
|
// ========================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse LLM output into structured ExtractionOutput.
|
||||||
|
*
|
||||||
|
* Supports 3 parsing modes:
|
||||||
|
* 1. Pure JSON: Output is a valid JSON object
|
||||||
|
* 2. Fenced JSON block: JSON wrapped in ```json ... ``` markers
|
||||||
|
* 3. Text extraction: Non-conforming output wrapped in fallback structure
|
||||||
|
*
|
||||||
|
* Applies secret redaction and size limit enforcement.
|
||||||
|
*
|
||||||
|
* @param llmOutput - Raw text output from the LLM
|
||||||
|
* @returns Validated ExtractionOutput with raw_memory and rollout_summary
|
||||||
|
*/
|
||||||
|
postProcess(llmOutput: string): ExtractionOutput {
|
||||||
|
let parsed: { raw_memory?: string; rollout_summary?: string } | null = null;
|
||||||
|
|
||||||
|
// Mode 1: Pure JSON
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(llmOutput);
|
||||||
|
} catch {
|
||||||
|
// Not pure JSON, try next mode
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mode 2: Fenced JSON block
|
||||||
|
if (!parsed) {
|
||||||
|
const fencedMatch = llmOutput.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/);
|
||||||
|
if (fencedMatch) {
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(fencedMatch[1]);
|
||||||
|
} catch {
|
||||||
|
// Fenced content is not valid JSON either
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mode 3: Text extraction fallback
|
||||||
|
if (!parsed || typeof parsed.raw_memory !== 'string') {
|
||||||
|
parsed = {
|
||||||
|
raw_memory: `# summary\n${llmOutput}\n\nMemory context:\n- Extracted from unstructured LLM output\n\nUser preferences:\n- (none detected)`,
|
||||||
|
rollout_summary: llmOutput.substring(0, 200).replace(/\n/g, ' ').trim(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply secret redaction
|
||||||
|
let rawMemory = redactSecrets(parsed.raw_memory || '');
|
||||||
|
let rolloutSummary = redactSecrets(parsed.rollout_summary || '');
|
||||||
|
|
||||||
|
// Enforce size limits
|
||||||
|
if (rawMemory.length > MAX_RAW_MEMORY_CHARS) {
|
||||||
|
rawMemory = rawMemory.substring(0, MAX_RAW_MEMORY_CHARS);
|
||||||
|
}
|
||||||
|
if (rolloutSummary.length > MAX_SUMMARY_CHARS) {
|
||||||
|
rolloutSummary = rolloutSummary.substring(0, MAX_SUMMARY_CHARS);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { raw_memory: rawMemory, rollout_summary: rolloutSummary };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ========================================================================
|
||||||
|
// Single session extraction
|
||||||
|
// ========================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run the full extraction pipeline for a single session.
|
||||||
|
*
|
||||||
|
* Pipeline stages: Filter -> Truncate -> LLM Extract -> PostProcess -> Store
|
||||||
|
*
|
||||||
|
* @param sessionId - The session to extract from
|
||||||
|
* @returns The stored Stage1Output, or null if extraction failed
|
||||||
|
*/
|
||||||
|
async runExtractionJob(sessionId: string): Promise<Stage1Output | null> {
|
||||||
|
const historyStore = getHistoryStore(this.projectPath);
|
||||||
|
const record = historyStore.getConversation(sessionId);
|
||||||
|
if (!record) {
|
||||||
|
throw new Error(`Session not found: ${sessionId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stage 1: Filter transcript
|
||||||
|
const transcript = this.filterTranscript(record);
|
||||||
|
if (!transcript.trim()) {
|
||||||
|
return null; // Empty transcript, nothing to extract
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stage 2: Truncate
|
||||||
|
const truncated = this.truncateTranscript(transcript);
|
||||||
|
|
||||||
|
// Stage 3: LLM extraction
|
||||||
|
const llmOutput = await this.extractMemory(sessionId, truncated);
|
||||||
|
if (!llmOutput) {
|
||||||
|
throw new Error(`LLM returned empty output for session: ${sessionId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stage 4: Post-process (parse + redact + validate)
|
||||||
|
const extracted = this.postProcess(llmOutput);
|
||||||
|
|
||||||
|
// Stage 5: Store result
|
||||||
|
const sourceUpdatedAt = Math.floor(new Date(record.updated_at).getTime() / 1000);
|
||||||
|
const generatedAt = Math.floor(Date.now() / 1000);
|
||||||
|
|
||||||
|
const output: Stage1Output = {
|
||||||
|
thread_id: sessionId,
|
||||||
|
source_updated_at: sourceUpdatedAt,
|
||||||
|
raw_memory: extracted.raw_memory,
|
||||||
|
rollout_summary: extracted.rollout_summary,
|
||||||
|
generated_at: generatedAt,
|
||||||
|
};
|
||||||
|
|
||||||
|
const store = getCoreMemoryStore(this.projectPath);
|
||||||
|
store.upsertStage1Output(output);
|
||||||
|
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ========================================================================
|
||||||
|
// Batch orchestration
|
||||||
|
// ========================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run extraction for all eligible sessions with concurrency control.
|
||||||
|
*
|
||||||
|
* Uses MemoryJobScheduler to claim jobs and enforce PHASE_ONE_CONCURRENCY.
|
||||||
|
* Failed extractions are recorded in the scheduler for retry.
|
||||||
|
*
|
||||||
|
* @returns BatchExtractionResult with counts and error details
|
||||||
|
*/
|
||||||
|
async runBatchExtraction(options?: { maxSessions?: number }): Promise<BatchExtractionResult> {
|
||||||
|
const store = getCoreMemoryStore(this.projectPath);
|
||||||
|
const scheduler = new MemoryJobScheduler(store.getDb());
|
||||||
|
|
||||||
|
// Scan eligible sessions
|
||||||
|
const eligibleSessions = this.scanEligibleSessions(options?.maxSessions);
|
||||||
|
|
||||||
|
const result: BatchExtractionResult = {
|
||||||
|
processed: 0,
|
||||||
|
succeeded: 0,
|
||||||
|
failed: 0,
|
||||||
|
skipped: 0,
|
||||||
|
errors: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
if (eligibleSessions.length === 0) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enqueue all eligible sessions
|
||||||
|
for (const session of eligibleSessions) {
|
||||||
|
const watermark = Math.floor(new Date(session.updated_at).getTime() / 1000);
|
||||||
|
scheduler.enqueueJob(JOB_KIND_EXTRACTION, session.id, watermark);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process with concurrency control using Promise.all with batching
|
||||||
|
const batchSize = PHASE_ONE_CONCURRENCY;
|
||||||
|
for (let i = 0; i < eligibleSessions.length; i += batchSize) {
|
||||||
|
const batch = eligibleSessions.slice(i, i + batchSize);
|
||||||
|
const promises = batch.map(async (session) => {
|
||||||
|
// Try to claim the job
|
||||||
|
const claim = scheduler.claimJob(JOB_KIND_EXTRACTION, session.id, batchSize);
|
||||||
|
if (!claim.claimed) {
|
||||||
|
result.skipped++;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
result.processed++;
|
||||||
|
const token = claim.ownership_token!;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const output = await this.runExtractionJob(session.id);
|
||||||
|
if (output) {
|
||||||
|
const watermark = output.source_updated_at;
|
||||||
|
scheduler.markSucceeded(JOB_KIND_EXTRACTION, session.id, token, watermark);
|
||||||
|
result.succeeded++;
|
||||||
|
} else {
|
||||||
|
// Empty transcript - mark as done (nothing to extract)
|
||||||
|
scheduler.markSucceeded(JOB_KIND_EXTRACTION, session.id, token, 0);
|
||||||
|
result.skipped++;
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
||||||
|
scheduler.markFailed(JOB_KIND_EXTRACTION, session.id, token, errorMsg);
|
||||||
|
result.failed++;
|
||||||
|
result.errors.push({ sessionId: session.id, error: errorMsg });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
await Promise.all(promises);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
91
ccw/src/core/memory-extraction-prompts.ts
Normal file
91
ccw/src/core/memory-extraction-prompts.ts
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
/**
|
||||||
|
* Memory Extraction Prompts - LLM prompt templates for Phase 1 extraction
|
||||||
|
*
|
||||||
|
* Provides system and user prompt templates for extracting structured memory
|
||||||
|
* from CLI session transcripts. The LLM output must conform to a JSON schema
|
||||||
|
* with raw_memory and rollout_summary fields.
|
||||||
|
*
|
||||||
|
* Design spec section 4.4: Prompt structure with outcome triage rules.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* System prompt for the extraction LLM call.
|
||||||
|
*
|
||||||
|
* Instructs the model to:
|
||||||
|
* - Produce a JSON object with raw_memory and rollout_summary
|
||||||
|
* - Follow structure markers in raw_memory (# summary, Memory context, etc.)
|
||||||
|
* - Apply outcome triage rules for categorizing task results
|
||||||
|
* - Keep rollout_summary concise (1-2 sentences)
|
||||||
|
*/
|
||||||
|
export const EXTRACTION_SYSTEM_PROMPT = `You are a memory extraction agent. Your job is to read a CLI session transcript and produce structured memory output.
|
||||||
|
|
||||||
|
You MUST respond with a valid JSON object containing exactly two fields:
|
||||||
|
|
||||||
|
{
|
||||||
|
"raw_memory": "<structured memory text>",
|
||||||
|
"rollout_summary": "<1-2 sentence summary>"
|
||||||
|
}
|
||||||
|
|
||||||
|
## raw_memory format
|
||||||
|
|
||||||
|
The raw_memory field must follow this structure:
|
||||||
|
|
||||||
|
# summary
|
||||||
|
<One paragraph high-level summary of what was accomplished in this session>
|
||||||
|
|
||||||
|
Memory context:
|
||||||
|
- Project: <project name or path if identifiable>
|
||||||
|
- Tools used: <CLI tools, frameworks, languages mentioned>
|
||||||
|
- Key files: <important files created or modified>
|
||||||
|
|
||||||
|
User preferences:
|
||||||
|
- <Any coding style preferences, conventions, or patterns the user demonstrated>
|
||||||
|
- <Tool preferences, workflow habits>
|
||||||
|
|
||||||
|
## Task: <task title or description>
|
||||||
|
Outcome: <success | partial | failed | abandoned>
|
||||||
|
<Detailed description of what was done, decisions made, and results>
|
||||||
|
|
||||||
|
### Key decisions
|
||||||
|
- <Important architectural or design decisions>
|
||||||
|
- <Trade-offs considered>
|
||||||
|
|
||||||
|
### Lessons learned
|
||||||
|
- <What worked well>
|
||||||
|
- <What did not work and why>
|
||||||
|
- <Gotchas or pitfalls discovered>
|
||||||
|
|
||||||
|
## Outcome Triage Rules
|
||||||
|
|
||||||
|
- **success**: Task was completed as intended, tests pass, code works
|
||||||
|
- **partial**: Some progress made but not fully complete; note what remains
|
||||||
|
- **failed**: Attempted but could not achieve the goal; document root cause
|
||||||
|
- **abandoned**: User switched direction or cancelled; note the reason
|
||||||
|
|
||||||
|
## rollout_summary format
|
||||||
|
|
||||||
|
A concise 1-2 sentence summary capturing:
|
||||||
|
- What the session was about (the goal)
|
||||||
|
- The outcome (success/partial/failed)
|
||||||
|
- The most important takeaway
|
||||||
|
|
||||||
|
Do NOT include markdown code fences in your response. Return raw JSON only.`;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the user prompt by injecting the session transcript.
|
||||||
|
*
|
||||||
|
* @param sessionId - The session/conversation ID for reference
|
||||||
|
* @param transcript - The filtered and truncated transcript text
|
||||||
|
* @returns The complete user prompt string
|
||||||
|
*/
|
||||||
|
export function buildExtractionUserPrompt(sessionId: string, transcript: string): string {
|
||||||
|
return `Extract structured memory from the following CLI session transcript.
|
||||||
|
|
||||||
|
Session ID: ${sessionId}
|
||||||
|
|
||||||
|
--- BEGIN TRANSCRIPT ---
|
||||||
|
${transcript}
|
||||||
|
--- END TRANSCRIPT ---
|
||||||
|
|
||||||
|
Respond with a JSON object containing "raw_memory" and "rollout_summary" fields.`;
|
||||||
|
}
|
||||||
335
ccw/src/core/memory-job-scheduler.ts
Normal file
335
ccw/src/core/memory-job-scheduler.ts
Normal file
@@ -0,0 +1,335 @@
|
|||||||
|
/**
|
||||||
|
* Memory Job Scheduler - Lease-based job scheduling backed by SQLite
|
||||||
|
*
|
||||||
|
* Provides atomic claim/release/heartbeat operations for coordinating
|
||||||
|
* concurrent memory extraction and consolidation jobs.
|
||||||
|
*
|
||||||
|
* All state lives in the `jobs` table of the CoreMemoryStore database.
|
||||||
|
* Concurrency control uses ownership_token + lease_until for distributed-safe
|
||||||
|
* (but single-process) job dispatch.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type Database from 'better-sqlite3';
|
||||||
|
import { randomUUID } from 'crypto';
|
||||||
|
import { LEASE_SECONDS, MAX_RETRIES, RETRY_DELAY_SECONDS } from './memory-v2-config.js';
|
||||||
|
|
||||||
|
// -- Types --
|
||||||
|
|
||||||
|
export type JobStatus = 'pending' | 'running' | 'done' | 'error';
|
||||||
|
|
||||||
|
export interface JobRecord {
|
||||||
|
kind: string;
|
||||||
|
job_key: string;
|
||||||
|
status: JobStatus;
|
||||||
|
worker_id?: string;
|
||||||
|
ownership_token?: string;
|
||||||
|
started_at?: number;
|
||||||
|
finished_at?: number;
|
||||||
|
lease_until?: number;
|
||||||
|
retry_at?: number;
|
||||||
|
retry_remaining: number;
|
||||||
|
last_error?: string;
|
||||||
|
input_watermark: number;
|
||||||
|
last_success_watermark: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ClaimResult {
|
||||||
|
claimed: boolean;
|
||||||
|
ownership_token?: string;
|
||||||
|
reason?: 'already_running' | 'retry_exhausted' | 'retry_pending' | 'concurrency_limit';
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- Scheduler --
|
||||||
|
|
||||||
|
export class MemoryJobScheduler {
|
||||||
|
private db: Database.Database;
|
||||||
|
|
||||||
|
constructor(db: Database.Database) {
|
||||||
|
this.db = db;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Atomically claim a job for processing.
|
||||||
|
*
|
||||||
|
* Logic:
|
||||||
|
* 1. If job does not exist, insert it as 'running' with a fresh token.
|
||||||
|
* 2. If job exists and is 'pending', transition to 'running'.
|
||||||
|
* 3. If job exists and is 'running' but lease expired, reclaim it.
|
||||||
|
* 4. If job exists and is 'error' with retry_remaining > 0 and retry_at <= now, reclaim it.
|
||||||
|
* 5. Otherwise, return not claimed with reason.
|
||||||
|
*
|
||||||
|
* Respects maxConcurrent: total running jobs of this `kind` must not exceed limit.
|
||||||
|
*/
|
||||||
|
claimJob(kind: string, jobKey: string, maxConcurrent: number, workerId?: string): ClaimResult {
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
const token = randomUUID();
|
||||||
|
const leaseUntil = now + LEASE_SECONDS;
|
||||||
|
|
||||||
|
// Use a transaction for atomicity
|
||||||
|
const result = this.db.transaction(() => {
|
||||||
|
// Check concurrency limit for this kind
|
||||||
|
const runningCount = this.db.prepare(
|
||||||
|
`SELECT COUNT(*) as cnt FROM jobs WHERE kind = ? AND status = 'running' AND lease_until > ?`
|
||||||
|
).get(kind, now) as { cnt: number };
|
||||||
|
|
||||||
|
const existing = this.db.prepare(
|
||||||
|
`SELECT * FROM jobs WHERE kind = ? AND job_key = ?`
|
||||||
|
).get(kind, jobKey) as any | undefined;
|
||||||
|
|
||||||
|
if (!existing) {
|
||||||
|
// No job row yet - check concurrency before inserting
|
||||||
|
if (runningCount.cnt >= maxConcurrent) {
|
||||||
|
return { claimed: false, reason: 'concurrency_limit' as const };
|
||||||
|
}
|
||||||
|
|
||||||
|
this.db.prepare(`
|
||||||
|
INSERT INTO jobs (kind, job_key, status, worker_id, ownership_token, started_at, lease_until, retry_remaining, input_watermark, last_success_watermark)
|
||||||
|
VALUES (?, ?, 'running', ?, ?, ?, ?, ?, 0, 0)
|
||||||
|
`).run(kind, jobKey, workerId || null, token, now, leaseUntil, MAX_RETRIES);
|
||||||
|
|
||||||
|
return { claimed: true, ownership_token: token };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Job exists - check status transitions
|
||||||
|
if (existing.status === 'done') {
|
||||||
|
// Already done - check dirty (input_watermark > last_success_watermark)
|
||||||
|
if (existing.input_watermark <= existing.last_success_watermark) {
|
||||||
|
return { claimed: false, reason: 'already_running' as const };
|
||||||
|
}
|
||||||
|
// Dirty - re-run
|
||||||
|
if (runningCount.cnt >= maxConcurrent) {
|
||||||
|
return { claimed: false, reason: 'concurrency_limit' as const };
|
||||||
|
}
|
||||||
|
this.db.prepare(`
|
||||||
|
UPDATE jobs SET status = 'running', worker_id = ?, ownership_token = ?,
|
||||||
|
started_at = ?, lease_until = ?, finished_at = NULL, last_error = NULL,
|
||||||
|
retry_remaining = ?
|
||||||
|
WHERE kind = ? AND job_key = ?
|
||||||
|
`).run(workerId || null, token, now, leaseUntil, MAX_RETRIES, kind, jobKey);
|
||||||
|
return { claimed: true, ownership_token: token };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (existing.status === 'running') {
|
||||||
|
// Running - check lease expiry
|
||||||
|
if (existing.lease_until > now) {
|
||||||
|
return { claimed: false, reason: 'already_running' as const };
|
||||||
|
}
|
||||||
|
// Lease expired - reclaim if concurrency allows
|
||||||
|
// The expired job doesn't count towards running total (lease_until <= now),
|
||||||
|
// so runningCount already excludes it.
|
||||||
|
if (runningCount.cnt >= maxConcurrent) {
|
||||||
|
return { claimed: false, reason: 'concurrency_limit' as const };
|
||||||
|
}
|
||||||
|
this.db.prepare(`
|
||||||
|
UPDATE jobs SET worker_id = ?, ownership_token = ?, started_at = ?,
|
||||||
|
lease_until = ?, last_error = NULL
|
||||||
|
WHERE kind = ? AND job_key = ?
|
||||||
|
`).run(workerId || null, token, now, leaseUntil, kind, jobKey);
|
||||||
|
return { claimed: true, ownership_token: token };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (existing.status === 'pending') {
|
||||||
|
if (runningCount.cnt >= maxConcurrent) {
|
||||||
|
return { claimed: false, reason: 'concurrency_limit' as const };
|
||||||
|
}
|
||||||
|
this.db.prepare(`
|
||||||
|
UPDATE jobs SET status = 'running', worker_id = ?, ownership_token = ?,
|
||||||
|
started_at = ?, lease_until = ?
|
||||||
|
WHERE kind = ? AND job_key = ?
|
||||||
|
`).run(workerId || null, token, now, leaseUntil, kind, jobKey);
|
||||||
|
return { claimed: true, ownership_token: token };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (existing.status === 'error') {
|
||||||
|
if (existing.retry_remaining <= 0) {
|
||||||
|
return { claimed: false, reason: 'retry_exhausted' as const };
|
||||||
|
}
|
||||||
|
if (existing.retry_at && existing.retry_at > now) {
|
||||||
|
return { claimed: false, reason: 'retry_pending' as const };
|
||||||
|
}
|
||||||
|
if (runningCount.cnt >= maxConcurrent) {
|
||||||
|
return { claimed: false, reason: 'concurrency_limit' as const };
|
||||||
|
}
|
||||||
|
this.db.prepare(`
|
||||||
|
UPDATE jobs SET status = 'running', worker_id = ?, ownership_token = ?,
|
||||||
|
started_at = ?, lease_until = ?, last_error = NULL,
|
||||||
|
retry_remaining = retry_remaining - 1
|
||||||
|
WHERE kind = ? AND job_key = ?
|
||||||
|
`).run(workerId || null, token, now, leaseUntil, kind, jobKey);
|
||||||
|
return { claimed: true, ownership_token: token };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { claimed: false, reason: 'already_running' as const };
|
||||||
|
})();
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Release a job, marking it as done or error.
|
||||||
|
* Only succeeds if the ownership_token matches.
|
||||||
|
*/
|
||||||
|
releaseJob(kind: string, jobKey: string, token: string, status: 'done' | 'error', error?: string): boolean {
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
|
||||||
|
const result = this.db.prepare(`
|
||||||
|
UPDATE jobs SET
|
||||||
|
status = ?,
|
||||||
|
finished_at = ?,
|
||||||
|
lease_until = NULL,
|
||||||
|
ownership_token = NULL,
|
||||||
|
worker_id = NULL,
|
||||||
|
last_error = ?
|
||||||
|
WHERE kind = ? AND job_key = ? AND ownership_token = ?
|
||||||
|
`).run(status, now, error || null, kind, jobKey, token);
|
||||||
|
|
||||||
|
return result.changes > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Renew the lease for an active job.
|
||||||
|
* Returns false if ownership_token does not match or job is not running.
|
||||||
|
*/
|
||||||
|
heartbeat(kind: string, jobKey: string, token: string, leaseSeconds: number = LEASE_SECONDS): boolean {
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
const newLeaseUntil = now + leaseSeconds;
|
||||||
|
|
||||||
|
const result = this.db.prepare(`
|
||||||
|
UPDATE jobs SET lease_until = ?
|
||||||
|
WHERE kind = ? AND job_key = ? AND ownership_token = ? AND status = 'running'
|
||||||
|
`).run(newLeaseUntil, kind, jobKey, token);
|
||||||
|
|
||||||
|
return result.changes > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Enqueue a job or update its input_watermark.
|
||||||
|
* Uses MAX(existing, new) for watermark to ensure monotonicity.
|
||||||
|
* If job doesn't exist, creates it in 'pending' status.
|
||||||
|
*/
|
||||||
|
enqueueJob(kind: string, jobKey: string, inputWatermark: number): void {
|
||||||
|
this.db.prepare(`
|
||||||
|
INSERT INTO jobs (kind, job_key, status, retry_remaining, input_watermark, last_success_watermark)
|
||||||
|
VALUES (?, ?, 'pending', ?, ?, 0)
|
||||||
|
ON CONFLICT(kind, job_key) DO UPDATE SET
|
||||||
|
input_watermark = MAX(jobs.input_watermark, excluded.input_watermark)
|
||||||
|
`).run(kind, jobKey, MAX_RETRIES, inputWatermark);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mark a job as successfully completed and update success watermark.
|
||||||
|
* Only succeeds if ownership_token matches.
|
||||||
|
*/
|
||||||
|
markSucceeded(kind: string, jobKey: string, token: string, watermark: number): boolean {
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
|
||||||
|
const result = this.db.prepare(`
|
||||||
|
UPDATE jobs SET
|
||||||
|
status = 'done',
|
||||||
|
finished_at = ?,
|
||||||
|
lease_until = NULL,
|
||||||
|
ownership_token = NULL,
|
||||||
|
worker_id = NULL,
|
||||||
|
last_error = NULL,
|
||||||
|
last_success_watermark = ?
|
||||||
|
WHERE kind = ? AND job_key = ? AND ownership_token = ?
|
||||||
|
`).run(now, watermark, kind, jobKey, token);
|
||||||
|
|
||||||
|
return result.changes > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mark a job as failed with error message and schedule retry.
|
||||||
|
* Only succeeds if ownership_token matches.
|
||||||
|
*/
|
||||||
|
markFailed(kind: string, jobKey: string, token: string, error: string, retryDelay: number = RETRY_DELAY_SECONDS): boolean {
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
const retryAt = now + retryDelay;
|
||||||
|
|
||||||
|
const result = this.db.prepare(`
|
||||||
|
UPDATE jobs SET
|
||||||
|
status = 'error',
|
||||||
|
finished_at = ?,
|
||||||
|
lease_until = NULL,
|
||||||
|
ownership_token = NULL,
|
||||||
|
worker_id = NULL,
|
||||||
|
last_error = ?,
|
||||||
|
retry_at = ?
|
||||||
|
WHERE kind = ? AND job_key = ? AND ownership_token = ?
|
||||||
|
`).run(now, error, retryAt, kind, jobKey, token);
|
||||||
|
|
||||||
|
return result.changes > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get current status of a specific job.
|
||||||
|
*/
|
||||||
|
getJobStatus(kind: string, jobKey: string): JobRecord | null {
|
||||||
|
const row = this.db.prepare(
|
||||||
|
`SELECT * FROM jobs WHERE kind = ? AND job_key = ?`
|
||||||
|
).get(kind, jobKey) as any;
|
||||||
|
|
||||||
|
if (!row) return null;
|
||||||
|
return this.rowToJobRecord(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* List jobs, optionally filtered by kind and/or status.
|
||||||
|
*/
|
||||||
|
listJobs(kind?: string, status?: JobStatus): JobRecord[] {
|
||||||
|
let query = 'SELECT * FROM jobs';
|
||||||
|
const params: any[] = [];
|
||||||
|
const conditions: string[] = [];
|
||||||
|
|
||||||
|
if (kind) {
|
||||||
|
conditions.push('kind = ?');
|
||||||
|
params.push(kind);
|
||||||
|
}
|
||||||
|
if (status) {
|
||||||
|
conditions.push('status = ?');
|
||||||
|
params.push(status);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (conditions.length > 0) {
|
||||||
|
query += ' WHERE ' + conditions.join(' AND ');
|
||||||
|
}
|
||||||
|
|
||||||
|
query += ' ORDER BY kind, job_key';
|
||||||
|
|
||||||
|
const rows = this.db.prepare(query).all(...params) as any[];
|
||||||
|
return rows.map(row => this.rowToJobRecord(row));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if a job is dirty (input_watermark > last_success_watermark).
|
||||||
|
*/
|
||||||
|
isDirty(kind: string, jobKey: string): boolean {
|
||||||
|
const row = this.db.prepare(
|
||||||
|
`SELECT input_watermark, last_success_watermark FROM jobs WHERE kind = ? AND job_key = ?`
|
||||||
|
).get(kind, jobKey) as any;
|
||||||
|
|
||||||
|
if (!row) return false;
|
||||||
|
return row.input_watermark > row.last_success_watermark;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert a database row to a typed JobRecord.
|
||||||
|
*/
|
||||||
|
private rowToJobRecord(row: any): JobRecord {
|
||||||
|
return {
|
||||||
|
kind: row.kind,
|
||||||
|
job_key: row.job_key,
|
||||||
|
status: row.status,
|
||||||
|
worker_id: row.worker_id || undefined,
|
||||||
|
ownership_token: row.ownership_token || undefined,
|
||||||
|
started_at: row.started_at || undefined,
|
||||||
|
finished_at: row.finished_at || undefined,
|
||||||
|
lease_until: row.lease_until || undefined,
|
||||||
|
retry_at: row.retry_at || undefined,
|
||||||
|
retry_remaining: row.retry_remaining,
|
||||||
|
last_error: row.last_error || undefined,
|
||||||
|
input_watermark: row.input_watermark ?? 0,
|
||||||
|
last_success_watermark: row.last_success_watermark ?? 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
84
ccw/src/core/memory-v2-config.ts
Normal file
84
ccw/src/core/memory-v2-config.ts
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
/**
|
||||||
|
* Memory V2 Configuration Constants
|
||||||
|
*
|
||||||
|
* All tuning parameters for the two-phase memory extraction and consolidation pipeline.
|
||||||
|
* Phase 1: Per-session extraction (transcript -> structured memory)
|
||||||
|
* Phase 2: Global consolidation (structured memories -> MEMORY.md)
|
||||||
|
*/
|
||||||
|
|
||||||
|
// -- Batch orchestration --
|
||||||
|
|
||||||
|
/** Maximum sessions to process per startup/trigger */
|
||||||
|
export const MAX_SESSIONS_PER_STARTUP = 64;
|
||||||
|
|
||||||
|
/** Maximum concurrent Phase 1 extraction jobs */
|
||||||
|
export const PHASE_ONE_CONCURRENCY = 64;
|
||||||
|
|
||||||
|
// -- Session eligibility --
|
||||||
|
|
||||||
|
/** Maximum session age in days to consider for extraction */
|
||||||
|
export const MAX_SESSION_AGE_DAYS = 30;
|
||||||
|
|
||||||
|
/** Minimum idle hours before a session becomes eligible */
|
||||||
|
export const MIN_IDLE_HOURS = 12;
|
||||||
|
|
||||||
|
// -- Job scheduler --
|
||||||
|
|
||||||
|
/** Default lease duration in seconds (1 hour) */
|
||||||
|
export const LEASE_SECONDS = 3600;
|
||||||
|
|
||||||
|
/** Delay in seconds before retrying a failed job */
|
||||||
|
export const RETRY_DELAY_SECONDS = 3600;
|
||||||
|
|
||||||
|
/** Maximum retry attempts for a failed job */
|
||||||
|
export const MAX_RETRIES = 3;
|
||||||
|
|
||||||
|
/** Interval in seconds between heartbeat renewals */
|
||||||
|
export const HEARTBEAT_INTERVAL_SECONDS = 30;
|
||||||
|
|
||||||
|
// -- Content size limits --
|
||||||
|
|
||||||
|
/** Maximum characters for raw_memory field in stage1_outputs */
|
||||||
|
export const MAX_RAW_MEMORY_CHARS = 300_000;
|
||||||
|
|
||||||
|
/** Maximum characters for rollout_summary field in stage1_outputs */
|
||||||
|
export const MAX_SUMMARY_CHARS = 1200;
|
||||||
|
|
||||||
|
/** Maximum bytes of transcript to send to LLM for extraction */
|
||||||
|
export const MAX_ROLLOUT_BYTES_FOR_PROMPT = 1_000_000;
|
||||||
|
|
||||||
|
/** Maximum number of raw memories included in global consolidation input */
|
||||||
|
export const MAX_RAW_MEMORIES_FOR_GLOBAL = 64;
|
||||||
|
|
||||||
|
// -- Typed configuration object --
|
||||||
|
|
||||||
|
export interface MemoryV2Config {
|
||||||
|
MAX_SESSIONS_PER_STARTUP: number;
|
||||||
|
PHASE_ONE_CONCURRENCY: number;
|
||||||
|
MAX_SESSION_AGE_DAYS: number;
|
||||||
|
MIN_IDLE_HOURS: number;
|
||||||
|
LEASE_SECONDS: number;
|
||||||
|
RETRY_DELAY_SECONDS: number;
|
||||||
|
MAX_RETRIES: number;
|
||||||
|
HEARTBEAT_INTERVAL_SECONDS: number;
|
||||||
|
MAX_RAW_MEMORY_CHARS: number;
|
||||||
|
MAX_SUMMARY_CHARS: number;
|
||||||
|
MAX_ROLLOUT_BYTES_FOR_PROMPT: number;
|
||||||
|
MAX_RAW_MEMORIES_FOR_GLOBAL: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Default configuration object - use individual exports for direct access */
|
||||||
|
export const MEMORY_V2_DEFAULTS: Readonly<MemoryV2Config> = {
|
||||||
|
MAX_SESSIONS_PER_STARTUP,
|
||||||
|
PHASE_ONE_CONCURRENCY,
|
||||||
|
MAX_SESSION_AGE_DAYS,
|
||||||
|
MIN_IDLE_HOURS,
|
||||||
|
LEASE_SECONDS,
|
||||||
|
RETRY_DELAY_SECONDS,
|
||||||
|
MAX_RETRIES,
|
||||||
|
MAX_RAW_MEMORY_CHARS,
|
||||||
|
MAX_SUMMARY_CHARS,
|
||||||
|
MAX_ROLLOUT_BYTES_FOR_PROMPT,
|
||||||
|
MAX_RAW_MEMORIES_FOR_GLOBAL,
|
||||||
|
HEARTBEAT_INTERVAL_SECONDS,
|
||||||
|
} as const;
|
||||||
@@ -4,6 +4,8 @@ import { getCoreMemoryStore } from '../core-memory-store.js';
|
|||||||
import type { CoreMemory, SessionCluster, ClusterMember, ClusterRelation } from '../core-memory-store.js';
|
import type { CoreMemory, SessionCluster, ClusterMember, ClusterRelation } from '../core-memory-store.js';
|
||||||
import { getEmbeddingStatus, generateEmbeddings } from '../memory-embedder-bridge.js';
|
import { getEmbeddingStatus, generateEmbeddings } from '../memory-embedder-bridge.js';
|
||||||
import { checkSemanticStatus } from '../../tools/codex-lens.js';
|
import { checkSemanticStatus } from '../../tools/codex-lens.js';
|
||||||
|
import { MemoryJobScheduler } from '../memory-job-scheduler.js';
|
||||||
|
import type { JobStatus } from '../memory-job-scheduler.js';
|
||||||
import { StoragePaths } from '../../config/storage-paths.js';
|
import { StoragePaths } from '../../config/storage-paths.js';
|
||||||
import { join } from 'path';
|
import { join } from 'path';
|
||||||
import { getDefaultTool } from '../../tools/claude-cli-tools.js';
|
import { getDefaultTool } from '../../tools/claude-cli-tools.js';
|
||||||
@@ -233,6 +235,199 @@ export async function handleCoreMemoryRoutes(ctx: RouteContext): Promise<boolean
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================
|
||||||
|
// Memory V2 Pipeline API Endpoints
|
||||||
|
// ============================================================
|
||||||
|
|
||||||
|
// API: Trigger batch extraction (fire-and-forget)
|
||||||
|
if (pathname === '/api/core-memory/extract' && req.method === 'POST') {
|
||||||
|
handlePostRequest(req, res, async (body) => {
|
||||||
|
const { maxSessions, path: projectPath } = body;
|
||||||
|
const basePath = projectPath || initialPath;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { MemoryExtractionPipeline } = await import('../memory-extraction-pipeline.js');
|
||||||
|
const pipeline = new MemoryExtractionPipeline(basePath);
|
||||||
|
|
||||||
|
// Broadcast start event
|
||||||
|
broadcastToClients({
|
||||||
|
type: 'MEMORY_EXTRACTION_STARTED',
|
||||||
|
payload: {
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
maxSessions: maxSessions || 'default',
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Fire-and-forget: trigger async, notify on completion
|
||||||
|
const batchPromise = pipeline.runBatchExtraction();
|
||||||
|
batchPromise.then(() => {
|
||||||
|
broadcastToClients({
|
||||||
|
type: 'MEMORY_EXTRACTION_COMPLETED',
|
||||||
|
payload: { timestamp: new Date().toISOString() }
|
||||||
|
});
|
||||||
|
}).catch((err: Error) => {
|
||||||
|
broadcastToClients({
|
||||||
|
type: 'MEMORY_EXTRACTION_FAILED',
|
||||||
|
payload: {
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
error: err.message,
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Scan eligible sessions to report count
|
||||||
|
const eligible = pipeline.scanEligibleSessions();
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
triggered: true,
|
||||||
|
eligibleCount: eligible.length,
|
||||||
|
message: `Extraction triggered for ${eligible.length} eligible sessions`,
|
||||||
|
};
|
||||||
|
} catch (error: unknown) {
|
||||||
|
return { error: (error as Error).message, status: 500 };
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// API: Get extraction pipeline status
|
||||||
|
if (pathname === '/api/core-memory/extract/status' && req.method === 'GET') {
|
||||||
|
const projectPath = url.searchParams.get('path') || initialPath;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const store = getCoreMemoryStore(projectPath);
|
||||||
|
const scheduler = new MemoryJobScheduler(store.getDb());
|
||||||
|
|
||||||
|
const stage1Count = store.countStage1Outputs();
|
||||||
|
const extractionJobs = scheduler.listJobs('phase1_extraction');
|
||||||
|
|
||||||
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||||
|
res.end(JSON.stringify({
|
||||||
|
success: true,
|
||||||
|
total_stage1: stage1Count,
|
||||||
|
jobs: extractionJobs.map(j => ({
|
||||||
|
job_key: j.job_key,
|
||||||
|
status: j.status,
|
||||||
|
started_at: j.started_at,
|
||||||
|
finished_at: j.finished_at,
|
||||||
|
last_error: j.last_error,
|
||||||
|
retry_remaining: j.retry_remaining,
|
||||||
|
})),
|
||||||
|
}));
|
||||||
|
} catch (error: unknown) {
|
||||||
|
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||||
|
res.end(JSON.stringify({ error: (error as Error).message }));
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// API: Trigger consolidation (fire-and-forget)
|
||||||
|
if (pathname === '/api/core-memory/consolidate' && req.method === 'POST') {
|
||||||
|
handlePostRequest(req, res, async (body) => {
|
||||||
|
const { path: projectPath } = body;
|
||||||
|
const basePath = projectPath || initialPath;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { MemoryConsolidationPipeline } = await import('../memory-consolidation-pipeline.js');
|
||||||
|
const pipeline = new MemoryConsolidationPipeline(basePath);
|
||||||
|
|
||||||
|
// Broadcast start event
|
||||||
|
broadcastToClients({
|
||||||
|
type: 'MEMORY_CONSOLIDATION_STARTED',
|
||||||
|
payload: { timestamp: new Date().toISOString() }
|
||||||
|
});
|
||||||
|
|
||||||
|
// Fire-and-forget
|
||||||
|
const consolidatePromise = pipeline.runConsolidation();
|
||||||
|
consolidatePromise.then(() => {
|
||||||
|
broadcastToClients({
|
||||||
|
type: 'MEMORY_CONSOLIDATION_COMPLETED',
|
||||||
|
payload: { timestamp: new Date().toISOString() }
|
||||||
|
});
|
||||||
|
}).catch((err: Error) => {
|
||||||
|
broadcastToClients({
|
||||||
|
type: 'MEMORY_CONSOLIDATION_FAILED',
|
||||||
|
payload: {
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
error: err.message,
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
triggered: true,
|
||||||
|
message: 'Consolidation triggered',
|
||||||
|
};
|
||||||
|
} catch (error: unknown) {
|
||||||
|
return { error: (error as Error).message, status: 500 };
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// API: Get consolidation status
|
||||||
|
if (pathname === '/api/core-memory/consolidate/status' && req.method === 'GET') {
|
||||||
|
const projectPath = url.searchParams.get('path') || initialPath;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { MemoryConsolidationPipeline } = await import('../memory-consolidation-pipeline.js');
|
||||||
|
const pipeline = new MemoryConsolidationPipeline(projectPath);
|
||||||
|
const status = pipeline.getStatus();
|
||||||
|
const memoryMd = pipeline.getMemoryMdContent();
|
||||||
|
|
||||||
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||||
|
res.end(JSON.stringify({
|
||||||
|
success: true,
|
||||||
|
status: status?.status || 'unknown',
|
||||||
|
memoryMdAvailable: !!memoryMd,
|
||||||
|
memoryMdPreview: memoryMd ? memoryMd.substring(0, 500) : undefined,
|
||||||
|
}));
|
||||||
|
} catch (error: unknown) {
|
||||||
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||||
|
res.end(JSON.stringify({
|
||||||
|
success: true,
|
||||||
|
status: 'unavailable',
|
||||||
|
memoryMdAvailable: false,
|
||||||
|
error: (error as Error).message,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// API: List all V2 pipeline jobs
|
||||||
|
if (pathname === '/api/core-memory/jobs' && req.method === 'GET') {
|
||||||
|
const projectPath = url.searchParams.get('path') || initialPath;
|
||||||
|
const kind = url.searchParams.get('kind') || undefined;
|
||||||
|
const statusFilter = url.searchParams.get('status') as JobStatus | undefined;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const store = getCoreMemoryStore(projectPath);
|
||||||
|
const scheduler = new MemoryJobScheduler(store.getDb());
|
||||||
|
|
||||||
|
const jobs = scheduler.listJobs(kind, statusFilter);
|
||||||
|
|
||||||
|
// Compute byStatus counts
|
||||||
|
const byStatus: Record<string, number> = {};
|
||||||
|
for (const job of jobs) {
|
||||||
|
byStatus[job.status] = (byStatus[job.status] || 0) + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||||
|
res.end(JSON.stringify({
|
||||||
|
success: true,
|
||||||
|
jobs,
|
||||||
|
total: jobs.length,
|
||||||
|
byStatus,
|
||||||
|
}));
|
||||||
|
} catch (error: unknown) {
|
||||||
|
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||||
|
res.end(JSON.stringify({ error: (error as Error).message }));
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
// ============================================================
|
// ============================================================
|
||||||
// Session Clustering API Endpoints
|
// Session Clustering API Endpoints
|
||||||
// ============================================================
|
// ============================================================
|
||||||
|
|||||||
@@ -7,12 +7,17 @@ import { z } from 'zod';
|
|||||||
import type { ToolSchema, ToolResult } from '../types/tool.js';
|
import type { ToolSchema, ToolResult } from '../types/tool.js';
|
||||||
import { getCoreMemoryStore, findMemoryAcrossProjects } from '../core/core-memory-store.js';
|
import { getCoreMemoryStore, findMemoryAcrossProjects } from '../core/core-memory-store.js';
|
||||||
import * as MemoryEmbedder from '../core/memory-embedder-bridge.js';
|
import * as MemoryEmbedder from '../core/memory-embedder-bridge.js';
|
||||||
|
import { MemoryJobScheduler } from '../core/memory-job-scheduler.js';
|
||||||
|
import type { JobRecord, JobStatus } from '../core/memory-job-scheduler.js';
|
||||||
import { StoragePaths } from '../config/storage-paths.js';
|
import { StoragePaths } from '../config/storage-paths.js';
|
||||||
import { join } from 'path';
|
import { join } from 'path';
|
||||||
import { getProjectRoot } from '../utils/path-validator.js';
|
import { getProjectRoot } from '../utils/path-validator.js';
|
||||||
|
|
||||||
// Zod schemas
|
// Zod schemas
|
||||||
const OperationEnum = z.enum(['list', 'import', 'export', 'summary', 'embed', 'search', 'embed_status']);
|
const OperationEnum = z.enum([
|
||||||
|
'list', 'import', 'export', 'summary', 'embed', 'search', 'embed_status',
|
||||||
|
'extract', 'extract_status', 'consolidate', 'consolidate_status', 'jobs',
|
||||||
|
]);
|
||||||
|
|
||||||
const ParamsSchema = z.object({
|
const ParamsSchema = z.object({
|
||||||
operation: OperationEnum,
|
operation: OperationEnum,
|
||||||
@@ -31,6 +36,11 @@ const ParamsSchema = z.object({
|
|||||||
source_id: z.string().optional(),
|
source_id: z.string().optional(),
|
||||||
batch_size: z.number().optional().default(8),
|
batch_size: z.number().optional().default(8),
|
||||||
force: z.boolean().optional().default(false),
|
force: z.boolean().optional().default(false),
|
||||||
|
// V2 extract parameters
|
||||||
|
max_sessions: z.number().optional(),
|
||||||
|
// V2 jobs parameters
|
||||||
|
kind: z.string().optional(),
|
||||||
|
status_filter: z.enum(['pending', 'running', 'done', 'error']).optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
type Params = z.infer<typeof ParamsSchema>;
|
type Params = z.infer<typeof ParamsSchema>;
|
||||||
@@ -105,7 +115,44 @@ interface EmbedStatusResult {
|
|||||||
by_type: Record<string, { total: number; embedded: number }>;
|
by_type: Record<string, { total: number; embedded: number }>;
|
||||||
}
|
}
|
||||||
|
|
||||||
type OperationResult = ListResult | ImportResult | ExportResult | SummaryResult | EmbedResult | SearchResult | EmbedStatusResult;
|
// -- Memory V2 operation result types --
|
||||||
|
|
||||||
|
interface ExtractResult {
|
||||||
|
operation: 'extract';
|
||||||
|
triggered: boolean;
|
||||||
|
jobIds: string[];
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ExtractStatusResult {
|
||||||
|
operation: 'extract_status';
|
||||||
|
total_stage1: number;
|
||||||
|
jobs: Array<{ job_key: string; status: string; last_error?: string }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ConsolidateResult {
|
||||||
|
operation: 'consolidate';
|
||||||
|
triggered: boolean;
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ConsolidateStatusResult {
|
||||||
|
operation: 'consolidate_status';
|
||||||
|
status: string;
|
||||||
|
memoryMdAvailable: boolean;
|
||||||
|
memoryMdPreview?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface JobsResult {
|
||||||
|
operation: 'jobs';
|
||||||
|
jobs: JobRecord[];
|
||||||
|
total: number;
|
||||||
|
byStatus: Record<string, number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
type OperationResult = ListResult | ImportResult | ExportResult | SummaryResult
|
||||||
|
| EmbedResult | SearchResult | EmbedStatusResult
|
||||||
|
| ExtractResult | ExtractStatusResult | ConsolidateResult | ConsolidateStatusResult | JobsResult;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get project path - uses explicit path if provided, otherwise falls back to current working directory
|
* Get project path - uses explicit path if provided, otherwise falls back to current working directory
|
||||||
@@ -333,6 +380,165 @@ async function executeEmbedStatus(params: Params): Promise<EmbedStatusResult> {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Memory V2 Operation Handlers
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Operation: extract
|
||||||
|
* Trigger batch extraction (fire-and-forget). Returns job IDs immediately.
|
||||||
|
*/
|
||||||
|
async function executeExtract(params: Params): Promise<ExtractResult> {
|
||||||
|
const { max_sessions, path } = params;
|
||||||
|
const projectPath = getProjectPath(path);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { MemoryExtractionPipeline } = await import('../core/memory-extraction-pipeline.js');
|
||||||
|
const pipeline = new MemoryExtractionPipeline(projectPath);
|
||||||
|
|
||||||
|
// Fire-and-forget: trigger batch extraction asynchronously
|
||||||
|
const batchPromise = pipeline.runBatchExtraction({ maxSessions: max_sessions });
|
||||||
|
|
||||||
|
// Don't await - let it run in background
|
||||||
|
batchPromise.catch((err: Error) => {
|
||||||
|
// Log errors but don't throw - fire-and-forget
|
||||||
|
console.error(`[memory-v2] Batch extraction error: ${err.message}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Scan eligible sessions to report count
|
||||||
|
const eligible = pipeline.scanEligibleSessions(max_sessions);
|
||||||
|
const sessionIds = eligible.map(s => s.id);
|
||||||
|
|
||||||
|
return {
|
||||||
|
operation: 'extract',
|
||||||
|
triggered: true,
|
||||||
|
jobIds: sessionIds,
|
||||||
|
message: `Extraction triggered for ${eligible.length} eligible sessions (max: ${max_sessions || 'default'})`,
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
return {
|
||||||
|
operation: 'extract',
|
||||||
|
triggered: false,
|
||||||
|
jobIds: [],
|
||||||
|
message: `Failed to trigger extraction: ${(err as Error).message}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Operation: extract_status
|
||||||
|
* Get extraction pipeline state.
|
||||||
|
*/
|
||||||
|
async function executeExtractStatus(params: Params): Promise<ExtractStatusResult> {
|
||||||
|
const { path } = params;
|
||||||
|
const projectPath = getProjectPath(path);
|
||||||
|
|
||||||
|
const store = getCoreMemoryStore(projectPath);
|
||||||
|
const scheduler = new MemoryJobScheduler(store.getDb());
|
||||||
|
|
||||||
|
const stage1Count = store.countStage1Outputs();
|
||||||
|
const extractionJobs = scheduler.listJobs('extraction');
|
||||||
|
|
||||||
|
return {
|
||||||
|
operation: 'extract_status',
|
||||||
|
total_stage1: stage1Count,
|
||||||
|
jobs: extractionJobs.map(j => ({
|
||||||
|
job_key: j.job_key,
|
||||||
|
status: j.status,
|
||||||
|
last_error: j.last_error,
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Operation: consolidate
|
||||||
|
* Trigger consolidation (fire-and-forget).
|
||||||
|
*/
|
||||||
|
async function executeConsolidate(params: Params): Promise<ConsolidateResult> {
|
||||||
|
const { path } = params;
|
||||||
|
const projectPath = getProjectPath(path);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { MemoryConsolidationPipeline } = await import('../core/memory-consolidation-pipeline.js');
|
||||||
|
const pipeline = new MemoryConsolidationPipeline(projectPath);
|
||||||
|
|
||||||
|
// Fire-and-forget: trigger consolidation asynchronously
|
||||||
|
const consolidatePromise = pipeline.runConsolidation();
|
||||||
|
|
||||||
|
consolidatePromise.catch((err: Error) => {
|
||||||
|
console.error(`[memory-v2] Consolidation error: ${err.message}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
operation: 'consolidate',
|
||||||
|
triggered: true,
|
||||||
|
message: 'Consolidation triggered',
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
return {
|
||||||
|
operation: 'consolidate',
|
||||||
|
triggered: false,
|
||||||
|
message: `Failed to trigger consolidation: ${(err as Error).message}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Operation: consolidate_status
|
||||||
|
* Get consolidation pipeline state.
|
||||||
|
*/
|
||||||
|
async function executeConsolidateStatus(params: Params): Promise<ConsolidateStatusResult> {
|
||||||
|
const { path } = params;
|
||||||
|
const projectPath = getProjectPath(path);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { MemoryConsolidationPipeline } = await import('../core/memory-consolidation-pipeline.js');
|
||||||
|
const pipeline = new MemoryConsolidationPipeline(projectPath);
|
||||||
|
const status = pipeline.getStatus();
|
||||||
|
const memoryMd = pipeline.getMemoryMdContent();
|
||||||
|
|
||||||
|
return {
|
||||||
|
operation: 'consolidate_status',
|
||||||
|
status: status?.status || 'unknown',
|
||||||
|
memoryMdAvailable: !!memoryMd,
|
||||||
|
memoryMdPreview: memoryMd ? memoryMd.substring(0, 500) : undefined,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return {
|
||||||
|
operation: 'consolidate_status',
|
||||||
|
status: 'unavailable',
|
||||||
|
memoryMdAvailable: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Operation: jobs
|
||||||
|
* List all V2 jobs with optional kind filter.
|
||||||
|
*/
|
||||||
|
function executeJobs(params: Params): JobsResult {
|
||||||
|
const { kind, status_filter, path } = params;
|
||||||
|
const projectPath = getProjectPath(path);
|
||||||
|
|
||||||
|
const store = getCoreMemoryStore(projectPath);
|
||||||
|
const scheduler = new MemoryJobScheduler(store.getDb());
|
||||||
|
|
||||||
|
const jobs = scheduler.listJobs(kind, status_filter as JobStatus | undefined);
|
||||||
|
|
||||||
|
// Compute byStatus counts
|
||||||
|
const byStatus: Record<string, number> = {};
|
||||||
|
for (const job of jobs) {
|
||||||
|
byStatus[job.status] = (byStatus[job.status] || 0) + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
operation: 'jobs',
|
||||||
|
jobs,
|
||||||
|
total: jobs.length,
|
||||||
|
byStatus,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Route to appropriate operation handler
|
* Route to appropriate operation handler
|
||||||
*/
|
*/
|
||||||
@@ -354,9 +560,19 @@ async function execute(params: Params): Promise<OperationResult> {
|
|||||||
return executeSearch(params);
|
return executeSearch(params);
|
||||||
case 'embed_status':
|
case 'embed_status':
|
||||||
return executeEmbedStatus(params);
|
return executeEmbedStatus(params);
|
||||||
|
case 'extract':
|
||||||
|
return executeExtract(params);
|
||||||
|
case 'extract_status':
|
||||||
|
return executeExtractStatus(params);
|
||||||
|
case 'consolidate':
|
||||||
|
return executeConsolidate(params);
|
||||||
|
case 'consolidate_status':
|
||||||
|
return executeConsolidateStatus(params);
|
||||||
|
case 'jobs':
|
||||||
|
return executeJobs(params);
|
||||||
default:
|
default:
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Unknown operation: ${operation}. Valid operations: list, import, export, summary, embed, search, embed_status`
|
`Unknown operation: ${operation}. Valid operations: list, import, export, summary, embed, search, embed_status, extract, extract_status, consolidate, consolidate_status, jobs`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -374,6 +590,11 @@ Usage:
|
|||||||
core_memory(operation="embed", source_id="CMEM-xxx") # Generate embeddings for memory
|
core_memory(operation="embed", source_id="CMEM-xxx") # Generate embeddings for memory
|
||||||
core_memory(operation="search", query="authentication") # Search memories semantically
|
core_memory(operation="search", query="authentication") # Search memories semantically
|
||||||
core_memory(operation="embed_status") # Check embedding status
|
core_memory(operation="embed_status") # Check embedding status
|
||||||
|
core_memory(operation="extract") # Trigger batch memory extraction (V2)
|
||||||
|
core_memory(operation="extract_status") # Check extraction pipeline status
|
||||||
|
core_memory(operation="consolidate") # Trigger memory consolidation (V2)
|
||||||
|
core_memory(operation="consolidate_status") # Check consolidation status
|
||||||
|
core_memory(operation="jobs") # List all V2 pipeline jobs
|
||||||
|
|
||||||
Path parameter (highest priority):
|
Path parameter (highest priority):
|
||||||
core_memory(operation="list", path="/path/to/project") # Use specific project path
|
core_memory(operation="list", path="/path/to/project") # Use specific project path
|
||||||
@@ -384,7 +605,10 @@ Memory IDs use format: CMEM-YYYYMMDD-HHMMSS`,
|
|||||||
properties: {
|
properties: {
|
||||||
operation: {
|
operation: {
|
||||||
type: 'string',
|
type: 'string',
|
||||||
enum: ['list', 'import', 'export', 'summary', 'embed', 'search', 'embed_status'],
|
enum: [
|
||||||
|
'list', 'import', 'export', 'summary', 'embed', 'search', 'embed_status',
|
||||||
|
'extract', 'extract_status', 'consolidate', 'consolidate_status', 'jobs',
|
||||||
|
],
|
||||||
description: 'Operation to perform',
|
description: 'Operation to perform',
|
||||||
},
|
},
|
||||||
path: {
|
path: {
|
||||||
@@ -437,6 +661,19 @@ Memory IDs use format: CMEM-YYYYMMDD-HHMMSS`,
|
|||||||
type: 'boolean',
|
type: 'boolean',
|
||||||
description: 'Force re-embedding even if embeddings exist (default: false)',
|
description: 'Force re-embedding even if embeddings exist (default: false)',
|
||||||
},
|
},
|
||||||
|
max_sessions: {
|
||||||
|
type: 'number',
|
||||||
|
description: 'Max sessions to extract in one batch (for extract operation)',
|
||||||
|
},
|
||||||
|
kind: {
|
||||||
|
type: 'string',
|
||||||
|
description: 'Filter jobs by kind (for jobs operation, e.g. "extraction" or "consolidation")',
|
||||||
|
},
|
||||||
|
status_filter: {
|
||||||
|
type: 'string',
|
||||||
|
enum: ['pending', 'running', 'done', 'error'],
|
||||||
|
description: 'Filter jobs by status (for jobs operation)',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
required: ['operation'],
|
required: ['operation'],
|
||||||
},
|
},
|
||||||
|
|||||||
50
ccw/src/utils/secret-redactor.ts
Normal file
50
ccw/src/utils/secret-redactor.ts
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
/**
|
||||||
|
* Secret Redactor - Regex-based secret pattern detection and replacement
|
||||||
|
*
|
||||||
|
* Scans text for common secret patterns (API keys, tokens, credentials)
|
||||||
|
* and replaces them with [REDACTED_SECRET] to prevent leakage into
|
||||||
|
* memory extraction outputs.
|
||||||
|
*
|
||||||
|
* Patterns are intentionally specific (prefix-based) to minimize false positives.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const REDACTED = '[REDACTED_SECRET]';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Secret patterns with named regex for each category.
|
||||||
|
* Each pattern targets a specific, well-known secret format.
|
||||||
|
*/
|
||||||
|
const SECRET_PATTERNS: ReadonlyArray<{ name: string; regex: RegExp }> = [
|
||||||
|
// OpenAI API keys: sk-<20+ alphanumeric chars>
|
||||||
|
{ name: 'openai_key', regex: /sk-[A-Za-z0-9]{20,}/g },
|
||||||
|
// AWS Access Key IDs: AKIA<16 uppercase alphanumeric chars>
|
||||||
|
{ name: 'aws_key', regex: /AKIA[0-9A-Z]{16}/g },
|
||||||
|
// Bearer tokens: Bearer <16+ token chars>
|
||||||
|
{ name: 'bearer_token', regex: /Bearer\s+[A-Za-z0-9._\-]{16,}/g },
|
||||||
|
// Secret assignments: key=value or key:value patterns for known secret variable names
|
||||||
|
{ name: 'secret_assignment', regex: /(?:api_key|token|secret|password)[:=]\S+/gi },
|
||||||
|
];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply regex-based secret pattern matching and replacement.
|
||||||
|
*
|
||||||
|
* Scans the input text for 4 pattern categories:
|
||||||
|
* 1. OpenAI API keys (sk-...)
|
||||||
|
* 2. AWS Access Key IDs (AKIA...)
|
||||||
|
* 3. Bearer tokens (Bearer ...)
|
||||||
|
* 4. Secret variable assignments (api_key=..., token:..., etc.)
|
||||||
|
*
|
||||||
|
* @param text - Input text to scan for secrets
|
||||||
|
* @returns Text with all matched secrets replaced by [REDACTED_SECRET]
|
||||||
|
*/
|
||||||
|
export function redactSecrets(text: string): string {
|
||||||
|
if (!text) return text;
|
||||||
|
|
||||||
|
let result = text;
|
||||||
|
for (const { regex } of SECRET_PATTERNS) {
|
||||||
|
// Reset lastIndex for global regexes to ensure fresh match on each call
|
||||||
|
regex.lastIndex = 0;
|
||||||
|
result = result.replace(regex, REDACTED);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
33
codex-lens/benchmarks/accuracy_queries_codexlens.jsonl
Normal file
33
codex-lens/benchmarks/accuracy_queries_codexlens.jsonl
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
{"query":"class StandaloneLspManager","relevant_paths":["codexlens/lsp/standalone_manager.py"]}
|
||||||
|
{"query":"def _open_document","relevant_paths":["codexlens/lsp/standalone_manager.py"]}
|
||||||
|
{"query":"def _read_message","relevant_paths":["codexlens/lsp/standalone_manager.py"]}
|
||||||
|
{"query":"how does textDocument/didOpen work","relevant_paths":["codexlens/lsp/standalone_manager.py"]}
|
||||||
|
{"query":"class LspBridge","relevant_paths":["codexlens/lsp/lsp_bridge.py"]}
|
||||||
|
{"query":"def get_document_symbols","relevant_paths":["codexlens/lsp/lsp_bridge.py"]}
|
||||||
|
{"query":"class KeepAliveLspBridge","relevant_paths":["codexlens/lsp/keepalive_bridge.py"]}
|
||||||
|
{"query":"LSP keepalive bridge","relevant_paths":["codexlens/lsp/keepalive_bridge.py"]}
|
||||||
|
{"query":"class LspGraphBuilder","relevant_paths":["codexlens/lsp/lsp_graph_builder.py"]}
|
||||||
|
{"query":"def build_from_seeds","relevant_paths":["codexlens/lsp/lsp_graph_builder.py"]}
|
||||||
|
{"query":"def _stage2_realtime_lsp_expand","relevant_paths":["codexlens/search/chain_search.py"]}
|
||||||
|
{"query":"def _stage3_cluster_prune","relevant_paths":["codexlens/search/chain_search.py"]}
|
||||||
|
{"query":"def _cross_encoder_rerank","relevant_paths":["codexlens/search/chain_search.py"]}
|
||||||
|
{"query":"def dense_rerank_cascade_search","relevant_paths":["codexlens/search/chain_search.py"]}
|
||||||
|
{"query":"def cascade_search","relevant_paths":["codexlens/search/chain_search.py"]}
|
||||||
|
{"query":"def _find_nearest_binary_mmap_root","relevant_paths":["codexlens/search/chain_search.py"]}
|
||||||
|
{"query":"class BinarySearcher","relevant_paths":["codexlens/search/binary_searcher.py"]}
|
||||||
|
{"query":"class GraphExpander","relevant_paths":["codexlens/search/graph_expander.py"]}
|
||||||
|
{"query":"def cross_encoder_rerank","relevant_paths":["codexlens/search/ranking.py"]}
|
||||||
|
{"query":"def group_similar_results","relevant_paths":["codexlens/search/ranking.py"]}
|
||||||
|
{"query":"class ConfigError","relevant_paths":["codexlens/errors.py"]}
|
||||||
|
{"query":"def load_settings","relevant_paths":["codexlens/config.py"]}
|
||||||
|
{"query":"BINARY_VECTORS_MMAP_NAME","relevant_paths":["codexlens/config.py"]}
|
||||||
|
{"query":"STAGED_CLUSTERING_STRATEGY","relevant_paths":["codexlens/config.py","codexlens/env_config.py"]}
|
||||||
|
{"query":"def apply_workspace_env","relevant_paths":["codexlens/env_config.py"]}
|
||||||
|
{"query":"def generate_env_example","relevant_paths":["codexlens/env_config.py"]}
|
||||||
|
{"query":"def get_reranker","relevant_paths":["codexlens/semantic/reranker/factory.py"]}
|
||||||
|
{"query":"class APIReranker","relevant_paths":["codexlens/semantic/reranker/api_reranker.py"]}
|
||||||
|
{"query":"class RegistryStore","relevant_paths":["codexlens/storage/registry.py"]}
|
||||||
|
{"query":"class PathMapper","relevant_paths":["codexlens/storage/path_mapper.py"]}
|
||||||
|
{"query":"def lsp_status","relevant_paths":["codexlens/cli/commands.py"]}
|
||||||
|
{"query":"graph_neighbors migration","relevant_paths":["codexlens/storage/migrations/migration_007_add_graph_neighbors.py"]}
|
||||||
|
{"query":"def get_model_config","relevant_paths":["codexlens/semantic/vector_store.py"]}
|
||||||
365
codex-lens/benchmarks/compare_accuracy_labeled.py
Normal file
365
codex-lens/benchmarks/compare_accuracy_labeled.py
Normal file
@@ -0,0 +1,365 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
"""Compare labeled accuracy: staged(realtime LSP graph) vs dense_rerank.
|
||||||
|
|
||||||
|
This script measures retrieval "accuracy" against a labeled query set.
|
||||||
|
Each query must provide a list of relevant file paths (relative to --source
|
||||||
|
or absolute). We report:
|
||||||
|
- Hit@K (any relevant file appears in top-K)
|
||||||
|
- MRR@K (reciprocal rank of first relevant file within top-K)
|
||||||
|
- Recall@K (fraction of relevant files present in top-K)
|
||||||
|
|
||||||
|
Example:
|
||||||
|
python benchmarks/compare_accuracy_labeled.py --source ./src
|
||||||
|
python benchmarks/compare_accuracy_labeled.py --queries-file benchmarks/accuracy_queries_codexlens.jsonl
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import gc
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import statistics
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
|
||||||
|
|
||||||
|
# Add src to path (match other benchmark scripts)
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||||
|
|
||||||
|
from codexlens.config import Config
|
||||||
|
from codexlens.search.chain_search import ChainSearchEngine, SearchOptions
|
||||||
|
from codexlens.storage.path_mapper import PathMapper
|
||||||
|
from codexlens.storage.registry import RegistryStore
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_QUERIES_FILE = Path(__file__).parent / "accuracy_queries_codexlens.jsonl"
|
||||||
|
|
||||||
|
|
||||||
|
def _now_ms() -> float:
|
||||||
|
return time.perf_counter() * 1000.0
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_path_key(path: str) -> str:
|
||||||
|
"""Normalize file paths for overlap/dedup metrics (Windows-safe)."""
|
||||||
|
try:
|
||||||
|
p = Path(path)
|
||||||
|
# Don't explode on non-files like "<memory>".
|
||||||
|
if str(p) and (p.is_absolute() or re.match(r"^[A-Za-z]:", str(p))):
|
||||||
|
norm = str(p.resolve())
|
||||||
|
else:
|
||||||
|
norm = str(p)
|
||||||
|
except Exception:
|
||||||
|
norm = path
|
||||||
|
norm = norm.replace("/", "\\")
|
||||||
|
if os.name == "nt":
|
||||||
|
norm = norm.lower()
|
||||||
|
return norm
|
||||||
|
|
||||||
|
|
||||||
|
def _load_labeled_queries(path: Path, limit: Optional[int]) -> List[Dict[str, Any]]:
|
||||||
|
if not path.is_file():
|
||||||
|
raise SystemExit(f"Queries file does not exist: {path}")
|
||||||
|
|
||||||
|
out: List[Dict[str, Any]] = []
|
||||||
|
for raw_line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||||
|
line = raw_line.strip()
|
||||||
|
if not line or line.startswith("#"):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
item = json.loads(line)
|
||||||
|
except Exception as exc:
|
||||||
|
raise SystemExit(f"Invalid JSONL line in {path}: {raw_line!r} ({exc})") from exc
|
||||||
|
if not isinstance(item, dict) or "query" not in item:
|
||||||
|
raise SystemExit(f"Invalid query item (expected object with 'query'): {item!r}")
|
||||||
|
out.append(item)
|
||||||
|
if limit is not None and len(out) >= limit:
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _dedup_topk(paths: Iterable[str], k: int) -> List[str]:
|
||||||
|
out: List[str] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for p in paths:
|
||||||
|
if p in seen:
|
||||||
|
continue
|
||||||
|
seen.add(p)
|
||||||
|
out.append(p)
|
||||||
|
if len(out) >= k:
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _first_hit_rank(topk_paths: Sequence[str], relevant: set[str]) -> Optional[int]:
|
||||||
|
for i, p in enumerate(topk_paths, start=1):
|
||||||
|
if p in relevant:
|
||||||
|
return i
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class StrategyRun:
|
||||||
|
strategy: str
|
||||||
|
latency_ms: float
|
||||||
|
topk_paths: List[str]
|
||||||
|
first_hit_rank: Optional[int]
|
||||||
|
hit_at_k: bool
|
||||||
|
recall_at_k: float
|
||||||
|
error: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class QueryEval:
|
||||||
|
query: str
|
||||||
|
relevant_paths: List[str]
|
||||||
|
staged: StrategyRun
|
||||||
|
dense_rerank: StrategyRun
|
||||||
|
|
||||||
|
|
||||||
|
def _run_strategy(
|
||||||
|
engine: ChainSearchEngine,
|
||||||
|
*,
|
||||||
|
strategy: str,
|
||||||
|
query: str,
|
||||||
|
source_path: Path,
|
||||||
|
k: int,
|
||||||
|
coarse_k: int,
|
||||||
|
relevant: set[str],
|
||||||
|
options: Optional[SearchOptions] = None,
|
||||||
|
) -> StrategyRun:
|
||||||
|
gc.collect()
|
||||||
|
start_ms = _now_ms()
|
||||||
|
try:
|
||||||
|
result = engine.cascade_search(
|
||||||
|
query=query,
|
||||||
|
source_path=source_path,
|
||||||
|
k=k,
|
||||||
|
coarse_k=coarse_k,
|
||||||
|
options=options,
|
||||||
|
strategy=strategy,
|
||||||
|
)
|
||||||
|
latency_ms = _now_ms() - start_ms
|
||||||
|
paths_raw = [r.path for r in (result.results or []) if getattr(r, "path", None)]
|
||||||
|
paths_norm = [_normalize_path_key(p) for p in paths_raw]
|
||||||
|
topk = _dedup_topk(paths_norm, k=k)
|
||||||
|
rank = _first_hit_rank(topk, relevant)
|
||||||
|
hit = rank is not None
|
||||||
|
recall = 0.0
|
||||||
|
if relevant:
|
||||||
|
recall = len(set(topk) & relevant) / float(len(relevant))
|
||||||
|
return StrategyRun(
|
||||||
|
strategy=strategy,
|
||||||
|
latency_ms=latency_ms,
|
||||||
|
topk_paths=topk,
|
||||||
|
first_hit_rank=rank,
|
||||||
|
hit_at_k=hit,
|
||||||
|
recall_at_k=recall,
|
||||||
|
error=None,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
latency_ms = _now_ms() - start_ms
|
||||||
|
return StrategyRun(
|
||||||
|
strategy=strategy,
|
||||||
|
latency_ms=latency_ms,
|
||||||
|
topk_paths=[],
|
||||||
|
first_hit_rank=None,
|
||||||
|
hit_at_k=False,
|
||||||
|
recall_at_k=0.0,
|
||||||
|
error=repr(exc),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _mrr(ranks: Sequence[Optional[int]]) -> float:
|
||||||
|
vals = []
|
||||||
|
for r in ranks:
|
||||||
|
if r is None or r <= 0:
|
||||||
|
vals.append(0.0)
|
||||||
|
else:
|
||||||
|
vals.append(1.0 / float(r))
|
||||||
|
return statistics.mean(vals) if vals else 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Compare labeled retrieval accuracy: staged(realtime) vs dense_rerank"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--source",
|
||||||
|
type=Path,
|
||||||
|
default=Path(__file__).parent.parent / "src",
|
||||||
|
help="Source directory to search (default: ./src)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--queries-file",
|
||||||
|
type=Path,
|
||||||
|
default=DEFAULT_QUERIES_FILE,
|
||||||
|
help="JSONL file with {query, relevant_paths[]} per line",
|
||||||
|
)
|
||||||
|
parser.add_argument("--queries", type=int, default=None, help="Limit number of queries")
|
||||||
|
parser.add_argument("--k", type=int, default=10, help="Top-K for evaluation (default 10)")
|
||||||
|
parser.add_argument("--coarse-k", type=int, default=100, help="Coarse candidates (default 100)")
|
||||||
|
parser.add_argument(
|
||||||
|
"--staged-cluster-strategy",
|
||||||
|
type=str,
|
||||||
|
default="path",
|
||||||
|
help="Config.staged_clustering_strategy override for staged (default: path)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--stage2-mode",
|
||||||
|
type=str,
|
||||||
|
default="realtime",
|
||||||
|
help="Config.staged_stage2_mode override for staged (default: realtime)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--output",
|
||||||
|
type=Path,
|
||||||
|
default=Path(__file__).parent / "results" / "accuracy_labeled.json",
|
||||||
|
help="Output JSON path",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if not args.source.exists():
|
||||||
|
raise SystemExit(f"Source path does not exist: {args.source}")
|
||||||
|
|
||||||
|
labeled = _load_labeled_queries(args.queries_file, args.queries)
|
||||||
|
if not labeled:
|
||||||
|
raise SystemExit("No queries to run")
|
||||||
|
|
||||||
|
source_root = args.source.expanduser().resolve()
|
||||||
|
|
||||||
|
# Match CLI behavior: load settings + apply global/workspace .env overrides.
|
||||||
|
config = Config.load()
|
||||||
|
config.cascade_strategy = "staged"
|
||||||
|
config.staged_stage2_mode = str(args.stage2_mode or "realtime").strip().lower()
|
||||||
|
config.enable_staged_rerank = True
|
||||||
|
config.staged_clustering_strategy = str(args.staged_cluster_strategy or "path").strip().lower()
|
||||||
|
# Stability: on some Windows setups, DirectML/ONNX can crash under load.
|
||||||
|
config.embedding_use_gpu = False
|
||||||
|
|
||||||
|
registry = RegistryStore()
|
||||||
|
registry.initialize()
|
||||||
|
mapper = PathMapper()
|
||||||
|
engine = ChainSearchEngine(registry=registry, mapper=mapper, config=config)
|
||||||
|
|
||||||
|
def resolve_expected(paths: Sequence[str]) -> set[str]:
|
||||||
|
out: set[str] = set()
|
||||||
|
for p in paths:
|
||||||
|
try:
|
||||||
|
cand = Path(p)
|
||||||
|
if not cand.is_absolute():
|
||||||
|
cand = (source_root / cand).resolve()
|
||||||
|
out.add(_normalize_path_key(str(cand)))
|
||||||
|
except Exception:
|
||||||
|
out.add(_normalize_path_key(p))
|
||||||
|
return out
|
||||||
|
|
||||||
|
evaluations: List[QueryEval] = []
|
||||||
|
|
||||||
|
try:
|
||||||
|
for i, item in enumerate(labeled, start=1):
|
||||||
|
query = str(item.get("query", "")).strip()
|
||||||
|
relevant_raw = item.get("relevant_paths") or []
|
||||||
|
if not query:
|
||||||
|
continue
|
||||||
|
if not isinstance(relevant_raw, list) or not relevant_raw:
|
||||||
|
raise SystemExit(f"Query item missing relevant_paths[]: {item!r}")
|
||||||
|
relevant = resolve_expected([str(p) for p in relevant_raw])
|
||||||
|
|
||||||
|
print(f"[{i}/{len(labeled)}] {query}")
|
||||||
|
|
||||||
|
staged = _run_strategy(
|
||||||
|
engine,
|
||||||
|
strategy="staged",
|
||||||
|
query=query,
|
||||||
|
source_path=source_root,
|
||||||
|
k=int(args.k),
|
||||||
|
coarse_k=int(args.coarse_k),
|
||||||
|
relevant=relevant,
|
||||||
|
options=None,
|
||||||
|
)
|
||||||
|
dense = _run_strategy(
|
||||||
|
engine,
|
||||||
|
strategy="dense_rerank",
|
||||||
|
query=query,
|
||||||
|
source_path=source_root,
|
||||||
|
k=int(args.k),
|
||||||
|
coarse_k=int(args.coarse_k),
|
||||||
|
relevant=relevant,
|
||||||
|
options=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
evaluations.append(
|
||||||
|
QueryEval(
|
||||||
|
query=query,
|
||||||
|
relevant_paths=[_normalize_path_key(str((source_root / p).resolve())) if not Path(p).is_absolute() else _normalize_path_key(p) for p in relevant_raw],
|
||||||
|
staged=staged,
|
||||||
|
dense_rerank=dense,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
engine.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
registry.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
staged_runs = [e.staged for e in evaluations]
|
||||||
|
dense_runs = [e.dense_rerank for e in evaluations]
|
||||||
|
|
||||||
|
def mean(xs: Sequence[float]) -> float:
|
||||||
|
return statistics.mean(xs) if xs else 0.0
|
||||||
|
|
||||||
|
staged_ranks = [r.first_hit_rank for r in staged_runs]
|
||||||
|
dense_ranks = [r.first_hit_rank for r in dense_runs]
|
||||||
|
|
||||||
|
summary = {
|
||||||
|
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||||
|
"source": str(source_root),
|
||||||
|
"queries_file": str(args.queries_file),
|
||||||
|
"query_count": len(evaluations),
|
||||||
|
"k": int(args.k),
|
||||||
|
"coarse_k": int(args.coarse_k),
|
||||||
|
"staged": {
|
||||||
|
"hit_at_k": mean([1.0 if r.hit_at_k else 0.0 for r in staged_runs]),
|
||||||
|
"mrr_at_k": _mrr(staged_ranks),
|
||||||
|
"avg_recall_at_k": mean([r.recall_at_k for r in staged_runs]),
|
||||||
|
"avg_latency_ms": mean([r.latency_ms for r in staged_runs if not r.error]),
|
||||||
|
"errors": sum(1 for r in staged_runs if r.error),
|
||||||
|
},
|
||||||
|
"dense_rerank": {
|
||||||
|
"hit_at_k": mean([1.0 if r.hit_at_k else 0.0 for r in dense_runs]),
|
||||||
|
"mrr_at_k": _mrr(dense_ranks),
|
||||||
|
"avg_recall_at_k": mean([r.recall_at_k for r in dense_runs]),
|
||||||
|
"avg_latency_ms": mean([r.latency_ms for r in dense_runs if not r.error]),
|
||||||
|
"errors": sum(1 for r in dense_runs if r.error),
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"staged_stage2_mode": config.staged_stage2_mode,
|
||||||
|
"staged_clustering_strategy": config.staged_clustering_strategy,
|
||||||
|
"enable_staged_rerank": bool(config.enable_staged_rerank),
|
||||||
|
"reranker_backend": config.reranker_backend,
|
||||||
|
"reranker_model": config.reranker_model,
|
||||||
|
"embedding_backend": config.embedding_backend,
|
||||||
|
"embedding_model": config.embedding_model,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
payload = {"summary": summary, "evaluations": [asdict(e) for e in evaluations]}
|
||||||
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
args.output.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
||||||
|
|
||||||
|
print("\n=== SUMMARY ===")
|
||||||
|
print(json.dumps(summary, indent=2))
|
||||||
|
print(f"\nSaved: {args.output}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
||||||
1308
codex-lens/benchmarks/results/accuracy_2026-02-11_codexlens.json
Normal file
1308
codex-lens/benchmarks/results/accuracy_2026-02-11_codexlens.json
Normal file
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -315,119 +315,117 @@ class Config:
|
|||||||
|
|
||||||
def load_settings(self) -> None:
|
def load_settings(self) -> None:
|
||||||
"""Load settings from file if exists."""
|
"""Load settings from file if exists."""
|
||||||
if not self.settings_path.exists():
|
if self.settings_path.exists():
|
||||||
return
|
try:
|
||||||
|
with open(self.settings_path, "r", encoding="utf-8") as f:
|
||||||
|
settings = json.load(f)
|
||||||
|
|
||||||
try:
|
# Load embedding settings
|
||||||
with open(self.settings_path, "r", encoding="utf-8") as f:
|
embedding = settings.get("embedding", {})
|
||||||
settings = json.load(f)
|
if "backend" in embedding:
|
||||||
|
backend = embedding["backend"]
|
||||||
|
# Support 'api' as alias for 'litellm'
|
||||||
|
if backend == "api":
|
||||||
|
backend = "litellm"
|
||||||
|
if backend in {"fastembed", "litellm"}:
|
||||||
|
self.embedding_backend = backend
|
||||||
|
else:
|
||||||
|
log.warning(
|
||||||
|
"Invalid embedding backend in %s: %r (expected 'fastembed' or 'litellm')",
|
||||||
|
self.settings_path,
|
||||||
|
embedding["backend"],
|
||||||
|
)
|
||||||
|
if "model" in embedding:
|
||||||
|
self.embedding_model = embedding["model"]
|
||||||
|
if "use_gpu" in embedding:
|
||||||
|
self.embedding_use_gpu = embedding["use_gpu"]
|
||||||
|
|
||||||
# Load embedding settings
|
# Load multi-endpoint configuration
|
||||||
embedding = settings.get("embedding", {})
|
if "endpoints" in embedding:
|
||||||
if "backend" in embedding:
|
self.embedding_endpoints = embedding["endpoints"]
|
||||||
backend = embedding["backend"]
|
if "pool_enabled" in embedding:
|
||||||
# Support 'api' as alias for 'litellm'
|
self.embedding_pool_enabled = embedding["pool_enabled"]
|
||||||
if backend == "api":
|
if "strategy" in embedding:
|
||||||
backend = "litellm"
|
self.embedding_strategy = embedding["strategy"]
|
||||||
if backend in {"fastembed", "litellm"}:
|
if "cooldown" in embedding:
|
||||||
self.embedding_backend = backend
|
self.embedding_cooldown = embedding["cooldown"]
|
||||||
else:
|
|
||||||
log.warning(
|
|
||||||
"Invalid embedding backend in %s: %r (expected 'fastembed' or 'litellm')",
|
|
||||||
self.settings_path,
|
|
||||||
embedding["backend"],
|
|
||||||
)
|
|
||||||
if "model" in embedding:
|
|
||||||
self.embedding_model = embedding["model"]
|
|
||||||
if "use_gpu" in embedding:
|
|
||||||
self.embedding_use_gpu = embedding["use_gpu"]
|
|
||||||
|
|
||||||
# Load multi-endpoint configuration
|
# Load LLM settings
|
||||||
if "endpoints" in embedding:
|
llm = settings.get("llm", {})
|
||||||
self.embedding_endpoints = embedding["endpoints"]
|
if "enabled" in llm:
|
||||||
if "pool_enabled" in embedding:
|
self.llm_enabled = llm["enabled"]
|
||||||
self.embedding_pool_enabled = embedding["pool_enabled"]
|
if "tool" in llm:
|
||||||
if "strategy" in embedding:
|
self.llm_tool = llm["tool"]
|
||||||
self.embedding_strategy = embedding["strategy"]
|
if "timeout_ms" in llm:
|
||||||
if "cooldown" in embedding:
|
self.llm_timeout_ms = llm["timeout_ms"]
|
||||||
self.embedding_cooldown = embedding["cooldown"]
|
if "batch_size" in llm:
|
||||||
|
self.llm_batch_size = llm["batch_size"]
|
||||||
|
|
||||||
# Load LLM settings
|
# Load reranker settings
|
||||||
llm = settings.get("llm", {})
|
reranker = settings.get("reranker", {})
|
||||||
if "enabled" in llm:
|
if "enabled" in reranker:
|
||||||
self.llm_enabled = llm["enabled"]
|
self.enable_cross_encoder_rerank = reranker["enabled"]
|
||||||
if "tool" in llm:
|
if "backend" in reranker:
|
||||||
self.llm_tool = llm["tool"]
|
backend = reranker["backend"]
|
||||||
if "timeout_ms" in llm:
|
if backend in {"fastembed", "onnx", "api", "litellm", "legacy"}:
|
||||||
self.llm_timeout_ms = llm["timeout_ms"]
|
self.reranker_backend = backend
|
||||||
if "batch_size" in llm:
|
else:
|
||||||
self.llm_batch_size = llm["batch_size"]
|
log.warning(
|
||||||
|
"Invalid reranker backend in %s: %r (expected 'fastembed', 'onnx', 'api', 'litellm', or 'legacy')",
|
||||||
|
self.settings_path,
|
||||||
|
backend,
|
||||||
|
)
|
||||||
|
if "model" in reranker:
|
||||||
|
self.reranker_model = reranker["model"]
|
||||||
|
if "top_k" in reranker:
|
||||||
|
self.reranker_top_k = reranker["top_k"]
|
||||||
|
if "max_input_tokens" in reranker:
|
||||||
|
self.reranker_max_input_tokens = reranker["max_input_tokens"]
|
||||||
|
if "pool_enabled" in reranker:
|
||||||
|
self.reranker_pool_enabled = reranker["pool_enabled"]
|
||||||
|
if "strategy" in reranker:
|
||||||
|
self.reranker_strategy = reranker["strategy"]
|
||||||
|
if "cooldown" in reranker:
|
||||||
|
self.reranker_cooldown = reranker["cooldown"]
|
||||||
|
|
||||||
# Load reranker settings
|
# Load cascade settings
|
||||||
reranker = settings.get("reranker", {})
|
cascade = settings.get("cascade", {})
|
||||||
if "enabled" in reranker:
|
if "strategy" in cascade:
|
||||||
self.enable_cross_encoder_rerank = reranker["enabled"]
|
strategy = cascade["strategy"]
|
||||||
if "backend" in reranker:
|
if strategy in {"binary", "binary_rerank", "dense_rerank", "staged"}:
|
||||||
backend = reranker["backend"]
|
self.cascade_strategy = strategy
|
||||||
if backend in {"fastembed", "onnx", "api", "litellm", "legacy"}:
|
else:
|
||||||
self.reranker_backend = backend
|
log.warning(
|
||||||
else:
|
"Invalid cascade strategy in %s: %r (expected 'binary', 'binary_rerank', 'dense_rerank', or 'staged')",
|
||||||
log.warning(
|
self.settings_path,
|
||||||
"Invalid reranker backend in %s: %r (expected 'fastembed', 'onnx', 'api', 'litellm', or 'legacy')",
|
strategy,
|
||||||
self.settings_path,
|
)
|
||||||
backend,
|
if "coarse_k" in cascade:
|
||||||
)
|
self.cascade_coarse_k = cascade["coarse_k"]
|
||||||
if "model" in reranker:
|
if "fine_k" in cascade:
|
||||||
self.reranker_model = reranker["model"]
|
self.cascade_fine_k = cascade["fine_k"]
|
||||||
if "top_k" in reranker:
|
|
||||||
self.reranker_top_k = reranker["top_k"]
|
|
||||||
if "max_input_tokens" in reranker:
|
|
||||||
self.reranker_max_input_tokens = reranker["max_input_tokens"]
|
|
||||||
if "pool_enabled" in reranker:
|
|
||||||
self.reranker_pool_enabled = reranker["pool_enabled"]
|
|
||||||
if "strategy" in reranker:
|
|
||||||
self.reranker_strategy = reranker["strategy"]
|
|
||||||
if "cooldown" in reranker:
|
|
||||||
self.reranker_cooldown = reranker["cooldown"]
|
|
||||||
|
|
||||||
# Load cascade settings
|
# Load API settings
|
||||||
cascade = settings.get("cascade", {})
|
api = settings.get("api", {})
|
||||||
if "strategy" in cascade:
|
if "max_workers" in api:
|
||||||
strategy = cascade["strategy"]
|
self.api_max_workers = api["max_workers"]
|
||||||
if strategy in {"binary", "binary_rerank", "dense_rerank", "staged"}:
|
if "batch_size" in api:
|
||||||
self.cascade_strategy = strategy
|
self.api_batch_size = api["batch_size"]
|
||||||
else:
|
if "batch_size_dynamic" in api:
|
||||||
log.warning(
|
self.api_batch_size_dynamic = api["batch_size_dynamic"]
|
||||||
"Invalid cascade strategy in %s: %r (expected 'binary', 'binary_rerank', 'dense_rerank', or 'staged')",
|
if "batch_size_utilization_factor" in api:
|
||||||
self.settings_path,
|
self.api_batch_size_utilization_factor = api["batch_size_utilization_factor"]
|
||||||
strategy,
|
if "batch_size_max" in api:
|
||||||
)
|
self.api_batch_size_max = api["batch_size_max"]
|
||||||
if "coarse_k" in cascade:
|
if "chars_per_token_estimate" in api:
|
||||||
self.cascade_coarse_k = cascade["coarse_k"]
|
self.chars_per_token_estimate = api["chars_per_token_estimate"]
|
||||||
if "fine_k" in cascade:
|
except Exception as exc:
|
||||||
self.cascade_fine_k = cascade["fine_k"]
|
log.warning(
|
||||||
|
"Failed to load settings from %s (%s): %s",
|
||||||
# Load API settings
|
self.settings_path,
|
||||||
api = settings.get("api", {})
|
type(exc).__name__,
|
||||||
if "max_workers" in api:
|
exc,
|
||||||
self.api_max_workers = api["max_workers"]
|
)
|
||||||
if "batch_size" in api:
|
|
||||||
self.api_batch_size = api["batch_size"]
|
|
||||||
if "batch_size_dynamic" in api:
|
|
||||||
self.api_batch_size_dynamic = api["batch_size_dynamic"]
|
|
||||||
if "batch_size_utilization_factor" in api:
|
|
||||||
self.api_batch_size_utilization_factor = api["batch_size_utilization_factor"]
|
|
||||||
if "batch_size_max" in api:
|
|
||||||
self.api_batch_size_max = api["batch_size_max"]
|
|
||||||
if "chars_per_token_estimate" in api:
|
|
||||||
self.chars_per_token_estimate = api["chars_per_token_estimate"]
|
|
||||||
except Exception as exc:
|
|
||||||
log.warning(
|
|
||||||
"Failed to load settings from %s (%s): %s",
|
|
||||||
self.settings_path,
|
|
||||||
type(exc).__name__,
|
|
||||||
exc,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Apply .env overrides (highest priority)
|
# Apply .env overrides (highest priority)
|
||||||
self._apply_env_overrides()
|
self._apply_env_overrides()
|
||||||
@@ -450,9 +448,9 @@ class Config:
|
|||||||
RERANKER_STRATEGY: Load balance strategy for reranker
|
RERANKER_STRATEGY: Load balance strategy for reranker
|
||||||
RERANKER_COOLDOWN: Rate limit cooldown for reranker
|
RERANKER_COOLDOWN: Rate limit cooldown for reranker
|
||||||
"""
|
"""
|
||||||
from .env_config import load_global_env
|
from .env_config import load_env_file
|
||||||
|
|
||||||
env_vars = load_global_env()
|
env_vars = load_env_file(self.data_dir / ".env")
|
||||||
if not env_vars:
|
if not env_vars:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -461,6 +459,43 @@ class Config:
|
|||||||
# Check prefixed version first (Dashboard format), then unprefixed
|
# Check prefixed version first (Dashboard format), then unprefixed
|
||||||
return env_vars.get(f"CODEXLENS_{key}") or env_vars.get(key)
|
return env_vars.get(f"CODEXLENS_{key}") or env_vars.get(key)
|
||||||
|
|
||||||
|
def _parse_bool(value: str) -> bool:
|
||||||
|
return value.strip().lower() in {"true", "1", "yes", "on"}
|
||||||
|
|
||||||
|
# Cascade overrides
|
||||||
|
cascade_enabled = get_env("ENABLE_CASCADE_SEARCH")
|
||||||
|
if cascade_enabled:
|
||||||
|
self.enable_cascade_search = _parse_bool(cascade_enabled)
|
||||||
|
log.debug(
|
||||||
|
"Overriding enable_cascade_search from .env: %s",
|
||||||
|
self.enable_cascade_search,
|
||||||
|
)
|
||||||
|
|
||||||
|
cascade_strategy = get_env("CASCADE_STRATEGY")
|
||||||
|
if cascade_strategy:
|
||||||
|
strategy = cascade_strategy.strip().lower()
|
||||||
|
if strategy in {"binary", "binary_rerank", "dense_rerank", "staged"}:
|
||||||
|
self.cascade_strategy = strategy
|
||||||
|
log.debug("Overriding cascade_strategy from .env: %s", self.cascade_strategy)
|
||||||
|
else:
|
||||||
|
log.warning("Invalid CASCADE_STRATEGY in .env: %r", cascade_strategy)
|
||||||
|
|
||||||
|
cascade_coarse_k = get_env("CASCADE_COARSE_K")
|
||||||
|
if cascade_coarse_k:
|
||||||
|
try:
|
||||||
|
self.cascade_coarse_k = int(cascade_coarse_k)
|
||||||
|
log.debug("Overriding cascade_coarse_k from .env: %s", self.cascade_coarse_k)
|
||||||
|
except ValueError:
|
||||||
|
log.warning("Invalid CASCADE_COARSE_K in .env: %r", cascade_coarse_k)
|
||||||
|
|
||||||
|
cascade_fine_k = get_env("CASCADE_FINE_K")
|
||||||
|
if cascade_fine_k:
|
||||||
|
try:
|
||||||
|
self.cascade_fine_k = int(cascade_fine_k)
|
||||||
|
log.debug("Overriding cascade_fine_k from .env: %s", self.cascade_fine_k)
|
||||||
|
except ValueError:
|
||||||
|
log.warning("Invalid CASCADE_FINE_K in .env: %r", cascade_fine_k)
|
||||||
|
|
||||||
# Embedding overrides
|
# Embedding overrides
|
||||||
embedding_model = get_env("EMBEDDING_MODEL")
|
embedding_model = get_env("EMBEDDING_MODEL")
|
||||||
if embedding_model:
|
if embedding_model:
|
||||||
@@ -583,6 +618,136 @@ class Config:
|
|||||||
self.chunk_strip_docstrings = strip_docstrings.lower() in ("true", "1", "yes")
|
self.chunk_strip_docstrings = strip_docstrings.lower() in ("true", "1", "yes")
|
||||||
log.debug("Overriding chunk_strip_docstrings from .env: %s", self.chunk_strip_docstrings)
|
log.debug("Overriding chunk_strip_docstrings from .env: %s", self.chunk_strip_docstrings)
|
||||||
|
|
||||||
|
# Staged cascade overrides
|
||||||
|
staged_stage2_mode = get_env("STAGED_STAGE2_MODE")
|
||||||
|
if staged_stage2_mode:
|
||||||
|
mode = staged_stage2_mode.strip().lower()
|
||||||
|
if mode in {"precomputed", "realtime"}:
|
||||||
|
self.staged_stage2_mode = mode
|
||||||
|
log.debug("Overriding staged_stage2_mode from .env: %s", self.staged_stage2_mode)
|
||||||
|
elif mode in {"live"}:
|
||||||
|
self.staged_stage2_mode = "realtime"
|
||||||
|
log.debug("Overriding staged_stage2_mode from .env: %s", self.staged_stage2_mode)
|
||||||
|
else:
|
||||||
|
log.warning("Invalid STAGED_STAGE2_MODE in .env: %r", staged_stage2_mode)
|
||||||
|
|
||||||
|
staged_clustering_strategy = get_env("STAGED_CLUSTERING_STRATEGY")
|
||||||
|
if staged_clustering_strategy:
|
||||||
|
strategy = staged_clustering_strategy.strip().lower()
|
||||||
|
if strategy in {"auto", "hdbscan", "dbscan", "frequency", "noop", "score", "dir_rr", "path"}:
|
||||||
|
self.staged_clustering_strategy = strategy
|
||||||
|
log.debug(
|
||||||
|
"Overriding staged_clustering_strategy from .env: %s",
|
||||||
|
self.staged_clustering_strategy,
|
||||||
|
)
|
||||||
|
elif strategy in {"none", "off"}:
|
||||||
|
self.staged_clustering_strategy = "noop"
|
||||||
|
log.debug(
|
||||||
|
"Overriding staged_clustering_strategy from .env: %s",
|
||||||
|
self.staged_clustering_strategy,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
log.warning(
|
||||||
|
"Invalid STAGED_CLUSTERING_STRATEGY in .env: %r",
|
||||||
|
staged_clustering_strategy,
|
||||||
|
)
|
||||||
|
|
||||||
|
staged_clustering_min_size = get_env("STAGED_CLUSTERING_MIN_SIZE")
|
||||||
|
if staged_clustering_min_size:
|
||||||
|
try:
|
||||||
|
self.staged_clustering_min_size = int(staged_clustering_min_size)
|
||||||
|
log.debug(
|
||||||
|
"Overriding staged_clustering_min_size from .env: %s",
|
||||||
|
self.staged_clustering_min_size,
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
log.warning(
|
||||||
|
"Invalid STAGED_CLUSTERING_MIN_SIZE in .env: %r",
|
||||||
|
staged_clustering_min_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
enable_staged_rerank = get_env("ENABLE_STAGED_RERANK")
|
||||||
|
if enable_staged_rerank:
|
||||||
|
self.enable_staged_rerank = _parse_bool(enable_staged_rerank)
|
||||||
|
log.debug("Overriding enable_staged_rerank from .env: %s", self.enable_staged_rerank)
|
||||||
|
|
||||||
|
rt_timeout = get_env("STAGED_REALTIME_LSP_TIMEOUT_S")
|
||||||
|
if rt_timeout:
|
||||||
|
try:
|
||||||
|
self.staged_realtime_lsp_timeout_s = float(rt_timeout)
|
||||||
|
log.debug(
|
||||||
|
"Overriding staged_realtime_lsp_timeout_s from .env: %s",
|
||||||
|
self.staged_realtime_lsp_timeout_s,
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
log.warning("Invalid STAGED_REALTIME_LSP_TIMEOUT_S in .env: %r", rt_timeout)
|
||||||
|
|
||||||
|
rt_depth = get_env("STAGED_REALTIME_LSP_DEPTH")
|
||||||
|
if rt_depth:
|
||||||
|
try:
|
||||||
|
self.staged_realtime_lsp_depth = int(rt_depth)
|
||||||
|
log.debug(
|
||||||
|
"Overriding staged_realtime_lsp_depth from .env: %s",
|
||||||
|
self.staged_realtime_lsp_depth,
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
log.warning("Invalid STAGED_REALTIME_LSP_DEPTH in .env: %r", rt_depth)
|
||||||
|
|
||||||
|
rt_max_nodes = get_env("STAGED_REALTIME_LSP_MAX_NODES")
|
||||||
|
if rt_max_nodes:
|
||||||
|
try:
|
||||||
|
self.staged_realtime_lsp_max_nodes = int(rt_max_nodes)
|
||||||
|
log.debug(
|
||||||
|
"Overriding staged_realtime_lsp_max_nodes from .env: %s",
|
||||||
|
self.staged_realtime_lsp_max_nodes,
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
log.warning("Invalid STAGED_REALTIME_LSP_MAX_NODES in .env: %r", rt_max_nodes)
|
||||||
|
|
||||||
|
rt_max_seeds = get_env("STAGED_REALTIME_LSP_MAX_SEEDS")
|
||||||
|
if rt_max_seeds:
|
||||||
|
try:
|
||||||
|
self.staged_realtime_lsp_max_seeds = int(rt_max_seeds)
|
||||||
|
log.debug(
|
||||||
|
"Overriding staged_realtime_lsp_max_seeds from .env: %s",
|
||||||
|
self.staged_realtime_lsp_max_seeds,
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
log.warning("Invalid STAGED_REALTIME_LSP_MAX_SEEDS in .env: %r", rt_max_seeds)
|
||||||
|
|
||||||
|
rt_max_concurrent = get_env("STAGED_REALTIME_LSP_MAX_CONCURRENT")
|
||||||
|
if rt_max_concurrent:
|
||||||
|
try:
|
||||||
|
self.staged_realtime_lsp_max_concurrent = int(rt_max_concurrent)
|
||||||
|
log.debug(
|
||||||
|
"Overriding staged_realtime_lsp_max_concurrent from .env: %s",
|
||||||
|
self.staged_realtime_lsp_max_concurrent,
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
log.warning(
|
||||||
|
"Invalid STAGED_REALTIME_LSP_MAX_CONCURRENT in .env: %r",
|
||||||
|
rt_max_concurrent,
|
||||||
|
)
|
||||||
|
|
||||||
|
rt_warmup = get_env("STAGED_REALTIME_LSP_WARMUP_S")
|
||||||
|
if rt_warmup:
|
||||||
|
try:
|
||||||
|
self.staged_realtime_lsp_warmup_s = float(rt_warmup)
|
||||||
|
log.debug(
|
||||||
|
"Overriding staged_realtime_lsp_warmup_s from .env: %s",
|
||||||
|
self.staged_realtime_lsp_warmup_s,
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
log.warning("Invalid STAGED_REALTIME_LSP_WARMUP_S in .env: %r", rt_warmup)
|
||||||
|
|
||||||
|
rt_resolve = get_env("STAGED_REALTIME_LSP_RESOLVE_SYMBOLS")
|
||||||
|
if rt_resolve:
|
||||||
|
self.staged_realtime_lsp_resolve_symbols = _parse_bool(rt_resolve)
|
||||||
|
log.debug(
|
||||||
|
"Overriding staged_realtime_lsp_resolve_symbols from .env: %s",
|
||||||
|
self.staged_realtime_lsp_resolve_symbols,
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def load(cls) -> "Config":
|
def load(cls) -> "Config":
|
||||||
"""Load config with settings from file."""
|
"""Load config with settings from file."""
|
||||||
|
|||||||
@@ -45,6 +45,22 @@ ENV_VARS = {
|
|||||||
# General configuration
|
# General configuration
|
||||||
"CODEXLENS_DATA_DIR": "Custom data directory path",
|
"CODEXLENS_DATA_DIR": "Custom data directory path",
|
||||||
"CODEXLENS_DEBUG": "Enable debug mode (true/false)",
|
"CODEXLENS_DEBUG": "Enable debug mode (true/false)",
|
||||||
|
# Cascade / staged pipeline configuration
|
||||||
|
"ENABLE_CASCADE_SEARCH": "Enable cascade search (true/false)",
|
||||||
|
"CASCADE_STRATEGY": "Cascade strategy: binary, binary_rerank, dense_rerank, staged",
|
||||||
|
"CASCADE_COARSE_K": "Cascade coarse_k candidate count (int)",
|
||||||
|
"CASCADE_FINE_K": "Cascade fine_k result count (int)",
|
||||||
|
"STAGED_STAGE2_MODE": "Staged Stage 2 mode: precomputed, realtime",
|
||||||
|
"STAGED_CLUSTERING_STRATEGY": "Staged clustering strategy: auto, score, path, dir_rr, noop, ...",
|
||||||
|
"STAGED_CLUSTERING_MIN_SIZE": "Staged clustering min cluster size (int)",
|
||||||
|
"ENABLE_STAGED_RERANK": "Enable staged reranking in Stage 4 (true/false)",
|
||||||
|
"STAGED_REALTIME_LSP_TIMEOUT_S": "Realtime LSP expansion timeout budget (float seconds)",
|
||||||
|
"STAGED_REALTIME_LSP_DEPTH": "Realtime LSP BFS depth (int)",
|
||||||
|
"STAGED_REALTIME_LSP_MAX_NODES": "Realtime LSP max nodes (int)",
|
||||||
|
"STAGED_REALTIME_LSP_MAX_SEEDS": "Realtime LSP max seeds (int)",
|
||||||
|
"STAGED_REALTIME_LSP_MAX_CONCURRENT": "Realtime LSP max concurrent requests (int)",
|
||||||
|
"STAGED_REALTIME_LSP_WARMUP_S": "Realtime LSP warmup wait after didOpen (float seconds)",
|
||||||
|
"STAGED_REALTIME_LSP_RESOLVE_SYMBOLS": "Resolve symbols via documentSymbol in realtime expansion (true/false)",
|
||||||
# Chunking configuration
|
# Chunking configuration
|
||||||
"CHUNK_STRIP_COMMENTS": "Strip comments from code chunks for embedding: true/false (default: true)",
|
"CHUNK_STRIP_COMMENTS": "Strip comments from code chunks for embedding: true/false (default: true)",
|
||||||
"CHUNK_STRIP_DOCSTRINGS": "Strip docstrings from code chunks for embedding: true/false (default: true)",
|
"CHUNK_STRIP_DOCSTRINGS": "Strip docstrings from code chunks for embedding: true/false (default: true)",
|
||||||
|
|||||||
117
codex-lens/tests/test_config_staged_env_overrides.py
Normal file
117
codex-lens/tests/test_config_staged_env_overrides.py
Normal file
@@ -0,0 +1,117 @@
|
|||||||
|
"""Unit tests for Config .env overrides for staged/cascade settings."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from codexlens.config import Config
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def temp_config_dir() -> Path:
|
||||||
|
"""Create temporary directory for config data_dir."""
|
||||||
|
tmpdir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
|
||||||
|
yield Path(tmpdir.name)
|
||||||
|
try:
|
||||||
|
tmpdir.cleanup()
|
||||||
|
except (PermissionError, OSError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def test_staged_env_overrides_apply(temp_config_dir: Path) -> None:
|
||||||
|
config = Config(data_dir=temp_config_dir)
|
||||||
|
|
||||||
|
env_path = temp_config_dir / ".env"
|
||||||
|
env_path.write_text(
|
||||||
|
"\n".join(
|
||||||
|
[
|
||||||
|
"ENABLE_CASCADE_SEARCH=true",
|
||||||
|
"CASCADE_STRATEGY=staged",
|
||||||
|
"CASCADE_COARSE_K=111",
|
||||||
|
"CASCADE_FINE_K=7",
|
||||||
|
"STAGED_STAGE2_MODE=realtime",
|
||||||
|
"STAGED_CLUSTERING_STRATEGY=path",
|
||||||
|
"STAGED_CLUSTERING_MIN_SIZE=5",
|
||||||
|
"ENABLE_STAGED_RERANK=false",
|
||||||
|
"STAGED_REALTIME_LSP_TIMEOUT_S=12.5",
|
||||||
|
"STAGED_REALTIME_LSP_DEPTH=2",
|
||||||
|
"STAGED_REALTIME_LSP_MAX_NODES=123",
|
||||||
|
"STAGED_REALTIME_LSP_MAX_SEEDS=3",
|
||||||
|
"STAGED_REALTIME_LSP_MAX_CONCURRENT=4",
|
||||||
|
"STAGED_REALTIME_LSP_WARMUP_S=0.25",
|
||||||
|
"STAGED_REALTIME_LSP_RESOLVE_SYMBOLS=yes",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
config.load_settings()
|
||||||
|
|
||||||
|
assert config.enable_cascade_search is True
|
||||||
|
assert config.cascade_strategy == "staged"
|
||||||
|
assert config.cascade_coarse_k == 111
|
||||||
|
assert config.cascade_fine_k == 7
|
||||||
|
|
||||||
|
assert config.staged_stage2_mode == "realtime"
|
||||||
|
assert config.staged_clustering_strategy == "path"
|
||||||
|
assert config.staged_clustering_min_size == 5
|
||||||
|
assert config.enable_staged_rerank is False
|
||||||
|
assert config.staged_realtime_lsp_timeout_s == 12.5
|
||||||
|
assert config.staged_realtime_lsp_depth == 2
|
||||||
|
assert config.staged_realtime_lsp_max_nodes == 123
|
||||||
|
assert config.staged_realtime_lsp_max_seeds == 3
|
||||||
|
assert config.staged_realtime_lsp_max_concurrent == 4
|
||||||
|
assert config.staged_realtime_lsp_warmup_s == 0.25
|
||||||
|
assert config.staged_realtime_lsp_resolve_symbols is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_staged_env_overrides_prefixed_wins(temp_config_dir: Path) -> None:
|
||||||
|
config = Config(data_dir=temp_config_dir)
|
||||||
|
|
||||||
|
env_path = temp_config_dir / ".env"
|
||||||
|
env_path.write_text(
|
||||||
|
"\n".join(
|
||||||
|
[
|
||||||
|
"STAGED_CLUSTERING_STRATEGY=score",
|
||||||
|
"CODEXLENS_STAGED_CLUSTERING_STRATEGY=path",
|
||||||
|
"STAGED_STAGE2_MODE=precomputed",
|
||||||
|
"CODEXLENS_STAGED_STAGE2_MODE=realtime",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
config.load_settings()
|
||||||
|
|
||||||
|
assert config.staged_clustering_strategy == "path"
|
||||||
|
assert config.staged_stage2_mode == "realtime"
|
||||||
|
|
||||||
|
|
||||||
|
def test_staged_env_overrides_invalid_ignored(temp_config_dir: Path) -> None:
|
||||||
|
config = Config(data_dir=temp_config_dir)
|
||||||
|
|
||||||
|
env_path = temp_config_dir / ".env"
|
||||||
|
env_path.write_text(
|
||||||
|
"\n".join(
|
||||||
|
[
|
||||||
|
"STAGED_STAGE2_MODE=bogus",
|
||||||
|
"STAGED_CLUSTERING_STRATEGY=embedding_remote",
|
||||||
|
"STAGED_REALTIME_LSP_TIMEOUT_S=nope",
|
||||||
|
"CASCADE_STRATEGY=???",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
config.load_settings()
|
||||||
|
|
||||||
|
assert config.cascade_strategy == "binary"
|
||||||
|
assert config.staged_stage2_mode == "precomputed"
|
||||||
|
assert config.staged_clustering_strategy == "auto"
|
||||||
|
assert config.staged_realtime_lsp_timeout_s == 30.0
|
||||||
Reference in New Issue
Block a user