catlog22/Claude-Code-Workflow
1
0
Fork 0
mirror of https://github.com/catlog22/Claude-Code-Workflow.git synced 2026-02-12 02:37:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: unified task.json schema migration and multi-module updates

Browse Source 
- Create task-schema.json (JSON Schema draft-07) with 10 field blocks fusing
  Unified JSONL, 6-field Task JSON, and Solution Schema advantages
- Migrate unified-execute-with-file from JSONL to .task/*.json directory scanning
- Migrate 3 producers (lite-plan, plan-converter, collaborative-plan) to
  .task/*.json multi-file output
- Add review-cycle Phase 7.5 export-to-tasks (FIX-*.json) and issue-resolve
  --export-tasks option
- Add schema compatibility annotations to action-planning-agent, workflow-plan,
  and tdd-plan
- Add spec-generator skill phases and templates
- Add memory v2 pipeline (consolidation, extraction, job scheduler, embedder)
- Add secret-redactor utility and core-memory enhancements
- Add codex-lens accuracy benchmarks and staged env config overrides
This commit is contained in:
catlog22
2026-02-11 17:40:56 +08:00
parent 7aa1038951
commit 99ee4e7d36
36 changed files with 7823 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files

379
.ccw/workflows/cli-templates/schemas/task-schema.json Normal file
View File

@@ -0,0 +1,379 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "task-schema.json",
"title": "Unified Task JSON Schema",
"description": "统一任务定义 schema v1.0 — 每个任务一个独立 JSON 文件，自包含所有执行所需信息。融合 Unified JSONL (convergence)、6-field Task JSON (execution_config)、Solution Schema (modification_points) 三者优势。",
"type": "object",
"required": ["id", "title", "description", "depends_on", "convergence"],
"properties": {
"_comment_IDENTITY": "IDENTITY 区块 (必填) — 任务基本信息",
"id": {
"type": "string",
"description": "任务ID，前缀由生产者决定 (TASK-001 / IMPL-001 / L0 / T1 / FIX-001 等)"
},
"title": {
"type": "string",
"description": "任务标题 (动词+目标，如 'Create unified task schema')"
},
"description": {
"type": "string",
"description": "目标+原因 (1-3 句，描述做什么和为什么)"
},
"_comment_CLASSIFICATION": "CLASSIFICATION 区块 (可选) — 任务分类",
"type": {
"type": "string",
"enum": ["infrastructure", "feature", "enhancement", "fix", "refactor", "testing", "docs", "chore"],
"description": "任务类型"
},
"priority": {
"type": "string",
"enum": ["critical", "high", "medium", "low"],
"description": "优先级"
},
"effort": {
"type": "string",
"enum": ["small", "medium", "large"],
"description": "工作量估算"
},
"action": {
"type": "string",
"enum": ["Create", "Update", "Implement", "Refactor", "Add", "Delete", "Configure", "Test", "Fix"],
"description": "操作动作 (便于 issue 系统分类)"
},
"_comment_SCOPE": "SCOPE 区块 (可选) — 覆盖范围",
"scope": {
"oneOf": [
{ "type": "string" },
{ "type": "array", "items": { "type": "string" } }
],
"description": "覆盖范围 (模块路径或功能区域)"
},
"excludes": {
"type": "array",
"items": { "type": "string" },
"description": "明确排除的范围"
},
"focus_paths": {
"type": "array",
"items": { "type": "string" },
"description": "重点文件/目录路径"
},
"_comment_DEPENDENCIES": "DEPENDENCIES 区块 (必填) — 依赖关系",
"depends_on": {
"type": "array",
"items": { "type": "string" },
"default": [],
"description": "依赖任务 ID 列表 (无依赖则 [])"
},
"parallel_group": {
"type": "number",
"description": "并行分组编号 (同组可并行执行)"
},
"_comment_CONVERGENCE": "CONVERGENCE 区块 (必填) — 完成标准",
"convergence": {
"type": "object",
"required": ["criteria"],
"properties": {
"criteria": {
"type": "array",
"items": { "type": "string" },
"minItems": 1,
"description": "可测试的完成条件 (可写为断言或手动步骤)"
},
"verification": {
"type": "string",
"description": "可执行的验证步骤 (命令或明确步骤)"
},
"definition_of_done": {
"type": "string",
"description": "业务语言完成定义 (非技术人员可判断)"
}
},
"additionalProperties": false
},
"_comment_FILES": "FILES 区块 (可选) — 文件级修改点",
"files": {
"type": "array",
"items": {
"type": "object",
"required": ["path"],
"properties": {
"path": {
"type": "string",
"description": "文件路径"
},
"action": {
"type": "string",
"enum": ["modify", "create", "delete"],
"description": "文件操作类型"
},
"target": {
"type": "string",
"description": "修改目标 (函数名/类名，来自 Solution Schema)"
},
"changes": {
"type": "array",
"items": { "type": "string" },
"description": "修改描述列表"
},
"conflict_risk": {
"type": "string",
"enum": ["low", "medium", "high"],
"description": "冲突风险等级"
}
},
"additionalProperties": false
},
"description": "涉及文件列表及修改详情"
},
"_comment_IMPLEMENTATION": "IMPLEMENTATION 区块 (可选) — 实施指南",
"implementation": {
"type": "array",
"items": { "type": "string" },
"description": "步骤化实施指南 (来自 Solution Schema)"
},
"test": {
"type": "object",
"properties": {
"commands": {
"type": "array",
"items": { "type": "string" },
"description": "测试命令"
},
"unit": {
"type": "array",
"items": { "type": "string" },
"description": "单元测试要求"
},
"integration": {
"type": "array",
"items": { "type": "string" },
"description": "集成测试要求"
},
"coverage_target": {
"type": "number",
"minimum": 0,
"maximum": 100,
"description": "覆盖率目标 (%)"
}
},
"additionalProperties": false,
"description": "测试要求"
},
"regression": {
"type": "array",
"items": { "type": "string" },
"description": "回归检查点"
},
"_comment_EXECUTION": "EXECUTION 区块 (可选) — 执行策略",
"meta": {
"type": "object",
"properties": {
"agent": {
"type": "string",
"description": "分配的 agent (@code-developer / @test-fix-agent 等)"
},
"module": {
"type": "string",
"description": "所属模块 (frontend/backend/shared)"
},
"execution_config": {
"type": "object",
"properties": {
"method": {
"type": "string",
"enum": ["agent", "cli"],
"description": "执行方式"
},
"cli_tool": {
"type": "string",
"enum": ["codex", "gemini", "qwen", "auto"],
"description": "CLI 工具选择"
},
"enable_resume": {
"type": "boolean",
"description": "是否启用会话恢复"
}
},
"additionalProperties": false
}
},
"additionalProperties": true,
"description": "执行元信息"
},
"cli_execution": {
"type": "object",
"properties": {
"id": {
"type": "string",
"description": "CLI 会话 ID (WFS-{session}-TASK-{id})"
},
"strategy": {
"type": "string",
"enum": ["new", "resume", "fork", "merge_fork"],
"description": "CLI 执行策略"
},
"resume_from": {
"type": "string",
"description": "父任务 CLI ID (用于 resume/fork)"
},
"merge_from": {
"type": "array",
"items": { "type": "string" },
"description": "合并来源 CLI ID 列表 (用于 merge_fork)"
}
},
"additionalProperties": false,
"description": "CLI 执行配置"
},
"_comment_CONTEXT": "CONTEXT 区块 (可选) — 来源与上下文",
"source": {
"type": "object",
"properties": {
"tool": {
"type": "string",
"description": "产出工具名 (workflow-plan / lite-plan / issue-resolve / review-cycle 等)"
},
"session_id": {
"type": "string",
"description": "来源 session ID"
},
"original_id": {
"type": "string",
"description": "转换前原始 ID"
},
"issue_id": {
"type": "string",
"description": "关联的 Issue ID"
}
},
"additionalProperties": false,
"description": "任务来源信息"
},
"context_package_path": {
"type": "string",
"description": "上下文包路径"
},
"evidence": {
"type": "array",
"description": "支撑证据"
},
"risk_items": {
"type": "array",
"items": { "type": "string" },
"description": "风险项"
},
"inputs": {
"type": "array",
"items": { "type": "string" },
"description": "消费的产物 (文件/资源)"
},
"outputs": {
"type": "array",
"items": { "type": "string" },
"description": "产出的产物 (文件/资源)"
},
"commit": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["feat", "fix", "refactor", "test", "docs", "chore"],
"description": "提交类型"
},
"scope": {
"type": "string",
"description": "提交范围"
},
"message_template": {
"type": "string",
"description": "提交消息模板"
}
},
"additionalProperties": false,
"description": "提交信息模板"
},
"_comment_RUNTIME": "RUNTIME 区块 (执行时填充) — 运行时状态",
"status": {
"type": "string",
"enum": ["pending", "in_progress", "completed", "failed", "skipped", "blocked"],
"default": "pending",
"description": "执行状态 (执行引擎填充)"
},
"executed_at": {
"type": "string",
"format": "date-time",
"description": "执行时间戳 (ISO 8601)"
},
"result": {
"type": "object",
"properties": {
"success": {
"type": "boolean",
"description": "是否成功"
},
"files_modified": {
"type": "array",
"items": { "type": "string" },
"description": "修改的文件列表"
},
"summary": {
"type": "string",
"description": "执行摘要"
},
"error": {
"type": "string",
"description": "错误信息 (失败时)"
},
"convergence_verified": {
"type": "array",
"items": { "type": "boolean" },
"description": "收敛标准验证结果 (对应 convergence.criteria 每项)"
},
"commit_hash": {
"type": "string",
"description": "提交哈希"
}
},
"additionalProperties": false,
"description": "执行结果 (执行引擎填充)"
}
},
"additionalProperties": true,
"_field_usage_by_producer": {
"workflow-plan": "IDENTITY + CLASSIFICATION + SCOPE + DEPENDENCIES + CONVERGENCE + FILES + EXECUTION(meta+cli_execution) + CONTEXT(context_package_path)",
"lite-plan": "IDENTITY + CLASSIFICATION + DEPENDENCIES + CONVERGENCE + FILES",
"req-plan": "IDENTITY + CLASSIFICATION + SCOPE + DEPENDENCIES + CONVERGENCE + CONTEXT(inputs/outputs/risk_items)",
"collaborative-plan": "IDENTITY + CLASSIFICATION + SCOPE + DEPENDENCIES + CONVERGENCE + FILES + CONTEXT(source)",
"issue-resolve": "IDENTITY + CLASSIFICATION + SCOPE + DEPENDENCIES + CONVERGENCE + FILES(with target) + IMPLEMENTATION + CONTEXT(commit/source)",
"review-cycle": "IDENTITY + CLASSIFICATION + FILES + CONVERGENCE + IMPLEMENTATION + CONTEXT(source/evidence)",
"tdd-plan": "IDENTITY + CLASSIFICATION + DEPENDENCIES + CONVERGENCE + EXECUTION + IMPLEMENTATION(test focused)",
"analyze-brainstorm": "IDENTITY + CLASSIFICATION + DEPENDENCIES + CONVERGENCE + CONTEXT(evidence/source)"
},
"_directory_convention": {
"standard_path": "{session_folder}/.task/",
"file_naming": "TASK-{id}.json (或保留原有 ID 前缀: IMPL-, L0-, T1-, FIX-)",
"examples": {
"workflow-plan": ".workflow/active/WFS-xxx/.task/IMPL-001.json",
"lite-plan": ".workflow/.lite-plan/{id}/.task/TASK-001.json",
"req-plan": ".workflow/.req-plan/RPLAN-{id}/.task/TASK-001.json",
"collab-plan": ".workflow/.planning/CPLAN-{id}/.task/TASK-001.json",
"review-cycle": ".workflow/active/WFS-{id}/.review/.task/FIX-001.json",
"issue-resolve": ".workflow/issues/{issue-id}/.task/TASK-001.json"
}
}
}

40
.claude/agents/action-planning-agent.md
View File

@@ -283,6 +283,34 @@ Generate individual `.task/IMPL-*.json` files with the following structure:
- `resume_from`: Parent task's cli_execution_id (for resume/fork)
- `merge_from`: Array of parent cli_execution_ids (for merge_fork)
#### Schema Compatibility
The 6-field task JSON is a **superset** of `task-schema.json` (the unified task schema at `.ccw/workflows/cli-templates/schemas/task-schema.json`). All generated `.task/IMPL-*.json` files are compatible with the unified schema via the following field mapping:
| 6-Field Task JSON (this schema) | task-schema.json (unified) | Notes |
|--------------------------------|---------------------------|-------|
| `id` | `id` | Direct mapping |
| `title` | `title` | Direct mapping |
| `status` | `status` | Direct mapping |
| `meta.type` | `type` | Flattened in unified schema |
| `meta.agent` | `meta.agent` | Same path, preserved |
| `meta.execution_config` | `meta.execution_config` | Same path, preserved |
| `context.requirements` | `description` + `implementation` | Unified schema splits into goal description and step-by-step guide |
| `context.acceptance` | `convergence.criteria` | **Key mapping**: acceptance criteria become convergence criteria |
| `context.focus_paths` | `focus_paths` | Moved to top-level in unified schema |
| `context.depends_on` | `depends_on` | Moved to top-level in unified schema |
| `context.shared_context` | _(no direct equivalent)_ | 6-field extension for tech stack and conventions |
| `context.artifacts` | `evidence` + `inputs` | Unified schema uses generic evidence/inputs arrays |
| `flow_control.target_files` | `files[].path` | Unified schema uses structured file objects |
| `flow_control.implementation_approach` | `implementation` | Unified schema uses flat string array |
| `flow_control.pre_analysis` | _(no direct equivalent)_ | 6-field extension for pre-execution analysis |
| `context_package_path` | `context_package_path` | Direct mapping |
| `cli_execution_id` | `cli_execution.id` | Nested in unified schema |
| `cli_execution` | `cli_execution` | Direct mapping |
**Backward Compatibility**: The 6-field schema retains all existing fields. The unified schema fields (`convergence`, `depends_on` at top-level, `files`, `implementation`) are accepted as **optional aliases** when present. Consumers SHOULD check both locations (e.g., `convergence.criteria` OR `context.acceptance`).
**CLI Execution Strategy Rules** (MANDATORY - apply to all tasks):
| Dependency Pattern | Strategy | CLI Command Pattern |
@@ -418,6 +446,14 @@ userConfig.executionMethod → meta.execution_config
"auth_strategy": "JWT with refresh tokens",
"conventions": ["Follow existing auth patterns in src/auth/legacy/"]
},
"convergence": {
"criteria": [
"3 features implemented: verify by npm test -- auth (exit code 0)",
"5 files created: verify by ls src/auth/*.ts | wc -l = 5"
],
"verification": "npm test -- auth && ls src/auth/*.ts | wc -l",
"definition_of_done": "Authentication module fully functional with all endpoints and tests passing"
},
"artifacts": [
{
"type": "feature_spec|cross_cutting_spec|synthesis_specification|topic_framework|individual_role_analysis",
@@ -437,6 +473,10 @@ userConfig.executionMethod → meta.execution_config
- `requirements`: **QUANTIFIED** implementation requirements (MUST include explicit counts and enumerated lists, e.g., "5 files: [list]")
- `focus_paths`: Target directories/files (concrete paths without wildcards)
- `acceptance`: **MEASURABLE** acceptance criteria (MUST include verification commands, e.g., "verify by ls ... | wc -l = N")
- `convergence`: _(Optional, unified schema alias)_ Structured completion criteria object following `task-schema.json` format. When present, `convergence.criteria` maps to `acceptance`. Use **either** `acceptance` (6-field native) **or** `convergence` (unified schema native), not both. See [Schema Compatibility](#schema-compatibility) for full mapping.
- `criteria`: Array of testable completion conditions (equivalent to `acceptance`)
- `verification`: Executable verification command or steps
- `definition_of_done`: Business-language completion definition (non-technical)
- `depends_on`: Prerequisite task IDs that must complete before this task starts
- `inherited`: Context, patterns, and dependencies passed from parent task
- `shared_context`: Tech stack, conventions, and architectural strategies for the task

81
.claude/skills/spec-generator/README.md Normal file
View File

@@ -0,0 +1,81 @@
# Spec Generator
Structured specification document generator producing a complete document chain (Product Brief -> PRD -> Architecture -> Epics).
## Usage
```bash
# Via workflow command
/workflow:spec "Build a task management system"
/workflow:spec -y "User auth with OAuth2" # Auto mode
/workflow:spec -c "task management" # Resume session
```
## Architecture
```
spec-generator/
|- SKILL.md # Entry point: metadata + architecture + flow
|- phases/
| |- 01-discovery.md # Seed analysis + codebase exploration
| |- 02-product-brief.md # Multi-CLI product brief generation
| |- 03-requirements.md # PRD with MoSCoW priorities
| |- 04-architecture.md # Architecture decisions + review
| |- 05-epics-stories.md # Epic/Story decomposition
| |- 06-readiness-check.md # Quality validation + handoff
|- specs/
| |- document-standards.md # Format, frontmatter, naming rules
| |- quality-gates.md # Per-phase quality criteria
|- templates/
| |- product-brief.md # Product brief template
| |- requirements-prd.md # PRD template
| |- architecture-doc.md # Architecture document template
| |- epics-template.md # Epic/Story template
|- README.md # This file
```
## 6-Phase Pipeline
| Phase | Name | Output | CLI Tools |
|-------|------|--------|-----------|
| 1 | Discovery | spec-config.json | Gemini (analysis) |
| 2 | Product Brief | product-brief.md | Gemini + Codex + Claude (parallel) |
| 3 | Requirements | requirements.md | Gemini (analysis) |
| 4 | Architecture | architecture.md | Gemini + Codex (sequential) |
| 5 | Epics & Stories | epics.md | Gemini (analysis) |
| 6 | Readiness Check | readiness-report.md, spec-summary.md | Gemini (validation) |
## Runtime Output
```
.workflow/.spec/SPEC-{slug}-{YYYY-MM-DD}/
|- spec-config.json # Session state
|- discovery-context.json # Codebase context (optional)
|- product-brief.md # Phase 2
|- requirements.md # Phase 3
|- architecture.md # Phase 4
|- epics.md # Phase 5
|- readiness-report.md # Phase 6
|- spec-summary.md # Phase 6
```
## Flags
- `-y|--yes`: Auto mode - skip all interactive confirmations
- `-c|--continue`: Resume from last completed phase
## Handoff
After Phase 6, choose execution path:
- `workflow:lite-plan` - Execute per Epic
- `workflow:req-plan-with-file` - Roadmap decomposition
- `workflow:plan` - Full planning
- `issue:new` - Create issues per Epic
## Design Principles
- **Document chain**: Each phase builds on previous outputs
- **Multi-perspective**: Gemini/Codex/Claude provide different viewpoints
- **Template-driven**: Consistent format via templates + frontmatter
- **Resumable**: spec-config.json tracks completed phases
- **Pure documentation**: No code generation - clean handoff to execution workflows

242
.claude/skills/spec-generator/phases/01-discovery.md Normal file
View File

@@ -0,0 +1,242 @@
# Phase 1: Discovery
Parse input, analyze the seed idea, optionally explore codebase, establish session configuration.
## Objective
- Generate session ID and create output directory
- Parse user input (text description or file reference)
- Analyze seed via Gemini CLI to extract problem space dimensions
- Conditionally explore codebase for existing patterns and constraints
- Gather user preferences (depth, focus areas) via interactive confirmation
- Write `spec-config.json` as the session state file
## Input
- Dependency: `$ARGUMENTS` (user input from command)
- Flags: `-y` (auto mode), `-c` (continue mode)
## Execution Steps
### Step 1: Session Initialization
```javascript
// Parse arguments
const args = $ARGUMENTS;
const autoMode = args.includes('-y') || args.includes('--yes');
const continueMode = args.includes('-c') || args.includes('--continue');
// Extract the idea/topic (remove flags)
const idea = args.replace(/(-y|--yes|-c|--continue)\s*/g, '').trim();
// Generate session ID
const slug = idea.toLowerCase()
.replace(/[^a-z0-9\u4e00-\u9fff]+/g, '-')
.replace(/^-|-$/g, '')
.slice(0, 40);
const date = new Date().toISOString().slice(0, 10);
const sessionId = `SPEC-${slug}-${date}`;
const workDir = `.workflow/.spec/${sessionId}`;
// Check for continue mode
if (continueMode) {
// Find existing session
const existingSessions = Glob('.workflow/.spec/SPEC-*/spec-config.json');
// If slug matches an existing session, load it and resume
// Read spec-config.json, find first incomplete phase, jump to that phase
return; // Resume logic handled by orchestrator
}
// Create output directory
Bash(`mkdir -p "${workDir}"`);
```
### Step 2: Input Parsing
```javascript
// Determine input type
if (idea.startsWith('@') || idea.endsWith('.md') || idea.endsWith('.txt')) {
// File reference - read and extract content
const filePath = idea.replace(/^@/, '');
const fileContent = Read(filePath);
// Use file content as the seed
inputType = 'file';
seedInput = fileContent;
} else {
// Direct text description
inputType = 'text';
seedInput = idea;
}
```
### Step 3: Seed Analysis via Gemini CLI
```javascript
Bash({
command: `ccw cli -p "PURPOSE: Analyze this seed idea/requirement to extract structured problem space dimensions.
Success: Clear problem statement, target users, domain identification, 3-5 exploration dimensions.
SEED INPUT:
${seedInput}
TASK:
- Extract a clear problem statement (what problem does this solve?)
- Identify target users (who benefits?)
- Determine the domain (technical, business, consumer, etc.)
- List constraints (budget, time, technical, regulatory)
- Generate 3-5 exploration dimensions (key areas to investigate)
- Assess complexity: simple (1-2 components), moderate (3-5 components), complex (6+ components)
MODE: analysis
EXPECTED: JSON output with fields: problem_statement, target_users[], domain, constraints[], dimensions[], complexity
CONSTRAINTS: Be specific and actionable, not vague
" --tool gemini --mode analysis`,
run_in_background: true
});
// Wait for CLI result before continuing
```
Parse the CLI output into structured `seedAnalysis`:
```javascript
const seedAnalysis = {
problem_statement: "...",
target_users: ["..."],
domain: "...",
constraints: ["..."],
dimensions: ["..."]
};
const complexity = "moderate"; // from CLI output
```
### Step 4: Codebase Exploration (Conditional)
```javascript
// Detect if running inside a project with code
const hasCodebase = Glob('**/*.{ts,js,py,java,go,rs}').length > 0
|| Glob('package.json').length > 0
|| Glob('Cargo.toml').length > 0;
if (hasCodebase) {
Task({
subagent_type: "cli-explore-agent",
run_in_background: false,
description: `Explore codebase for spec: ${slug}`,
prompt: `
## Spec Generator Context
Topic: ${seedInput}
Dimensions: ${seedAnalysis.dimensions.join(', ')}
Session: ${workDir}
## MANDATORY FIRST STEPS
1. Search for code related to topic keywords
2. Read project config files (package.json, pyproject.toml, etc.) if they exist
## Exploration Focus
- Identify existing implementations related to the topic
- Find patterns that could inform architecture decisions
- Map current architecture constraints
- Locate integration points and dependencies
## Output
Write findings to: ${workDir}/discovery-context.json
Schema:
{
"relevant_files": [{"path": "...", "relevance": "high|medium|low", "rationale": "..."}],
"existing_patterns": ["pattern descriptions"],
"architecture_constraints": ["constraint descriptions"],
"integration_points": ["integration point descriptions"],
"tech_stack": {"languages": [], "frameworks": [], "databases": []},
"_metadata": { "exploration_type": "spec-discovery", "timestamp": "ISO8601" }
}
`
});
}
```
### Step 5: User Confirmation (Interactive)
```javascript
if (!autoMode) {
// Confirm problem statement and select depth
AskUserQuestion({
questions: [
{
question: `Problem statement: "${seedAnalysis.problem_statement}" - Is this accurate?`,
header: "Problem",
multiSelect: false,
options: [
{ label: "Accurate", description: "Proceed with this problem statement" },
{ label: "Needs adjustment", description: "I'll refine the problem statement" }
]
},
{
question: "What specification depth do you need?",
header: "Depth",
multiSelect: false,
options: [
{ label: "Light", description: "Quick overview - key decisions only" },
{ label: "Standard (Recommended)", description: "Balanced detail for most projects" },
{ label: "Comprehensive", description: "Maximum detail for complex/critical projects" }
]
},
{
question: "Which areas should we focus on?",
header: "Focus",
multiSelect: true,
options: seedAnalysis.dimensions.map(d => ({ label: d, description: `Explore ${d} in depth` }))
}
]
});
} else {
// Auto mode defaults
depth = "standard";
focusAreas = seedAnalysis.dimensions;
}
```
### Step 6: Write spec-config.json
```javascript
const specConfig = {
session_id: sessionId,
seed_input: seedInput,
input_type: inputType,
timestamp: new Date().toISOString(),
mode: autoMode ? "auto" : "interactive",
complexity: complexity,
depth: depth,
focus_areas: focusAreas,
seed_analysis: seedAnalysis,
has_codebase: hasCodebase,
phasesCompleted: [
{
phase: 1,
name: "discovery",
output_file: "spec-config.json",
completed_at: new Date().toISOString()
}
]
};
Write(`${workDir}/spec-config.json`, JSON.stringify(specConfig, null, 2));
```
## Output
- **File**: `spec-config.json`
- **File**: `discovery-context.json` (optional, if codebase detected)
- **Format**: JSON
## Quality Checklist
- [ ] Session ID matches `SPEC-{slug}-{date}` format
- [ ] Problem statement exists and is >= 20 characters
- [ ] Target users identified (>= 1)
- [ ] 3-5 exploration dimensions generated
- [ ] spec-config.json written with all required fields
- [ ] Output directory created
## Next Phase
Proceed to [Phase 2: Product Brief](02-product-brief.md) with the generated spec-config.json.

226
.claude/skills/spec-generator/phases/02-product-brief.md Normal file
View File

@@ -0,0 +1,226 @@
# Phase 2: Product Brief
Generate a product brief through multi-perspective CLI analysis, establishing "what" and "why".
## Objective
- Read Phase 1 outputs (spec-config.json, discovery-context.json)
- Launch 3 parallel CLI analyses from product, technical, and user perspectives
- Synthesize convergent themes and conflicting views
- Optionally refine with user input
- Generate product-brief.md using template
## Input
- Dependency: `{workDir}/spec-config.json`
- Optional: `{workDir}/discovery-context.json`
- Config: `{workDir}/spec-config.json`
- Template: `templates/product-brief.md`
## Execution Steps
### Step 1: Load Phase 1 Context
```javascript
const specConfig = JSON.parse(Read(`${workDir}/spec-config.json`));
const { seed_analysis, seed_input, has_codebase, depth, focus_areas } = specConfig;
let discoveryContext = null;
if (has_codebase) {
try {
discoveryContext = JSON.parse(Read(`${workDir}/discovery-context.json`));
} catch (e) {
// No discovery context available, proceed without
}
}
// Build shared context string for CLI prompts
const sharedContext = `
SEED: ${seed_input}
PROBLEM: ${seed_analysis.problem_statement}
TARGET USERS: ${seed_analysis.target_users.join(', ')}
DOMAIN: ${seed_analysis.domain}
CONSTRAINTS: ${seed_analysis.constraints.join(', ')}
FOCUS AREAS: ${focus_areas.join(', ')}
${discoveryContext ? `
CODEBASE CONTEXT:
- Existing patterns: ${discoveryContext.existing_patterns?.slice(0,5).join(', ') || 'none'}
- Architecture constraints: ${discoveryContext.architecture_constraints?.slice(0,3).join(', ') || 'none'}
- Tech stack: ${JSON.stringify(discoveryContext.tech_stack || {})}
` : ''}`;
```
### Step 2: Multi-CLI Parallel Analysis (3 perspectives)
Launch 3 CLI calls in parallel:
**Product Perspective (Gemini)**:
```javascript
Bash({
command: `ccw cli -p "PURPOSE: Product analysis for specification - identify market fit, user value, and success criteria.
Success: Clear vision, measurable goals, competitive positioning.
${sharedContext}
TASK:
- Define product vision (1-3 sentences, aspirational)
- Analyze market/competitive landscape
- Define 3-5 measurable success metrics
- Identify scope boundaries (in-scope vs out-of-scope)
- Assess user value proposition
- List assumptions that need validation
MODE: analysis
EXPECTED: Structured product analysis with: vision, goals with metrics, scope, competitive positioning, assumptions
CONSTRAINTS: Focus on 'what' and 'why', not 'how'
" --tool gemini --mode analysis`,
run_in_background: true
});
```
**Technical Perspective (Codex)**:
```javascript
Bash({
command: `ccw cli -p "PURPOSE: Technical feasibility analysis for specification - assess implementation viability and constraints.
Success: Clear technical constraints, integration complexity, technology recommendations.
${sharedContext}
TASK:
- Assess technical feasibility of the core concept
- Identify technical constraints and blockers
- Evaluate integration complexity with existing systems
- Recommend technology approach (high-level)
- Identify technical risks and dependencies
- Estimate complexity: simple/moderate/complex
MODE: analysis
EXPECTED: Technical analysis with: feasibility assessment, constraints, integration complexity, tech recommendations, risks
CONSTRAINTS: Focus on feasibility and constraints, not detailed architecture
" --tool codex --mode analysis`,
run_in_background: true
});
```
**User Perspective (Claude)**:
```javascript
Bash({
command: `ccw cli -p "PURPOSE: User experience analysis for specification - understand user journeys, pain points, and UX considerations.
Success: Clear user personas, journey maps, UX requirements.
${sharedContext}
TASK:
- Elaborate user personas with goals and frustrations
- Map primary user journey (happy path)
- Identify key pain points in current experience
- Define UX success criteria
- List accessibility and usability considerations
- Suggest interaction patterns
MODE: analysis
EXPECTED: User analysis with: personas, journey map, pain points, UX criteria, interaction recommendations
CONSTRAINTS: Focus on user needs and experience, not implementation
" --tool claude --mode analysis`,
run_in_background: true
});
// STOP: Wait for all 3 CLI results before continuing
```
### Step 3: Synthesize Perspectives
```javascript
// After receiving all 3 CLI results:
// Extract convergent themes (all agree)
// Identify conflicting views (need resolution)
// Note unique contributions from each perspective
const synthesis = {
convergent_themes: [], // themes all 3 perspectives agree on
conflicts: [], // areas where perspectives differ
product_insights: [], // unique from product perspective
technical_insights: [], // unique from technical perspective
user_insights: [] // unique from user perspective
};
```
### Step 4: Interactive Refinement (Optional)
```javascript
if (!autoMode) {
// Present synthesis summary to user
// AskUserQuestion with:
// - Confirm vision statement
// - Resolve any conflicts between perspectives
// - Adjust scope if needed
AskUserQuestion({
questions: [
{
question: "Review the synthesized product brief. Any adjustments needed?",
header: "Review",
multiSelect: false,
options: [
{ label: "Looks good", description: "Proceed to PRD generation" },
{ label: "Adjust scope", description: "Narrow or expand the scope" },
{ label: "Revise vision", description: "Refine the vision statement" }
]
}
]
});
}
```
### Step 5: Generate product-brief.md
```javascript
// Read template
const template = Read('templates/product-brief.md');
// Fill template with synthesized content
// Apply document-standards.md formatting rules
// Write with YAML frontmatter
const frontmatter = `---
session_id: ${specConfig.session_id}
phase: 2
document_type: product-brief
status: ${autoMode ? 'complete' : 'draft'}
generated_at: ${new Date().toISOString()}
stepsCompleted: ["load-context", "multi-cli-analysis", "synthesis", "generation"]
version: 1
dependencies:
- spec-config.json
---`;
// Combine frontmatter + filled template content
Write(`${workDir}/product-brief.md`, `${frontmatter}\n\n${filledContent}`);
// Update spec-config.json
specConfig.phasesCompleted.push({
phase: 2,
name: "product-brief",
output_file: "product-brief.md",
completed_at: new Date().toISOString()
});
Write(`${workDir}/spec-config.json`, JSON.stringify(specConfig, null, 2));
```
## Output
- **File**: `product-brief.md`
- **Format**: Markdown with YAML frontmatter
## Quality Checklist
- [ ] Vision statement: clear, 1-3 sentences
- [ ] Problem statement: specific and measurable
- [ ] Target users: >= 1 persona with needs
- [ ] Goals: >= 2 with measurable metrics
- [ ] Scope: in-scope and out-of-scope defined
- [ ] Multi-perspective synthesis included
- [ ] YAML frontmatter valid
## Next Phase
Proceed to [Phase 3: Requirements](03-requirements.md) with the generated product-brief.md.

192
.claude/skills/spec-generator/specs/document-standards.md Normal file
View File

@@ -0,0 +1,192 @@
# Document Standards
Defines format conventions, YAML frontmatter schema, naming rules, and content structure for all spec-generator outputs.
## When to Use
| Phase | Usage | Section |
|-------|-------|---------|
| All Phases | Frontmatter format | YAML Frontmatter Schema |
| All Phases | File naming | Naming Conventions |
| Phase 2-5 | Document structure | Content Structure |
| Phase 6 | Validation reference | All sections |
---
## YAML Frontmatter Schema
Every generated document MUST begin with YAML frontmatter:
```yaml
---
session_id: SPEC-{slug}-{YYYY-MM-DD}
phase: {1-6}
document_type: {product-brief|requirements|architecture|epics|readiness-report|spec-summary}
status: draft|review|complete
generated_at: {ISO8601 timestamp}
stepsCompleted: []
version: 1
dependencies:
- {list of input documents used}
---
```
### Field Definitions
| Field | Type | Required | Description |
|-------|------|----------|-------------|
| `session_id` | string | Yes | Session identifier matching spec-config.json |
| `phase` | number | Yes | Phase number that generated this document (1-6) |
| `document_type` | string | Yes | One of: product-brief, requirements, architecture, epics, readiness-report, spec-summary |
| `status` | enum | Yes | draft (initial), review (user reviewed), complete (finalized) |
| `generated_at` | string | Yes | ISO8601 timestamp of generation |
| `stepsCompleted` | array | Yes | List of step IDs completed during generation |
| `version` | number | Yes | Document version, incremented on re-generation |
| `dependencies` | array | No | List of input files this document depends on |
### Status Transitions
```
draft -> review -> complete
| ^
+-------------------+ (direct promotion in auto mode)
```
- **draft**: Initial generation, not yet user-reviewed
- **review**: User has reviewed and provided feedback
- **complete**: Finalized, ready for downstream consumption
In auto mode (`-y`), documents are promoted directly from `draft` to `complete`.
---
## Naming Conventions
### Session ID Format
```
SPEC-{slug}-{YYYY-MM-DD}
```
- **slug**: Lowercase, alphanumeric + Chinese characters, hyphens as separators, max 40 chars
- **date**: UTC+8 date in YYYY-MM-DD format
Examples:
- `SPEC-task-management-system-2026-02-11`
- `SPEC-user-auth-oauth-2026-02-11`
### Output Files
| File | Phase | Description |
|------|-------|-------------|
| `spec-config.json` | 1 | Session configuration and state |
| `discovery-context.json` | 1 | Codebase exploration results (optional) |
| `product-brief.md` | 2 | Product brief document |
| `requirements.md` | 3 | PRD document |
| `architecture.md` | 4 | Architecture decisions document |
| `epics.md` | 5 | Epic/Story breakdown document |
| `readiness-report.md` | 6 | Quality validation report |
| `spec-summary.md` | 6 | One-page executive summary |
### Output Directory
```
.workflow/.spec/{session-id}/
```
---
## Content Structure
### Heading Hierarchy
- `#` (H1): Document title only (one per document)
- `##` (H2): Major sections
- `###` (H3): Subsections
- `####` (H4): Detail items (use sparingly)
Maximum depth: 4 levels. Prefer flat structures.
### Section Ordering
Every document follows this general pattern:
1. **YAML Frontmatter** (mandatory)
2. **Title** (H1)
3. **Executive Summary** (2-3 sentences)
4. **Core Content Sections** (H2, document-specific)
5. **Open Questions / Risks** (if applicable)
6. **References / Traceability** (links to upstream/downstream docs)
### Formatting Rules
| Element | Format | Example |
|---------|--------|---------|
| Requirements | `REQ-{NNN}` prefix | REQ-001: User login |
| Acceptance criteria | Checkbox list | `- [ ] User can log in with email` |
| Architecture decisions | `ADR-{NNN}` prefix | ADR-001: Use PostgreSQL |
| Epics | `EPIC-{NNN}` prefix | EPIC-001: Authentication |
| Stories | `STORY-{EPIC}-{NNN}` prefix | STORY-001-001: Login form |
| Priority tags | MoSCoW labels | `[Must]`, `[Should]`, `[Could]`, `[Won't]` |
| Mermaid diagrams | Fenced code blocks | ````mermaid ... ``` `` |
| Code examples | Language-tagged blocks | ````typescript ... ``` `` |
### Cross-Reference Format
Use relative references between documents:
```markdown
See [Product Brief](product-brief.md#section-name) for details.
Derived from [REQ-001](requirements.md#req-001).
```
### Language
- Document body: Follow user's input language (Chinese or English)
- Technical identifiers: Always English (REQ-001, ADR-001, EPIC-001)
- YAML frontmatter keys: Always English
---
## spec-config.json Schema
```json
{
"session_id": "string (required)",
"seed_input": "string (required) - original user input",
"input_type": "text|file (required)",
"timestamp": "ISO8601 (required)",
"mode": "interactive|auto (required)",
"complexity": "simple|moderate|complex (required)",
"depth": "light|standard|comprehensive (required)",
"focus_areas": ["string array"],
"seed_analysis": {
"problem_statement": "string",
"target_users": ["string array"],
"domain": "string",
"constraints": ["string array"],
"dimensions": ["string array - 3-5 exploration dimensions"]
},
"has_codebase": "boolean",
"phasesCompleted": [
{
"phase": "number (1-6)",
"name": "string (phase name)",
"output_file": "string (primary output file)",
"completed_at": "ISO8601"
}
]
}
```
---
## Validation Checklist
- [ ] Every document starts with valid YAML frontmatter
- [ ] `session_id` matches across all documents in a session
- [ ] `status` field reflects current document state
- [ ] All cross-references resolve to valid targets
- [ ] Heading hierarchy is correct (no skipped levels)
- [ ] Technical identifiers use correct prefixes
- [ ] Output files are in the correct directory

207
.claude/skills/spec-generator/specs/quality-gates.md Normal file
View File

@@ -0,0 +1,207 @@
# Quality Gates
Per-phase quality gate criteria and scoring dimensions for spec-generator outputs.
## When to Use
| Phase | Usage | Section |
|-------|-------|---------|
| Phase 2-5 | Post-generation self-check | Per-Phase Gates |
| Phase 6 | Cross-document validation | Cross-Document Validation |
| Phase 6 | Final scoring | Scoring Dimensions |
---
## Quality Thresholds
| Gate | Score | Action |
|------|-------|--------|
| **Pass** | >= 80% | Continue to next phase |
| **Review** | 60-79% | Log warnings, continue with caveats |
| **Fail** | < 60% | Must address issues before continuing |
In auto mode (`-y`), Review-level issues are logged but do not block progress.
---
## Scoring Dimensions
### 1. Completeness (25%)
All required sections present with substantive content.
| Score | Criteria |
|-------|----------|
| 100% | All template sections filled with detailed content |
| 75% | All sections present, some lack detail |
| 50% | Major sections present but minor sections missing |
| 25% | Multiple major sections missing or empty |
| 0% | Document is a skeleton only |
### 2. Consistency (25%)
Terminology, formatting, and references are uniform across documents.
| Score | Criteria |
|-------|----------|
| 100% | All terms consistent, all references valid, formatting uniform |
| 75% | Minor terminology variations, all references valid |
| 50% | Some inconsistent terms, 1-2 broken references |
| 25% | Frequent inconsistencies, multiple broken references |
| 0% | Documents contradict each other |
### 3. Traceability (25%)
Requirements, architecture decisions, and stories trace back to goals.
| Score | Criteria |
|-------|----------|
| 100% | Every story traces to a requirement, every requirement traces to a goal |
| 75% | Most items traceable, few orphans |
| 50% | Partial traceability, some disconnected items |
| 25% | Weak traceability, many orphan items |
| 0% | No traceability between documents |
### 4. Depth (25%)
Content provides sufficient detail for execution teams.
| Score | Criteria |
|-------|----------|
| 100% | Acceptance criteria specific and testable, architecture decisions justified, stories estimable |
| 75% | Most items detailed enough, few vague areas |
| 50% | Mix of detailed and vague content |
| 25% | Mostly high-level, lacking actionable detail |
| 0% | Too abstract for execution |
---
## Per-Phase Quality Gates
### Phase 1: Discovery
| Check | Criteria | Severity |
|-------|----------|----------|
| Session ID valid | Matches `SPEC-{slug}-{date}` format | Error |
| Problem statement exists | Non-empty, >= 20 characters | Error |
| Target users identified | >= 1 user group | Error |
| Dimensions generated | 3-5 exploration dimensions | Warning |
| Constraints listed | >= 0 (can be empty with justification) | Info |
### Phase 2: Product Brief
| Check | Criteria | Severity |
|-------|----------|----------|
| Vision statement | Clear, 1-3 sentences | Error |
| Problem statement | Specific and measurable | Error |
| Target users | >= 1 persona with needs described | Error |
| Goals defined | >= 2 measurable goals | Error |
| Success metrics | >= 2 quantifiable metrics | Warning |
| Scope boundaries | In-scope and out-of-scope listed | Warning |
| Multi-perspective | >= 2 CLI perspectives synthesized | Info |
### Phase 3: Requirements (PRD)
| Check | Criteria | Severity |
|-------|----------|----------|
| Functional requirements | >= 3 with REQ-NNN IDs | Error |
| Acceptance criteria | Every requirement has >= 1 criterion | Error |
| MoSCoW priority | Every requirement tagged | Error |
| Non-functional requirements | >= 1 (performance, security, etc.) | Warning |
| User stories | >= 1 per Must-have requirement | Warning |
| Traceability | Requirements trace to product brief goals | Warning |
### Phase 4: Architecture
| Check | Criteria | Severity |
|-------|----------|----------|
| Component diagram | Present (Mermaid or ASCII) | Error |
| Tech stack specified | Languages, frameworks, key libraries | Error |
| ADR present | >= 1 Architecture Decision Record | Error |
| ADR has alternatives | Each ADR lists >= 2 options considered | Warning |
| Integration points | External systems/APIs identified | Warning |
| Data model | Key entities and relationships described | Warning |
| Codebase mapping | Mapped to existing code (if has_codebase) | Info |
### Phase 5: Epics & Stories
| Check | Criteria | Severity |
|-------|----------|----------|
| Epics defined | 3-7 epics with EPIC-NNN IDs | Error |
| MVP subset | >= 1 epic tagged as MVP | Error |
| Stories per epic | 2-5 stories per epic | Error |
| Story format | "As a...I want...So that..." pattern | Warning |
| Dependency map | Cross-epic dependencies documented | Warning |
| Estimation hints | Relative sizing (S/M/L/XL) per story | Info |
| Traceability | Stories trace to requirements | Warning |
### Phase 6: Readiness Check
| Check | Criteria | Severity |
|-------|----------|----------|
| All documents exist | product-brief, requirements, architecture, epics | Error |
| Frontmatter valid | All YAML frontmatter parseable and correct | Error |
| Cross-references valid | All document links resolve | Error |
| Overall score >= 60% | Weighted average across 4 dimensions | Error |
| No unresolved Errors | All Error-severity issues addressed | Error |
| Summary generated | spec-summary.md created | Warning |
---
## Cross-Document Validation
Checks performed during Phase 6 across all documents:
### Completeness Matrix
```
Product Brief goals -> Requirements (each goal has >= 1 requirement)
Requirements -> Architecture (each Must requirement has design coverage)
Requirements -> Epics (each Must requirement appears in >= 1 story)
Architecture ADRs -> Epics (tech choices reflected in implementation stories)
```
### Consistency Checks
| Check | Documents | Rule |
|-------|-----------|------|
| Terminology | All | Same term used consistently (no synonyms for same concept) |
| User personas | Brief + PRD + Epics | Same user names/roles throughout |
| Scope | Brief + PRD | PRD scope does not exceed brief scope |
| Tech stack | Architecture + Epics | Stories reference correct technologies |
### Traceability Matrix Format
```markdown
| Goal | Requirements | Architecture | Epics |
|------|-------------|--------------|-------|
| G-001: ... | REQ-001, REQ-002 | ADR-001 | EPIC-001 |
| G-002: ... | REQ-003 | ADR-002 | EPIC-002, EPIC-003 |
```
---
## Issue Classification
### Error (Must Fix)
- Missing required document or section
- Broken cross-references
- Contradictory information between documents
- Empty acceptance criteria on Must-have requirements
- No MVP subset defined in epics
### Warning (Should Fix)
- Vague acceptance criteria
- Missing non-functional requirements
- No success metrics defined
- Incomplete traceability
- Missing architecture review notes
### Info (Nice to Have)
- Could add more detailed personas
- Consider additional ADR alternatives
- Story estimation hints missing
- Mermaid diagrams could be more detailed

133
.claude/skills/spec-generator/templates/product-brief.md Normal file
View File

@@ -0,0 +1,133 @@
# Product Brief Template
Template for generating product brief documents in Phase 2.
## Usage Context
| Phase | Usage |
|-------|-------|
| Phase 2 (Product Brief) | Generate product-brief.md from multi-CLI analysis |
| Output Location | `{workDir}/product-brief.md` |
---
## Template
```markdown
---
session_id: {session_id}
phase: 2
document_type: product-brief
status: draft
generated_at: {timestamp}
stepsCompleted: []
version: 1
dependencies:
- spec-config.json
---
# Product Brief: {product_name}
{executive_summary - 2-3 sentences capturing the essence of the product/feature}
## Vision
{vision_statement - clear, aspirational 1-3 sentence statement of what success looks like}
## Problem Statement
### Current Situation
{description of the current state and pain points}
### Impact
{quantified impact of the problem - who is affected, how much, how often}
## Target Users
{for each user persona:}
### {Persona Name}
- **Role**: {user's role/context}
- **Needs**: {primary needs related to this product}
- **Pain Points**: {current frustrations}
- **Success Criteria**: {what success looks like for this user}
## Goals & Success Metrics
| Goal ID | Goal | Success Metric | Target |
|---------|------|----------------|--------|
| G-001 | {goal description} | {measurable metric} | {specific target} |
| G-002 | {goal description} | {measurable metric} | {specific target} |
## Scope
### In Scope
- {feature/capability 1}
- {feature/capability 2}
- {feature/capability 3}
### Out of Scope
- {explicitly excluded item 1}
- {explicitly excluded item 2}
### Assumptions
- {key assumption 1}
- {key assumption 2}
## Competitive Landscape
| Aspect | Current State | Proposed Solution | Advantage |
|--------|--------------|-------------------|-----------|
| {aspect} | {how it's done now} | {our approach} | {differentiator} |
## Constraints & Dependencies
### Technical Constraints
- {constraint 1}
- {constraint 2}
### Business Constraints
- {constraint 1}
### Dependencies
- {external dependency 1}
- {external dependency 2}
## Multi-Perspective Synthesis
### Product Perspective
{summary of product/market analysis findings}
### Technical Perspective
{summary of technical feasibility and constraints}
### User Perspective
{summary of user journey and UX considerations}
### Convergent Themes
{themes where all perspectives agree}
### Conflicting Views
{areas where perspectives differ, with notes on resolution approach}
## Open Questions
- [ ] {unresolved question 1}
- [ ] {unresolved question 2}
## References
- Derived from: [spec-config.json](spec-config.json)
- Next: [Requirements PRD](requirements.md)
```
## Variable Descriptions
| Variable | Source | Description |
|----------|--------|-------------|
| `{session_id}` | spec-config.json | Session identifier |
| `{timestamp}` | Runtime | ISO8601 generation timestamp |
| `{product_name}` | Seed analysis | Product/feature name |
| `{executive_summary}` | CLI synthesis | 2-3 sentence summary |
| `{vision_statement}` | CLI product perspective | Aspirational vision |
| All `{...}` fields | CLI analysis outputs | Filled from multi-perspective analysis |

89
.codex/skills/collaborative-plan-with-file/SKILL.md
View File

@@ -55,11 +55,11 @@ The key innovation is the **Plan Note** architecture — a shared collaborative
│ │
│ Phase 2: Serial Domain Planning │
│ ┌──────────────┐ │
│ │ Domain 1 │→ Explore codebase → Generate tasks.jsonl │
│ │ Domain 1 │→ Explore codebase → Generate .task/TASK-*.json │
│ │ Section 1 │→ Fill task pool + evidence in plan-note.md │
│ └──────┬───────┘ │
│ ┌──────▼───────┐ │
│ │ Domain 2 │→ Explore codebase → Generate tasks.jsonl │
│ │ Domain 2 │→ Explore codebase → Generate .task/TASK-*.json │
│ │ Section 2 │→ Fill task pool + evidence in plan-note.md │
│ └──────┬───────┘ │
│ ┌──────▼───────┐ │
@@ -72,7 +72,7 @@ The key innovation is the **Plan Note** architecture — a shared collaborative
│ └─ Update plan-note.md conflict section │
│ │
│ Phase 4: Completion (No Merge) │
│ ├─ Merge domain tasks.jsonl → session tasks.jsonl │
│ ├─ Collect domain .task/*.json → session .task/*.json │
│ ├─ Generate plan.md (human-readable) │
│ └─ Ready for execution │
│ │
@@ -81,17 +81,26 @@ The key innovation is the **Plan Note** architecture — a shared collaborative
## Output Structure
> **Schema**: `cat ~/.ccw/workflows/cli-templates/schemas/task-schema.json`
```
{projectRoot}/.workflow/.planning/CPLAN-{slug}-{date}/
├── plan-note.md # ⭐ Core: Requirements + Tasks + Conflicts
├── requirement-analysis.json # Phase 1: Sub-domain assignments
├── domains/ # Phase 2: Per-domain plans
│ ├── {domain-1}/
│ │ └── tasks.jsonl # Unified JSONL (one task per line)
│ │ └── .task/ # Per-domain task JSON files
│ │ ├── TASK-001.json
│ │ └── ...
│ ├── {domain-2}/
│ │ └── tasks.jsonl
│ │ └── .task/
│ │ ├── TASK-101.json
│ │ └── ...
│ └── ...
├── .task/ # ⭐ Merged task JSON files (all domains)
│ ├── TASK-001.json
│ ├── TASK-101.json
│ └── ...
├── tasks.jsonl # ⭐ Merged unified JSONL (all domains)
├── conflicts.json # Phase 3: Conflict report
└── plan.md # Phase 4: Human-readable summary
```
@@ -109,7 +118,7 @@ The key innovation is the **Plan Note** architecture — a shared collaborative
| Artifact | Purpose |
|----------|---------|
| `domains/{domain}/tasks.jsonl` | Unified JSONL per domain (one task per line with convergence) |
| `domains/{domain}/.task/TASK-*.json` | Task JSON files per domain (one file per task with convergence) |
| Updated `plan-note.md` | Task pool and evidence sections filled for each domain |
### Phase 3: Conflict Detection
@@ -123,7 +132,7 @@ The key innovation is the **Plan Note** architecture — a shared collaborative
| Artifact | Purpose |
|----------|---------|
| `tasks.jsonl` | Merged unified JSONL from all domains (consumable by unified-execute) |
| `.task/TASK-*.json` | Merged task JSON files from all domains (consumable by unified-execute) |
| `plan.md` | Human-readable summary with requirements, tasks, and conflicts |
---
@@ -294,8 +303,8 @@ For each sub-domain, execute the full planning cycle inline:
```javascript
for (const sub of subDomains) {
// 1. Create domain directory
Bash(`mkdir -p ${sessionFolder}/domains/${sub.focus_area}`)
// 1. Create domain directory with .task/ subfolder
Bash(`mkdir -p ${sessionFolder}/domains/${sub.focus_area}/.task`)
// 2. Explore codebase for domain-relevant context
// Use: mcp__ace-tool__search_context, Grep, Glob, Read
@@ -305,7 +314,7 @@ for (const sub of subDomains) {
// - Integration points with other domains
// - Architecture constraints
// 3. Generate unified JSONL tasks (one task per line)
// 3. Generate task JSON records (following task-schema.json)
const domainTasks = [
// For each task within the assigned ID range:
{
@@ -339,9 +348,11 @@ for (const sub of subDomains) {
// ... more tasks
]
// 4. Write domain tasks.jsonl (one task per line)
const jsonlContent = domainTasks.map(t => JSON.stringify(t)).join('\n')
Write(`${sessionFolder}/domains/${sub.focus_area}/tasks.jsonl`, jsonlContent)
// 4. Write individual task JSON files (one per task)
domainTasks.forEach(task => {
Write(`${sessionFolder}/domains/${sub.focus_area}/.task/${task.id}.json`,
JSON.stringify(task, null, 2))
})
// 5. Sync summary to plan-note.md
// Read current plan-note.md
@@ -399,7 +410,7 @@ After all domains are planned, verify the shared document.
5. Check for any section format inconsistencies
**Success Criteria**:
- `domains/{domain}/tasks.jsonl` created for each domain (unified JSONL format)
- `domains/{domain}/.task/TASK-*.json` created for each domain (one file per task)
- Each task has convergence (criteria + verification + definition_of_done)
- `plan-note.md` updated with all task pools and evidence sections
- Task summaries follow consistent format
@@ -413,7 +424,7 @@ After all domains are planned, verify the shared document.
### Step 3.1: Parse plan-note.md
Extract all tasks from all "任务池" sections and domain tasks.jsonl files.
Extract all tasks from all "任务池" sections and domain .task/*.json files.
```javascript
// parsePlanNote(markdown)
@@ -422,13 +433,14 @@ Extract all tasks from all "任务池" sections and domain tasks.jsonl files.
// - Build sections array: { level, heading, start, content }
// - Return: { frontmatter, sections }
// Also load all domain tasks.jsonl for detailed data
// Also load all domain .task/*.json for detailed data
// loadDomainTasks(sessionFolder, subDomains):
// const allTasks = []
// for (const sub of subDomains) {
// const jsonl = Read(`${sessionFolder}/domains/${sub.focus_area}/tasks.jsonl`)
// jsonl.split('\n').filter(l => l.trim()).forEach(line => {
// allTasks.push(JSON.parse(line))
// const taskDir = `${sessionFolder}/domains/${sub.focus_area}/.task`
// const taskFiles = Glob(`${taskDir}/TASK-*.json`)
// taskFiles.forEach(file => {
// allTasks.push(JSON.parse(Read(file)))
// })
// }
// return allTasks
@@ -543,23 +555,24 @@ Write(`${sessionFolder}/conflicts.json`, JSON.stringify({
**Objective**: Generate human-readable plan summary and finalize workflow.
### Step 4.1: Merge Domain tasks.jsonl
### Step 4.1: Collect Domain .task/*.json to Session .task/
Merge all per-domain JSONL files into a single session-level `tasks.jsonl`.
Copy all per-domain task JSON files into a single session-level `.task/` directory.
```javascript
// Collect all domain tasks
const allDomainTasks = []
// Create session-level .task/ directory
Bash(`mkdir -p ${sessionFolder}/.task`)
// Collect all domain task files
for (const sub of subDomains) {
const jsonl = Read(`${sessionFolder}/domains/${sub.focus_area}/tasks.jsonl`)
jsonl.split('\n').filter(l => l.trim()).forEach(line => {
allDomainTasks.push(JSON.parse(line))
const taskDir = `${sessionFolder}/domains/${sub.focus_area}/.task`
const taskFiles = Glob(`${taskDir}/TASK-*.json`)
taskFiles.forEach(file => {
const filename = path.basename(file)
// Copy domain task file to session .task/ directory
Bash(`cp ${file} ${sessionFolder}/.task/${filename}`)
})
}
// Write merged tasks.jsonl at session root
const mergedJsonl = allDomainTasks.map(t => JSON.stringify(t)).join('\n')
Write(`${sessionFolder}/tasks.jsonl`, mergedJsonl)
```
### Step 4.2: Generate plan.md
@@ -613,7 +626,7 @@ ${allConflicts.length === 0
## 执行
\`\`\`bash
/workflow:unified-execute-with-file PLAN="${sessionFolder}/tasks.jsonl"
/workflow:unified-execute-with-file PLAN="${sessionFolder}/.task/"
\`\`\`
**Session artifacts**: \`${sessionFolder}/\`
@@ -652,14 +665,14 @@ if (!autoMode) {
| Selection | Action |
|-----------|--------|
| Execute Plan | `Skill(skill="workflow:unified-execute-with-file", args="PLAN=\"${sessionFolder}/tasks.jsonl\"")` |
| Execute Plan | `Skill(skill="workflow:unified-execute-with-file", args="PLAN=\"${sessionFolder}/.task/\"")` |
| Review Conflicts | Display conflicts.json content for manual resolution |
| Export | Copy plan.md + plan-note.md to user-specified location |
| Done | Display artifact paths, end workflow |
**Success Criteria**:
- `plan.md` generated with complete summary
- `tasks.jsonl` merged at session root (consumable by unified-execute)
- `.task/TASK-*.json` collected at session root (consumable by unified-execute)
- All artifacts present in session directory
- User informed of completion and next steps
@@ -685,11 +698,11 @@ User initiates: TASK="task description"
├─ Generate requirement-analysis.json
│
├─ Serial domain planning:
│ ├─ Domain 1: explore → tasks.jsonl → fill plan-note.md
│ ├─ Domain 2: explore → tasks.jsonl → fill plan-note.md
│ ├─ Domain 1: explore → .task/TASK-*.json → fill plan-note.md
│ ├─ Domain 2: explore → .task/TASK-*.json → fill plan-note.md
│ └─ Domain N: ...
│
├─ Merge domain tasks.jsonl → session tasks.jsonl
├─ Collect domain .task/*.json → session .task/
│
├─ Verify plan-note.md consistency
├─ Detect conflicts
@@ -738,7 +751,7 @@ User resumes: TASK="same task"
1. **Review Plan Note**: Check plan-note.md between domains to verify progress
2. **Verify Independence**: Ensure sub-domains are truly independent and have minimal overlap
3. **Check Dependencies**: Cross-domain dependencies should be documented explicitly
4. **Inspect Details**: Review `domains/{domain}/tasks.jsonl` for specifics when needed
4. **Inspect Details**: Review `domains/{domain}/.task/TASK-*.json` for specifics when needed
5. **Consistent Format**: Follow task summary format strictly across all domains
6. **TASK ID Isolation**: Use pre-assigned non-overlapping ranges to prevent ID conflicts

32
.codex/skills/issue-resolve/SKILL.md
View File

@@ -1,6 +1,6 @@
---
name: issue-resolve
description: Unified issue resolution pipeline with source selection. Plan issues via AI exploration, convert from artifacts, import from brainstorm sessions, or form execution queues. Triggers on "issue:plan", "issue:queue", "issue:convert-to-plan", "issue:from-brainstorm", "resolve issue", "plan issue", "queue issues", "convert plan to issue".
description: Unified issue resolution pipeline with source selection. Plan issues via AI exploration, convert from artifacts, import from brainstorm sessions, form execution queues, or export solutions to task JSON. Triggers on "issue:plan", "issue:queue", "issue:convert-to-plan", "issue:from-brainstorm", "export-to-tasks", "resolve issue", "plan issue", "queue issues", "convert plan to issue".
allowed-tools: spawn_agent, wait, send_input, close_agent, AskUserQuestion, Read, Write, Edit, Bash, Glob, Grep
---
@@ -28,6 +28,10 @@ Unified issue resolution pipeline that orchestrates solution creation from multi
↓ ↓ ↓ ↓ │
Solutions Solutions Issue+Sol Exec Queue │
(bound) (bound) (bound) (ordered) │
│ │ │ │
└─────┬─────┘───────────┘ │
↓ (optional --export-tasks) │
.task/TASK-*.json │
│
┌────────────────────────────────┘
↓
@@ -119,6 +123,7 @@ codex -p "@.codex/prompts/issue-resolve.md [FLAGS] \"<input>\""
--issue <id> Bind to existing issue (convert mode)
--supplement Add tasks to existing solution (convert mode)
--queues <n> Number of parallel queues (queue mode, default: 1)
--export-tasks Export solution tasks to .task/TASK-*.json (task-schema.json format)
# Examples
codex -p "@.codex/prompts/issue-resolve.md GH-123,GH-124" # Explore & plan issues
@@ -151,6 +156,8 @@ Phase Execution (load one phase):
└─ Phase 4: Form Queue → phases/04-issue-queue.md
Post-Phase:
├─ Export to Task JSON (optional, with --export-tasks flag)
│ └─ For each solution.tasks[] → write .task/TASK-{T-id}.json
└─ Summary + Next steps recommendation
```
@@ -263,6 +270,29 @@ User Input (issue IDs / artifact path / session ID / flags)
[Summary + Next Steps]
├─ After Plan/Convert/Brainstorm → Suggest /issue:queue or /issue:execute
└─ After Queue → Suggest /issue:execute
(Optional) Export to Task JSON (when --export-tasks flag is set):
├─ For each solution.tasks[] entry:
│ ├─ solution.task.id → id (prefixed as TASK-{T-id})
│ ├─ solution.task.title → title
│ ├─ solution.task.description → description
│ ├─ solution.task.action → action
│ ├─ solution.task.scope → scope
│ ├─ solution.task.modification_points[] → files[]
│ │ ├─ mp.file → files[].path
│ │ ├─ mp.target → files[].target
│ │ └─ mp.change → files[].changes[]
│ ├─ solution.task.acceptance → convergence
│ │ ├─ acceptance.criteria[] → convergence.criteria[]
│ │ └─ acceptance.verification[]→ convergence.verification (joined)
│ ├─ solution.task.implementation → implementation[]
│ ├─ solution.task.test → test
│ ├─ solution.task.depends_on → depends_on
│ ├─ solution.task.commit → commit
│ └─ solution.task.priority → priority (1→critical, 2→high, 3→medium, 4-5→low)
├─ Output path: .workflow/issues/{issue-id}/.task/TASK-{T-id}.json
├─ Each file follows task-schema.json (IDENTITY + CONVERGENCE + FILES required)
└─ source.tool = "issue-resolve", source.issue_id = {issue-id}
```
## Task Tracking Pattern

105
.codex/skills/plan-converter/SKILL.md
View File

@@ -1,6 +1,6 @@
---
name: plan-converter
description: Convert any planning/analysis/brainstorm output to unified JSONL task format. Supports roadmap.jsonl, plan.json, plan-note.md, conclusions.json, synthesis.json.
description: Convert any planning/analysis/brainstorm output to .task/*.json multi-file format. Supports roadmap.jsonl, plan.json, plan-note.md, conclusions.json, synthesis.json.
argument-hint: "<input-file> [-o <output-file>]"
---
@@ -8,70 +8,46 @@ argument-hint: "<input-file> [-o <output-file>]"
## Overview
Converts any planning artifact to **unified JSONL task format** — the single standard consumed by `unified-execute-with-file`.
Converts any planning artifact to **`.task/*.json` multi-file format** — the single standard consumed by `unified-execute-with-file`.
> **Schema**: `cat ~/.ccw/workflows/cli-templates/schemas/task-schema.json`
```bash
# Auto-detect format, output to same directory
# Auto-detect format, output to same directory .task/
/codex:plan-converter ".workflow/.req-plan/RPLAN-auth-2025-01-21/roadmap.jsonl"
# Specify output path
/codex:plan-converter ".workflow/.planning/CPLAN-xxx/plan-note.md" -o tasks.jsonl
# Specify output directory
/codex:plan-converter ".workflow/.planning/CPLAN-xxx/plan-note.md" -o .task/
# Convert brainstorm synthesis
/codex:plan-converter ".workflow/.brainstorm/BS-xxx/synthesis.json"
```
**Supported inputs**: roadmap.jsonl, tasks.jsonl (per-domain), plan-note.md, conclusions.json, synthesis.json
**Supported inputs**: roadmap.jsonl, .task/*.json (per-domain), plan-note.md, conclusions.json, synthesis.json
**Output**: Unified JSONL (`tasks.jsonl` in same directory, or specified `-o` path)
**Output**: `.task/*.json` (one file per task, in same directory's `.task/` subfolder, or specified `-o` path)
## Unified JSONL Schema
## Task JSON Schema
每行一条自包含任务记录：
每个任务一个独立 JSON 文件 (`.task/TASK-{id}.json`)，遵循统一 schema：
> **Schema 定义**: `cat ~/.ccw/workflows/cli-templates/schemas/task-schema.json`
**Producer 使用的字段集** (plan-converter 输出):
```
┌─ IDENTITY (必填) ──────────────────────────────────────────┐
│ id string 任务 ID (L0/T1/TASK-001) │
│ title string 任务标题 │
│ description string 目标 + 原因 │
├─ CLASSIFICATION (可选) ────────────────────────────────────┤
│ type enum infrastructure|feature|enhancement│
│ enum |fix|refactor|testing │
│ priority enum high|medium|low │
│ effort enum small|medium|large │
├─ SCOPE (可选) ─────────────────────────────────────────────┤
│ scope string|[] 覆盖范围 │
│ excludes string[] 明确排除 │
├─ DEPENDENCIES (depends_on 必填) ───────────────────────────┤
│ depends_on string[] 依赖任务 ID（无依赖则 []） │
│ parallel_group number 并行分组（同组可并行） │
│ inputs string[] 消费的产物 │
│ outputs string[] 产出的产物 │
├─ CONVERGENCE (必填) ───────────────────────────────────────┤
│ convergence.criteria string[] 可测试的完成条件 │
│ convergence.verification string 可执行的验证步骤 │
│ convergence.definition_of_done string 业务语言完成定义 │
├─ FILES (可选，渐进详细) ───────────────────────────────────┤
│ files[].path string 文件路径 (必填*) │
│ files[].action enum modify|create|delete │
│ files[].changes string[] 修改描述 │
│ files[].conflict_risk enum low|medium|high │
├─ CONTEXT (可选) ───────────────────────────────────────────┤
│ source.tool string 产出工具名 │
│ source.session_id string 来源 session │
│ source.original_id string 转换前原始 ID │
│ evidence any[] 支撑证据 │
│ risk_items string[] 风险项 │
├─ EXECUTION (执行时填充，规划时不存在) ─────────────────────┤
│ _execution.status enum pending|in_progress| │
│ completed|failed|skipped │
│ _execution.executed_at string ISO 时间戳 │
│ _execution.result object { success, files_modified, │
│ summary, error, │
│ convergence_verified[] } │
└────────────────────────────────────────────────────────────┘
IDENTITY (必填): id, title, description
CLASSIFICATION (可选): type, priority, effort
SCOPE (可选): scope, excludes
DEPENDENCIES (必填): depends_on, parallel_group, inputs, outputs
CONVERGENCE (必填): convergence.criteria, convergence.verification, convergence.definition_of_done
FILES (可选): files[].path, files[].action, files[].changes, files[].conflict_risk
CONTEXT (可选): source.tool, source.session_id, source.original_id, evidence, risk_items
RUNTIME (执行时填充): status, executed_at, result
```
**文件命名**: `TASK-{id}.json` (保留原有 ID 前缀: L0-, T1-, IDEA- 等)
## Target Input
**$ARGUMENTS**
@@ -82,9 +58,9 @@ Converts any planning artifact to **unified JSONL task format** — the single s
Step 0: Parse arguments, resolve input path
Step 1: Detect input format
Step 2: Parse input → extract raw records
Step 3: Transform → unified JSONL records
Step 3: Transform → unified task records
Step 4: Validate convergence quality
Step 5: Write output + display summary
Step 5: Write .task/*.json output + display summary
```
## Implementation
@@ -110,7 +86,7 @@ const content = Read(resolvedInput)
function detectFormat(filename, content) {
if (filename === 'roadmap.jsonl') return 'roadmap-jsonl'
if (filename === 'tasks.jsonl') return 'tasks-jsonl' // already unified or per-domain
if (filename === 'tasks.jsonl') return 'tasks-jsonl' // legacy JSONL or per-domain
if (filename === 'plan-note.md') return 'plan-note-md'
if (filename === 'conclusions.json') return 'conclusions-json'
if (filename === 'synthesis.json') return 'synthesis-json'
@@ -132,7 +108,7 @@ function detectFormat(filename, content) {
| Filename | Format ID | Source Tool |
|----------|-----------|------------|
| `roadmap.jsonl` | roadmap-jsonl | req-plan-with-file |
| `tasks.jsonl` (per-domain) | tasks-jsonl | collaborative-plan-with-file |
| `tasks.jsonl` (legacy) / `.task/*.json` | tasks-jsonl / task-json | collaborative-plan-with-file |
| `plan-note.md` | plan-note-md | collaborative-plan-with-file |
| `conclusions.json` | conclusions-json | analyze-with-file |
| `synthesis.json` | synthesis-json | brainstorm-with-file |
@@ -394,12 +370,15 @@ function validateConvergence(records) {
// | Technical DoD | Rewrite in business language |
```
### Step 5: Write Output & Summary
### Step 5: Write .task/*.json Output & Summary
```javascript
// Determine output path
const outputFile = outputPath
|| `${path.dirname(resolvedInput)}/tasks.jsonl`
// Determine output directory
const outputDir = outputPath
|| `${path.dirname(resolvedInput)}/.task`
// Create output directory
Bash(`mkdir -p ${outputDir}`)
// Clean records: remove undefined/null optional fields
const cleanedRecords = records.map(rec => {
@@ -411,16 +390,18 @@ const cleanedRecords = records.map(rec => {
return clean
})
// Write JSONL
const jsonlContent = cleanedRecords.map(r => JSON.stringify(r)).join('\n')
Write(outputFile, jsonlContent)
// Write individual task JSON files
cleanedRecords.forEach(record => {
const filename = `${record.id}.json`
Write(`${outputDir}/${filename}`, JSON.stringify(record, null, 2))
})
// Display summary
// | Source | Format | Records | Issues |
// |-----------------|-------------------|---------|--------|
// | roadmap.jsonl | roadmap-jsonl | 4 | 0 |
//
// Output: .workflow/.req-plan/RPLAN-xxx/tasks.jsonl
// Output: .workflow/.req-plan/RPLAN-xxx/.task/ (4 files)
// Records: 4 tasks with convergence criteria
// Quality: All convergence checks passed
```
@@ -433,7 +414,7 @@ Write(outputFile, jsonlContent)
|--------|-----------|------------|-----------------|-----------|--------------|-----------|
| roadmap.jsonl (progressive) | req-plan | L0-L3 | **Yes** | No | No | **Yes** |
| roadmap.jsonl (direct) | req-plan | T1-TN | **Yes** | No | No | **Yes** |
| tasks.jsonl (per-domain) | collaborative-plan | TASK-NNN | **Yes** | **Yes** (detailed) | Optional | **Yes** |
| .task/TASK-*.json (per-domain) | collaborative-plan | TASK-NNN | **Yes** | **Yes** (detailed) | Optional | **Yes** |
| plan-note.md | collaborative-plan | TASK-NNN | **Generate** | **Yes** (from 修改文件) | From effort | No |
| conclusions.json | analyze | TASK-NNN | **Generate** | No | **Yes** | No |
| synthesis.json | brainstorm | IDEA-NNN | **Generate** | No | From score | No |

49
.codex/skills/review-cycle/SKILL.md
View File

@@ -34,14 +34,14 @@ Unified multi-dimensional code review orchestrator with dual-mode (session/modul
┌─────────────────────────────┼─────────────────────────────────┐
│ Fix Pipeline (Phase 6-9) │
│ │
│ ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌─────────┐
│ │ Phase 6 │→ │ Phase 7 │→ │ Phase 8 │→ │ Phase 9 │
│ │Discovery│ │Parallel │ │Execution│ │Complete │
│ │Batching │ │Planning │ │Orchestr.│ │ │
│ └─────────┘ └─────────┘ └─────────┘ └─────────┘
│ grouping N agents M agents aggregate
│ + batch ×cli-plan ×cli-exec + summary
└────────────────────────────────────────────────────────────────┘
│ ┌─────────┐ ┌─────────┐ ┌──────────┐ ┌─────────┐ ┌─────────┐
│ │ Phase 6 │→ │ Phase 7 │→ │Phase 7.5 │→ │ Phase 8 │→ │ Phase 9 │
│ │Discovery│ │Parallel │ │Export to │ │Execution│ │Complete │
│ │Batching │ │Planning │ │Task JSON │ │Orchestr.│ │ │
│ └─────────┘ └─────────┘ └──────────┘ └─────────┘ └─────────┘
│ grouping N agents fix-plan → M agents aggregate
│ + batch ×cli-plan .task/FIX-* ×cli-exec + summary
└────────────────────────────────────────────────────────────────────┘
```
## Key Design Principles
@@ -73,6 +73,7 @@ review-cycle --fix <review-dir> [FLAGS] # Fix with flags
--fix Enter fix pipeline after review or standalone
--resume Resume interrupted fix session
--batch-size=N Findings per planning batch (default: 5, fix mode only)
--export-tasks Export fix-plan findings to .task/FIX-*.json (auto-enabled with --fix)
# Examples
review-cycle src/auth/** # Module: review auth
@@ -159,6 +160,20 @@ Phase 7: Fix Parallel Planning
├─ Lifecycle: spawn_agent → batch wait → close_agent
└─ Orchestrator aggregates → fix-plan.json
Phase 7.5: Export to Task JSON (auto with --fix, or explicit --export-tasks)
└─ Convert fix-plan.json findings → .task/FIX-{seq}.json
├─ For each finding in fix-plan.json:
│ ├─ finding.file → files[].path (action: "modify")
│ ├─ finding.severity → priority (critical|high|medium|low)
│ ├─ finding.fix_description → description
│ ├─ finding.dimension → scope
│ ├─ finding.verification → convergence.verification
│ ├─ finding.changes[] → convergence.criteria[]
│ └─ finding.fix_steps[] → implementation[]
├─ Output path: {projectRoot}/.workflow/active/WFS-{id}/.review/.task/FIX-{seq}.json
├─ Each file follows task-schema.json (IDENTITY + CONVERGENCE + FILES required)
└─ source.tool = "review-cycle", source.session_id = WFS-{id}
Phase 8: Fix Execution
└─ Ref: phases/08-fix-execution.md
├─ Stage-based execution per aggregated timeline
@@ -185,7 +200,8 @@ Complete: Review reports + optional fix results
| 5 | [phases/05-review-completion.md](phases/05-review-completion.md) | No more iterations needed | Shared from both review commands Phase 5 |
| 6 | [phases/06-fix-discovery-batching.md](phases/06-fix-discovery-batching.md) | Fix mode entry | review-cycle-fix Phase 1 + 1.5 |
| 7 | [phases/07-fix-parallel-planning.md](phases/07-fix-parallel-planning.md) | Phase 6 complete | review-cycle-fix Phase 2 |
| 8 | [phases/08-fix-execution.md](phases/08-fix-execution.md) | Phase 7 complete | review-cycle-fix Phase 3 |
| 7.5 | _(inline in SKILL.md)_ | Phase 7 complete | Export fix-plan findings to .task/FIX-*.json |
| 8 | [phases/08-fix-execution.md](phases/08-fix-execution.md) | Phase 7.5 complete | review-cycle-fix Phase 3 |
| 9 | [phases/09-fix-completion.md](phases/09-fix-completion.md) | Phase 8 complete | review-cycle-fix Phase 4 + 5 |
## Core Rules
@@ -225,6 +241,8 @@ Phase 6: Fix Discovery & Batching
↓ Output: finding batches (in-memory)
Phase 7: Fix Parallel Planning
↓ Output: partial-plan-*.json → fix-plan.json (aggregated)
Phase 7.5: Export to Task JSON
↓ Output: .task/FIX-{seq}.json (per finding, follows task-schema.json)
Phase 8: Fix Execution
↓ Output: fix-progress-*.json, git commits
Phase 9: Fix Completion
@@ -324,10 +342,11 @@ Phase 5: Review Completion → pending
**Fix Pipeline (added after Phase 5 if triggered)**:
```
Phase 6: Fix Discovery & Batching → pending
Phase 7: Parallel Planning → pending
Phase 8: Execution → pending
Phase 9: Fix Completion → pending
Phase 6: Fix Discovery & Batching → pending
Phase 7: Parallel Planning → pending
Phase 7.5: Export to Task JSON → pending
Phase 8: Execution → pending
Phase 9: Fix Completion → pending
```
## Error Handling
@@ -386,6 +405,10 @@ Gemini → Qwen → Codex → degraded mode
│ ├── security-cli-output.txt
│ ├── deep-dive-1-{uuid}.md
│ └── ...
├── .task/ # Task JSON exports (Phase 7.5)
│ ├── FIX-001.json # Per-finding task (task-schema.json)
│ ├── FIX-002.json
│ └── ...
└── fixes/{fix-session-id}/ # Fix results (Phase 6-9)
├── partial-plan-*.json
├── fix-plan.json

172
.codex/skills/unified-execute-with-file/SKILL.md
View File

@@ -1,40 +1,41 @@
---
name: unified-execute-with-file
description: Universal execution engine consuming unified JSONL task format. Serial task execution with convergence verification, progress tracking via execution.md + execution-events.md.
argument-hint: "PLAN=\"<path/to/tasks.jsonl>\" [--auto-commit] [--dry-run]"
description: Universal execution engine consuming .task/*.json directory format. Serial task execution with convergence verification, progress tracking via execution.md + execution-events.md.
argument-hint: "PLAN=\"<path/to/.task/>\" [--auto-commit] [--dry-run]"
---
# Unified-Execute-With-File Workflow
## Quick Start
Universal execution engine consuming **unified JSONL** (`tasks.jsonl`) and executing tasks serially with convergence verification and progress tracking.
Universal execution engine consuming **`.task/*.json`** directory and executing tasks serially with convergence verification and progress tracking.
```bash
# Execute from req-plan output
/codex:unified-execute-with-file PLAN=".workflow/.req-plan/RPLAN-auth-2025-01-21/tasks.jsonl"
# Execute from lite-plan output
/codex:unified-execute-with-file PLAN=".workflow/.lite-plan/LPLAN-auth-2025-01-21/.task/"
# Execute from collaborative-plan output
/codex:unified-execute-with-file PLAN=".workflow/.planning/CPLAN-xxx/tasks.jsonl" --auto-commit
# Execute from workflow session output
/codex:unified-execute-with-file PLAN=".workflow/active/WFS-xxx/.task/" --auto-commit
# Dry-run mode
/codex:unified-execute-with-file PLAN="tasks.jsonl" --dry-run
# Execute a single task JSON file
/codex:unified-execute-with-file PLAN=".workflow/active/WFS-xxx/.task/IMPL-001.json" --dry-run
# Auto-detect from .workflow/ directories
/codex:unified-execute-with-file
```
**Core workflow**: Load JSONL → Validate → Pre-Execution Analysis → Execute → Verify Convergence → Track Progress
**Core workflow**: Scan .task/*.json → Validate → Pre-Execution Analysis → Execute → Verify Convergence → Track Progress
**Key features**:
- **Single format**: Only consumes unified JSONL (`tasks.jsonl`)
- **Directory-based**: Consumes `.task/` directory containing individual task JSON files
- **Convergence-driven**: Verifies each task's convergence criteria after execution
- **Serial execution**: Process tasks in topological order with dependency tracking
- **Dual progress tracking**: `execution.md` (overview) + `execution-events.md` (event stream)
- **Auto-commit**: Optional conventional commits per task
- **Dry-run mode**: Simulate execution without changes
- **Flexible input**: Accepts `.task/` directory path or a single `.json` file path
**Input format**: Use `plan-converter` to convert other formats (roadmap.jsonl, plan-note.md, conclusions.json, synthesis.json) to unified JSONL first.
**Input format**: Each task is a standalone JSON file in `.task/` directory (e.g., `IMPL-001.json`). Use `plan-converter` to convert other formats to `.task/*.json` first.
## Overview
@@ -44,7 +45,7 @@ Universal execution engine consuming **unified JSONL** (`tasks.jsonl`) and execu
├─────────────────────────────────────────────────────────────┤
│ │
│ Phase 1: Load & Validate │
│ ├─ Parse tasks.jsonl (one task per line) │
│ ├─ Scan .task/*.json (one task per file) │
│ ├─ Validate schema (id, title, depends_on, convergence) │
│ ├─ Detect cycles, build topological order │
│ └─ Initialize execution.md + execution-events.md │
@@ -63,13 +64,13 @@ Universal execution engine consuming **unified JSONL** (`tasks.jsonl`) and execu
│ ├─ Verify convergence.criteria[] │
│ ├─ Run convergence.verification command │
│ ├─ Record COMPLETE/FAIL event with verification results │
│ ├─ Update _execution state in JSONL │
│ ├─ Update _execution state in task JSON file │
│ └─ Auto-commit if enabled │
│ │
│ Phase 4: Completion │
│ ├─ Finalize execution.md with summary statistics │
│ ├─ Finalize execution-events.md with session footer │
│ ├─ Write back tasks.jsonl with _execution states │
│ ├─ Write back .task/*.json with _execution states │
│ └─ Offer follow-up actions │
│ │
└─────────────────────────────────────────────────────────────┘
@@ -83,7 +84,7 @@ ${projectRoot}/.workflow/.execution/EXEC-{slug}-{date}-{random}/
└── execution-events.md # ⭐ Unified event log (single source of truth)
```
Additionally, the source `tasks.jsonl` is updated in-place with `_execution` states.
Additionally, each source `.task/*.json` file is updated in-place with `_execution` states.
---
@@ -105,12 +106,12 @@ let planPath = planMatch ? planMatch[1] : null
// Auto-detect if no PLAN specified
if (!planPath) {
// Search in order:
// .workflow/.req-plan/*/tasks.jsonl
// .workflow/.planning/*/tasks.jsonl
// .workflow/.analysis/*/tasks.jsonl
// .workflow/.brainstorm/*/tasks.jsonl
// Use most recently modified
// Search in order (most recent first):
// .workflow/active/*/.task/
// .workflow/.lite-plan/*/.task/
// .workflow/.req-plan/*/.task/
// .workflow/.planning/*/.task/
// Use most recently modified directory containing *.json files
}
// Resolve path
@@ -130,41 +131,85 @@ Bash(`mkdir -p ${sessionFolder}`)
## Phase 1: Load & Validate
**Objective**: Parse unified JSONL, validate schema and dependencies, build execution order.
**Objective**: Scan `.task/` directory, parse individual task JSON files, validate schema and dependencies, build execution order.
### Step 1.1: Parse Unified JSONL
### Step 1.1: Scan .task/ Directory and Parse Task Files
```javascript
const content = Read(planPath)
const tasks = content.split('\n')
.filter(line => line.trim())
.map((line, i) => {
try { return JSON.parse(line) }
catch (e) { throw new Error(`Line ${i + 1}: Invalid JSON — ${e.message}`) }
})
// Determine if planPath is a directory or single file
const isDirectory = planPath.endsWith('/') || Bash(`test -d "${planPath}" && echo dir || echo file`).trim() === 'dir'
if (tasks.length === 0) throw new Error('No tasks found in JSONL file')
let taskFiles, tasks
if (isDirectory) {
// Directory mode: scan for all *.json files
taskFiles = Glob('*.json', planPath)
if (taskFiles.length === 0) throw new Error(`No .json files found in ${planPath}`)
tasks = taskFiles.map(filePath => {
try {
const content = Read(filePath)
const task = JSON.parse(content)
task._source_file = filePath // Track source file for write-back
return task
} catch (e) {
throw new Error(`${path.basename(filePath)}: Invalid JSON - ${e.message}`)
}
})
} else {
// Single file mode: parse one task JSON
try {
const content = Read(planPath)
const task = JSON.parse(content)
task._source_file = planPath
tasks = [task]
} catch (e) {
throw new Error(`${path.basename(planPath)}: Invalid JSON - ${e.message}`)
}
}
if (tasks.length === 0) throw new Error('No tasks found')
```
### Step 1.2: Validate Schema
Validate against unified task schema: `~/.ccw/workflows/cli-templates/schemas/task-schema.json`
```javascript
const errors = []
tasks.forEach((task, i) => {
// Required fields
if (!task.id) errors.push(`Task ${i + 1}: missing 'id'`)
if (!task.title) errors.push(`Task ${i + 1}: missing 'title'`)
if (!task.description) errors.push(`Task ${i + 1}: missing 'description'`)
if (!Array.isArray(task.depends_on)) errors.push(`${task.id}: missing 'depends_on' array`)
const src = task._source_file ? path.basename(task._source_file) : `Task ${i + 1}`
// Convergence required
// Required fields (per task-schema.json)
if (!task.id) errors.push(`${src}: missing 'id'`)
if (!task.title) errors.push(`${src}: missing 'title'`)
if (!task.description) errors.push(`${src}: missing 'description'`)
if (!Array.isArray(task.depends_on)) errors.push(`${task.id || src}: missing 'depends_on' array`)
// Context block (optional but validated if present)
if (task.context) {
if (task.context.requirements && !Array.isArray(task.context.requirements))
errors.push(`${task.id}: context.requirements must be array`)
if (task.context.acceptance && !Array.isArray(task.context.acceptance))
errors.push(`${task.id}: context.acceptance must be array`)
if (task.context.focus_paths && !Array.isArray(task.context.focus_paths))
errors.push(`${task.id}: context.focus_paths must be array`)
}
// Convergence (required for execution verification)
if (!task.convergence) {
errors.push(`${task.id}: missing 'convergence'`)
errors.push(`${task.id || src}: missing 'convergence'`)
} else {
if (!task.convergence.criteria?.length) errors.push(`${task.id}: empty convergence.criteria`)
if (!task.convergence.verification) errors.push(`${task.id}: missing convergence.verification`)
if (!task.convergence.definition_of_done) errors.push(`${task.id}: missing convergence.definition_of_done`)
}
// Flow control (optional but validated if present)
if (task.flow_control) {
if (task.flow_control.target_files && !Array.isArray(task.flow_control.target_files))
errors.push(`${task.id}: flow_control.target_files must be array`)
}
})
if (errors.length) {
@@ -610,14 +655,29 @@ appendToEvents(`
`)
```
### Step 4.3: Write Back tasks.jsonl with _execution
### Step 4.3: Write Back .task/*.json with _execution
Update the source JSONL file with execution states:
Update each source task JSON file with execution states:
```javascript
const updatedJsonl = tasks.map(task => JSON.stringify(task)).join('\n')
Write(planPath, updatedJsonl)
// Each task now has _execution: { status, executed_at, result }
tasks.forEach(task => {
const filePath = task._source_file
if (!filePath) return
// Read current file to preserve formatting and non-execution fields
const current = JSON.parse(Read(filePath))
// Update _execution status and result
current._execution = {
status: task._execution?.status || 'pending',
executed_at: task._execution?.executed_at || null,
result: task._execution?.result || null
}
// Write back individual task file
Write(filePath, JSON.stringify(current, null, 2))
})
// Each task JSON file now has _execution: { status, executed_at, result }
```
### Step 4.4: Post-Completion Options
@@ -651,20 +711,20 @@ AskUserQuestion({
| Flag | Default | Description |
|------|---------|-------------|
| `PLAN="..."` | auto-detect | Path to unified JSONL file (`tasks.jsonl`) |
| `PLAN="..."` | auto-detect | Path to `.task/` directory or single task `.json` file |
| `--auto-commit` | false | Commit changes after each successful task |
| `--dry-run` | false | Simulate execution without making changes |
### Plan Auto-Detection Order
When no `PLAN` specified, search in order (most recent first):
When no `PLAN` specified, search for `.task/` directories in order (most recent first):
1. `.workflow/.req-plan/*/tasks.jsonl`
2. `.workflow/.planning/*/tasks.jsonl`
3. `.workflow/.analysis/*/tasks.jsonl`
4. `.workflow/.brainstorm/*/tasks.jsonl`
1. `.workflow/active/*/.task/`
2. `.workflow/.lite-plan/*/.task/`
3. `.workflow/.req-plan/*/.task/`
4. `.workflow/.planning/*/.task/`
**If source is not unified JSONL**: Run `plan-converter` first.
**If source is not `.task/*.json`**: Run `plan-converter` first to generate `.task/` directory.
---
@@ -672,10 +732,10 @@ When no `PLAN` specified, search in order (most recent first):
| Situation | Action | Recovery |
|-----------|--------|----------|
| JSONL file not found | Report error with path | Check path, run plan-converter |
| Invalid JSON line | Report line number and error | Fix JSONL file manually |
| .task/ directory not found | Report error with path | Check path, run plan-converter |
| Invalid JSON in task file | Report filename and error | Fix task JSON file manually |
| Missing convergence | Report validation error | Run plan-converter to add convergence |
| Circular dependency | Stop, report cycle path | Fix dependencies in JSONL |
| Circular dependency | Stop, report cycle path | Fix dependencies in task JSON |
| Task execution fails | Record in events, ask user | Retry, skip, accept, or abort |
| Convergence verification fails | Mark task failed, ask user | Fix code and retry, or accept |
| Verification command timeout | Mark as unverified | Manual verification needed |
@@ -692,7 +752,7 @@ When no `PLAN` specified, search in order (most recent first):
2. **Check Convergence**: Ensure all tasks have meaningful convergence criteria
3. **Review Dependencies**: Verify execution order makes sense
4. **Backup**: Commit pending changes before starting
5. **Convert First**: Use `plan-converter` for non-JSONL sources
5. **Convert First**: Use `plan-converter` for non-.task/ sources
### During Execution
@@ -704,7 +764,7 @@ When no `PLAN` specified, search in order (most recent first):
1. **Review Summary**: Check execution.md statistics and failed tasks
2. **Verify Changes**: Inspect modified files match expectations
3. **Check JSONL**: Review `_execution` states in tasks.jsonl
3. **Check Task Files**: Review `_execution` states in `.task/*.json` files
4. **Next Steps**: Use completion options for follow-up
---

72
.codex/skills/workflow-lite-plan-execute/SKILL.md
View File

@@ -1,19 +1,21 @@
---
name: workflow-lite-plan-execute
description: Lightweight planning + execution workflow. Serial CLI exploration → Search verification → Clarification → Planning → Unified JSONL output → Execution via unified-execute.
description: Lightweight planning + execution workflow. Serial CLI exploration → Search verification → Clarification → Planning → .task/*.json multi-file output → Execution via unified-execute.
allowed-tools: AskUserQuestion, Read, Write, Edit, Bash, Glob, Grep, mcp__ace-tool__search_context
---
# Planning Workflow
Lite Plan produces a unified JSONL (`tasks.jsonl`) implementation plan via serial CLI exploration and direct planning, then hands off to unified-execute-with-file for task execution.
Lite Plan produces `.task/TASK-*.json` (one file per task) implementation plan via serial CLI exploration and direct planning, then hands off to unified-execute-with-file for task execution.
> **Schema**: `cat ~/.ccw/workflows/cli-templates/schemas/task-schema.json`
## Key Design Principles
1. **Serial Execution**: All phases execute serially inline, no agent delegation
2. **CLI Exploration**: Multi-angle codebase exploration via `ccw cli` calls (default gemini, fallback claude)
3. **Search Verification**: Verify CLI findings with ACE search / Grep / Glob before incorporating
4. **Unified JSONL Output**: Produces `tasks.jsonl` compatible with `collaborative-plan-with-file` and `unified-execute-with-file`
4. **Multi-File Task Output**: Produces `.task/TASK-*.json` (one file per task) compatible with `collaborative-plan-with-file` and `unified-execute-with-file`
5. **Progressive Phase Loading**: Only load phase docs when about to execute
## Auto Mode
@@ -49,7 +51,7 @@ $workflow-lite-plan-execute "docs/todo.md"
| Phase | Document | Purpose |
|-------|----------|---------|
| 1 | `phases/01-lite-plan.md` | Serial CLI exploration, clarification, plan generation → tasks.jsonl |
| 1 | `phases/01-lite-plan.md` | Serial CLI exploration, clarification, plan generation → .task/TASK-*.json |
| 2 | `phases/02-lite-execute.md` | Handoff to unified-execute-with-file for task execution |
## Orchestrator Logic
@@ -72,7 +74,7 @@ function extractTaskDescription(args) {
const taskDescription = extractTaskDescription($ARGUMENTS)
// Phase 1: Lite Plan → tasks.jsonl
// Phase 1: Lite Plan → .task/TASK-*.json
Read('phases/01-lite-plan.md')
// Execute planning phase...
@@ -84,12 +86,14 @@ if (planResult?.userSelection?.confirmation !== 'Allow' && !autoYes) {
// Phase 2: Handoff to unified-execute-with-file
Read('phases/02-lite-execute.md')
// Invoke unified-execute-with-file with tasks.jsonl path
// Invoke unified-execute-with-file with .task/ directory path
```
## Output Contract
Phase 1 produces `tasks.jsonl` (unified JSONL format) — compatible with `collaborative-plan-with-file` and consumable by `unified-execute-with-file`.
Phase 1 produces `.task/TASK-*.json` (one file per task) — compatible with `collaborative-plan-with-file` and consumable by `unified-execute-with-file`.
> **Schema**: `cat ~/.ccw/workflows/cli-templates/schemas/task-schema.json`
**Output Directory**: `{projectRoot}/.workflow/.lite-plan/{session-id}/`
@@ -100,37 +104,41 @@ Phase 1 produces `tasks.jsonl` (unified JSONL format) — compatible with `colla
├── explorations-manifest.json # Exploration index
├── exploration-notes.md # Synthesized exploration notes
├── requirement-analysis.json # Complexity assessment
├── tasks.jsonl # ⭐ Unified JSONL (collaborative-plan-with-file compatible)
├── .task/ # ⭐ Task JSON files (one per task)
│ ├── TASK-001.json # Individual task definition
│ ├── TASK-002.json
│ └── ...
└── plan.md # Human-readable summary
```
**Unified JSONL Task Format** (one task per line):
**Task JSON Format** (one file per task, following task-schema.json):
```javascript
// File: .task/TASK-001.json
{
id: "TASK-001",
title: string,
description: string,
type: "feature|fix|refactor|enhancement|testing|infrastructure",
priority: "high|medium|low",
effort: "small|medium|large",
scope: string,
depends_on: ["TASK-xxx"],
convergence: {
criteria: string[], // Testable conditions
verification: string, // Executable command or manual steps
definition_of_done: string // Business language
"id": "TASK-001",
"title": "string",
"description": "string",
"type": "feature|fix|refactor|enhancement|testing|infrastructure",
"priority": "high|medium|low",
"effort": "small|medium|large",
"scope": "string",
"depends_on": ["TASK-xxx"],
"convergence": {
"criteria": ["string"], // Testable conditions
"verification": "string", // Executable command or manual steps
"definition_of_done": "string" // Business language
},
files: [{
path: string,
action: "modify|create|delete",
changes: string[],
conflict_risk: "low|medium|high"
"files": [{
"path": "string",
"action": "modify|create|delete",
"changes": ["string"],
"conflict_risk": "low|medium|high"
}],
source: {
tool: "workflow-lite-plan-execute",
session_id: string,
original_id: string
"source": {
"tool": "workflow-lite-plan-execute",
"session_id": "string",
"original_id": "string"
}
}
```
@@ -158,7 +166,7 @@ After planning completes:
1. **Planning phase NEVER modifies project code** — it may write planning artifacts, but all implementation is delegated to unified-execute
2. **All phases serial, no agent delegation** — everything runs inline, no spawn_agent
3. **CLI exploration with search verification** — CLI calls produce findings, ACE/Grep/Glob verify them
4. **tasks.jsonl is the output contract** — unified JSONL format passed to unified-execute-with-file
4. **`.task/*.json` is the output contract** — individual task JSON files passed to unified-execute-with-file
5. **Progressive loading**: Read phase doc only when about to execute
6. **File-path detection**: Treat input as a file path only if the path exists; do not infer from file extensions
@@ -168,7 +176,7 @@ After planning completes:
|-------|------------|
| CLI exploration failure | Skip angle, continue with remaining; fallback gemini → claude |
| Planning phase failure | Display error, offer retry |
| tasks.jsonl missing | Error: planning phase did not produce output |
| .task/ directory empty | Error: planning phase did not produce output |
| Phase file not found | Error with file path for debugging |
## Related Skills

15
.codex/skills/workflow-plan-execute/SKILL.md
View File

@@ -77,6 +77,7 @@ Phase 2: Context Gathering & Conflict Resolution
Phase 3: Task Generation
└─ Ref: phases/03-task-generation.md
└─ Output: IMPL_PLAN.md, task JSONs, TODO_LIST.md
└─ Schema: .task/IMPL-*.json follows 6-field superset of task-schema.json
User Decision (or --yes auto):
└─ "Start Execution" → Phase 4
@@ -426,6 +427,20 @@ if (autoYes) {
- After each phase, automatically continue to next phase based on TodoList status
- **Always close_agent after wait completes**
## Task JSON Schema Compatibility
Phase 3 generates `.task/IMPL-*.json` files using the **6-field schema** defined in `action-planning-agent.md`. These task JSONs are a **superset** of the unified `task-schema.json` (located at `.ccw/workflows/cli-templates/schemas/task-schema.json`).
**Key field mappings** (6-field → unified schema):
- `context.acceptance` → `convergence.criteria`
- `context.requirements` → `description` + `implementation`
- `context.depends_on` → `depends_on` (top-level)
- `context.focus_paths` → `focus_paths` (top-level)
- `meta.type` → `type` (top-level)
- `flow_control.target_files` → `files[].path`
All existing 6-field schema fields are preserved. The unified schema fields are accepted as optional aliases for cross-tool interoperability. See `action-planning-agent.md` Section 2.1 "Schema Compatibility" for the full mapping table.
## Related Commands
**Prerequisite Commands**:

15
.codex/skills/workflow-tdd-plan/SKILL.md
View File

@@ -233,6 +233,7 @@ Phase 5: TDD Task Generation ← ATTACHED (3 tasks)
├─ Phase 5.2: Planning - design Red-Green-Refactor cycles
└─ Phase 5.3: Output - generate IMPL tasks with internal TDD phases
└─ Output: IMPL-*.json, IMPL_PLAN.md ← COLLAPSED
└─ Schema: .task/IMPL-*.json follows 6-field superset of task-schema.json
Phase 6: TDD Structure Validation (inline)
└─ Internal validation + summary returned
@@ -766,6 +767,20 @@ Read and execute: `phases/03-tdd-verify.md` with `--session [sessionId]`
This generates a comprehensive TDD_COMPLIANCE_REPORT.md with quality gate recommendation.
## Task JSON Schema Compatibility
Phase 5 generates `.task/IMPL-*.json` files using the **6-field schema** defined in `action-planning-agent.md`. These task JSONs are a **superset** of the unified `task-schema.json` (located at `.ccw/workflows/cli-templates/schemas/task-schema.json`).
**Key field mappings** (6-field → unified schema):
- `context.acceptance` → `convergence.criteria`
- `context.requirements` → `description` + `implementation`
- `context.depends_on` → `depends_on` (top-level)
- `context.focus_paths` → `focus_paths` (top-level)
- `meta.type` → `type` (top-level)
- `flow_control.target_files` → `files[].path`
All existing 6-field schema fields are preserved. TDD-specific extensions (`meta.tdd_workflow`, `tdd_phase` in implementation_approach steps) are additive and do not conflict with the unified schema. See `action-planning-agent.md` Section 2.1 "Schema Compatibility" for the full mapping table.
## Related Skills
**Prerequisite**:

207
ccw/src/commands/core-memory.ts
View File

@@ -11,6 +11,7 @@ import {
exportMemories,
importMemories
} from '../core/core-memory-store.js';
import { MemoryJobScheduler } from '../core/memory-job-scheduler.js';
import { notifyRefreshRequired } from '../tools/notifier.js';
interface CommandOptions {
@@ -664,6 +665,185 @@ async function searchAction(keyword: string, options: CommandOptions): Promise<v
}
}
// ============================================================================
// Memory V2 CLI Subcommands
// ============================================================================
/**
* Run batch extraction
*/
async function extractAction(options: CommandOptions): Promise<void> {
try {
const projectPath = getProjectPath();
console.log(chalk.cyan('\n Triggering memory extraction...\n'));
const { MemoryExtractionPipeline } = await import('../core/memory-extraction-pipeline.js');
const pipeline = new MemoryExtractionPipeline(projectPath);
// Scan eligible sessions first
const eligible = await pipeline.scanEligibleSessions();
console.log(chalk.white(` Eligible sessions: ${eligible.length}`));
if (eligible.length === 0) {
console.log(chalk.yellow(' No eligible sessions for extraction.\n'));
return;
}
// Run extraction (synchronous for CLI - shows progress)
console.log(chalk.cyan(' Running batch extraction...'));
await pipeline.runBatchExtraction();
const store = getCoreMemoryStore(projectPath);
const stage1Count = store.countStage1Outputs();
console.log(chalk.green(`\n Extraction complete.`));
console.log(chalk.white(` Total stage1 outputs: ${stage1Count}\n`));
notifyRefreshRequired('memory').catch(() => { /* ignore */ });
} catch (error) {
console.error(chalk.red(`Error: ${(error as Error).message}`));
process.exit(1);
}
}
/**
* Show extraction status
*/
async function extractStatusAction(options: CommandOptions): Promise<void> {
try {
const projectPath = getProjectPath();
const store = getCoreMemoryStore(projectPath);
const scheduler = new MemoryJobScheduler(store.getDb());
const stage1Count = store.countStage1Outputs();
const extractionJobs = scheduler.listJobs('extraction');
if (options.json) {
console.log(JSON.stringify({ total_stage1: stage1Count, jobs: extractionJobs }, null, 2));
return;
}
console.log(chalk.bold.cyan('\n Extraction Pipeline Status\n'));
console.log(chalk.white(` Stage 1 outputs: ${stage1Count}`));
console.log(chalk.white(` Extraction jobs: ${extractionJobs.length}`));
if (extractionJobs.length > 0) {
console.log(chalk.gray('\n ─────────────────────────────────────────────────────────────────'));
for (const job of extractionJobs) {
const statusColor = job.status === 'done' ? chalk.green
: job.status === 'error' ? chalk.red
: job.status === 'running' ? chalk.yellow
: chalk.gray;
console.log(chalk.cyan(` ${job.job_key}`) + chalk.white(` [${statusColor(job.status)}]`));
if (job.last_error) console.log(chalk.red(` Error: ${job.last_error}`));
if (job.started_at) console.log(chalk.gray(` Started: ${new Date(job.started_at * 1000).toLocaleString()}`));
if (job.finished_at) console.log(chalk.gray(` Finished: ${new Date(job.finished_at * 1000).toLocaleString()}`));
console.log(chalk.gray(' ─────────────────────────────────────────────────────────────────'));
}
}
console.log();
} catch (error) {
console.error(chalk.red(`Error: ${(error as Error).message}`));
process.exit(1);
}
}
/**
* Run consolidation
*/
async function consolidateAction(options: CommandOptions): Promise<void> {
try {
const projectPath = getProjectPath();
console.log(chalk.cyan('\n Triggering memory consolidation...\n'));
const { MemoryConsolidationPipeline } = await import('../core/memory-consolidation-pipeline.js');
const pipeline = new MemoryConsolidationPipeline(projectPath);
await pipeline.runConsolidation();
const memoryMd = pipeline.getMemoryMdContent();
console.log(chalk.green(' Consolidation complete.'));
if (memoryMd) {
console.log(chalk.white(` MEMORY.md generated (${memoryMd.length} chars)`));
}
console.log();
notifyRefreshRequired('memory').catch(() => { /* ignore */ });
} catch (error) {
console.error(chalk.red(`Error: ${(error as Error).message}`));
process.exit(1);
}
}
/**
* List all V2 pipeline jobs
*/
async function jobsAction(options: CommandOptions): Promise<void> {
try {
const projectPath = getProjectPath();
const store = getCoreMemoryStore(projectPath);
const scheduler = new MemoryJobScheduler(store.getDb());
const kind = options.type || undefined;
const jobs = scheduler.listJobs(kind);
if (options.json) {
console.log(JSON.stringify({ jobs, total: jobs.length }, null, 2));
return;
}
console.log(chalk.bold.cyan('\n Memory V2 Pipeline Jobs\n'));
if (jobs.length === 0) {
console.log(chalk.yellow(' No jobs found.\n'));
return;
}
// Summary counts
const byStatus: Record<string, number> = {};
for (const job of jobs) {
byStatus[job.status] = (byStatus[job.status] || 0) + 1;
}
const statusParts = Object.entries(byStatus)
.map(([s, c]) => `${s}: ${c}`)
.join(' | ');
console.log(chalk.white(` Summary: ${statusParts}`));
console.log(chalk.gray(' ─────────────────────────────────────────────────────────────────'));
for (const job of jobs) {
const statusColor = job.status === 'done' ? chalk.green
: job.status === 'error' ? chalk.red
: job.status === 'running' ? chalk.yellow
: chalk.gray;
console.log(
chalk.cyan(` [${job.kind}]`) +
chalk.white(` ${job.job_key}`) +
` [${statusColor(job.status)}]` +
chalk.gray(` retries: ${job.retry_remaining}`)
);
if (job.last_error) console.log(chalk.red(` Error: ${job.last_error}`));
console.log(chalk.gray(' ─────────────────────────────────────────────────────────────────'));
}
console.log(chalk.gray(`\n Total: ${jobs.length}\n`));
} catch (error) {
console.error(chalk.red(`Error: ${(error as Error).message}`));
process.exit(1);
}
}
/**
* Core Memory command entry point
*/
@@ -724,6 +904,23 @@ export async function coreMemoryCommand(
await listFromAction(textArg, options);
break;
// Memory V2 subcommands
case 'extract':
await extractAction(options);
break;
case 'extract-status':
await extractStatusAction(options);
break;
case 'consolidate':
await consolidateAction(options);
break;
case 'jobs':
await jobsAction(options);
break;
default:
console.log(chalk.bold.cyan('\n CCW Core Memory\n'));
console.log(' Manage core memory entries and session clusters.\n');
@@ -749,6 +946,12 @@ export async function coreMemoryCommand(
console.log(chalk.white(' load-cluster <id> ') + chalk.gray('Load cluster context'));
console.log(chalk.white(' search <keyword> ') + chalk.gray('Search sessions'));
console.log();
console.log(chalk.bold(' Memory V2 Pipeline:'));
console.log(chalk.white(' extract ') + chalk.gray('Run batch memory extraction'));
console.log(chalk.white(' extract-status ') + chalk.gray('Show extraction pipeline status'));
console.log(chalk.white(' consolidate ') + chalk.gray('Run memory consolidation'));
console.log(chalk.white(' jobs ') + chalk.gray('List all pipeline jobs'));
console.log();
console.log(chalk.bold(' Options:'));
console.log(chalk.gray(' --id <id> Memory ID (for export/summary)'));
console.log(chalk.gray(' --tool gemini|qwen AI tool for summary (default: gemini)'));
@@ -765,6 +968,10 @@ export async function coreMemoryCommand(
console.log(chalk.gray(' ccw core-memory list-from d--other-project'));
console.log(chalk.gray(' ccw core-memory cluster --auto'));
console.log(chalk.gray(' ccw core-memory cluster --dedup'));
console.log(chalk.gray(' ccw core-memory extract # Run memory extraction'));
console.log(chalk.gray(' ccw core-memory extract-status # Check extraction state'));
console.log(chalk.gray(' ccw core-memory consolidate # Run consolidation'));
console.log(chalk.gray(' ccw core-memory jobs # List pipeline jobs'));
console.log();
}
}

30
ccw/src/config/storage-paths.ts
View File

@@ -375,6 +375,19 @@ export interface ProjectPaths {
config: string;
/** CLI config file */
cliConfig: string;
/** Memory V2 paths */
memoryV2: {
/** Root: <projectRoot>/core-memory/v2/ */
root: string;
/** Rollout summaries directory */
rolloutSummaries: string;
/** Concatenated raw memories file */
rawMemories: string;
/** Final consolidated memory file */
memoryMd: string;
/** Skills directory */
skills: string;
};
}
/**
@@ -434,6 +447,13 @@ export function getProjectPaths(projectPath: string): ProjectPaths {
dashboardCache: join(projectDir, 'cache', 'dashboard-data.json'),
config: join(projectDir, 'config'),
cliConfig: join(projectDir, 'config', 'cli-config.json'),
memoryV2: {
root: join(projectDir, 'core-memory', 'v2'),
rolloutSummaries: join(projectDir, 'core-memory', 'v2', 'rollout_summaries'),
rawMemories: join(projectDir, 'core-memory', 'v2', 'raw_memories.md'),
memoryMd: join(projectDir, 'core-memory', 'v2', 'MEMORY.md'),
skills: join(projectDir, 'core-memory', 'v2', 'skills'),
},
};
}
@@ -456,6 +476,13 @@ export function getProjectPathsById(projectId: string): ProjectPaths {
dashboardCache: join(projectDir, 'cache', 'dashboard-data.json'),
config: join(projectDir, 'config'),
cliConfig: join(projectDir, 'config', 'cli-config.json'),
memoryV2: {
root: join(projectDir, 'core-memory', 'v2'),
rolloutSummaries: join(projectDir, 'core-memory', 'v2', 'rollout_summaries'),
rawMemories: join(projectDir, 'core-memory', 'v2', 'raw_memories.md'),
memoryMd: join(projectDir, 'core-memory', 'v2', 'MEMORY.md'),
skills: join(projectDir, 'core-memory', 'v2', 'skills'),
},
};
}
@@ -682,4 +709,7 @@ export function initializeProjectStorage(projectPath: string): void {
ensureStorageDir(paths.memory);
ensureStorageDir(paths.cache);
ensureStorageDir(paths.config);
ensureStorageDir(paths.memoryV2.root);
ensureStorageDir(paths.memoryV2.rolloutSummaries);
ensureStorageDir(paths.memoryV2.skills);
}

135
ccw/src/core/core-memory-store.ts
View File

@@ -83,6 +83,17 @@ export interface ClaudeUpdateRecord {
metadata?: string;
}
/**
* Memory V2: Phase 1 extraction output row
*/
export interface Stage1Output {
thread_id: string;
source_updated_at: number;
raw_memory: string;
rollout_summary: string;
generated_at: number;
}
/**
* Core Memory Store using SQLite
*/
@@ -215,6 +226,40 @@ export class CoreMemoryStore {
CREATE INDEX IF NOT EXISTS idx_claude_history_path ON claude_update_history(file_path);
CREATE INDEX IF NOT EXISTS idx_claude_history_updated ON claude_update_history(updated_at DESC);
CREATE INDEX IF NOT EXISTS idx_claude_history_module ON claude_update_history(module_path);
-- Memory V2: Phase 1 extraction outputs
CREATE TABLE IF NOT EXISTS stage1_outputs (
thread_id TEXT PRIMARY KEY,
source_updated_at INTEGER NOT NULL,
raw_memory TEXT NOT NULL,
rollout_summary TEXT NOT NULL,
generated_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_stage1_generated ON stage1_outputs(generated_at DESC);
CREATE INDEX IF NOT EXISTS idx_stage1_source_updated ON stage1_outputs(source_updated_at DESC);
-- Memory V2: Job scheduler
CREATE TABLE IF NOT EXISTS jobs (
kind TEXT NOT NULL,
job_key TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'running', 'done', 'error')),
worker_id TEXT,
ownership_token TEXT,
started_at INTEGER,
finished_at INTEGER,
lease_until INTEGER,
retry_at INTEGER,
retry_remaining INTEGER NOT NULL DEFAULT 3,
last_error TEXT,
input_watermark INTEGER DEFAULT 0,
last_success_watermark INTEGER DEFAULT 0,
PRIMARY KEY (kind, job_key)
);
CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status);
CREATE INDEX IF NOT EXISTS idx_jobs_kind_status ON jobs(kind, status);
CREATE INDEX IF NOT EXISTS idx_jobs_lease ON jobs(lease_until);
`);
}
@@ -275,6 +320,14 @@ export class CoreMemoryStore {
}
}
/**
* Get the underlying database instance.
* Used by MemoryJobScheduler and other V2 components that share this DB.
*/
getDb(): Database.Database {
return this.db;
}
/**
* Generate timestamp-based ID for core memory
*/
@@ -1255,6 +1308,88 @@ ${memory.content}
return result.changes;
}
// ============================================================================
// Memory V2: Stage 1 Output CRUD Operations
// ============================================================================
/**
* Upsert a Phase 1 extraction output (idempotent by thread_id)
*/
upsertStage1Output(output: Stage1Output): void {
const stmt = this.db.prepare(`
INSERT INTO stage1_outputs (thread_id, source_updated_at, raw_memory, rollout_summary, generated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(thread_id) DO UPDATE SET
source_updated_at = excluded.source_updated_at,
raw_memory = excluded.raw_memory,
rollout_summary = excluded.rollout_summary,
generated_at = excluded.generated_at
`);
stmt.run(
output.thread_id,
output.source_updated_at,
output.raw_memory,
output.rollout_summary,
output.generated_at
);
}
/**
* Get a Phase 1 output by thread_id
*/
getStage1Output(threadId: string): Stage1Output | null {
const stmt = this.db.prepare(`SELECT * FROM stage1_outputs WHERE thread_id = ?`);
const row = stmt.get(threadId) as any;
if (!row) return null;
return {
thread_id: row.thread_id,
source_updated_at: row.source_updated_at,
raw_memory: row.raw_memory,
rollout_summary: row.rollout_summary,
generated_at: row.generated_at,
};
}
/**
* List all Phase 1 outputs, ordered by generated_at descending
*/
listStage1Outputs(limit?: number): Stage1Output[] {
const query = limit
? `SELECT * FROM stage1_outputs ORDER BY generated_at DESC LIMIT ?`
: `SELECT * FROM stage1_outputs ORDER BY generated_at DESC`;
const stmt = this.db.prepare(query);
const rows = (limit ? stmt.all(limit) : stmt.all()) as any[];
return rows.map(row => ({
thread_id: row.thread_id,
source_updated_at: row.source_updated_at,
raw_memory: row.raw_memory,
rollout_summary: row.rollout_summary,
generated_at: row.generated_at,
}));
}
/**
* Count Phase 1 outputs
*/
countStage1Outputs(): number {
const stmt = this.db.prepare(`SELECT COUNT(*) as count FROM stage1_outputs`);
const row = stmt.get() as { count: number };
return row.count;
}
/**
* Delete a Phase 1 output by thread_id
*/
deleteStage1Output(threadId: string): boolean {
const stmt = this.db.prepare(`DELETE FROM stage1_outputs WHERE thread_id = ?`);
const result = stmt.run(threadId);
return result.changes > 0;
}
/**
* Close database connection
*/

474
ccw/src/core/memory-consolidation-pipeline.ts Normal file
View File

@@ -0,0 +1,474 @@
/**
* Memory Consolidation Pipeline - Phase 2 Global Consolidation
*
* Orchestrates the global memory consolidation process:
* Lock -> Materialize -> Agent -> Monitor -> Done
*
* Phase 1 outputs (per-session extractions stored in stage1_outputs DB table)
* are materialized to disk as rollout_summaries/*.md + raw_memories.md,
* then a CLI agent (--mode write) reads those files and produces MEMORY.md.
*
* The pipeline uses lease-based locking via MemoryJobScheduler to ensure
* only one consolidation runs at a time, with heartbeat-based lease renewal.
*/
import { existsSync, readFileSync, readdirSync, unlinkSync, writeFileSync } from 'fs';
import { join } from 'path';
import { MemoryJobScheduler, type ClaimResult } from './memory-job-scheduler.js';
import { getCoreMemoryStore, type Stage1Output } from './core-memory-store.js';
import { getProjectPaths, ensureStorageDir } from '../config/storage-paths.js';
import {
HEARTBEAT_INTERVAL_SECONDS,
MAX_RAW_MEMORIES_FOR_GLOBAL,
} from './memory-v2-config.js';
import {
CONSOLIDATION_SYSTEM_PROMPT,
buildConsolidationPrompt,
} from './memory-consolidation-prompts.js';
// -- Types --
export interface ConsolidationStatus {
status: 'idle' | 'running' | 'completed' | 'error';
lastRun?: number;
memoryMdExists: boolean;
inputCount: number;
lastError?: string;
}
export interface MaterializationResult {
summariesWritten: number;
summariesPruned: number;
rawMemoriesSize: number;
}
// -- Constants --
const JOB_KIND = 'memory_consolidate_global';
const JOB_KEY = 'global';
const MAX_CONCURRENT = 1;
const AGENT_TIMEOUT_MS = 300_000; // 5 minutes
const DEFAULT_CLI_TOOL = 'gemini';
// -- Utility --
/**
* Sanitize a thread ID for use as a filename.
* Replaces filesystem-unsafe characters with underscores.
*/
function sanitizeFilename(name: string): string {
return name.replace(/[<>:"/\\|?*\x00-\x1f]/g, '_');
}
// -- Standalone Functions --
/**
* Write one .md file per stage1_output to rollout_summaries/, prune orphans.
*
* Each file is named {sanitized_thread_id}.md and contains the rollout_summary.
* Files in the directory that do not correspond to any DB row are deleted.
*/
export function syncRolloutSummaries(
memoryHome: string,
outputs: Stage1Output[]
): MaterializationResult {
const summariesDir = join(memoryHome, 'rollout_summaries');
ensureStorageDir(summariesDir);
// Build set of expected filenames
const expectedFiles = new Set<string>();
let summariesWritten = 0;
for (const output of outputs) {
const filename = `${sanitizeFilename(output.thread_id)}.md`;
expectedFiles.add(filename);
const filePath = join(summariesDir, filename);
// Write summary content with thread header
const content = [
`# Session: ${output.thread_id}`,
`> Generated: ${new Date(output.generated_at * 1000).toISOString()}`,
`> Source updated: ${new Date(output.source_updated_at * 1000).toISOString()}`,
'',
output.rollout_summary,
].join('\n');
writeFileSync(filePath, content, 'utf-8');
summariesWritten++;
}
// Prune orphan files not in DB
let summariesPruned = 0;
const existingFiles = readdirSync(summariesDir);
for (const file of existingFiles) {
if (file.endsWith('.md') && !expectedFiles.has(file)) {
unlinkSync(join(summariesDir, file));
summariesPruned++;
}
}
return {
summariesWritten,
summariesPruned,
rawMemoriesSize: 0, // Not applicable for this function
};
}
/**
* Concatenate the latest raw_memory entries into raw_memories.md with thread headers.
*
* Entries are sorted by generated_at descending, limited to maxCount.
* Format per entry:
* ## Thread: {thread_id}
* {raw_memory content}
* ---
*
* @returns The byte size of the written raw_memories.md file.
*/
export function rebuildRawMemories(
memoryHome: string,
outputs: Stage1Output[],
maxCount: number
): number {
ensureStorageDir(memoryHome);
// Sort by generated_at descending, take up to maxCount
const sorted = [...outputs]
.sort((a, b) => b.generated_at - a.generated_at)
.slice(0, maxCount);
const sections: string[] = [];
for (const output of sorted) {
sections.push(
`## Thread: ${output.thread_id}`,
`> Generated: ${new Date(output.generated_at * 1000).toISOString()}`,
'',
output.raw_memory,
'',
'---',
'',
);
}
const content = sections.join('\n');
const filePath = join(memoryHome, 'raw_memories.md');
writeFileSync(filePath, content, 'utf-8');
return Buffer.byteLength(content, 'utf-8');
}
/**
* Read MEMORY.md content for session prompt injection.
*
* @param projectPath - Project root path (used to resolve storage paths)
* @returns MEMORY.md content string, or null if the file does not exist
*/
export function getMemoryMdContent(projectPath: string): string | null {
const paths = getProjectPaths(projectPath);
const memoryMdPath = paths.memoryV2.memoryMd;
if (!existsSync(memoryMdPath)) {
return null;
}
try {
return readFileSync(memoryMdPath, 'utf-8');
} catch {
return null;
}
}
// -- Pipeline Class --
/**
* MemoryConsolidationPipeline orchestrates global memory consolidation:
* 1. Claim global lock via job scheduler
* 2. Materialize Phase 1 outputs to disk (rollout_summaries/ + raw_memories.md)
* 3. Invoke consolidation agent via executeCliTool --mode write
* 4. Monitor with heartbeat lease renewal
* 5. Mark job as succeeded or failed
*/
export class MemoryConsolidationPipeline {
private projectPath: string;
private store: ReturnType<typeof getCoreMemoryStore>;
private scheduler: MemoryJobScheduler;
private memoryHome: string;
private cliTool: string;
constructor(projectPath: string, cliTool?: string) {
this.projectPath = projectPath;
this.store = getCoreMemoryStore(projectPath);
this.scheduler = new MemoryJobScheduler(this.store.getDb());
this.memoryHome = getProjectPaths(projectPath).memoryV2.root;
this.cliTool = cliTool || DEFAULT_CLI_TOOL;
}
/**
* Attempt to claim the global consolidation lock.
*
* Before claiming, ensures the job row exists and checks dirtiness.
* The job scheduler handles the actual concurrency control.
*/
claimGlobalLock(): ClaimResult {
return this.scheduler.claimJob(JOB_KIND, JOB_KEY, MAX_CONCURRENT);
}
/**
* Materialize rollout summaries from DB to disk.
* Writes one .md file per stage1_output row, prunes orphan files.
*/
materializeSummaries(): MaterializationResult {
const outputs = this.store.listStage1Outputs();
return syncRolloutSummaries(this.memoryHome, outputs);
}
/**
* Rebuild raw_memories.md from DB entries.
* Concatenates the latest entries up to MAX_RAW_MEMORIES_FOR_GLOBAL.
*/
materializeRawMemories(): number {
const outputs = this.store.listStage1Outputs();
return rebuildRawMemories(
this.memoryHome,
outputs,
MAX_RAW_MEMORIES_FOR_GLOBAL
);
}
/**
* Run the consolidation agent via executeCliTool with --mode write.
* Starts a heartbeat timer to keep the lease alive during agent execution.
*
* @param token - Ownership token from claimGlobalLock
* @returns true if agent completed successfully, false otherwise
*/
async runConsolidationAgent(token: string): Promise<boolean> {
// Lazy import to avoid circular dependencies at module load time
const { executeCliTool } = await import('../tools/cli-executor-core.js');
// Determine input state for prompt
const summariesDir = join(this.memoryHome, 'rollout_summaries');
let summaryCount = 0;
if (existsSync(summariesDir)) {
summaryCount = readdirSync(summariesDir).filter(f => f.endsWith('.md')).length;
}
const hasExistingMemoryMd = existsSync(join(this.memoryHome, 'MEMORY.md'));
// Ensure skills directory exists
ensureStorageDir(join(this.memoryHome, 'skills'));
// Build the full prompt
const userPrompt = buildConsolidationPrompt(summaryCount, hasExistingMemoryMd);
const fullPrompt = `${CONSOLIDATION_SYSTEM_PROMPT}\n\n${userPrompt}`;
// Start heartbeat timer
const heartbeatMs = HEARTBEAT_INTERVAL_SECONDS * 1000;
const heartbeatTimer = setInterval(() => {
const renewed = this.scheduler.heartbeat(JOB_KIND, JOB_KEY, token);
if (!renewed) {
// Heartbeat rejected - lease was lost
clearInterval(heartbeatTimer);
}
}, heartbeatMs);
try {
// Execute the consolidation agent
const result = await Promise.race([
executeCliTool({
tool: this.cliTool,
prompt: fullPrompt,
mode: 'write',
cd: this.memoryHome,
category: 'internal',
}),
new Promise<never>((_, reject) =>
setTimeout(
() => reject(new Error('Consolidation agent timed out')),
AGENT_TIMEOUT_MS
)
),
]);
return result.success;
} finally {
clearInterval(heartbeatTimer);
}
}
/**
* Verify that Phase 1 artifacts were not modified by the agent.
* Compares file count in rollout_summaries/ before and after agent run.
*
* @param expectedSummaryCount - Number of summaries before agent run
* @returns true if artifacts are intact
*/
verifyPhase1Integrity(expectedSummaryCount: number): boolean {
const summariesDir = join(this.memoryHome, 'rollout_summaries');
if (!existsSync(summariesDir)) return expectedSummaryCount === 0;
const currentCount = readdirSync(summariesDir).filter(f => f.endsWith('.md')).length;
return currentCount === expectedSummaryCount;
}
/**
* Get the current consolidation status.
*/
getStatus(): ConsolidationStatus {
const jobStatus = this.scheduler.getJobStatus(JOB_KIND, JOB_KEY);
const inputCount = this.store.countStage1Outputs();
const memoryMdExists = existsSync(join(this.memoryHome, 'MEMORY.md'));
if (!jobStatus) {
return {
status: 'idle',
memoryMdExists,
inputCount,
};
}
const statusMap: Record<string, ConsolidationStatus['status']> = {
pending: 'idle',
running: 'running',
done: 'completed',
error: 'error',
};
return {
status: statusMap[jobStatus.status] || 'idle',
lastRun: jobStatus.finished_at,
memoryMdExists,
inputCount,
lastError: jobStatus.last_error,
};
}
/**
* Read MEMORY.md content for session prompt injection.
* Convenience method that delegates to the standalone function.
*/
getMemoryMdContent(): string | null {
return getMemoryMdContent(this.projectPath);
}
/**
* Run the full consolidation pipeline.
*
* Pipeline flow:
* 1. Check if there are inputs to process
* 2. Claim global lock
* 3. Materialize Phase 1 outputs to disk
* 4. Run consolidation agent with heartbeat
* 5. Verify Phase 1 integrity
* 6. Mark job as succeeded or failed
*
* @returns Final consolidation status
*/
async runConsolidation(): Promise<ConsolidationStatus> {
// Step 1: Check inputs
const inputCount = this.store.countStage1Outputs();
if (inputCount === 0) {
return {
status: 'idle',
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
inputCount: 0,
};
}
// Step 2: Claim global lock
const claim = this.claimGlobalLock();
if (!claim.claimed || !claim.ownership_token) {
return {
status: claim.reason === 'already_running' ? 'running' : 'idle',
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
inputCount,
lastError: claim.reason
? `Lock not acquired: ${claim.reason}`
: undefined,
};
}
const token = claim.ownership_token;
try {
// Step 3: Materialize Phase 1 outputs to disk
const matResult = this.materializeSummaries();
const rawMemoriesSize = this.materializeRawMemories();
const expectedSummaryCount = matResult.summariesWritten;
// Step 4: Run consolidation agent with heartbeat
const agentSuccess = await this.runConsolidationAgent(token);
if (!agentSuccess) {
this.scheduler.markFailed(
JOB_KIND,
JOB_KEY,
token,
'Consolidation agent returned failure'
);
return {
status: 'error',
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
inputCount,
lastError: 'Consolidation agent returned failure',
};
}
// Step 5: Verify Phase 1 integrity
if (!this.verifyPhase1Integrity(expectedSummaryCount)) {
this.scheduler.markFailed(
JOB_KIND,
JOB_KEY,
token,
'Phase 1 artifacts were modified during consolidation'
);
return {
status: 'error',
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
inputCount,
lastError: 'Phase 1 artifacts were modified during consolidation',
};
}
// Step 6: Check that MEMORY.md was actually produced
const memoryMdExists = existsSync(join(this.memoryHome, 'MEMORY.md'));
if (!memoryMdExists) {
this.scheduler.markFailed(
JOB_KIND,
JOB_KEY,
token,
'Agent completed but MEMORY.md was not produced'
);
return {
status: 'error',
memoryMdExists: false,
inputCount,
lastError: 'Agent completed but MEMORY.md was not produced',
};
}
// Step 7: Mark success with watermark
// Use the current input count as the success watermark
const watermark = inputCount;
this.scheduler.markSucceeded(JOB_KIND, JOB_KEY, token, watermark);
return {
status: 'completed',
lastRun: Math.floor(Date.now() / 1000),
memoryMdExists: true,
inputCount,
};
} catch (err) {
const errorMessage =
err instanceof Error ? err.message : String(err);
this.scheduler.markFailed(JOB_KIND, JOB_KEY, token, errorMessage);
return {
status: 'error',
memoryMdExists: existsSync(join(this.memoryHome, 'MEMORY.md')),
inputCount,
lastError: errorMessage,
};
}
}
}

108
ccw/src/core/memory-consolidation-prompts.ts Normal file
View File

@@ -0,0 +1,108 @@
/**
* Memory Consolidation Prompts - Phase 2 LLM Agent Prompt Templates
*
* System prompt and instruction templates for the consolidation agent that
* reads Phase 1 outputs (rollout_summaries/ + raw_memories.md) and produces
* MEMORY.md (and optional skills/ files).
*
* Design: The agent runs with --mode write in the MEMORY_HOME directory,
* using standard file read/write tools to produce output.
*/
/**
* System-level instructions for the consolidation agent.
* This is prepended to every consolidation prompt.
*/
export const CONSOLIDATION_SYSTEM_PROMPT = `You are a memory consolidation agent. Your job is to read phase-1 extraction artifacts and produce a consolidated MEMORY.md file that captures the most important, actionable knowledge from recent coding sessions.
## CRITICAL RULES
1. **Do NOT modify phase-1 artifacts.** The files in rollout_summaries/ and raw_memories.md are READ-ONLY inputs. Never edit, delete, or overwrite them.
2. **Output only to MEMORY.md** (and optionally skills/ directory). Do not create files outside these paths.
3. **Be concise and actionable.** Every line in MEMORY.md should help a future coding session be more productive.
4. **Resolve conflicts.** When rollout summaries and raw memories disagree, prefer the more recent or more specific information.
5. **Deduplicate.** Merge overlapping knowledge from different sessions into single authoritative entries.
6. **Preserve attribution.** When a piece of knowledge comes from a specific session thread, note it briefly.`;
/**
* Build the full consolidation prompt given the MEMORY_HOME directory path.
*
* The agent will be invoked with --cd pointing to memoryHome, so all file
* references are relative to that directory.
*
* @param inputSummaryCount - Number of rollout summary files available
* @param hasExistingMemoryMd - Whether a previous MEMORY.md exists to update
* @returns Complete prompt string for executeCliTool
*/
export function buildConsolidationPrompt(
inputSummaryCount: number,
hasExistingMemoryMd: boolean
): string {
const action = hasExistingMemoryMd ? 'UPDATE' : 'CREATE';
return `## Task: ${action} MEMORY.md
You have access to the following phase-1 artifacts in the current directory:
### Input Files (READ-ONLY - do NOT modify these)
- **rollout_summaries/*.md** - ${inputSummaryCount} per-session summary files. Each contains a concise summary of what was accomplished in that coding session. Use these for high-level routing and prioritization.
- **raw_memories.md** - Concatenated detailed memories from recent sessions, organized by thread. Use this for detailed knowledge extraction and cross-referencing.
${hasExistingMemoryMd ? '- **MEMORY.md** - The existing consolidated memory file. Update it with new knowledge while preserving still-relevant existing content.' : ''}
### Your Process
1. Read all files in rollout_summaries/ to understand the scope and themes of recent sessions.
2. Read raw_memories.md to extract detailed, actionable knowledge.
3. Cross-reference summaries with raw memories to identify:
- High-signal patterns and conventions discovered
- Architecture decisions and their rationale
- Common pitfalls and their solutions
- Key APIs, interfaces, and integration points
- Testing patterns and debugging approaches
${hasExistingMemoryMd ? '4. Read the existing MEMORY.md and merge new knowledge, removing stale entries.' : '4. Organize extracted knowledge into the output structure below.'}
5. Write the consolidated MEMORY.md file.
6. Optionally, if there are reusable code patterns or workflows worth extracting, create files in the skills/ directory.
### Output: MEMORY.md Structure
Write MEMORY.md with the following sections. Omit any section that has no relevant content.
\`\`\`markdown
# Project Memory
> Auto-generated by memory consolidation. Last updated: [current date]
> Sources: [number] session summaries, [number] raw memory entries
## Architecture & Structure
<!-- Key architectural decisions, module boundaries, data flow patterns -->
## Code Conventions
<!-- Naming conventions, import patterns, error handling approaches, formatting rules -->
## Common Patterns
<!-- Reusable patterns discovered across sessions: state management, API calls, testing approaches -->
## Key APIs & Interfaces
<!-- Important interfaces, function signatures, configuration schemas that are frequently referenced -->
## Known Issues & Gotchas
<!-- Pitfalls, edge cases, platform-specific behaviors, workarounds -->
## Recent Decisions
<!-- Decisions made in recent sessions with brief rationale. Remove when no longer relevant. -->
## Session Insights
<!-- High-value observations from individual sessions worth preserving -->
\`\`\`
### Output: skills/ Directory (Optional)
If you identify reusable code snippets, shell commands, or workflow templates that appear across multiple sessions, create files in the skills/ directory:
- skills/[name].md - Each file should be a self-contained, copy-paste ready reference
### Quality Criteria
- Every entry should be actionable (helps write better code or avoid mistakes)
- No vague platitudes - be specific with file paths, function names, config values
- Prefer concrete examples over abstract descriptions
- Keep total MEMORY.md under 5000 words - be ruthlessly concise
- Remove outdated information that contradicts newer findings`;
}

112
ccw/src/core/memory-embedder-bridge.ts
View File

@@ -9,6 +9,7 @@
* - JSON protocol communication
* - Three commands: embed, search, status
* - Automatic availability checking
* - Stage1 output embedding for V2 pipeline
*/
import { spawn } from 'child_process';
@@ -16,6 +17,9 @@ import { join, dirname } from 'path';
import { existsSync } from 'fs';
import { fileURLToPath } from 'url';
import { getCodexLensPython } from '../utils/codexlens-path.js';
import { getCoreMemoryStore } from './core-memory-store.js';
import type { Stage1Output } from './core-memory-store.js';
import { StoragePaths } from '../config/storage-paths.js';
// Get directory of this module
const __filename = fileURLToPath(import.meta.url);
@@ -256,3 +260,111 @@ export async function getEmbeddingStatus(dbPath: string): Promise<EmbeddingStatu
};
}
}
// ============================================================================
// Memory V2: Stage1 Output Embedding
// ============================================================================
/** Result of stage1 embedding operation */
export interface Stage1EmbedResult {
success: boolean;
chunksCreated: number;
chunksEmbedded: number;
error?: string;
}
/**
* Chunk and embed stage1_outputs (raw_memory + rollout_summary) for semantic search.
*
* Reads all stage1_outputs from the DB, chunks their raw_memory and rollout_summary
* content, inserts chunks into memory_chunks with source_type='cli_history' and
* metadata indicating the V2 origin, then triggers embedding generation.
*
* Uses source_id format: "s1:{thread_id}" to differentiate from regular cli_history chunks.
*
* @param projectPath - Project root path
* @param force - Force re-chunking even if chunks exist
* @returns Embedding result
*/
export async function embedStage1Outputs(
projectPath: string,
force: boolean = false
): Promise<Stage1EmbedResult> {
try {
const store = getCoreMemoryStore(projectPath);
const stage1Outputs = store.listStage1Outputs();
if (stage1Outputs.length === 0) {
return { success: true, chunksCreated: 0, chunksEmbedded: 0 };
}
let totalChunksCreated = 0;
for (const output of stage1Outputs) {
const sourceId = `s1:${output.thread_id}`;
// Check if already chunked
const existingChunks = store.getChunks(sourceId);
if (existingChunks.length > 0 && !force) continue;
// Delete old chunks if force
if (force && existingChunks.length > 0) {
store.deleteChunks(sourceId);
}
// Combine raw_memory and rollout_summary for richer semantic content
const combinedContent = [
output.rollout_summary ? `## Summary\n${output.rollout_summary}` : '',
output.raw_memory ? `## Raw Memory\n${output.raw_memory}` : '',
].filter(Boolean).join('\n\n');
if (!combinedContent.trim()) continue;
// Chunk using the store's built-in chunking
const chunks = store.chunkContent(combinedContent, sourceId, 'cli_history');
// Insert chunks with V2 metadata
for (let i = 0; i < chunks.length; i++) {
store.insertChunk({
source_id: sourceId,
source_type: 'cli_history',
chunk_index: i,
content: chunks[i],
metadata: JSON.stringify({
v2_source: 'stage1_output',
thread_id: output.thread_id,
generated_at: output.generated_at,
}),
created_at: new Date().toISOString(),
});
totalChunksCreated++;
}
}
// If we created chunks, generate embeddings
let chunksEmbedded = 0;
if (totalChunksCreated > 0) {
const paths = StoragePaths.project(projectPath);
const dbPath = join(paths.root, 'core-memory', 'core_memory.db');
const embedResult = await generateEmbeddings(dbPath, { force: false });
if (embedResult.success) {
chunksEmbedded = embedResult.chunks_processed;
}
}
return {
success: true,
chunksCreated: totalChunksCreated,
chunksEmbedded,
};
} catch (err) {
return {
success: false,
chunksCreated: 0,
chunksEmbedded: 0,
error: (err as Error).message,
};
}
}

466
ccw/src/core/memory-extraction-pipeline.ts Normal file
View File

@@ -0,0 +1,466 @@
/**
* Memory Extraction Pipeline - Phase 1 per-session extraction
*
* Orchestrates the full extraction flow for each CLI session:
* Filter transcript -> Truncate -> LLM Extract -> SecretRedact -> PostProcess -> Store
*
* Uses CliHistoryStore for transcript access, executeCliTool for LLM invocation,
* CoreMemoryStore for stage1_outputs storage, and MemoryJobScheduler for
* concurrency control.
*/
import type { ConversationRecord } from '../tools/cli-history-store.js';
import { getHistoryStore } from '../tools/cli-history-store.js';
import { getCoreMemoryStore, type Stage1Output } from './core-memory-store.js';
import { MemoryJobScheduler } from './memory-job-scheduler.js';
import {
MAX_SESSION_AGE_DAYS,
MIN_IDLE_HOURS,
MAX_ROLLOUT_BYTES_FOR_PROMPT,
MAX_RAW_MEMORY_CHARS,
MAX_SUMMARY_CHARS,
MAX_SESSIONS_PER_STARTUP,
PHASE_ONE_CONCURRENCY,
} from './memory-v2-config.js';
import { EXTRACTION_SYSTEM_PROMPT, buildExtractionUserPrompt } from './memory-extraction-prompts.js';
import { redactSecrets } from '../utils/secret-redactor.js';
// -- Types --
export interface ExtractionInput {
sessionId: string;
transcript: string;
sourceUpdatedAt: number;
}
export interface ExtractionOutput {
raw_memory: string;
rollout_summary: string;
}
export interface TranscriptFilterOptions {
/** Bitmask for turn type selection. Default ALL = 0x7FF */
bitmask: number;
/** Maximum bytes for the transcript sent to LLM */
maxBytes: number;
}
export interface BatchExtractionResult {
processed: number;
succeeded: number;
failed: number;
skipped: number;
errors: Array<{ sessionId: string; error: string }>;
}
// -- Turn type bitmask constants --
/** All turn types included */
export const TURN_TYPE_ALL = 0x7FF;
// Individual turn type bits (for future filtering granularity)
export const TURN_TYPE_USER_PROMPT = 0x001;
export const TURN_TYPE_STDOUT = 0x002;
export const TURN_TYPE_STDERR = 0x004;
export const TURN_TYPE_PARSED = 0x008;
// -- Truncation marker --
const TRUNCATION_MARKER = '\n\n[... CONTENT TRUNCATED ...]\n\n';
// -- Job kind constant --
const JOB_KIND_EXTRACTION = 'phase1_extraction';
// -- Pipeline --
export class MemoryExtractionPipeline {
private projectPath: string;
/** Optional: override the LLM tool used for extraction. Defaults to 'gemini'. */
private tool: string;
/** Optional: current session ID to exclude from scanning */
private currentSessionId?: string;
constructor(projectPath: string, options?: { tool?: string; currentSessionId?: string }) {
this.projectPath = projectPath;
this.tool = options?.tool || 'gemini';
this.currentSessionId = options?.currentSessionId;
}
// ========================================================================
// Eligibility scanning
// ========================================================================
/**
* Scan CLI history for sessions eligible for memory extraction.
*
* Eligibility criteria (from design spec section 4.1):
* - Session age <= MAX_SESSION_AGE_DAYS (30 days)
* - Session idle >= MIN_IDLE_HOURS (12 hours) since last update
* - Not an ephemeral/internal session (category !== 'internal')
* - Not the currently active session
* - Has at least one turn with content
*
* @returns Array of eligible ConversationRecord objects, capped at MAX_SESSIONS_PER_STARTUP
*/
scanEligibleSessions(maxSessions?: number): ConversationRecord[] {
const historyStore = getHistoryStore(this.projectPath);
const now = Date.now();
const maxAgeMs = MAX_SESSION_AGE_DAYS * 24 * 60 * 60 * 1000;
const minIdleMs = MIN_IDLE_HOURS * 60 * 60 * 1000;
// Fetch recent conversations (generous limit to filter in-memory)
const { executions } = historyStore.getHistory({ limit: 500 });
const eligible: ConversationRecord[] = [];
for (const entry of executions) {
// Skip current session
if (this.currentSessionId && entry.id === this.currentSessionId) continue;
// Age check: created within MAX_SESSION_AGE_DAYS
const createdAt = new Date(entry.timestamp).getTime();
if (now - createdAt > maxAgeMs) continue;
// Idle check: last updated at least MIN_IDLE_HOURS ago
const updatedAt = new Date(entry.updated_at || entry.timestamp).getTime();
if (now - updatedAt < minIdleMs) continue;
// Skip internal/ephemeral sessions
if (entry.category === 'internal') continue;
// Must have at least 1 turn
if (!entry.turn_count || entry.turn_count < 1) continue;
// Load full conversation to include in result
const conv = historyStore.getConversation(entry.id);
if (!conv) continue;
eligible.push(conv);
if (eligible.length >= (maxSessions || MAX_SESSIONS_PER_STARTUP)) break;
}
return eligible;
}
// ========================================================================
// Transcript filtering
// ========================================================================
/**
* Extract transcript text from a ConversationRecord, keeping only turn types
* that match the given bitmask.
*
* Default bitmask (ALL=0x7FF) includes all turn content: prompt, stdout, stderr, parsed.
*
* @param record - The conversation record to filter
* @param bitmask - Bitmask for type selection (default: TURN_TYPE_ALL)
* @returns Combined transcript text
*/
filterTranscript(record: ConversationRecord, bitmask: number = TURN_TYPE_ALL): string {
const parts: string[] = [];
for (const turn of record.turns) {
const turnParts: string[] = [];
if (bitmask & TURN_TYPE_USER_PROMPT) {
if (turn.prompt) {
turnParts.push(`[USER] ${turn.prompt}`);
}
}
if (bitmask & TURN_TYPE_STDOUT) {
const stdout = turn.output?.parsed_output || turn.output?.stdout;
if (stdout) {
turnParts.push(`[ASSISTANT] ${stdout}`);
}
}
if (bitmask & TURN_TYPE_STDERR) {
if (turn.output?.stderr) {
turnParts.push(`[STDERR] ${turn.output.stderr}`);
}
}
if (bitmask & TURN_TYPE_PARSED) {
// Use final_output if available and not already captured
if (turn.output?.final_output && !(bitmask & TURN_TYPE_STDOUT)) {
turnParts.push(`[FINAL] ${turn.output.final_output}`);
}
}
if (turnParts.length > 0) {
parts.push(`--- Turn ${turn.turn} ---\n${turnParts.join('\n')}`);
}
}
return parts.join('\n\n');
}
// ========================================================================
// Truncation
// ========================================================================
/**
* Truncate transcript content to fit within LLM context limit.
*
* Strategy: Keep head 33% + truncation marker + tail 67%.
* This preserves the session opening context and the most recent work.
*
* @param content - The full transcript text
* @param maxBytes - Maximum allowed size in bytes (default: MAX_ROLLOUT_BYTES_FOR_PROMPT)
* @returns Truncated content, or original if within limit
*/
truncateTranscript(content: string, maxBytes: number = MAX_ROLLOUT_BYTES_FOR_PROMPT): string {
const contentBytes = Buffer.byteLength(content, 'utf-8');
if (contentBytes <= maxBytes) {
return content;
}
// Calculate split sizes accounting for the marker
const markerBytes = Buffer.byteLength(TRUNCATION_MARKER, 'utf-8');
const availableBytes = maxBytes - markerBytes;
const headBytes = Math.floor(availableBytes * 0.33);
const tailBytes = availableBytes - headBytes;
// Convert to character-based approximation (safe for multi-byte)
// Use Buffer slicing for byte-accurate truncation
const buf = Buffer.from(content, 'utf-8');
const headBuf = buf.subarray(0, headBytes);
const tailBuf = buf.subarray(buf.length - tailBytes);
// Decode back to strings, trimming at character boundaries
const head = headBuf.toString('utf-8').replace(/[\uFFFD]$/, '');
const tail = tailBuf.toString('utf-8').replace(/^[\uFFFD]/, '');
return head + TRUNCATION_MARKER + tail;
}
// ========================================================================
// LLM extraction
// ========================================================================
/**
* Call the LLM to extract structured memory from a session transcript.
*
* Uses executeCliTool with the extraction prompts. The LLM is expected
* to return a JSON object with raw_memory and rollout_summary fields.
*
* @param sessionId - Session ID for prompt context
* @param transcript - The filtered and truncated transcript
* @returns Raw LLM output string
*/
async extractMemory(sessionId: string, transcript: string): Promise<string> {
const { executeCliTool } = await import('../tools/cli-executor-core.js');
const userPrompt = buildExtractionUserPrompt(sessionId, transcript);
const fullPrompt = `${EXTRACTION_SYSTEM_PROMPT}\n\n${userPrompt}`;
const result = await executeCliTool({
tool: this.tool,
prompt: fullPrompt,
mode: 'analysis',
cd: this.projectPath,
category: 'internal',
});
// Prefer parsedOutput (extracted text from stream JSON) over raw stdout
const output = result.parsedOutput?.trim() || result.stdout?.trim() || '';
return output;
}
// ========================================================================
// Post-processing
// ========================================================================
/**
* Parse LLM output into structured ExtractionOutput.
*
* Supports 3 parsing modes:
* 1. Pure JSON: Output is a valid JSON object
* 2. Fenced JSON block: JSON wrapped in ```json ... ``` markers
* 3. Text extraction: Non-conforming output wrapped in fallback structure
*
* Applies secret redaction and size limit enforcement.
*
* @param llmOutput - Raw text output from the LLM
* @returns Validated ExtractionOutput with raw_memory and rollout_summary
*/
postProcess(llmOutput: string): ExtractionOutput {
let parsed: { raw_memory?: string; rollout_summary?: string } | null = null;
// Mode 1: Pure JSON
try {
parsed = JSON.parse(llmOutput);
} catch {
// Not pure JSON, try next mode
}
// Mode 2: Fenced JSON block
if (!parsed) {
const fencedMatch = llmOutput.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/);
if (fencedMatch) {
try {
parsed = JSON.parse(fencedMatch[1]);
} catch {
// Fenced content is not valid JSON either
}
}
}
// Mode 3: Text extraction fallback
if (!parsed || typeof parsed.raw_memory !== 'string') {
parsed = {
raw_memory: `# summary\n${llmOutput}\n\nMemory context:\n- Extracted from unstructured LLM output\n\nUser preferences:\n- (none detected)`,
rollout_summary: llmOutput.substring(0, 200).replace(/\n/g, ' ').trim(),
};
}
// Apply secret redaction
let rawMemory = redactSecrets(parsed.raw_memory || '');
let rolloutSummary = redactSecrets(parsed.rollout_summary || '');
// Enforce size limits
if (rawMemory.length > MAX_RAW_MEMORY_CHARS) {
rawMemory = rawMemory.substring(0, MAX_RAW_MEMORY_CHARS);
}
if (rolloutSummary.length > MAX_SUMMARY_CHARS) {
rolloutSummary = rolloutSummary.substring(0, MAX_SUMMARY_CHARS);
}
return { raw_memory: rawMemory, rollout_summary: rolloutSummary };
}
// ========================================================================
// Single session extraction
// ========================================================================
/**
* Run the full extraction pipeline for a single session.
*
* Pipeline stages: Filter -> Truncate -> LLM Extract -> PostProcess -> Store
*
* @param sessionId - The session to extract from
* @returns The stored Stage1Output, or null if extraction failed
*/
async runExtractionJob(sessionId: string): Promise<Stage1Output | null> {
const historyStore = getHistoryStore(this.projectPath);
const record = historyStore.getConversation(sessionId);
if (!record) {
throw new Error(`Session not found: ${sessionId}`);
}
// Stage 1: Filter transcript
const transcript = this.filterTranscript(record);
if (!transcript.trim()) {
return null; // Empty transcript, nothing to extract
}
// Stage 2: Truncate
const truncated = this.truncateTranscript(transcript);
// Stage 3: LLM extraction
const llmOutput = await this.extractMemory(sessionId, truncated);
if (!llmOutput) {
throw new Error(`LLM returned empty output for session: ${sessionId}`);
}
// Stage 4: Post-process (parse + redact + validate)
const extracted = this.postProcess(llmOutput);
// Stage 5: Store result
const sourceUpdatedAt = Math.floor(new Date(record.updated_at).getTime() / 1000);
const generatedAt = Math.floor(Date.now() / 1000);
const output: Stage1Output = {
thread_id: sessionId,
source_updated_at: sourceUpdatedAt,
raw_memory: extracted.raw_memory,
rollout_summary: extracted.rollout_summary,
generated_at: generatedAt,
};
const store = getCoreMemoryStore(this.projectPath);
store.upsertStage1Output(output);
return output;
}
// ========================================================================
// Batch orchestration
// ========================================================================
/**
* Run extraction for all eligible sessions with concurrency control.
*
* Uses MemoryJobScheduler to claim jobs and enforce PHASE_ONE_CONCURRENCY.
* Failed extractions are recorded in the scheduler for retry.
*
* @returns BatchExtractionResult with counts and error details
*/
async runBatchExtraction(options?: { maxSessions?: number }): Promise<BatchExtractionResult> {
const store = getCoreMemoryStore(this.projectPath);
const scheduler = new MemoryJobScheduler(store.getDb());
// Scan eligible sessions
const eligibleSessions = this.scanEligibleSessions(options?.maxSessions);
const result: BatchExtractionResult = {
processed: 0,
succeeded: 0,
failed: 0,
skipped: 0,
errors: [],
};
if (eligibleSessions.length === 0) {
return result;
}
// Enqueue all eligible sessions
for (const session of eligibleSessions) {
const watermark = Math.floor(new Date(session.updated_at).getTime() / 1000);
scheduler.enqueueJob(JOB_KIND_EXTRACTION, session.id, watermark);
}
// Process with concurrency control using Promise.all with batching
const batchSize = PHASE_ONE_CONCURRENCY;
for (let i = 0; i < eligibleSessions.length; i += batchSize) {
const batch = eligibleSessions.slice(i, i + batchSize);
const promises = batch.map(async (session) => {
// Try to claim the job
const claim = scheduler.claimJob(JOB_KIND_EXTRACTION, session.id, batchSize);
if (!claim.claimed) {
result.skipped++;
return;
}
result.processed++;
const token = claim.ownership_token!;
try {
const output = await this.runExtractionJob(session.id);
if (output) {
const watermark = output.source_updated_at;
scheduler.markSucceeded(JOB_KIND_EXTRACTION, session.id, token, watermark);
result.succeeded++;
} else {
// Empty transcript - mark as done (nothing to extract)
scheduler.markSucceeded(JOB_KIND_EXTRACTION, session.id, token, 0);
result.skipped++;
}
} catch (err) {
const errorMsg = err instanceof Error ? err.message : String(err);
scheduler.markFailed(JOB_KIND_EXTRACTION, session.id, token, errorMsg);
result.failed++;
result.errors.push({ sessionId: session.id, error: errorMsg });
}
});
await Promise.all(promises);
}
return result;
}
}

91
ccw/src/core/memory-extraction-prompts.ts Normal file
View File

@@ -0,0 +1,91 @@
/**
* Memory Extraction Prompts - LLM prompt templates for Phase 1 extraction
*
* Provides system and user prompt templates for extracting structured memory
* from CLI session transcripts. The LLM output must conform to a JSON schema
* with raw_memory and rollout_summary fields.
*
* Design spec section 4.4: Prompt structure with outcome triage rules.
*/
/**
* System prompt for the extraction LLM call.
*
* Instructs the model to:
* - Produce a JSON object with raw_memory and rollout_summary
* - Follow structure markers in raw_memory (# summary, Memory context, etc.)
* - Apply outcome triage rules for categorizing task results
* - Keep rollout_summary concise (1-2 sentences)
*/
export const EXTRACTION_SYSTEM_PROMPT = `You are a memory extraction agent. Your job is to read a CLI session transcript and produce structured memory output.
You MUST respond with a valid JSON object containing exactly two fields:
{
"raw_memory": "<structured memory text>",
"rollout_summary": "<1-2 sentence summary>"
}
## raw_memory format
The raw_memory field must follow this structure:
# summary
<One paragraph high-level summary of what was accomplished in this session>
Memory context:
- Project: <project name or path if identifiable>
- Tools used: <CLI tools, frameworks, languages mentioned>
- Key files: <important files created or modified>
User preferences:
- <Any coding style preferences, conventions, or patterns the user demonstrated>
- <Tool preferences, workflow habits>
## Task: <task title or description>
Outcome: <success | partial | failed | abandoned>
<Detailed description of what was done, decisions made, and results>
### Key decisions
- <Important architectural or design decisions>
- <Trade-offs considered>
### Lessons learned
- <What worked well>
- <What did not work and why>
- <Gotchas or pitfalls discovered>
## Outcome Triage Rules
- **success**: Task was completed as intended, tests pass, code works
- **partial**: Some progress made but not fully complete; note what remains
- **failed**: Attempted but could not achieve the goal; document root cause
- **abandoned**: User switched direction or cancelled; note the reason
## rollout_summary format
A concise 1-2 sentence summary capturing:
- What the session was about (the goal)
- The outcome (success/partial/failed)
- The most important takeaway
Do NOT include markdown code fences in your response. Return raw JSON only.`;
/**
* Build the user prompt by injecting the session transcript.
*
* @param sessionId - The session/conversation ID for reference
* @param transcript - The filtered and truncated transcript text
* @returns The complete user prompt string
*/
export function buildExtractionUserPrompt(sessionId: string, transcript: string): string {
return `Extract structured memory from the following CLI session transcript.
Session ID: ${sessionId}
--- BEGIN TRANSCRIPT ---
${transcript}
--- END TRANSCRIPT ---
Respond with a JSON object containing "raw_memory" and "rollout_summary" fields.`;
}

335
ccw/src/core/memory-job-scheduler.ts Normal file
View File

@@ -0,0 +1,335 @@
/**
* Memory Job Scheduler - Lease-based job scheduling backed by SQLite
*
* Provides atomic claim/release/heartbeat operations for coordinating
* concurrent memory extraction and consolidation jobs.
*
* All state lives in the `jobs` table of the CoreMemoryStore database.
* Concurrency control uses ownership_token + lease_until for distributed-safe
* (but single-process) job dispatch.
*/
import type Database from 'better-sqlite3';
import { randomUUID } from 'crypto';
import { LEASE_SECONDS, MAX_RETRIES, RETRY_DELAY_SECONDS } from './memory-v2-config.js';
// -- Types --
export type JobStatus = 'pending' | 'running' | 'done' | 'error';
export interface JobRecord {
kind: string;
job_key: string;
status: JobStatus;
worker_id?: string;
ownership_token?: string;
started_at?: number;
finished_at?: number;
lease_until?: number;
retry_at?: number;
retry_remaining: number;
last_error?: string;
input_watermark: number;
last_success_watermark: number;
}
export interface ClaimResult {
claimed: boolean;
ownership_token?: string;
reason?: 'already_running' | 'retry_exhausted' | 'retry_pending' | 'concurrency_limit';
}
// -- Scheduler --
export class MemoryJobScheduler {
private db: Database.Database;
constructor(db: Database.Database) {
this.db = db;
}
/**
* Atomically claim a job for processing.
*
* Logic:
* 1. If job does not exist, insert it as 'running' with a fresh token.
* 2. If job exists and is 'pending', transition to 'running'.
* 3. If job exists and is 'running' but lease expired, reclaim it.
* 4. If job exists and is 'error' with retry_remaining > 0 and retry_at <= now, reclaim it.
* 5. Otherwise, return not claimed with reason.
*
* Respects maxConcurrent: total running jobs of this `kind` must not exceed limit.
*/
claimJob(kind: string, jobKey: string, maxConcurrent: number, workerId?: string): ClaimResult {
const now = Math.floor(Date.now() / 1000);
const token = randomUUID();
const leaseUntil = now + LEASE_SECONDS;
// Use a transaction for atomicity
const result = this.db.transaction(() => {
// Check concurrency limit for this kind
const runningCount = this.db.prepare(
`SELECT COUNT(*) as cnt FROM jobs WHERE kind = ? AND status = 'running' AND lease_until > ?`
).get(kind, now) as { cnt: number };
const existing = this.db.prepare(
`SELECT * FROM jobs WHERE kind = ? AND job_key = ?`
).get(kind, jobKey) as any | undefined;
if (!existing) {
// No job row yet - check concurrency before inserting
if (runningCount.cnt >= maxConcurrent) {
return { claimed: false, reason: 'concurrency_limit' as const };
}
this.db.prepare(`
INSERT INTO jobs (kind, job_key, status, worker_id, ownership_token, started_at, lease_until, retry_remaining, input_watermark, last_success_watermark)
VALUES (?, ?, 'running', ?, ?, ?, ?, ?, 0, 0)
`).run(kind, jobKey, workerId || null, token, now, leaseUntil, MAX_RETRIES);
return { claimed: true, ownership_token: token };
}
// Job exists - check status transitions
if (existing.status === 'done') {
// Already done - check dirty (input_watermark > last_success_watermark)
if (existing.input_watermark <= existing.last_success_watermark) {
return { claimed: false, reason: 'already_running' as const };
}
// Dirty - re-run
if (runningCount.cnt >= maxConcurrent) {
return { claimed: false, reason: 'concurrency_limit' as const };
}
this.db.prepare(`
UPDATE jobs SET status = 'running', worker_id = ?, ownership_token = ?,
started_at = ?, lease_until = ?, finished_at = NULL, last_error = NULL,
retry_remaining = ?
WHERE kind = ? AND job_key = ?
`).run(workerId || null, token, now, leaseUntil, MAX_RETRIES, kind, jobKey);
return { claimed: true, ownership_token: token };
}
if (existing.status === 'running') {
// Running - check lease expiry
if (existing.lease_until > now) {
return { claimed: false, reason: 'already_running' as const };
}
// Lease expired - reclaim if concurrency allows
// The expired job doesn't count towards running total (lease_until <= now),
// so runningCount already excludes it.
if (runningCount.cnt >= maxConcurrent) {
return { claimed: false, reason: 'concurrency_limit' as const };
}
this.db.prepare(`
UPDATE jobs SET worker_id = ?, ownership_token = ?, started_at = ?,
lease_until = ?, last_error = NULL
WHERE kind = ? AND job_key = ?
`).run(workerId || null, token, now, leaseUntil, kind, jobKey);
return { claimed: true, ownership_token: token };
}
if (existing.status === 'pending') {
if (runningCount.cnt >= maxConcurrent) {
return { claimed: false, reason: 'concurrency_limit' as const };
}
this.db.prepare(`
UPDATE jobs SET status = 'running', worker_id = ?, ownership_token = ?,
started_at = ?, lease_until = ?
WHERE kind = ? AND job_key = ?
`).run(workerId || null, token, now, leaseUntil, kind, jobKey);
return { claimed: true, ownership_token: token };
}
if (existing.status === 'error') {
if (existing.retry_remaining <= 0) {
return { claimed: false, reason: 'retry_exhausted' as const };
}
if (existing.retry_at && existing.retry_at > now) {
return { claimed: false, reason: 'retry_pending' as const };
}
if (runningCount.cnt >= maxConcurrent) {
return { claimed: false, reason: 'concurrency_limit' as const };
}
this.db.prepare(`
UPDATE jobs SET status = 'running', worker_id = ?, ownership_token = ?,
started_at = ?, lease_until = ?, last_error = NULL,
retry_remaining = retry_remaining - 1
WHERE kind = ? AND job_key = ?
`).run(workerId || null, token, now, leaseUntil, kind, jobKey);
return { claimed: true, ownership_token: token };
}
return { claimed: false, reason: 'already_running' as const };
})();
return result;
}
/**
* Release a job, marking it as done or error.
* Only succeeds if the ownership_token matches.
*/
releaseJob(kind: string, jobKey: string, token: string, status: 'done' | 'error', error?: string): boolean {
const now = Math.floor(Date.now() / 1000);
const result = this.db.prepare(`
UPDATE jobs SET
status = ?,
finished_at = ?,
lease_until = NULL,
ownership_token = NULL,
worker_id = NULL,
last_error = ?
WHERE kind = ? AND job_key = ? AND ownership_token = ?
`).run(status, now, error || null, kind, jobKey, token);
return result.changes > 0;
}
/**
* Renew the lease for an active job.
* Returns false if ownership_token does not match or job is not running.
*/
heartbeat(kind: string, jobKey: string, token: string, leaseSeconds: number = LEASE_SECONDS): boolean {
const now = Math.floor(Date.now() / 1000);
const newLeaseUntil = now + leaseSeconds;
const result = this.db.prepare(`
UPDATE jobs SET lease_until = ?
WHERE kind = ? AND job_key = ? AND ownership_token = ? AND status = 'running'
`).run(newLeaseUntil, kind, jobKey, token);
return result.changes > 0;
}
/**
* Enqueue a job or update its input_watermark.
* Uses MAX(existing, new) for watermark to ensure monotonicity.
* If job doesn't exist, creates it in 'pending' status.
*/
enqueueJob(kind: string, jobKey: string, inputWatermark: number): void {
this.db.prepare(`
INSERT INTO jobs (kind, job_key, status, retry_remaining, input_watermark, last_success_watermark)
VALUES (?, ?, 'pending', ?, ?, 0)
ON CONFLICT(kind, job_key) DO UPDATE SET
input_watermark = MAX(jobs.input_watermark, excluded.input_watermark)
`).run(kind, jobKey, MAX_RETRIES, inputWatermark);
}
/**
* Mark a job as successfully completed and update success watermark.
* Only succeeds if ownership_token matches.
*/
markSucceeded(kind: string, jobKey: string, token: string, watermark: number): boolean {
const now = Math.floor(Date.now() / 1000);
const result = this.db.prepare(`
UPDATE jobs SET
status = 'done',
finished_at = ?,
lease_until = NULL,
ownership_token = NULL,
worker_id = NULL,
last_error = NULL,
last_success_watermark = ?
WHERE kind = ? AND job_key = ? AND ownership_token = ?
`).run(now, watermark, kind, jobKey, token);
return result.changes > 0;
}
/**
* Mark a job as failed with error message and schedule retry.
* Only succeeds if ownership_token matches.
*/
markFailed(kind: string, jobKey: string, token: string, error: string, retryDelay: number = RETRY_DELAY_SECONDS): boolean {
const now = Math.floor(Date.now() / 1000);
const retryAt = now + retryDelay;
const result = this.db.prepare(`
UPDATE jobs SET
status = 'error',
finished_at = ?,
lease_until = NULL,
ownership_token = NULL,
worker_id = NULL,
last_error = ?,
retry_at = ?
WHERE kind = ? AND job_key = ? AND ownership_token = ?
`).run(now, error, retryAt, kind, jobKey, token);
return result.changes > 0;
}
/**
* Get current status of a specific job.
*/
getJobStatus(kind: string, jobKey: string): JobRecord | null {
const row = this.db.prepare(
`SELECT * FROM jobs WHERE kind = ? AND job_key = ?`
).get(kind, jobKey) as any;
if (!row) return null;
return this.rowToJobRecord(row);
}
/**
* List jobs, optionally filtered by kind and/or status.
*/
listJobs(kind?: string, status?: JobStatus): JobRecord[] {
let query = 'SELECT * FROM jobs';
const params: any[] = [];
const conditions: string[] = [];
if (kind) {
conditions.push('kind = ?');
params.push(kind);
}
if (status) {
conditions.push('status = ?');
params.push(status);
}
if (conditions.length > 0) {
query += ' WHERE ' + conditions.join(' AND ');
}
query += ' ORDER BY kind, job_key';
const rows = this.db.prepare(query).all(...params) as any[];
return rows.map(row => this.rowToJobRecord(row));
}
/**
* Check if a job is dirty (input_watermark > last_success_watermark).
*/
isDirty(kind: string, jobKey: string): boolean {
const row = this.db.prepare(
`SELECT input_watermark, last_success_watermark FROM jobs WHERE kind = ? AND job_key = ?`
).get(kind, jobKey) as any;
if (!row) return false;
return row.input_watermark > row.last_success_watermark;
}
/**
* Convert a database row to a typed JobRecord.
*/
private rowToJobRecord(row: any): JobRecord {
return {
kind: row.kind,
job_key: row.job_key,
status: row.status,
worker_id: row.worker_id || undefined,
ownership_token: row.ownership_token || undefined,
started_at: row.started_at || undefined,
finished_at: row.finished_at || undefined,
lease_until: row.lease_until || undefined,
retry_at: row.retry_at || undefined,
retry_remaining: row.retry_remaining,
last_error: row.last_error || undefined,
input_watermark: row.input_watermark ?? 0,
last_success_watermark: row.last_success_watermark ?? 0,
};
}
}

84
ccw/src/core/memory-v2-config.ts Normal file
View File

@@ -0,0 +1,84 @@
/**
* Memory V2 Configuration Constants
*
* All tuning parameters for the two-phase memory extraction and consolidation pipeline.
* Phase 1: Per-session extraction (transcript -> structured memory)
* Phase 2: Global consolidation (structured memories -> MEMORY.md)
*/
// -- Batch orchestration --
/** Maximum sessions to process per startup/trigger */
export const MAX_SESSIONS_PER_STARTUP = 64;
/** Maximum concurrent Phase 1 extraction jobs */
export const PHASE_ONE_CONCURRENCY = 64;
// -- Session eligibility --
/** Maximum session age in days to consider for extraction */
export const MAX_SESSION_AGE_DAYS = 30;
/** Minimum idle hours before a session becomes eligible */
export const MIN_IDLE_HOURS = 12;
// -- Job scheduler --
/** Default lease duration in seconds (1 hour) */
export const LEASE_SECONDS = 3600;
/** Delay in seconds before retrying a failed job */
export const RETRY_DELAY_SECONDS = 3600;
/** Maximum retry attempts for a failed job */
export const MAX_RETRIES = 3;
/** Interval in seconds between heartbeat renewals */
export const HEARTBEAT_INTERVAL_SECONDS = 30;
// -- Content size limits --
/** Maximum characters for raw_memory field in stage1_outputs */
export const MAX_RAW_MEMORY_CHARS = 300_000;
/** Maximum characters for rollout_summary field in stage1_outputs */
export const MAX_SUMMARY_CHARS = 1200;
/** Maximum bytes of transcript to send to LLM for extraction */
export const MAX_ROLLOUT_BYTES_FOR_PROMPT = 1_000_000;
/** Maximum number of raw memories included in global consolidation input */
export const MAX_RAW_MEMORIES_FOR_GLOBAL = 64;
// -- Typed configuration object --
export interface MemoryV2Config {
MAX_SESSIONS_PER_STARTUP: number;
PHASE_ONE_CONCURRENCY: number;
MAX_SESSION_AGE_DAYS: number;
MIN_IDLE_HOURS: number;
LEASE_SECONDS: number;
RETRY_DELAY_SECONDS: number;
MAX_RETRIES: number;
HEARTBEAT_INTERVAL_SECONDS: number;
MAX_RAW_MEMORY_CHARS: number;
MAX_SUMMARY_CHARS: number;
MAX_ROLLOUT_BYTES_FOR_PROMPT: number;
MAX_RAW_MEMORIES_FOR_GLOBAL: number;
}
/** Default configuration object - use individual exports for direct access */
export const MEMORY_V2_DEFAULTS: Readonly<MemoryV2Config> = {
MAX_SESSIONS_PER_STARTUP,
PHASE_ONE_CONCURRENCY,
MAX_SESSION_AGE_DAYS,
MIN_IDLE_HOURS,
LEASE_SECONDS,
RETRY_DELAY_SECONDS,
MAX_RETRIES,
MAX_RAW_MEMORY_CHARS,
MAX_SUMMARY_CHARS,
MAX_ROLLOUT_BYTES_FOR_PROMPT,
MAX_RAW_MEMORIES_FOR_GLOBAL,
HEARTBEAT_INTERVAL_SECONDS,
} as const;

195
ccw/src/core/routes/core-memory-routes.ts
View File

@@ -4,6 +4,8 @@ import { getCoreMemoryStore } from '../core-memory-store.js';
import type { CoreMemory, SessionCluster, ClusterMember, ClusterRelation } from '../core-memory-store.js';
import { getEmbeddingStatus, generateEmbeddings } from '../memory-embedder-bridge.js';
import { checkSemanticStatus } from '../../tools/codex-lens.js';
import { MemoryJobScheduler } from '../memory-job-scheduler.js';
import type { JobStatus } from '../memory-job-scheduler.js';
import { StoragePaths } from '../../config/storage-paths.js';
import { join } from 'path';
import { getDefaultTool } from '../../tools/claude-cli-tools.js';
@@ -233,6 +235,199 @@ export async function handleCoreMemoryRoutes(ctx: RouteContext): Promise<boolean
return true;
}
// ============================================================
// Memory V2 Pipeline API Endpoints
// ============================================================
// API: Trigger batch extraction (fire-and-forget)
if (pathname === '/api/core-memory/extract' && req.method === 'POST') {
handlePostRequest(req, res, async (body) => {
const { maxSessions, path: projectPath } = body;
const basePath = projectPath || initialPath;
try {
const { MemoryExtractionPipeline } = await import('../memory-extraction-pipeline.js');
const pipeline = new MemoryExtractionPipeline(basePath);
// Broadcast start event
broadcastToClients({
type: 'MEMORY_EXTRACTION_STARTED',
payload: {
timestamp: new Date().toISOString(),
maxSessions: maxSessions || 'default',
}
});
// Fire-and-forget: trigger async, notify on completion
const batchPromise = pipeline.runBatchExtraction();
batchPromise.then(() => {
broadcastToClients({
type: 'MEMORY_EXTRACTION_COMPLETED',
payload: { timestamp: new Date().toISOString() }
});
}).catch((err: Error) => {
broadcastToClients({
type: 'MEMORY_EXTRACTION_FAILED',
payload: {
timestamp: new Date().toISOString(),
error: err.message,
}
});
});
// Scan eligible sessions to report count
const eligible = pipeline.scanEligibleSessions();
return {
success: true,
triggered: true,
eligibleCount: eligible.length,
message: `Extraction triggered for ${eligible.length} eligible sessions`,
};
} catch (error: unknown) {
return { error: (error as Error).message, status: 500 };
}
});
return true;
}
// API: Get extraction pipeline status
if (pathname === '/api/core-memory/extract/status' && req.method === 'GET') {
const projectPath = url.searchParams.get('path') || initialPath;
try {
const store = getCoreMemoryStore(projectPath);
const scheduler = new MemoryJobScheduler(store.getDb());
const stage1Count = store.countStage1Outputs();
const extractionJobs = scheduler.listJobs('phase1_extraction');
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
success: true,
total_stage1: stage1Count,
jobs: extractionJobs.map(j => ({
job_key: j.job_key,
status: j.status,
started_at: j.started_at,
finished_at: j.finished_at,
last_error: j.last_error,
retry_remaining: j.retry_remaining,
})),
}));
} catch (error: unknown) {
res.writeHead(500, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: (error as Error).message }));
}
return true;
}
// API: Trigger consolidation (fire-and-forget)
if (pathname === '/api/core-memory/consolidate' && req.method === 'POST') {
handlePostRequest(req, res, async (body) => {
const { path: projectPath } = body;
const basePath = projectPath || initialPath;
try {
const { MemoryConsolidationPipeline } = await import('../memory-consolidation-pipeline.js');
const pipeline = new MemoryConsolidationPipeline(basePath);
// Broadcast start event
broadcastToClients({
type: 'MEMORY_CONSOLIDATION_STARTED',
payload: { timestamp: new Date().toISOString() }
});
// Fire-and-forget
const consolidatePromise = pipeline.runConsolidation();
consolidatePromise.then(() => {
broadcastToClients({
type: 'MEMORY_CONSOLIDATION_COMPLETED',
payload: { timestamp: new Date().toISOString() }
});
}).catch((err: Error) => {
broadcastToClients({
type: 'MEMORY_CONSOLIDATION_FAILED',
payload: {
timestamp: new Date().toISOString(),
error: err.message,
}
});
});
return {
success: true,
triggered: true,
message: 'Consolidation triggered',
};
} catch (error: unknown) {
return { error: (error as Error).message, status: 500 };
}
});
return true;
}
// API: Get consolidation status
if (pathname === '/api/core-memory/consolidate/status' && req.method === 'GET') {
const projectPath = url.searchParams.get('path') || initialPath;
try {
const { MemoryConsolidationPipeline } = await import('../memory-consolidation-pipeline.js');
const pipeline = new MemoryConsolidationPipeline(projectPath);
const status = pipeline.getStatus();
const memoryMd = pipeline.getMemoryMdContent();
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
success: true,
status: status?.status || 'unknown',
memoryMdAvailable: !!memoryMd,
memoryMdPreview: memoryMd ? memoryMd.substring(0, 500) : undefined,
}));
} catch (error: unknown) {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
success: true,
status: 'unavailable',
memoryMdAvailable: false,
error: (error as Error).message,
}));
}
return true;
}
// API: List all V2 pipeline jobs
if (pathname === '/api/core-memory/jobs' && req.method === 'GET') {
const projectPath = url.searchParams.get('path') || initialPath;
const kind = url.searchParams.get('kind') || undefined;
const statusFilter = url.searchParams.get('status') as JobStatus | undefined;
try {
const store = getCoreMemoryStore(projectPath);
const scheduler = new MemoryJobScheduler(store.getDb());
const jobs = scheduler.listJobs(kind, statusFilter);
// Compute byStatus counts
const byStatus: Record<string, number> = {};
for (const job of jobs) {
byStatus[job.status] = (byStatus[job.status] || 0) + 1;
}
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
success: true,
jobs,
total: jobs.length,
byStatus,
}));
} catch (error: unknown) {
res.writeHead(500, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: (error as Error).message }));
}
return true;
}
// ============================================================
// Session Clustering API Endpoints
// ============================================================

245
ccw/src/tools/core-memory.ts
View File

@@ -7,12 +7,17 @@ import { z } from 'zod';
import type { ToolSchema, ToolResult } from '../types/tool.js';
import { getCoreMemoryStore, findMemoryAcrossProjects } from '../core/core-memory-store.js';
import * as MemoryEmbedder from '../core/memory-embedder-bridge.js';
import { MemoryJobScheduler } from '../core/memory-job-scheduler.js';
import type { JobRecord, JobStatus } from '../core/memory-job-scheduler.js';
import { StoragePaths } from '../config/storage-paths.js';
import { join } from 'path';
import { getProjectRoot } from '../utils/path-validator.js';
// Zod schemas
const OperationEnum = z.enum(['list', 'import', 'export', 'summary', 'embed', 'search', 'embed_status']);
const OperationEnum = z.enum([
'list', 'import', 'export', 'summary', 'embed', 'search', 'embed_status',
'extract', 'extract_status', 'consolidate', 'consolidate_status', 'jobs',
]);
const ParamsSchema = z.object({
operation: OperationEnum,
@@ -31,6 +36,11 @@ const ParamsSchema = z.object({
source_id: z.string().optional(),
batch_size: z.number().optional().default(8),
force: z.boolean().optional().default(false),
// V2 extract parameters
max_sessions: z.number().optional(),
// V2 jobs parameters
kind: z.string().optional(),
status_filter: z.enum(['pending', 'running', 'done', 'error']).optional(),
});
type Params = z.infer<typeof ParamsSchema>;
@@ -105,7 +115,44 @@ interface EmbedStatusResult {
by_type: Record<string, { total: number; embedded: number }>;
}
type OperationResult = ListResult | ImportResult | ExportResult | SummaryResult | EmbedResult | SearchResult | EmbedStatusResult;
// -- Memory V2 operation result types --
interface ExtractResult {
operation: 'extract';
triggered: boolean;
jobIds: string[];
message: string;
}
interface ExtractStatusResult {
operation: 'extract_status';
total_stage1: number;
jobs: Array<{ job_key: string; status: string; last_error?: string }>;
}
interface ConsolidateResult {
operation: 'consolidate';
triggered: boolean;
message: string;
}
interface ConsolidateStatusResult {
operation: 'consolidate_status';
status: string;
memoryMdAvailable: boolean;
memoryMdPreview?: string;
}
interface JobsResult {
operation: 'jobs';
jobs: JobRecord[];
total: number;
byStatus: Record<string, number>;
}
type OperationResult = ListResult | ImportResult | ExportResult | SummaryResult
| EmbedResult | SearchResult | EmbedStatusResult
| ExtractResult | ExtractStatusResult | ConsolidateResult | ConsolidateStatusResult | JobsResult;
/**
* Get project path - uses explicit path if provided, otherwise falls back to current working directory
@@ -333,6 +380,165 @@ async function executeEmbedStatus(params: Params): Promise<EmbedStatusResult> {
};
}
// ============================================================================
// Memory V2 Operation Handlers
// ============================================================================
/**
* Operation: extract
* Trigger batch extraction (fire-and-forget). Returns job IDs immediately.
*/
async function executeExtract(params: Params): Promise<ExtractResult> {
const { max_sessions, path } = params;
const projectPath = getProjectPath(path);
try {
const { MemoryExtractionPipeline } = await import('../core/memory-extraction-pipeline.js');
const pipeline = new MemoryExtractionPipeline(projectPath);
// Fire-and-forget: trigger batch extraction asynchronously
const batchPromise = pipeline.runBatchExtraction({ maxSessions: max_sessions });
// Don't await - let it run in background
batchPromise.catch((err: Error) => {
// Log errors but don't throw - fire-and-forget
console.error(`[memory-v2] Batch extraction error: ${err.message}`);
});
// Scan eligible sessions to report count
const eligible = pipeline.scanEligibleSessions(max_sessions);
const sessionIds = eligible.map(s => s.id);
return {
operation: 'extract',
triggered: true,
jobIds: sessionIds,
message: `Extraction triggered for ${eligible.length} eligible sessions (max: ${max_sessions || 'default'})`,
};
} catch (err) {
return {
operation: 'extract',
triggered: false,
jobIds: [],
message: `Failed to trigger extraction: ${(err as Error).message}`,
};
}
}
/**
* Operation: extract_status
* Get extraction pipeline state.
*/
async function executeExtractStatus(params: Params): Promise<ExtractStatusResult> {
const { path } = params;
const projectPath = getProjectPath(path);
const store = getCoreMemoryStore(projectPath);
const scheduler = new MemoryJobScheduler(store.getDb());
const stage1Count = store.countStage1Outputs();
const extractionJobs = scheduler.listJobs('extraction');
return {
operation: 'extract_status',
total_stage1: stage1Count,
jobs: extractionJobs.map(j => ({
job_key: j.job_key,
status: j.status,
last_error: j.last_error,
})),
};
}
/**
* Operation: consolidate
* Trigger consolidation (fire-and-forget).
*/
async function executeConsolidate(params: Params): Promise<ConsolidateResult> {
const { path } = params;
const projectPath = getProjectPath(path);
try {
const { MemoryConsolidationPipeline } = await import('../core/memory-consolidation-pipeline.js');
const pipeline = new MemoryConsolidationPipeline(projectPath);
// Fire-and-forget: trigger consolidation asynchronously
const consolidatePromise = pipeline.runConsolidation();
consolidatePromise.catch((err: Error) => {
console.error(`[memory-v2] Consolidation error: ${err.message}`);
});
return {
operation: 'consolidate',
triggered: true,
message: 'Consolidation triggered',
};
} catch (err) {
return {
operation: 'consolidate',
triggered: false,
message: `Failed to trigger consolidation: ${(err as Error).message}`,
};
}
}
/**
* Operation: consolidate_status
* Get consolidation pipeline state.
*/
async function executeConsolidateStatus(params: Params): Promise<ConsolidateStatusResult> {
const { path } = params;
const projectPath = getProjectPath(path);
try {
const { MemoryConsolidationPipeline } = await import('../core/memory-consolidation-pipeline.js');
const pipeline = new MemoryConsolidationPipeline(projectPath);
const status = pipeline.getStatus();
const memoryMd = pipeline.getMemoryMdContent();
return {
operation: 'consolidate_status',
status: status?.status || 'unknown',
memoryMdAvailable: !!memoryMd,
memoryMdPreview: memoryMd ? memoryMd.substring(0, 500) : undefined,
};
} catch {
return {
operation: 'consolidate_status',
status: 'unavailable',
memoryMdAvailable: false,
};
}
}
/**
* Operation: jobs
* List all V2 jobs with optional kind filter.
*/
function executeJobs(params: Params): JobsResult {
const { kind, status_filter, path } = params;
const projectPath = getProjectPath(path);
const store = getCoreMemoryStore(projectPath);
const scheduler = new MemoryJobScheduler(store.getDb());
const jobs = scheduler.listJobs(kind, status_filter as JobStatus | undefined);
// Compute byStatus counts
const byStatus: Record<string, number> = {};
for (const job of jobs) {
byStatus[job.status] = (byStatus[job.status] || 0) + 1;
}
return {
operation: 'jobs',
jobs,
total: jobs.length,
byStatus,
};
}
/**
* Route to appropriate operation handler
*/
@@ -354,9 +560,19 @@ async function execute(params: Params): Promise<OperationResult> {
return executeSearch(params);
case 'embed_status':
return executeEmbedStatus(params);
case 'extract':
return executeExtract(params);
case 'extract_status':
return executeExtractStatus(params);
case 'consolidate':
return executeConsolidate(params);
case 'consolidate_status':
return executeConsolidateStatus(params);
case 'jobs':
return executeJobs(params);
default:
throw new Error(
`Unknown operation: ${operation}. Valid operations: list, import, export, summary, embed, search, embed_status`
`Unknown operation: ${operation}. Valid operations: list, import, export, summary, embed, search, embed_status, extract, extract_status, consolidate, consolidate_status, jobs`
);
}
}
@@ -374,6 +590,11 @@ Usage:
core_memory(operation="embed", source_id="CMEM-xxx") # Generate embeddings for memory
core_memory(operation="search", query="authentication") # Search memories semantically
core_memory(operation="embed_status") # Check embedding status
core_memory(operation="extract") # Trigger batch memory extraction (V2)
core_memory(operation="extract_status") # Check extraction pipeline status
core_memory(operation="consolidate") # Trigger memory consolidation (V2)
core_memory(operation="consolidate_status") # Check consolidation status
core_memory(operation="jobs") # List all V2 pipeline jobs
Path parameter (highest priority):
core_memory(operation="list", path="/path/to/project") # Use specific project path
@@ -384,7 +605,10 @@ Memory IDs use format: CMEM-YYYYMMDD-HHMMSS`,
properties: {
operation: {
type: 'string',
enum: ['list', 'import', 'export', 'summary', 'embed', 'search', 'embed_status'],
enum: [
'list', 'import', 'export', 'summary', 'embed', 'search', 'embed_status',
'extract', 'extract_status', 'consolidate', 'consolidate_status', 'jobs',
],
description: 'Operation to perform',
},
path: {
@@ -437,6 +661,19 @@ Memory IDs use format: CMEM-YYYYMMDD-HHMMSS`,
type: 'boolean',
description: 'Force re-embedding even if embeddings exist (default: false)',
},
max_sessions: {
type: 'number',
description: 'Max sessions to extract in one batch (for extract operation)',
},
kind: {
type: 'string',
description: 'Filter jobs by kind (for jobs operation, e.g. "extraction" or "consolidation")',
},
status_filter: {
type: 'string',
enum: ['pending', 'running', 'done', 'error'],
description: 'Filter jobs by status (for jobs operation)',
},
},
required: ['operation'],
},

50
ccw/src/utils/secret-redactor.ts Normal file
View File

@@ -0,0 +1,50 @@
/**
* Secret Redactor - Regex-based secret pattern detection and replacement
*
* Scans text for common secret patterns (API keys, tokens, credentials)
* and replaces them with [REDACTED_SECRET] to prevent leakage into
* memory extraction outputs.
*
* Patterns are intentionally specific (prefix-based) to minimize false positives.
*/
const REDACTED = '[REDACTED_SECRET]';
/**
* Secret patterns with named regex for each category.
* Each pattern targets a specific, well-known secret format.
*/
const SECRET_PATTERNS: ReadonlyArray<{ name: string; regex: RegExp }> = [
// OpenAI API keys: sk-<20+ alphanumeric chars>
{ name: 'openai_key', regex: /sk-[A-Za-z0-9]{20,}/g },
// AWS Access Key IDs: AKIA<16 uppercase alphanumeric chars>
{ name: 'aws_key', regex: /AKIA[0-9A-Z]{16}/g },
// Bearer tokens: Bearer <16+ token chars>
{ name: 'bearer_token', regex: /Bearer\s+[A-Za-z0-9._\-]{16,}/g },
// Secret assignments: key=value or key:value patterns for known secret variable names
{ name: 'secret_assignment', regex: /(?:api_key|token|secret|password)[:=]\S+/gi },
];
/**
* Apply regex-based secret pattern matching and replacement.
*
* Scans the input text for 4 pattern categories:
* 1. OpenAI API keys (sk-...)
* 2. AWS Access Key IDs (AKIA...)
* 3. Bearer tokens (Bearer ...)
* 4. Secret variable assignments (api_key=..., token:..., etc.)
*
* @param text - Input text to scan for secrets
* @returns Text with all matched secrets replaced by [REDACTED_SECRET]
*/
export function redactSecrets(text: string): string {
if (!text) return text;
let result = text;
for (const { regex } of SECRET_PATTERNS) {
// Reset lastIndex for global regexes to ensure fresh match on each call
regex.lastIndex = 0;
result = result.replace(regex, REDACTED);
}
return result;
}

33
codex-lens/benchmarks/accuracy_queries_codexlens.jsonl Normal file
View File

@@ -0,0 +1,33 @@
{"query":"class StandaloneLspManager","relevant_paths":["codexlens/lsp/standalone_manager.py"]}
{"query":"def _open_document","relevant_paths":["codexlens/lsp/standalone_manager.py"]}
{"query":"def _read_message","relevant_paths":["codexlens/lsp/standalone_manager.py"]}
{"query":"how does textDocument/didOpen work","relevant_paths":["codexlens/lsp/standalone_manager.py"]}
{"query":"class LspBridge","relevant_paths":["codexlens/lsp/lsp_bridge.py"]}
{"query":"def get_document_symbols","relevant_paths":["codexlens/lsp/lsp_bridge.py"]}
{"query":"class KeepAliveLspBridge","relevant_paths":["codexlens/lsp/keepalive_bridge.py"]}
{"query":"LSP keepalive bridge","relevant_paths":["codexlens/lsp/keepalive_bridge.py"]}
{"query":"class LspGraphBuilder","relevant_paths":["codexlens/lsp/lsp_graph_builder.py"]}
{"query":"def build_from_seeds","relevant_paths":["codexlens/lsp/lsp_graph_builder.py"]}
{"query":"def _stage2_realtime_lsp_expand","relevant_paths":["codexlens/search/chain_search.py"]}
{"query":"def _stage3_cluster_prune","relevant_paths":["codexlens/search/chain_search.py"]}
{"query":"def _cross_encoder_rerank","relevant_paths":["codexlens/search/chain_search.py"]}
{"query":"def dense_rerank_cascade_search","relevant_paths":["codexlens/search/chain_search.py"]}
{"query":"def cascade_search","relevant_paths":["codexlens/search/chain_search.py"]}
{"query":"def _find_nearest_binary_mmap_root","relevant_paths":["codexlens/search/chain_search.py"]}
{"query":"class BinarySearcher","relevant_paths":["codexlens/search/binary_searcher.py"]}
{"query":"class GraphExpander","relevant_paths":["codexlens/search/graph_expander.py"]}
{"query":"def cross_encoder_rerank","relevant_paths":["codexlens/search/ranking.py"]}
{"query":"def group_similar_results","relevant_paths":["codexlens/search/ranking.py"]}
{"query":"class ConfigError","relevant_paths":["codexlens/errors.py"]}
{"query":"def load_settings","relevant_paths":["codexlens/config.py"]}
{"query":"BINARY_VECTORS_MMAP_NAME","relevant_paths":["codexlens/config.py"]}
{"query":"STAGED_CLUSTERING_STRATEGY","relevant_paths":["codexlens/config.py","codexlens/env_config.py"]}
{"query":"def apply_workspace_env","relevant_paths":["codexlens/env_config.py"]}
{"query":"def generate_env_example","relevant_paths":["codexlens/env_config.py"]}
{"query":"def get_reranker","relevant_paths":["codexlens/semantic/reranker/factory.py"]}
{"query":"class APIReranker","relevant_paths":["codexlens/semantic/reranker/api_reranker.py"]}
{"query":"class RegistryStore","relevant_paths":["codexlens/storage/registry.py"]}
{"query":"class PathMapper","relevant_paths":["codexlens/storage/path_mapper.py"]}
{"query":"def lsp_status","relevant_paths":["codexlens/cli/commands.py"]}
{"query":"graph_neighbors migration","relevant_paths":["codexlens/storage/migrations/migration_007_add_graph_neighbors.py"]}
{"query":"def get_model_config","relevant_paths":["codexlens/semantic/vector_store.py"]}

365
codex-lens/benchmarks/compare_accuracy_labeled.py Normal file
View File

@@ -0,0 +1,365 @@
#!/usr/bin/env python
"""Compare labeled accuracy: staged(realtime LSP graph) vs dense_rerank.
This script measures retrieval "accuracy" against a labeled query set.
Each query must provide a list of relevant file paths (relative to --source
or absolute). We report:
- Hit@K (any relevant file appears in top-K)
- MRR@K (reciprocal rank of first relevant file within top-K)
- Recall@K (fraction of relevant files present in top-K)
Example:
python benchmarks/compare_accuracy_labeled.py --source ./src
python benchmarks/compare_accuracy_labeled.py --queries-file benchmarks/accuracy_queries_codexlens.jsonl
"""
from __future__ import annotations
import argparse
import gc
import json
import os
import re
import statistics
import sys
import time
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
# Add src to path (match other benchmark scripts)
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
from codexlens.config import Config
from codexlens.search.chain_search import ChainSearchEngine, SearchOptions
from codexlens.storage.path_mapper import PathMapper
from codexlens.storage.registry import RegistryStore
DEFAULT_QUERIES_FILE = Path(__file__).parent / "accuracy_queries_codexlens.jsonl"
def _now_ms() -> float:
return time.perf_counter() * 1000.0
def _normalize_path_key(path: str) -> str:
"""Normalize file paths for overlap/dedup metrics (Windows-safe)."""
try:
p = Path(path)
# Don't explode on non-files like "<memory>".
if str(p) and (p.is_absolute() or re.match(r"^[A-Za-z]:", str(p))):
norm = str(p.resolve())
else:
norm = str(p)
except Exception:
norm = path
norm = norm.replace("/", "\\")
if os.name == "nt":
norm = norm.lower()
return norm
def _load_labeled_queries(path: Path, limit: Optional[int]) -> List[Dict[str, Any]]:
if not path.is_file():
raise SystemExit(f"Queries file does not exist: {path}")
out: List[Dict[str, Any]] = []
for raw_line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
line = raw_line.strip()
if not line or line.startswith("#"):
continue
try:
item = json.loads(line)
except Exception as exc:
raise SystemExit(f"Invalid JSONL line in {path}: {raw_line!r} ({exc})") from exc
if not isinstance(item, dict) or "query" not in item:
raise SystemExit(f"Invalid query item (expected object with 'query'): {item!r}")
out.append(item)
if limit is not None and len(out) >= limit:
break
return out
def _dedup_topk(paths: Iterable[str], k: int) -> List[str]:
out: List[str] = []
seen: set[str] = set()
for p in paths:
if p in seen:
continue
seen.add(p)
out.append(p)
if len(out) >= k:
break
return out
def _first_hit_rank(topk_paths: Sequence[str], relevant: set[str]) -> Optional[int]:
for i, p in enumerate(topk_paths, start=1):
if p in relevant:
return i
return None
@dataclass
class StrategyRun:
strategy: str
latency_ms: float
topk_paths: List[str]
first_hit_rank: Optional[int]
hit_at_k: bool
recall_at_k: float
error: Optional[str] = None
@dataclass
class QueryEval:
query: str
relevant_paths: List[str]
staged: StrategyRun
dense_rerank: StrategyRun
def _run_strategy(
engine: ChainSearchEngine,
*,
strategy: str,
query: str,
source_path: Path,
k: int,
coarse_k: int,
relevant: set[str],
options: Optional[SearchOptions] = None,
) -> StrategyRun:
gc.collect()
start_ms = _now_ms()
try:
result = engine.cascade_search(
query=query,
source_path=source_path,
k=k,
coarse_k=coarse_k,
options=options,
strategy=strategy,
)
latency_ms = _now_ms() - start_ms
paths_raw = [r.path for r in (result.results or []) if getattr(r, "path", None)]
paths_norm = [_normalize_path_key(p) for p in paths_raw]
topk = _dedup_topk(paths_norm, k=k)
rank = _first_hit_rank(topk, relevant)
hit = rank is not None
recall = 0.0
if relevant:
recall = len(set(topk) & relevant) / float(len(relevant))
return StrategyRun(
strategy=strategy,
latency_ms=latency_ms,
topk_paths=topk,
first_hit_rank=rank,
hit_at_k=hit,
recall_at_k=recall,
error=None,
)
except Exception as exc:
latency_ms = _now_ms() - start_ms
return StrategyRun(
strategy=strategy,
latency_ms=latency_ms,
topk_paths=[],
first_hit_rank=None,
hit_at_k=False,
recall_at_k=0.0,
error=repr(exc),
)
def _mrr(ranks: Sequence[Optional[int]]) -> float:
vals = []
for r in ranks:
if r is None or r <= 0:
vals.append(0.0)
else:
vals.append(1.0 / float(r))
return statistics.mean(vals) if vals else 0.0
def main() -> None:
parser = argparse.ArgumentParser(
description="Compare labeled retrieval accuracy: staged(realtime) vs dense_rerank"
)
parser.add_argument(
"--source",
type=Path,
default=Path(__file__).parent.parent / "src",
help="Source directory to search (default: ./src)",
)
parser.add_argument(
"--queries-file",
type=Path,
default=DEFAULT_QUERIES_FILE,
help="JSONL file with {query, relevant_paths[]} per line",
)
parser.add_argument("--queries", type=int, default=None, help="Limit number of queries")
parser.add_argument("--k", type=int, default=10, help="Top-K for evaluation (default 10)")
parser.add_argument("--coarse-k", type=int, default=100, help="Coarse candidates (default 100)")
parser.add_argument(
"--staged-cluster-strategy",
type=str,
default="path",
help="Config.staged_clustering_strategy override for staged (default: path)",
)
parser.add_argument(
"--stage2-mode",
type=str,
default="realtime",
help="Config.staged_stage2_mode override for staged (default: realtime)",
)
parser.add_argument(
"--output",
type=Path,
default=Path(__file__).parent / "results" / "accuracy_labeled.json",
help="Output JSON path",
)
args = parser.parse_args()
if not args.source.exists():
raise SystemExit(f"Source path does not exist: {args.source}")
labeled = _load_labeled_queries(args.queries_file, args.queries)
if not labeled:
raise SystemExit("No queries to run")
source_root = args.source.expanduser().resolve()
# Match CLI behavior: load settings + apply global/workspace .env overrides.
config = Config.load()
config.cascade_strategy = "staged"
config.staged_stage2_mode = str(args.stage2_mode or "realtime").strip().lower()
config.enable_staged_rerank = True
config.staged_clustering_strategy = str(args.staged_cluster_strategy or "path").strip().lower()
# Stability: on some Windows setups, DirectML/ONNX can crash under load.
config.embedding_use_gpu = False
registry = RegistryStore()
registry.initialize()
mapper = PathMapper()
engine = ChainSearchEngine(registry=registry, mapper=mapper, config=config)
def resolve_expected(paths: Sequence[str]) -> set[str]:
out: set[str] = set()
for p in paths:
try:
cand = Path(p)
if not cand.is_absolute():
cand = (source_root / cand).resolve()
out.add(_normalize_path_key(str(cand)))
except Exception:
out.add(_normalize_path_key(p))
return out
evaluations: List[QueryEval] = []
try:
for i, item in enumerate(labeled, start=1):
query = str(item.get("query", "")).strip()
relevant_raw = item.get("relevant_paths") or []
if not query:
continue
if not isinstance(relevant_raw, list) or not relevant_raw:
raise SystemExit(f"Query item missing relevant_paths[]: {item!r}")
relevant = resolve_expected([str(p) for p in relevant_raw])
print(f"[{i}/{len(labeled)}] {query}")
staged = _run_strategy(
engine,
strategy="staged",
query=query,
source_path=source_root,
k=int(args.k),
coarse_k=int(args.coarse_k),
relevant=relevant,
options=None,
)
dense = _run_strategy(
engine,
strategy="dense_rerank",
query=query,
source_path=source_root,
k=int(args.k),
coarse_k=int(args.coarse_k),
relevant=relevant,
options=None,
)
evaluations.append(
QueryEval(
query=query,
relevant_paths=[_normalize_path_key(str((source_root / p).resolve())) if not Path(p).is_absolute() else _normalize_path_key(p) for p in relevant_raw],
staged=staged,
dense_rerank=dense,
)
)
finally:
try:
engine.close()
except Exception:
pass
try:
registry.close()
except Exception:
pass
staged_runs = [e.staged for e in evaluations]
dense_runs = [e.dense_rerank for e in evaluations]
def mean(xs: Sequence[float]) -> float:
return statistics.mean(xs) if xs else 0.0
staged_ranks = [r.first_hit_rank for r in staged_runs]
dense_ranks = [r.first_hit_rank for r in dense_runs]
summary = {
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
"source": str(source_root),
"queries_file": str(args.queries_file),
"query_count": len(evaluations),
"k": int(args.k),
"coarse_k": int(args.coarse_k),
"staged": {
"hit_at_k": mean([1.0 if r.hit_at_k else 0.0 for r in staged_runs]),
"mrr_at_k": _mrr(staged_ranks),
"avg_recall_at_k": mean([r.recall_at_k for r in staged_runs]),
"avg_latency_ms": mean([r.latency_ms for r in staged_runs if not r.error]),
"errors": sum(1 for r in staged_runs if r.error),
},
"dense_rerank": {
"hit_at_k": mean([1.0 if r.hit_at_k else 0.0 for r in dense_runs]),
"mrr_at_k": _mrr(dense_ranks),
"avg_recall_at_k": mean([r.recall_at_k for r in dense_runs]),
"avg_latency_ms": mean([r.latency_ms for r in dense_runs if not r.error]),
"errors": sum(1 for r in dense_runs if r.error),
},
"config": {
"staged_stage2_mode": config.staged_stage2_mode,
"staged_clustering_strategy": config.staged_clustering_strategy,
"enable_staged_rerank": bool(config.enable_staged_rerank),
"reranker_backend": config.reranker_backend,
"reranker_model": config.reranker_model,
"embedding_backend": config.embedding_backend,
"embedding_model": config.embedding_model,
},
}
payload = {"summary": summary, "evaluations": [asdict(e) for e in evaluations]}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(payload, indent=2), encoding="utf-8")
print("\n=== SUMMARY ===")
print(json.dumps(summary, indent=2))
print(f"\nSaved: {args.output}")
if __name__ == "__main__":
main()

1308
codex-lens/benchmarks/results/accuracy_2026-02-11_codexlens.json Normal file
View File

File diff suppressed because it is too large Load Diff

1335
codex-lens/benchmarks/results/accuracy_2026-02-11_codexlens_precomputed.json Normal file
View File

File diff suppressed because it is too large Load Diff

383
codex-lens/src/codexlens/config.py
View File

@@ -315,119 +315,117 @@ class Config:
def load_settings(self) -> None:
"""Load settings from file if exists."""
if not self.settings_path.exists():
return
if self.settings_path.exists():
try:
with open(self.settings_path, "r", encoding="utf-8") as f:
settings = json.load(f)
try:
with open(self.settings_path, "r", encoding="utf-8") as f:
settings = json.load(f)
# Load embedding settings
embedding = settings.get("embedding", {})
if "backend" in embedding:
backend = embedding["backend"]
# Support 'api' as alias for 'litellm'
if backend == "api":
backend = "litellm"
if backend in {"fastembed", "litellm"}:
self.embedding_backend = backend
else:
log.warning(
"Invalid embedding backend in %s: %r (expected 'fastembed' or 'litellm')",
self.settings_path,
embedding["backend"],
)
if "model" in embedding:
self.embedding_model = embedding["model"]
if "use_gpu" in embedding:
self.embedding_use_gpu = embedding["use_gpu"]
# Load embedding settings
embedding = settings.get("embedding", {})
if "backend" in embedding:
backend = embedding["backend"]
# Support 'api' as alias for 'litellm'
if backend == "api":
backend = "litellm"
if backend in {"fastembed", "litellm"}:
self.embedding_backend = backend
else:
log.warning(
"Invalid embedding backend in %s: %r (expected 'fastembed' or 'litellm')",
self.settings_path,
embedding["backend"],
)
if "model" in embedding:
self.embedding_model = embedding["model"]
if "use_gpu" in embedding:
self.embedding_use_gpu = embedding["use_gpu"]
# Load multi-endpoint configuration
if "endpoints" in embedding:
self.embedding_endpoints = embedding["endpoints"]
if "pool_enabled" in embedding:
self.embedding_pool_enabled = embedding["pool_enabled"]
if "strategy" in embedding:
self.embedding_strategy = embedding["strategy"]
if "cooldown" in embedding:
self.embedding_cooldown = embedding["cooldown"]
# Load multi-endpoint configuration
if "endpoints" in embedding:
self.embedding_endpoints = embedding["endpoints"]
if "pool_enabled" in embedding:
self.embedding_pool_enabled = embedding["pool_enabled"]
if "strategy" in embedding:
self.embedding_strategy = embedding["strategy"]
if "cooldown" in embedding:
self.embedding_cooldown = embedding["cooldown"]
# Load LLM settings
llm = settings.get("llm", {})
if "enabled" in llm:
self.llm_enabled = llm["enabled"]
if "tool" in llm:
self.llm_tool = llm["tool"]
if "timeout_ms" in llm:
self.llm_timeout_ms = llm["timeout_ms"]
if "batch_size" in llm:
self.llm_batch_size = llm["batch_size"]
# Load LLM settings
llm = settings.get("llm", {})
if "enabled" in llm:
self.llm_enabled = llm["enabled"]
if "tool" in llm:
self.llm_tool = llm["tool"]
if "timeout_ms" in llm:
self.llm_timeout_ms = llm["timeout_ms"]
if "batch_size" in llm:
self.llm_batch_size = llm["batch_size"]
# Load reranker settings
reranker = settings.get("reranker", {})
if "enabled" in reranker:
self.enable_cross_encoder_rerank = reranker["enabled"]
if "backend" in reranker:
backend = reranker["backend"]
if backend in {"fastembed", "onnx", "api", "litellm", "legacy"}:
self.reranker_backend = backend
else:
log.warning(
"Invalid reranker backend in %s: %r (expected 'fastembed', 'onnx', 'api', 'litellm', or 'legacy')",
self.settings_path,
backend,
)
if "model" in reranker:
self.reranker_model = reranker["model"]
if "top_k" in reranker:
self.reranker_top_k = reranker["top_k"]
if "max_input_tokens" in reranker:
self.reranker_max_input_tokens = reranker["max_input_tokens"]
if "pool_enabled" in reranker:
self.reranker_pool_enabled = reranker["pool_enabled"]
if "strategy" in reranker:
self.reranker_strategy = reranker["strategy"]
if "cooldown" in reranker:
self.reranker_cooldown = reranker["cooldown"]
# Load reranker settings
reranker = settings.get("reranker", {})
if "enabled" in reranker:
self.enable_cross_encoder_rerank = reranker["enabled"]
if "backend" in reranker:
backend = reranker["backend"]
if backend in {"fastembed", "onnx", "api", "litellm", "legacy"}:
self.reranker_backend = backend
else:
log.warning(
"Invalid reranker backend in %s: %r (expected 'fastembed', 'onnx', 'api', 'litellm', or 'legacy')",
self.settings_path,
backend,
)
if "model" in reranker:
self.reranker_model = reranker["model"]
if "top_k" in reranker:
self.reranker_top_k = reranker["top_k"]
if "max_input_tokens" in reranker:
self.reranker_max_input_tokens = reranker["max_input_tokens"]
if "pool_enabled" in reranker:
self.reranker_pool_enabled = reranker["pool_enabled"]
if "strategy" in reranker:
self.reranker_strategy = reranker["strategy"]
if "cooldown" in reranker:
self.reranker_cooldown = reranker["cooldown"]
# Load cascade settings
cascade = settings.get("cascade", {})
if "strategy" in cascade:
strategy = cascade["strategy"]
if strategy in {"binary", "binary_rerank", "dense_rerank", "staged"}:
self.cascade_strategy = strategy
else:
log.warning(
"Invalid cascade strategy in %s: %r (expected 'binary', 'binary_rerank', 'dense_rerank', or 'staged')",
self.settings_path,
strategy,
)
if "coarse_k" in cascade:
self.cascade_coarse_k = cascade["coarse_k"]
if "fine_k" in cascade:
self.cascade_fine_k = cascade["fine_k"]
# Load cascade settings
cascade = settings.get("cascade", {})
if "strategy" in cascade:
strategy = cascade["strategy"]
if strategy in {"binary", "binary_rerank", "dense_rerank", "staged"}:
self.cascade_strategy = strategy
else:
log.warning(
"Invalid cascade strategy in %s: %r (expected 'binary', 'binary_rerank', 'dense_rerank', or 'staged')",
self.settings_path,
strategy,
)
if "coarse_k" in cascade:
self.cascade_coarse_k = cascade["coarse_k"]
if "fine_k" in cascade:
self.cascade_fine_k = cascade["fine_k"]
# Load API settings
api = settings.get("api", {})
if "max_workers" in api:
self.api_max_workers = api["max_workers"]
if "batch_size" in api:
self.api_batch_size = api["batch_size"]
if "batch_size_dynamic" in api:
self.api_batch_size_dynamic = api["batch_size_dynamic"]
if "batch_size_utilization_factor" in api:
self.api_batch_size_utilization_factor = api["batch_size_utilization_factor"]
if "batch_size_max" in api:
self.api_batch_size_max = api["batch_size_max"]
if "chars_per_token_estimate" in api:
self.chars_per_token_estimate = api["chars_per_token_estimate"]
except Exception as exc:
log.warning(
"Failed to load settings from %s (%s): %s",
self.settings_path,
type(exc).__name__,
exc,
)
# Load API settings
api = settings.get("api", {})
if "max_workers" in api:
self.api_max_workers = api["max_workers"]
if "batch_size" in api:
self.api_batch_size = api["batch_size"]
if "batch_size_dynamic" in api:
self.api_batch_size_dynamic = api["batch_size_dynamic"]
if "batch_size_utilization_factor" in api:
self.api_batch_size_utilization_factor = api["batch_size_utilization_factor"]
if "batch_size_max" in api:
self.api_batch_size_max = api["batch_size_max"]
if "chars_per_token_estimate" in api:
self.chars_per_token_estimate = api["chars_per_token_estimate"]
except Exception as exc:
log.warning(
"Failed to load settings from %s (%s): %s",
self.settings_path,
type(exc).__name__,
exc,
)
# Apply .env overrides (highest priority)
self._apply_env_overrides()
@@ -450,9 +448,9 @@ class Config:
RERANKER_STRATEGY: Load balance strategy for reranker
RERANKER_COOLDOWN: Rate limit cooldown for reranker
"""
from .env_config import load_global_env
from .env_config import load_env_file
env_vars = load_global_env()
env_vars = load_env_file(self.data_dir / ".env")
if not env_vars:
return
@@ -461,6 +459,43 @@ class Config:
# Check prefixed version first (Dashboard format), then unprefixed
return env_vars.get(f"CODEXLENS_{key}") or env_vars.get(key)
def _parse_bool(value: str) -> bool:
return value.strip().lower() in {"true", "1", "yes", "on"}
# Cascade overrides
cascade_enabled = get_env("ENABLE_CASCADE_SEARCH")
if cascade_enabled:
self.enable_cascade_search = _parse_bool(cascade_enabled)
log.debug(
"Overriding enable_cascade_search from .env: %s",
self.enable_cascade_search,
)
cascade_strategy = get_env("CASCADE_STRATEGY")
if cascade_strategy:
strategy = cascade_strategy.strip().lower()
if strategy in {"binary", "binary_rerank", "dense_rerank", "staged"}:
self.cascade_strategy = strategy
log.debug("Overriding cascade_strategy from .env: %s", self.cascade_strategy)
else:
log.warning("Invalid CASCADE_STRATEGY in .env: %r", cascade_strategy)
cascade_coarse_k = get_env("CASCADE_COARSE_K")
if cascade_coarse_k:
try:
self.cascade_coarse_k = int(cascade_coarse_k)
log.debug("Overriding cascade_coarse_k from .env: %s", self.cascade_coarse_k)
except ValueError:
log.warning("Invalid CASCADE_COARSE_K in .env: %r", cascade_coarse_k)
cascade_fine_k = get_env("CASCADE_FINE_K")
if cascade_fine_k:
try:
self.cascade_fine_k = int(cascade_fine_k)
log.debug("Overriding cascade_fine_k from .env: %s", self.cascade_fine_k)
except ValueError:
log.warning("Invalid CASCADE_FINE_K in .env: %r", cascade_fine_k)
# Embedding overrides
embedding_model = get_env("EMBEDDING_MODEL")
if embedding_model:
@@ -583,6 +618,136 @@ class Config:
self.chunk_strip_docstrings = strip_docstrings.lower() in ("true", "1", "yes")
log.debug("Overriding chunk_strip_docstrings from .env: %s", self.chunk_strip_docstrings)
# Staged cascade overrides
staged_stage2_mode = get_env("STAGED_STAGE2_MODE")
if staged_stage2_mode:
mode = staged_stage2_mode.strip().lower()
if mode in {"precomputed", "realtime"}:
self.staged_stage2_mode = mode
log.debug("Overriding staged_stage2_mode from .env: %s", self.staged_stage2_mode)
elif mode in {"live"}:
self.staged_stage2_mode = "realtime"
log.debug("Overriding staged_stage2_mode from .env: %s", self.staged_stage2_mode)
else:
log.warning("Invalid STAGED_STAGE2_MODE in .env: %r", staged_stage2_mode)
staged_clustering_strategy = get_env("STAGED_CLUSTERING_STRATEGY")
if staged_clustering_strategy:
strategy = staged_clustering_strategy.strip().lower()
if strategy in {"auto", "hdbscan", "dbscan", "frequency", "noop", "score", "dir_rr", "path"}:
self.staged_clustering_strategy = strategy
log.debug(
"Overriding staged_clustering_strategy from .env: %s",
self.staged_clustering_strategy,
)
elif strategy in {"none", "off"}:
self.staged_clustering_strategy = "noop"
log.debug(
"Overriding staged_clustering_strategy from .env: %s",
self.staged_clustering_strategy,
)
else:
log.warning(
"Invalid STAGED_CLUSTERING_STRATEGY in .env: %r",
staged_clustering_strategy,
)
staged_clustering_min_size = get_env("STAGED_CLUSTERING_MIN_SIZE")
if staged_clustering_min_size:
try:
self.staged_clustering_min_size = int(staged_clustering_min_size)
log.debug(
"Overriding staged_clustering_min_size from .env: %s",
self.staged_clustering_min_size,
)
except ValueError:
log.warning(
"Invalid STAGED_CLUSTERING_MIN_SIZE in .env: %r",
staged_clustering_min_size,
)
enable_staged_rerank = get_env("ENABLE_STAGED_RERANK")
if enable_staged_rerank:
self.enable_staged_rerank = _parse_bool(enable_staged_rerank)
log.debug("Overriding enable_staged_rerank from .env: %s", self.enable_staged_rerank)
rt_timeout = get_env("STAGED_REALTIME_LSP_TIMEOUT_S")
if rt_timeout:
try:
self.staged_realtime_lsp_timeout_s = float(rt_timeout)
log.debug(
"Overriding staged_realtime_lsp_timeout_s from .env: %s",
self.staged_realtime_lsp_timeout_s,
)
except ValueError:
log.warning("Invalid STAGED_REALTIME_LSP_TIMEOUT_S in .env: %r", rt_timeout)
rt_depth = get_env("STAGED_REALTIME_LSP_DEPTH")
if rt_depth:
try:
self.staged_realtime_lsp_depth = int(rt_depth)
log.debug(
"Overriding staged_realtime_lsp_depth from .env: %s",
self.staged_realtime_lsp_depth,
)
except ValueError:
log.warning("Invalid STAGED_REALTIME_LSP_DEPTH in .env: %r", rt_depth)
rt_max_nodes = get_env("STAGED_REALTIME_LSP_MAX_NODES")
if rt_max_nodes:
try:
self.staged_realtime_lsp_max_nodes = int(rt_max_nodes)
log.debug(
"Overriding staged_realtime_lsp_max_nodes from .env: %s",
self.staged_realtime_lsp_max_nodes,
)
except ValueError:
log.warning("Invalid STAGED_REALTIME_LSP_MAX_NODES in .env: %r", rt_max_nodes)
rt_max_seeds = get_env("STAGED_REALTIME_LSP_MAX_SEEDS")
if rt_max_seeds:
try:
self.staged_realtime_lsp_max_seeds = int(rt_max_seeds)
log.debug(
"Overriding staged_realtime_lsp_max_seeds from .env: %s",
self.staged_realtime_lsp_max_seeds,
)
except ValueError:
log.warning("Invalid STAGED_REALTIME_LSP_MAX_SEEDS in .env: %r", rt_max_seeds)
rt_max_concurrent = get_env("STAGED_REALTIME_LSP_MAX_CONCURRENT")
if rt_max_concurrent:
try:
self.staged_realtime_lsp_max_concurrent = int(rt_max_concurrent)
log.debug(
"Overriding staged_realtime_lsp_max_concurrent from .env: %s",
self.staged_realtime_lsp_max_concurrent,
)
except ValueError:
log.warning(
"Invalid STAGED_REALTIME_LSP_MAX_CONCURRENT in .env: %r",
rt_max_concurrent,
)
rt_warmup = get_env("STAGED_REALTIME_LSP_WARMUP_S")
if rt_warmup:
try:
self.staged_realtime_lsp_warmup_s = float(rt_warmup)
log.debug(
"Overriding staged_realtime_lsp_warmup_s from .env: %s",
self.staged_realtime_lsp_warmup_s,
)
except ValueError:
log.warning("Invalid STAGED_REALTIME_LSP_WARMUP_S in .env: %r", rt_warmup)
rt_resolve = get_env("STAGED_REALTIME_LSP_RESOLVE_SYMBOLS")
if rt_resolve:
self.staged_realtime_lsp_resolve_symbols = _parse_bool(rt_resolve)
log.debug(
"Overriding staged_realtime_lsp_resolve_symbols from .env: %s",
self.staged_realtime_lsp_resolve_symbols,
)
@classmethod
def load(cls) -> "Config":
"""Load config with settings from file."""

16
codex-lens/src/codexlens/env_config.py
View File

@@ -45,6 +45,22 @@ ENV_VARS = {
# General configuration
"CODEXLENS_DATA_DIR": "Custom data directory path",
"CODEXLENS_DEBUG": "Enable debug mode (true/false)",
# Cascade / staged pipeline configuration
"ENABLE_CASCADE_SEARCH": "Enable cascade search (true/false)",
"CASCADE_STRATEGY": "Cascade strategy: binary, binary_rerank, dense_rerank, staged",
"CASCADE_COARSE_K": "Cascade coarse_k candidate count (int)",
"CASCADE_FINE_K": "Cascade fine_k result count (int)",
"STAGED_STAGE2_MODE": "Staged Stage 2 mode: precomputed, realtime",
"STAGED_CLUSTERING_STRATEGY": "Staged clustering strategy: auto, score, path, dir_rr, noop, ...",
"STAGED_CLUSTERING_MIN_SIZE": "Staged clustering min cluster size (int)",
"ENABLE_STAGED_RERANK": "Enable staged reranking in Stage 4 (true/false)",
"STAGED_REALTIME_LSP_TIMEOUT_S": "Realtime LSP expansion timeout budget (float seconds)",
"STAGED_REALTIME_LSP_DEPTH": "Realtime LSP BFS depth (int)",
"STAGED_REALTIME_LSP_MAX_NODES": "Realtime LSP max nodes (int)",
"STAGED_REALTIME_LSP_MAX_SEEDS": "Realtime LSP max seeds (int)",
"STAGED_REALTIME_LSP_MAX_CONCURRENT": "Realtime LSP max concurrent requests (int)",
"STAGED_REALTIME_LSP_WARMUP_S": "Realtime LSP warmup wait after didOpen (float seconds)",
"STAGED_REALTIME_LSP_RESOLVE_SYMBOLS": "Resolve symbols via documentSymbol in realtime expansion (true/false)",
# Chunking configuration
"CHUNK_STRIP_COMMENTS": "Strip comments from code chunks for embedding: true/false (default: true)",
"CHUNK_STRIP_DOCSTRINGS": "Strip docstrings from code chunks for embedding: true/false (default: true)",

117
codex-lens/tests/test_config_staged_env_overrides.py Normal file
View File

@@ -0,0 +1,117 @@
"""Unit tests for Config .env overrides for staged/cascade settings."""
from __future__ import annotations
import tempfile
from pathlib import Path
import pytest
from codexlens.config import Config
@pytest.fixture
def temp_config_dir() -> Path:
"""Create temporary directory for config data_dir."""
tmpdir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
yield Path(tmpdir.name)
try:
tmpdir.cleanup()
except (PermissionError, OSError):
pass
def test_staged_env_overrides_apply(temp_config_dir: Path) -> None:
config = Config(data_dir=temp_config_dir)
env_path = temp_config_dir / ".env"
env_path.write_text(
"\n".join(
[
"ENABLE_CASCADE_SEARCH=true",
"CASCADE_STRATEGY=staged",
"CASCADE_COARSE_K=111",
"CASCADE_FINE_K=7",
"STAGED_STAGE2_MODE=realtime",
"STAGED_CLUSTERING_STRATEGY=path",
"STAGED_CLUSTERING_MIN_SIZE=5",
"ENABLE_STAGED_RERANK=false",
"STAGED_REALTIME_LSP_TIMEOUT_S=12.5",
"STAGED_REALTIME_LSP_DEPTH=2",
"STAGED_REALTIME_LSP_MAX_NODES=123",
"STAGED_REALTIME_LSP_MAX_SEEDS=3",
"STAGED_REALTIME_LSP_MAX_CONCURRENT=4",
"STAGED_REALTIME_LSP_WARMUP_S=0.25",
"STAGED_REALTIME_LSP_RESOLVE_SYMBOLS=yes",
"",
]
),
encoding="utf-8",
)
config.load_settings()
assert config.enable_cascade_search is True
assert config.cascade_strategy == "staged"
assert config.cascade_coarse_k == 111
assert config.cascade_fine_k == 7
assert config.staged_stage2_mode == "realtime"
assert config.staged_clustering_strategy == "path"
assert config.staged_clustering_min_size == 5
assert config.enable_staged_rerank is False
assert config.staged_realtime_lsp_timeout_s == 12.5
assert config.staged_realtime_lsp_depth == 2
assert config.staged_realtime_lsp_max_nodes == 123
assert config.staged_realtime_lsp_max_seeds == 3
assert config.staged_realtime_lsp_max_concurrent == 4
assert config.staged_realtime_lsp_warmup_s == 0.25
assert config.staged_realtime_lsp_resolve_symbols is True
def test_staged_env_overrides_prefixed_wins(temp_config_dir: Path) -> None:
config = Config(data_dir=temp_config_dir)
env_path = temp_config_dir / ".env"
env_path.write_text(
"\n".join(
[
"STAGED_CLUSTERING_STRATEGY=score",
"CODEXLENS_STAGED_CLUSTERING_STRATEGY=path",
"STAGED_STAGE2_MODE=precomputed",
"CODEXLENS_STAGED_STAGE2_MODE=realtime",
"",
]
),
encoding="utf-8",
)
config.load_settings()
assert config.staged_clustering_strategy == "path"
assert config.staged_stage2_mode == "realtime"
def test_staged_env_overrides_invalid_ignored(temp_config_dir: Path) -> None:
config = Config(data_dir=temp_config_dir)
env_path = temp_config_dir / ".env"
env_path.write_text(
"\n".join(
[
"STAGED_STAGE2_MODE=bogus",
"STAGED_CLUSTERING_STRATEGY=embedding_remote",
"STAGED_REALTIME_LSP_TIMEOUT_S=nope",
"CASCADE_STRATEGY=???",
"",
]
),
encoding="utf-8",
)
config.load_settings()
assert config.cascade_strategy == "binary"
assert config.staged_stage2_mode == "precomputed"
assert config.staged_clustering_strategy == "auto"
assert config.staged_realtime_lsp_timeout_s == 30.0