diff --git a/ccw/docs/cli-output-converter-usage.md b/ccw/docs/cli-output-converter-usage.md new file mode 100644 index 00000000..dcbf0c91 --- /dev/null +++ b/ccw/docs/cli-output-converter-usage.md @@ -0,0 +1,338 @@ +# CLI Output Converter Usage Guide + +## Overview + +The CLI Output Converter provides a unified Intermediate Representation (IR) layer for CLI tool output, enabling clean separation between output parsing and consumption scenarios. + +## Architecture + +``` +┌─────────────────┐ +│ CLI Tool │ +│ (stdout/stderr)│ +└────────┬────────┘ + │ Buffer chunks + ▼ +┌─────────────────┐ +│ Output Parser │ +│ (text/json-lines)│ +└────────┬────────┘ + │ CliOutputUnit[] + ▼ +┌─────────────────────────────────────┐ +│ Intermediate Representation (IR) │ +│ - type: stdout|stderr|thought|... │ +│ - content: string | object │ +│ - timestamp: ISO 8601 │ +└────┬────────────────────────────┬───┘ + │ │ + ▼ ▼ +┌─────────────┐ ┌─────────────┐ +│ Storage │ │ View │ +│ (SQLite) │ │ (Dashboard)│ +└─────────────┘ └─────────────┘ + │ + ▼ + ┌─────────────┐ + │ Resume │ + │ (Flatten) │ + └─────────────┘ +``` + +## Basic Usage + +### 1. Creating Parsers + +```typescript +import { createOutputParser } from './cli-output-converter.js'; + +// For plain text output (e.g., Gemini/Qwen plain mode) +const textParser = createOutputParser('text'); + +// For JSON Lines output (e.g., Codex JSONL format) +const jsonParser = createOutputParser('json-lines'); +``` + +### 2. Parsing Stream Chunks + +```typescript +import { spawn } from 'child_process'; +import { createOutputParser } from './cli-output-converter.js'; + +const parser = createOutputParser('json-lines'); +const allUnits: CliOutputUnit[] = []; + +const child = spawn('codex', ['run', 'task']); + +child.stdout.on('data', (chunk: Buffer) => { + const units = parser.parse(chunk, 'stdout'); + allUnits.push(...units); + + // Real-time processing + for (const unit of units) { + console.log(`[${unit.type}] ${unit.content}`); + } +}); + +child.stderr.on('data', (chunk: Buffer) => { + const units = parser.parse(chunk, 'stderr'); + allUnits.push(...units); +}); + +child.on('close', () => { + // Flush remaining buffer + const remaining = parser.flush(); + allUnits.push(...remaining); + + // Save to storage + saveToDatabase(allUnits); +}); +``` + +### 3. Integration with CLI Executor + +```typescript +// In cli-executor-core.ts +import { createOutputParser, type CliOutputUnit } from './cli-output-converter.js'; + +async function executeCliTool(params: CliParams) { + const parser = createOutputParser(params.format === 'json-lines' ? 'json-lines' : 'text'); + const structuredOutput: CliOutputUnit[] = []; + + child.stdout.on('data', (data) => { + const text = data.toString(); + stdout += text; + + // Parse into IR + const units = parser.parse(data, 'stdout'); + structuredOutput.push(...units); + + // Existing streaming logic + if (onOutput) { + onOutput({ type: 'stdout', data: text }); + } + }); + + child.on('close', () => { + // Flush parser + structuredOutput.push(...parser.flush()); + + // Create turn with structured output + const newTurn: ConversationTurn = { + turn: turnNumber, + timestamp: new Date().toISOString(), + prompt, + duration_ms, + status, + exit_code: code, + output: { + stdout: stdout.substring(0, 10240), + stderr: stderr.substring(0, 2048), + truncated: stdout.length > 10240 || stderr.length > 2048, + cached: shouldCache, + stdout_full: shouldCache ? stdout : undefined, + stderr_full: shouldCache ? stderr : undefined, + structured: structuredOutput.length > 0 ? structuredOutput : undefined + } + }; + }); +} +``` + +## Scenario-Specific Usage + +### Scenario 1: View (Dashboard Display) + +```typescript +// In dashboard rendering logic +import { type CliOutputUnit } from '../tools/cli-output-converter.js'; + +function renderOutputUnits(units: CliOutputUnit[]) { + return units.map(unit => { + switch (unit.type) { + case 'thought': + return `
${unit.content}
`; + case 'code': + return `
${unit.content}
`; + case 'file_diff': + return renderDiff(unit.content); + case 'progress': + return renderProgress(unit.content); + default: + return `
${unit.content}
`; + } + }).join('\n'); +} +``` + +### Scenario 2: Storage (Backward Compatible) + +```typescript +// In cli-history-store.ts +function saveConversation(conversation: ConversationRecord) { + for (const turn of conversation.turns) { + // Save traditional fields + db.run(` + INSERT INTO turns (stdout, stderr, truncated, ...) + VALUES (?, ?, ?, ...) + `, turn.output.stdout, turn.output.stderr, turn.output.truncated); + + // Optionally save structured output + if (turn.output.structured) { + db.run(` + INSERT INTO turn_structured_output (turn_id, units) + VALUES (?, ?) + `, turnId, JSON.stringify(turn.output.structured)); + } + } +} +``` + +### Scenario 3: Resume (Context Concatenation) + +```typescript +// In resume-strategy.ts or cli-prompt-builder.ts +import { flattenOutputUnits } from './cli-output-converter.js'; + +function buildContextFromTurns(turns: ConversationTurn[]): string { + const lines: string[] = []; + + for (const turn of turns) { + lines.push(`USER: ${turn.prompt}`); + + // Use structured output if available + if (turn.output.structured) { + const assistantText = flattenOutputUnits(turn.output.structured, { + excludeTypes: ['metadata', 'system'], // Skip noise + includeTypes: ['stdout', 'thought', 'code'] // Keep meaningful content + }); + lines.push(`ASSISTANT: ${assistantText}`); + } else { + // Fallback to plain stdout + lines.push(`ASSISTANT: ${turn.output.stdout}`); + } + } + + return lines.join('\n\n'); +} +``` + +## Advanced Features + +### Filtering by Type + +```typescript +import { flattenOutputUnits, extractContent } from './cli-output-converter.js'; + +// Extract only AI thoughts for analysis +const thoughts = extractContent(units, 'thought'); + +// Get only code blocks +const codeBlocks = extractContent(units, 'code'); + +// Create clean context (exclude system noise) +const cleanContext = flattenOutputUnits(units, { + excludeTypes: ['metadata', 'system', 'progress'] +}); +``` + +### Analytics + +```typescript +import { getOutputStats } from './cli-output-converter.js'; + +const stats = getOutputStats(units); +console.log(`Total units: ${stats.total}`); +console.log(`Thoughts: ${stats.byType.thought || 0}`); +console.log(`Code blocks: ${stats.byType.code || 0}`); +console.log(`Duration: ${stats.firstTimestamp} - ${stats.lastTimestamp}`); +``` + +### Custom Processing + +```typescript +function processUnits(units: CliOutputUnit[]) { + for (const unit of units) { + switch (unit.type) { + case 'file_diff': + // Apply diff to file system + applyDiff(unit.content.path, unit.content.diff); + break; + case 'progress': + // Update progress bar + updateProgress(unit.content.progress, unit.content.total); + break; + case 'thought': + // Log reasoning + logThought(unit.content); + break; + } + } +} +``` + +## Type Reference + +### CliOutputUnitType + +```typescript +type CliOutputUnitType = + | 'stdout' // Standard output text + | 'stderr' // Standard error text + | 'thought' // AI reasoning/thinking + | 'code' // Code block content + | 'file_diff' // File modification diff + | 'progress' // Progress updates + | 'metadata' // Session/execution metadata + | 'system'; // System events/messages +``` + +### CliOutputUnit + +```typescript +interface CliOutputUnit { + type: CliOutputUnitType; + content: T; // string for text types, object for structured types + timestamp: string; // ISO 8601 format +} +``` + +## Migration Path + +### Phase 1: Optional Enhancement (Current) +- Add `structured` field to `ConversationTurn.output` +- Populate during parsing (optional) +- Existing code ignores it (backward compatible) + +### Phase 2: View Integration +- Dashboard uses `structured` when available +- Falls back to plain `stdout` when not present +- Better rendering for thoughts, code, diffs + +### Phase 3: Resume Optimization +- Resume logic prefers `structured` for cleaner context +- Filters out noise (metadata, system events) +- Reduces token usage while preserving semantics + +### Phase 4: Full Adoption +- All CLI tools use converters +- Storage optimized for structured data +- Analytics and insights from IR layer + +## Best Practices + +1. **Always flush the parser** when stream ends to capture incomplete lines +2. **Filter by type** for Resume scenarios to reduce token usage +3. **Use structured content** when available for better display +4. **Keep backward compatibility** by making `structured` optional +5. **Handle missing fields** gracefully (not all tools output JSON) +6. **Validate JSON** before parsing to avoid crashes + +## Future Enhancements + +1. **Streaming transformers** - Apply transformations during parsing +2. **Custom type mappers** - Register custom JSON-to-IR mappings +3. **Compression** - Store structured output more efficiently +4. **Semantic search** - Index thoughts and code separately +5. **Diff viewer** - Interactive file_diff rendering +6. **Progress tracking** - Aggregate progress events across tools diff --git a/ccw/src/commands/cli.ts b/ccw/src/commands/cli.ts index 55d749e1..25d3bf13 100644 --- a/ccw/src/commands/cli.ts +++ b/ccw/src/commands/cli.ts @@ -6,6 +6,7 @@ import chalk from 'chalk'; import http from 'http'; import inquirer from 'inquirer'; +import type { CliOutputUnit } from '../tools/cli-output-converter.js'; import { cliExecutorTool, getCliToolsStatus, @@ -725,7 +726,8 @@ async function execAction(positionalPrompt: string | undefined, options: CliExec // Generate execution ID for streaming (use custom ID or timestamp-based) const executionId = id || `${Date.now()}-${tool}`; const startTime = Date.now(); - const spinnerBaseText = `Executing ${tool} (${mode} mode${resumeInfo}${nativeMode})${idInfo}...`; + const modelInfo = model ? ` @${model}` : ''; + const spinnerBaseText = `Executing ${tool}${modelInfo} (${mode} mode${resumeInfo}${nativeMode})${idInfo}...`; console.log(); const spinner = stream ? null : createSpinner(` ${spinnerBaseText}`).start(); @@ -787,20 +789,49 @@ async function execAction(positionalPrompt: string | undefined, options: CliExec }); // Streaming output handler - broadcasts to dashboard AND writes to stdout - const onOutput = (chunk: any) => { + const onOutput = (unit: CliOutputUnit) => { // Always broadcast to dashboard for real-time viewing // Note: /api/hook wraps extraData into payload, so send fields directly + // Maintain backward compatibility with frontend expecting { chunkType, data } + const content = typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content); broadcastStreamEvent('CLI_OUTPUT', { executionId, - chunkType: chunk.type, - data: chunk.data + chunkType: unit.type, // For backward compatibility + data: content, // For backward compatibility + unit // New structured format }); + // Write to terminal only when --stream flag is passed if (stream) { - process.stdout.write(chunk.data); + switch (unit.type) { + case 'stdout': + case 'code': + process.stdout.write(typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content)); + break; + case 'stderr': + process.stderr.write(typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content)); + break; + case 'thought': + // Optional: display thinking process with different color + // For now, skip to reduce noise + break; + case 'progress': + // Optional: update progress bar + // For now, skip + break; + default: + // Other types: output content if available + if (unit.content) { + process.stdout.write(typeof unit.content === 'string' ? unit.content : ''); + } + } } }; + // Use JSON-lines parsing by default to enable type badges (thought, code, file_diff, etc.) + // All CLI tools may output structured JSON that can be parsed for richer UI + const outputFormat = 'json-lines'; + try { const result = await cliExecutorTool.execute({ tool, @@ -813,7 +844,8 @@ async function execAction(positionalPrompt: string | undefined, options: CliExec resume, id, // custom execution ID noNative, - stream: !!stream // stream=true → streaming enabled (no cache), stream=false → cache output (default) + stream: !!stream, // stream=true → streaming enabled (no cache), stream=false → cache output (default) + outputFormat // Enable JSONL parsing for tools that support it }, onOutput); // Always pass onOutput for real-time dashboard streaming if (elapsedInterval) clearInterval(elapsedInterval); diff --git a/ccw/src/core/auth/csrf-middleware.ts b/ccw/src/core/auth/csrf-middleware.ts index d0c85d32..119d1fba 100644 --- a/ccw/src/core/auth/csrf-middleware.ts +++ b/ccw/src/core/auth/csrf-middleware.ts @@ -118,8 +118,9 @@ export async function csrfValidation(ctx: CsrfMiddlewareContext): Promise { stream: false, category: 'internal', id: syncId - }, onOutput); + }, (unit) => { + // CliOutputUnit handler: convert to string content for broadcast + const content = typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content); + broadcastToClients({ + type: 'CLI_OUTPUT', + payload: { + executionId: syncId, + chunkType: unit.type, + data: content + } + }); + }); // Broadcast CLI_EXECUTION_COMPLETED event broadcastToClients({ diff --git a/ccw/src/core/routes/cli-routes.ts b/ccw/src/core/routes/cli-routes.ts index 01f1506f..c6b4e618 100644 --- a/ccw/src/core/routes/cli-routes.ts +++ b/ccw/src/core/routes/cli-routes.ts @@ -195,7 +195,7 @@ export async function handleCliRoutes(ctx: RouteContext): Promise { } // API: Get/Update Tool Config - const configMatch = pathname.match(/^\/api\/cli\/config\/(gemini|qwen|codex)$/); + const configMatch = pathname.match(/^\/api\/cli\/config\/(gemini|qwen|codex|claude|opencode)$/); if (configMatch) { const tool = configMatch[1]; @@ -216,7 +216,7 @@ export async function handleCliRoutes(ctx: RouteContext): Promise { if (req.method === 'PUT') { handlePostRequest(req, res, async (body: unknown) => { try { - const updates = body as { enabled?: boolean; primaryModel?: string; secondaryModel?: string }; + const updates = body as { enabled?: boolean; primaryModel?: string; secondaryModel?: string; tags?: string[] }; const updated = updateToolConfig(initialPath, tool, updates); // Broadcast config updated event @@ -559,19 +559,22 @@ export async function handleCliRoutes(ctx: RouteContext): Promise { category: category || 'user', parentExecutionId, stream: true - }, (chunk) => { - // Append chunk to active execution buffer + }, (unit) => { + // CliOutputUnit handler: convert to string content + const content = typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content); + + // Append to active execution buffer const activeExec = activeExecutions.get(executionId); if (activeExec) { - activeExec.output += chunk.data || ''; + activeExec.output += content || ''; } broadcastToClients({ type: 'CLI_OUTPUT', payload: { executionId, - chunkType: chunk.type, - data: chunk.data + chunkType: unit.type, + data: content } }); }); diff --git a/ccw/src/core/routes/memory-routes.ts b/ccw/src/core/routes/memory-routes.ts index cf8c9a45..24191289 100644 --- a/ccw/src/core/routes/memory-routes.ts +++ b/ccw/src/core/routes/memory-routes.ts @@ -1007,7 +1007,18 @@ RULES: Be concise. Focus on practical understanding. Include function signatures stream: false, category: 'internal', id: syncId - }, onOutput); + }, (unit) => { + // CliOutputUnit handler: convert to string content for broadcast + const content = typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content); + broadcastToClients({ + type: 'CLI_OUTPUT', + payload: { + executionId: syncId, + chunkType: unit.type, + data: content + } + }); + }); // Broadcast CLI_EXECUTION_COMPLETED event broadcastToClients({ diff --git a/ccw/src/core/routes/rules-routes.ts b/ccw/src/core/routes/rules-routes.ts index f794e443..1e3003fd 100644 --- a/ccw/src/core/routes/rules-routes.ts +++ b/ccw/src/core/routes/rules-routes.ts @@ -661,13 +661,15 @@ FILE NAME: ${fileName}`; // Create onOutput callback for real-time streaming const onOutput = broadcastToClients - ? (chunk: { type: string; data: string }) => { + ? (unit: import('../../tools/cli-output-converter.js').CliOutputUnit) => { + // CliOutputUnit handler: convert to string content for broadcast + const content = typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content); broadcastToClients({ type: 'CLI_OUTPUT', payload: { executionId, - chunkType: chunk.type, - data: chunk.data + chunkType: unit.type, + data: content } }); } @@ -746,13 +748,15 @@ FILE NAME: ${fileName}`; // Create onOutput callback for review step const reviewOnOutput = broadcastToClients - ? (chunk: { type: string; data: string }) => { + ? (unit: import('../../tools/cli-output-converter.js').CliOutputUnit) => { + // CliOutputUnit handler: convert to string content for broadcast + const content = typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content); broadcastToClients({ type: 'CLI_OUTPUT', payload: { executionId: reviewExecutionId, - chunkType: chunk.type, - data: chunk.data + chunkType: unit.type, + data: content } }); } diff --git a/ccw/src/core/routes/skills-routes.ts b/ccw/src/core/routes/skills-routes.ts index 0354fb7d..89a702ce 100644 --- a/ccw/src/core/routes/skills-routes.ts +++ b/ccw/src/core/routes/skills-routes.ts @@ -579,13 +579,15 @@ Create a new Claude Code skill with the following specifications: // Create onOutput callback for real-time streaming const onOutput = broadcastToClients - ? (chunk: { type: string; data: string }) => { + ? (unit: import('../../tools/cli-output-converter.js').CliOutputUnit) => { + // CliOutputUnit handler: convert to string content for broadcast + const content = typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content); broadcastToClients({ type: 'CLI_OUTPUT', payload: { executionId, - chunkType: chunk.type, - data: chunk.data + chunkType: unit.type, + data: content } }); } diff --git a/ccw/src/core/server.ts b/ccw/src/core/server.ts index 2580af8a..6d8a8b8c 100644 --- a/ccw/src/core/server.ts +++ b/ccw/src/core/server.ts @@ -357,7 +357,7 @@ export async function startServer(options: ServerOptions = {}): Promise(['/api/auth/token', '/api/csrf-token']); + const unauthenticatedPaths = new Set(['/api/auth/token', '/api/csrf-token', '/api/hook']); const server = http.createServer(async (req, res) => { const url = new URL(req.url ?? '/', `http://localhost:${serverPort}`); diff --git a/ccw/src/templates/dashboard-css/10-cli-status.css b/ccw/src/templates/dashboard-css/10-cli-status.css index 46bb22c4..8522a709 100644 --- a/ccw/src/templates/dashboard-css/10-cli-status.css +++ b/ccw/src/templates/dashboard-css/10-cli-status.css @@ -203,6 +203,168 @@ color: hsl(142 76% 36%); } +/* Tool Tags - displayed in tool cards */ +.tool-tags { + display: flex; + flex-wrap: wrap; + gap: 0.25rem; + margin-top: 0.25rem; +} + +.tool-tag { + font-size: 0.5625rem; + font-weight: 500; + padding: 0.125rem 0.375rem; + background: hsl(var(--primary) / 0.1); + color: hsl(var(--primary)); + border-radius: 0.25rem; +} + +/* Tags Input - used in config modal */ +.tags-input-group { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +/* Unified tag input - tags and input in one container */ +.tags-unified-input { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0.375rem; + min-height: 2.5rem; + padding: 0.375rem 0.5rem; + background: hsl(var(--background)); + border: 1px solid hsl(var(--border)); + border-radius: 0.5rem; + cursor: text; + transition: border-color 0.15s ease; +} + +.tags-unified-input:focus-within { + border-color: hsl(var(--primary)); + box-shadow: 0 0 0 2px hsl(var(--primary) / 0.1); +} + +.tag-inline-input { + flex: 1; + min-width: 120px; + border: none; + background: transparent; + outline: none; + font-size: 0.8125rem; + color: hsl(var(--foreground)); + padding: 0.25rem 0; +} + +.tag-inline-input::placeholder { + color: hsl(var(--muted-foreground) / 0.6); +} + +.tags-container { + display: flex; + flex-wrap: wrap; + gap: 0.375rem; + min-height: 1.75rem; + padding: 0.25rem; + background: hsl(var(--muted) / 0.3); + border-radius: 0.375rem; +} + +.tag-item { + display: inline-flex; + align-items: center; + gap: 0.25rem; + font-size: 0.75rem; + font-weight: 500; + padding: 0.25rem 0.5rem; + background: hsl(var(--primary) / 0.15); + color: hsl(var(--primary)); + border-radius: 0.25rem; +} + +.tag-remove { + display: flex; + align-items: center; + justify-content: center; + width: 1rem; + height: 1rem; + padding: 0; + border: none; + background: transparent; + color: hsl(var(--primary) / 0.6); + cursor: pointer; + font-size: 0.875rem; + line-height: 1; + border-radius: 0.125rem; +} + +.tag-remove:hover { + background: hsl(var(--destructive) / 0.2); + color: hsl(var(--destructive)); +} + +/* Predefined Tags Row - prominent quick add buttons */ +.predefined-tags-row { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + margin-top: 0.75rem; +} + +.predefined-tag-btn { + display: inline-flex; + align-items: center; + gap: 0.375rem; + font-size: 0.75rem; + font-weight: 500; + padding: 0.375rem 0.625rem; + background: hsl(var(--muted)); + color: hsl(var(--foreground)); + border: 1px solid hsl(var(--border)); + border-radius: 0.375rem; + cursor: pointer; + transition: all 0.15s ease; +} + +.predefined-tag-btn:hover { + background: hsl(var(--primary) / 0.15); + color: hsl(var(--primary)); + border-color: hsl(var(--primary) / 0.3); +} + +.predefined-tag-btn i { + opacity: 0.7; +} + +/* Legacy predefined tags */ +.predefined-tags { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0.375rem; + margin-top: 0.5rem; +} + +.predefined-tag { + font-size: 0.6875rem; + font-weight: 500; + padding: 0.25rem 0.5rem; + background: hsl(var(--muted) / 0.5); + color: hsl(var(--muted-foreground)); + border: 1px dashed hsl(var(--border)); + border-radius: 0.25rem; + cursor: pointer; + transition: all 0.15s ease; +} + +.predefined-tag:hover { + background: hsl(var(--primary) / 0.1); + color: hsl(var(--primary)); + border-color: hsl(var(--primary) / 0.3); +} + .tool-item-right { display: flex; align-items: center; diff --git a/ccw/src/templates/dashboard-css/33-cli-stream-viewer.css b/ccw/src/templates/dashboard-css/33-cli-stream-viewer.css index cf94f041..b309caf2 100644 --- a/ccw/src/templates/dashboard-css/33-cli-stream-viewer.css +++ b/ccw/src/templates/dashboard-css/33-cli-stream-viewer.css @@ -549,6 +549,77 @@ color: hsl(200 70% 70%); } +/* ===== Backend ChunkType Badges (CliOutputUnit.type) ===== */ + +/* Thought/Thinking Message (from JSONL parser) */ +.cli-stream-line.formatted.thought { + background: hsl(280 50% 20% / 0.3); + border-left: 3px solid hsl(280 70% 65%); + font-style: italic; +} + +.cli-msg-badge.cli-msg-thought { + background: hsl(280 70% 65% / 0.2); + color: hsl(280 70% 75%); +} + +/* Code Block Message */ +.cli-stream-line.formatted.code { + background: hsl(220 40% 18% / 0.4); + border-left: 3px solid hsl(220 60% 55%); + font-family: var(--font-mono, 'Consolas', 'Monaco', 'Courier New', monospace); +} + +.cli-msg-badge.cli-msg-code { + background: hsl(220 60% 55% / 0.25); + color: hsl(220 60% 70%); +} + +/* File Diff Message */ +.cli-stream-line.formatted.file_diff { + background: hsl(35 50% 18% / 0.4); + border-left: 3px solid hsl(35 80% 55%); +} + +.cli-msg-badge.cli-msg-file_diff { + background: hsl(35 80% 55% / 0.25); + color: hsl(35 80% 65%); +} + +/* Progress Message */ +.cli-stream-line.formatted.progress { + background: hsl(190 40% 18% / 0.3); + border-left: 3px solid hsl(190 70% 50%); +} + +.cli-msg-badge.cli-msg-progress { + background: hsl(190 70% 50% / 0.2); + color: hsl(190 70% 65%); +} + +/* Metadata Message */ +.cli-stream-line.formatted.metadata { + background: hsl(250 30% 18% / 0.3); + border-left: 3px solid hsl(250 50% 60%); + font-size: 0.85em; +} + +.cli-msg-badge.cli-msg-metadata { + background: hsl(250 50% 60% / 0.2); + color: hsl(250 50% 75%); +} + +/* Stderr Message (Error) */ +.cli-stream-line.formatted.stderr { + background: hsl(0 50% 20% / 0.4); + border-left: 3px solid hsl(0 70% 55%); +} + +.cli-msg-badge.cli-msg-stderr { + background: hsl(0 70% 55% / 0.25); + color: hsl(0 70% 70%); +} + /* Inline Code */ .cli-inline-code { padding: 1px 5px; diff --git a/ccw/src/templates/dashboard-js/components/cli-stream-viewer.js b/ccw/src/templates/dashboard-js/components/cli-stream-viewer.js index 9c87efc4..aa967fa4 100644 --- a/ccw/src/templates/dashboard-js/components/cli-stream-viewer.js +++ b/ccw/src/templates/dashboard-js/components/cli-stream-viewer.js @@ -346,16 +346,50 @@ function renderFormattedLine(line, searchFilter) { // Format inline code content = content.replace(/`([^`]+)`/g, '$1'); - // Build type badge if has prefix - const typeBadge = parsed.hasPrefix ? - ` + // Type badge icons for backend chunkType (CliOutputUnit.type) + const CHUNK_TYPE_ICONS = { + thought: 'brain', + code: 'code', + file_diff: 'git-compare', + progress: 'loader', + system: 'settings', + stderr: 'alert-circle', + metadata: 'info' + }; + + // Type badge labels for backend chunkType + const CHUNK_TYPE_LABELS = { + thought: 'Thinking', + code: 'Code', + file_diff: 'Diff', + progress: 'Progress', + system: 'System', + stderr: 'Error', + metadata: 'Info' + }; + + // Build type badge - prioritize content prefix, then fall back to chunkType + let typeBadge = ''; + let lineClass = ''; + + if (parsed.hasPrefix) { + // Content has Chinese prefix like [系统], [思考], etc. + typeBadge = ` ${parsed.label} - ` : ''; - - // Determine line class based on original type and parsed type - const lineClass = parsed.hasPrefix ? `cli-stream-line formatted ${parsed.type}` : - `cli-stream-line ${line.type}`; + `; + lineClass = `cli-stream-line formatted ${parsed.type}`; + } else if (line.type && line.type !== 'stdout' && CHUNK_TYPE_LABELS[line.type]) { + // No content prefix, but backend sent a meaningful chunkType + typeBadge = ` + + ${CHUNK_TYPE_LABELS[line.type]} + `; + lineClass = `cli-stream-line formatted ${line.type}`; + } else { + // Plain stdout, no badge + lineClass = `cli-stream-line ${line.type || 'stdout'}`; + } return `
${typeBadge}${content}
`; } diff --git a/ccw/src/templates/dashboard-js/i18n.js b/ccw/src/templates/dashboard-js/i18n.js index 6fd91450..bdc3a6ee 100644 --- a/ccw/src/templates/dashboard-js/i18n.js +++ b/ccw/src/templates/dashboard-js/i18n.js @@ -1721,6 +1721,28 @@ const i18n = { 'apiSettings.modelIdExists': 'Model ID already exists', 'apiSettings.useModelTreeToManage': 'Use the model tree to manage individual models', + // CLI Settings + 'apiSettings.cliSettings': 'CLI Settings', + 'apiSettings.addCliSettings': 'Add CLI Settings', + 'apiSettings.editCliSettings': 'Edit CLI Settings', + 'apiSettings.noCliSettings': 'No CLI settings configured', + 'apiSettings.noCliSettingsSelected': 'No CLI Settings Selected', + 'apiSettings.cliSettingsHint': 'Select a CLI settings endpoint or create a new one', + 'apiSettings.cliProviderHint': 'Select an Anthropic provider to use its API key and base URL', + 'apiSettings.noAnthropicProviders': 'No Anthropic providers configured. Please add one in the Providers tab first.', + 'apiSettings.selectProviderFirst': 'Select a provider first', + 'apiSettings.providerRequired': 'Provider is required', + 'apiSettings.modelRequired': 'Model is required', + 'apiSettings.providerNotFound': 'Provider not found', + 'apiSettings.settingsSaved': 'Settings saved successfully', + 'apiSettings.settingsDeleted': 'Settings deleted successfully', + 'apiSettings.confirmDeleteSettings': 'Are you sure you want to delete this CLI settings?', + 'apiSettings.endpointName': 'Endpoint Name', + 'apiSettings.envSettings': 'Environment Settings', + 'apiSettings.settingsFilePath': 'Settings File Path', + 'apiSettings.nameRequired': 'Name is required', + 'apiSettings.status': 'Status', + // Common 'common.cancel': 'Cancel', 'common.optional': '(Optional)', @@ -3777,6 +3799,29 @@ const i18n = { 'apiSettings.modelIdExists': '模型 ID 已存在', 'apiSettings.useModelTreeToManage': '使用模型树管理各个模型', + // CLI Settings + 'apiSettings.cliSettings': 'CLI 配置', + 'apiSettings.addCliSettings': '添加 CLI 配置', + 'apiSettings.editCliSettings': '编辑 CLI 配置', + 'apiSettings.noCliSettings': '未配置 CLI 设置', + 'apiSettings.noCliSettingsSelected': '未选择 CLI 配置', + 'apiSettings.cliSettingsHint': '选择一个 CLI 配置端点或创建新的', + 'apiSettings.cliProviderHint': '选择一个 Anthropic 供应商以使用其 API 密钥和基础 URL', + 'apiSettings.noAnthropicProviders': '未配置 Anthropic 供应商。请先在供应商标签页中添加。', + 'apiSettings.selectProviderFirst': '请先选择供应商', + 'apiSettings.providerRequired': '供应商为必填项', + 'apiSettings.modelRequired': '模型为必填项', + 'apiSettings.providerNotFound': '未找到供应商', + 'apiSettings.settingsSaved': '设置保存成功', + 'apiSettings.settingsDeleted': '设置删除成功', + 'apiSettings.confirmDeleteSettings': '确定要删除此 CLI 配置吗？', + 'apiSettings.endpointName': '端点名称', + 'apiSettings.envSettings': '环境变量设置', + 'apiSettings.settingsFilePath': '配置文件路径', + 'apiSettings.nameRequired': '名称为必填项', + 'apiSettings.tokenRequired': 'API 令牌为必填项', + 'apiSettings.status': '状态', + // Common 'common.cancel': '取消', 'common.optional': '(可选)', diff --git a/ccw/src/templates/dashboard-js/views/api-settings.js b/ccw/src/templates/dashboard-js/views/api-settings.js index 2bc5a266..10895641 100644 --- a/ccw/src/templates/dashboard-js/views/api-settings.js +++ b/ccw/src/templates/dashboard-js/views/api-settings.js @@ -3737,23 +3737,94 @@ function renderCliSettingsEmptyState() { if (window.lucide) lucide.createIcons(); } +/** + * Get available Anthropic providers + */ +function getAvailableAnthropicProviders() { + if (!apiSettingsData || !apiSettingsData.providers) { + return []; + } + + return apiSettingsData.providers.filter(function(p) { + return p.type === 'anthropic' && p.enabled; + }); +} + +/** + * Build provider options HTML for CLI Settings + */ +function buildCliProviderOptions(selectedProviderId) { + var providers = getAvailableAnthropicProviders(); + var optionsHtml = ''; + + providers.forEach(function(provider) { + var isSelected = provider.id === selectedProviderId ? ' selected' : ''; + optionsHtml += ''; + }); + + return optionsHtml; +} + +/** + * Build model options HTML for CLI Settings based on selected provider + */ +function buildCliModelOptions(providerId, selectedModel) { + var providers = getAvailableAnthropicProviders(); + var provider = providers.find(function(p) { return p.id === providerId; }); + + if (!provider || !provider.llmModels || provider.llmModels.length === 0) { + return ''; + } + + var optionsHtml = ''; + provider.llmModels.forEach(function(model) { + var isSelected = model.id === selectedModel ? ' selected' : ''; + optionsHtml += ''; + }); + + return optionsHtml; +} + +/** + * Update CLI Settings model dropdown when provider changes + */ +function onCliProviderChange() { + var providerId = document.getElementById('cli-settings-provider').value; + var modelSelect = document.getElementById('cli-settings-model'); + + if (modelSelect) { + modelSelect.innerHTML = buildCliModelOptions(providerId, ''); + } +} + /** * Show Add CLI Settings Modal */ function showAddCliSettingsModal(existingEndpoint) { var isEdit = !!existingEndpoint; - var settings = existingEndpoint ? existingEndpoint.settings : { env: {}, model: 'sonnet' }; - var env = settings.env || {}; + var settings = existingEndpoint ? existingEndpoint.settings : { env: {}, model: '' }; + var selectedProviderId = settings.providerId || ''; + var providerOptionsHtml = buildCliProviderOptions(selectedProviderId); + var modelOptionsHtml = buildCliModelOptions(selectedProviderId, settings.model); + + // Check if any Anthropic providers are configured + var hasProviders = getAvailableAnthropicProviders().length > 0; + var noProvidersWarning = !hasProviders ? + '
' + + '' + + '' + t('apiSettings.noAnthropicProviders') + '' + + '
' : ''; var modalHtml = - '
' + - '
' + - '
' + - '

' + (isEdit ? t('apiSettings.editCliSettings') : t('apiSettings.addCliSettings')) + '

' + - '' + + '
' + + '
' + + '
' + + '

' + (isEdit ? t('apiSettings.editCliSettings') : t('apiSettings.addCliSettings')) + '

' + + '' + '
' + - '
' + - '
' + + '
' + + noProvidersWarning + + '' + (isEdit ? '' : '') + '
' + '' + @@ -3764,20 +3835,17 @@ function showAddCliSettingsModal(existingEndpoint) { '' + '
' + '
' + - '' + - '' + + providerOptionsHtml + '' + + '' + t('apiSettings.cliProviderHint') + '' + '
' + '
' + - '' + - '' + - '
' + - '
' + - '' + - '' + + '' + + '' + '
' + '
' + '' + '
' + '' + + '
' + + '' + + '' + '
' + - '
' + - '' + - '' + '
' + '
' + '
'; - // Append modal to body - var modalsContainer = document.getElementById('modals'); - if (!modalsContainer) { - modalsContainer = document.createElement('div'); - modalsContainer.id = 'modals'; - document.body.appendChild(modalsContainer); - } - modalsContainer.innerHTML = modalHtml; + document.body.insertAdjacentHTML('beforeend', modalHtml); + + if (window.lucide) lucide.createIcons(); } /** @@ -3821,10 +3884,8 @@ function editCliSettings(endpointId) { * Close CLI Settings Modal */ function closeCliSettingsModal() { - var modalsContainer = document.getElementById('modals'); - if (modalsContainer) { - modalsContainer.innerHTML = ''; - } + var modal = document.getElementById('cliSettingsModal'); + if (modal) modal.remove(); } /** @@ -3833,9 +3894,8 @@ function closeCliSettingsModal() { async function submitCliSettings() { var name = document.getElementById('cli-settings-name').value.trim(); var description = document.getElementById('cli-settings-description').value.trim(); + var providerId = document.getElementById('cli-settings-provider').value; var model = document.getElementById('cli-settings-model').value; - var token = document.getElementById('cli-settings-token').value.trim(); - var baseUrl = document.getElementById('cli-settings-base-url').value.trim(); var enabled = document.getElementById('cli-settings-enabled').checked; var idInput = document.getElementById('cli-settings-id'); var id = idInput ? idInput.value : null; @@ -3845,26 +3905,42 @@ async function submitCliSettings() { return; } - if (!token) { - showRefreshToast(t('apiSettings.tokenRequired'), 'error'); + if (!providerId) { + showRefreshToast(t('apiSettings.providerRequired'), 'error'); return; } + if (!model) { + showRefreshToast(t('apiSettings.modelRequired'), 'error'); + return; + } + + // Get provider configuration + var providers = getAvailableAnthropicProviders(); + var provider = providers.find(function(p) { return p.id === providerId; }); + + if (!provider) { + showRefreshToast(t('apiSettings.providerNotFound'), 'error'); + return; + } + + // Build settings from provider var data = { name: name, description: description, enabled: enabled, settings: { env: { - ANTHROPIC_AUTH_TOKEN: token, + ANTHROPIC_AUTH_TOKEN: provider.apiKey || '', DISABLE_AUTOUPDATER: '1' }, - model: model + model: model, + providerId: providerId // Store for editing } }; - if (baseUrl) { - data.settings.env.ANTHROPIC_BASE_URL = baseUrl; + if (provider.apiBase) { + data.settings.env.ANTHROPIC_BASE_URL = provider.apiBase; } if (id) { @@ -3889,6 +3965,7 @@ window.showAddCliSettingsModal = showAddCliSettingsModal; window.editCliSettings = editCliSettings; window.closeCliSettingsModal = closeCliSettingsModal; window.submitCliSettings = submitCliSettings; +window.onCliProviderChange = onCliProviderChange; // ========== Utility Functions ========== diff --git a/ccw/src/templates/dashboard-js/views/cli-manager.js b/ccw/src/templates/dashboard-js/views/cli-manager.js index 7acdbccd..d2e8af1e 100644 --- a/ccw/src/templates/dashboard-js/views/cli-manager.js +++ b/ccw/src/templates/dashboard-js/views/cli-manager.js @@ -330,6 +330,25 @@ function buildToolConfigModalContent(tool, config, models, status) { '
' + '
' + + // Tags Section - Unified input with inline tags + '
' + + '

Tags (optional labels)

' + + '
' + + (config.tags || []).map(function(tag) { + return '' + escapeHtml(tag) + ''; + }).join('') + + '' + + '
' + + '
' + + '' + + '' + + '' + + '' + + '' + + '' + + '
' + + '
' + + // Footer '
' + '' + @@ -341,6 +360,79 @@ function buildToolConfigModalContent(tool, config, models, status) { } function initToolConfigModalEvents(tool, currentConfig, models) { + // Local tags state (copy from config) + var currentTags = (currentConfig.tags || []).slice(); + + // Helper to render tags inline with input + function renderTags() { + var container = document.getElementById('tagsUnifiedInput'); + var input = document.getElementById('tagInput'); + if (!container) return; + + // Remove existing tag items but keep the input + container.querySelectorAll('.tag-item').forEach(function(el) { el.remove(); }); + + // Insert tags before the input + currentTags.forEach(function(tag) { + var tagEl = document.createElement('span'); + tagEl.className = 'tag-item'; + tagEl.innerHTML = escapeHtml(tag) + ''; + container.insertBefore(tagEl, input); + }); + + // Re-attach remove handlers + container.querySelectorAll('.tag-remove').forEach(function(btn) { + btn.onclick = function(e) { + e.stopPropagation(); + var tagToRemove = this.getAttribute('data-tag'); + currentTags = currentTags.filter(function(t) { return t !== tagToRemove; }); + renderTags(); + }; + }); + } + + // Click on unified input container focuses the input + var unifiedInput = document.getElementById('tagsUnifiedInput'); + if (unifiedInput) { + unifiedInput.onclick = function(e) { + if (e.target === this) { + document.getElementById('tagInput').focus(); + } + }; + } + + // Tag input handler + var tagInput = document.getElementById('tagInput'); + if (tagInput) { + tagInput.onkeydown = function(e) { + if (e.key === 'Enter') { + e.preventDefault(); + var newTag = this.value.trim(); + if (newTag && currentTags.indexOf(newTag) === -1) { + currentTags.push(newTag); + renderTags(); + } + this.value = ''; + } + }; + } + + // Predefined tag click handlers + document.querySelectorAll('.predefined-tag-btn').forEach(function(btn) { + btn.onclick = function() { + var tag = this.getAttribute('data-tag'); + if (tag && currentTags.indexOf(tag) === -1) { + currentTags.push(tag); + renderTags(); + } + }; + }); + + // Initialize tags display + renderTags(); + // Initialize lucide icons for predefined buttons + if (window.lucide) lucide.createIcons(); + // Toggle Enable/Disable var toggleBtn = document.getElementById('toggleEnableBtn'); if (toggleBtn) { @@ -426,10 +518,15 @@ function initToolConfigModalEvents(tool, currentConfig, models) { try { await updateCliToolConfig(tool, { primaryModel: primaryModel, - secondaryModel: secondaryModel + secondaryModel: secondaryModel, + tags: currentTags }); + // Reload config to reflect changes + await loadCliToolConfig(); showRefreshToast('Configuration saved', 'success'); closeModal(); + renderToolsSection(); + if (window.lucide) lucide.createIcons(); } catch (err) { showRefreshToast('Failed to save: ' + err.message, 'error'); } @@ -554,35 +651,42 @@ function renderToolsSection() { var toolDescriptions = { gemini: t('cli.geminiDesc'), qwen: t('cli.qwenDesc'), - codex: t('cli.codexDesc') + codex: t('cli.codexDesc'), + claude: t('cli.claudeDesc') || 'Anthropic Claude Code CLI for AI-assisted development', + opencode: t('cli.opencodeDesc') || 'OpenCode CLI - Multi-provider AI coding assistant' }; - var tools = ['gemini', 'qwen', 'codex']; + var tools = ['gemini', 'qwen', 'codex', 'claude', 'opencode']; var available = Object.values(cliToolStatus).filter(function(t) { return t.available; }).length; var toolsHtml = tools.map(function(tool) { var status = cliToolStatus[tool] || {}; var isAvailable = status.available; - var isDefault = defaultCliTool === tool; + var toolConfig = cliToolConfig && cliToolConfig.tools ? cliToolConfig.tools[tool] : null; + var tags = toolConfig && toolConfig.tags ? toolConfig.tags : []; + + // Build tags HTML + var tagsHtml = tags.length > 0 + ? '
' + tags.map(function(tag) { + return '' + escapeHtml(tag) + ''; + }).join('') + '
' + : ''; return '
' + '
' + '' + '
' + '
' + tool.charAt(0).toUpperCase() + tool.slice(1) + - (isDefault ? '' + t('cli.default') + '' : '') + '' + '
' + '
' + toolDescriptions[tool] + '
' + + tagsHtml + '
' + '
' + '
' + (isAvailable ? ' ' + t('cli.ready') + '' : ' ' + t('cli.notInstalled') + '') + - (isAvailable && !isDefault - ? '' - : '') + '
' + '
'; }).join(''); diff --git a/ccw/src/tools/cli-config-manager.ts b/ccw/src/tools/cli-config-manager.ts index a57cb0f1..71847bd8 100644 --- a/ccw/src/tools/cli-config-manager.ts +++ b/ccw/src/tools/cli-config-manager.ts @@ -13,6 +13,7 @@ export interface CliToolConfig { enabled: boolean; primaryModel: string; // For CLI endpoint calls (ccw cli -p) secondaryModel: string; // For internal calls (llm_enhancer, generate_module_docs) + tags?: string[]; // User-defined tags/labels for the tool } export interface CliConfig { @@ -204,7 +205,8 @@ export function updateToolConfig( const updatedToolConfig: CliToolConfig = { enabled: updates.enabled !== undefined ? updates.enabled : currentToolConfig.enabled, primaryModel: updates.primaryModel || currentToolConfig.primaryModel, - secondaryModel: updates.secondaryModel || currentToolConfig.secondaryModel + secondaryModel: updates.secondaryModel || currentToolConfig.secondaryModel, + tags: updates.tags !== undefined ? updates.tags : currentToolConfig.tags }; // Save updated config diff --git a/ccw/src/tools/cli-executor-core.ts b/ccw/src/tools/cli-executor-core.ts index 3e274188..1035a394 100644 --- a/ccw/src/tools/cli-executor-core.ts +++ b/ccw/src/tools/cli-executor-core.ts @@ -10,6 +10,12 @@ import { validatePath } from '../utils/path-resolver.js'; import { escapeWindowsArg } from '../utils/shell-escape.js'; import { buildCommand, checkToolAvailability, clearToolCache, debugLog, errorLog, type NativeResumeConfig, type ToolAvailability } from './cli-executor-utils.js'; import type { ConversationRecord, ConversationTurn, ExecutionOutput, ExecutionRecord } from './cli-executor-state.js'; +import { + createOutputParser, + type CliOutputUnit, + type IOutputParser, + flattenOutputUnits +} from './cli-output-converter.js'; import { buildMergedPrompt, buildMultiTurnPrompt, @@ -110,6 +116,7 @@ const ParamsSchema = z.object({ category: z.enum(['user', 'internal', 'insight']).default('user'), // Execution category for tracking parentExecutionId: z.string().optional(), // Parent execution ID for fork/retry scenarios stream: z.boolean().default(false), // false = cache full output (default), true = stream output via callback + outputFormat: z.enum(['text', 'json-lines']).optional().default('json-lines'), // Output parsing format (default: json-lines for type badges) }); type Params = z.infer; @@ -127,14 +134,14 @@ function assertNonEmptyArray(items: T[], message: string): asserts items is N */ async function executeCliTool( params: Record, - onOutput?: ((data: { type: string; data: string }) => void) | null + onOutput?: ((unit: CliOutputUnit) => void) | null ): Promise { const parsed = ParamsSchema.safeParse(params); if (!parsed.success) { throw new Error(`Invalid params: ${parsed.error.message}`); } - const { tool, prompt, mode, format, model, cd, includeDirs, timeout, resume, id: customId, noNative, category, parentExecutionId } = parsed.data; + const { tool, prompt, mode, format, model, cd, includeDirs, timeout, resume, id: customId, noNative, category, parentExecutionId, outputFormat } = parsed.data; // Validate and determine working directory early (needed for conversation lookup) let workingDir: string; @@ -155,7 +162,11 @@ async function executeCliTool( if (endpoint) { // Route to LiteLLM executor if (onOutput) { - onOutput({ type: 'stderr', data: `[Routing to LiteLLM endpoint: ${model}]\n` }); + onOutput({ + type: 'stderr', + content: `[Routing to LiteLLM endpoint: ${model}]\n`, + timestamp: new Date().toISOString() + }); } const result = await executeLiteLLMEndpoint({ @@ -363,7 +374,11 @@ async function executeCliTool( if (resumeDecision) { const modeDesc = getResumeModeDescription(resumeDecision); if (onOutput) { - onOutput({ type: 'stderr', data: `[Resume mode: ${modeDesc}]\n` }); + onOutput({ + type: 'stderr', + content: `[Resume mode: ${modeDesc}]\n`, + timestamp: new Date().toISOString() + }); } } @@ -381,6 +396,10 @@ async function executeCliTool( nativeResume: nativeResumeConfig }); + // Create output parser and IR storage + const parser = createOutputParser(outputFormat); + const allOutputUnits: CliOutputUnit[] = []; + const startTime = Date.now(); debugLog('EXEC', `Starting CLI execution`, { @@ -390,7 +409,8 @@ async function executeCliTool( conversationId, promptLength: finalPrompt.length, hasResume: !!resume, - hasCustomId: !!customId + hasCustomId: !!customId, + outputFormat }); return new Promise((resolve, reject) => { @@ -436,20 +456,36 @@ async function executeCliTool( let timedOut = false; // Handle stdout - child.stdout!.on('data', (data) => { + child.stdout!.on('data', (data: Buffer) => { const text = data.toString(); stdout += text; + + // Parse into IR units + const units = parser.parse(data, 'stdout'); + allOutputUnits.push(...units); + if (onOutput) { - onOutput({ type: 'stdout', data: text }); + // Send each IR unit to callback + for (const unit of units) { + onOutput(unit); + } } }); // Handle stderr - child.stderr!.on('data', (data) => { + child.stderr!.on('data', (data: Buffer) => { const text = data.toString(); stderr += text; + + // Parse into IR units + const units = parser.parse(data, 'stderr'); + allOutputUnits.push(...units); + if (onOutput) { - onOutput({ type: 'stderr', data: text }); + // Send each IR unit to callback + for (const unit of units) { + onOutput(unit); + } } }); @@ -464,6 +500,15 @@ async function executeCliTool( // Clear current child process reference currentChildProcess = null; + // Flush remaining buffer from parser + const remainingUnits = parser.flush(); + allOutputUnits.push(...remainingUnits); + if (onOutput) { + for (const unit of remainingUnits) { + onOutput(unit); + } + } + const endTime = Date.now(); const duration = endTime - startTime; @@ -472,7 +517,8 @@ async function executeCliTool( duration: `${duration}ms`, timedOut, stdoutLength: stdout.length, - stderrLength: stderr.length + stderrLength: stderr.length, + outputUnitsCount: allOutputUnits.length }); // Determine status - prioritize output content over exit code @@ -524,7 +570,8 @@ async function executeCliTool( truncated: stdout.length > 10240 || stderr.length > 2048, cached: shouldCache, stdout_full: shouldCache ? stdout : undefined, - stderr_full: shouldCache ? stderr : undefined + stderr_full: shouldCache ? stderr : undefined, + structured: allOutputUnits // Save structured IR units }; // Determine base turn number for merge scenarios @@ -677,13 +724,16 @@ async function executeCliTool( id: conversationId, timestamp: new Date(startTime).toISOString(), tool, - model: model || 'default', + model: effectiveModel || 'default', mode, prompt, status, exit_code: code, duration_ms: duration, - output: newTurnOutput + output: newTurnOutput, + parsedOutput: flattenOutputUnits(allOutputUnits, { + excludeTypes: ['stderr', 'progress', 'metadata', 'system'] + }) }; resolve({ @@ -691,7 +741,8 @@ async function executeCliTool( execution, conversation, stdout, - stderr + stderr, + parsedOutput: execution.parsedOutput }); }); diff --git a/ccw/src/tools/cli-executor-state.ts b/ccw/src/tools/cli-executor-state.ts index c7700d52..a41511b8 100644 --- a/ccw/src/tools/cli-executor-state.ts +++ b/ccw/src/tools/cli-executor-state.ts @@ -5,6 +5,7 @@ import type { HistoryIndexEntry } from './cli-history-store.js'; import { StoragePaths, ensureStorageDir } from '../config/storage-paths.js'; +import type { CliOutputUnit } from './cli-output-converter.js'; // Lazy-loaded SQLite store module let sqliteStoreModule: typeof import('./cli-history-store.js') | null = null; @@ -44,6 +45,10 @@ export interface ConversationTurn { stdout: string; stderr: string; truncated: boolean; + cached?: boolean; + stdout_full?: string; + stderr_full?: string; + structured?: CliOutputUnit[]; // Structured IR sequence for advanced parsing }; } @@ -79,6 +84,7 @@ export interface ExecutionRecord { stderr: string; truncated: boolean; }; + parsedOutput?: string; // Extracted clean text from structured output units } interface HistoryIndex { diff --git a/ccw/src/tools/cli-history-store.ts b/ccw/src/tools/cli-history-store.ts index c748c103..649f2870 100644 --- a/ccw/src/tools/cli-history-store.ts +++ b/ccw/src/tools/cli-history-store.ts @@ -8,6 +8,7 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, unlinkSync, import { join } from 'path'; import { parseSessionFile, formatConversation, extractConversationPairs, type ParsedSession, type ParsedTurn } from './session-content-parser.js'; import { StoragePaths, ensureStorageDir, getProjectId } from '../config/storage-paths.js'; +import type { CliOutputUnit } from './cli-output-converter.js'; // Types export interface ConversationTurn { @@ -26,6 +27,7 @@ export interface ConversationTurn { cached?: boolean; stdout_full?: string; stderr_full?: string; + structured?: CliOutputUnit[]; // Structured IR sequence for advanced parsing }; } diff --git a/ccw/src/tools/cli-output-converter.ts b/ccw/src/tools/cli-output-converter.ts new file mode 100644 index 00000000..a12cc0a6 --- /dev/null +++ b/ccw/src/tools/cli-output-converter.ts @@ -0,0 +1,474 @@ +/** + * CLI Output Converter + * Converts raw CLI tool output into structured Intermediate Representation (IR) + * + * Purpose: Decouple output parsing from consumption scenarios (View, Storage, Resume) + * Supports: Plain text, JSON Lines, and other structured formats + */ + +// ========== Type Definitions ========== + +/** + * Unified output unit types for the intermediate representation layer + */ +export type CliOutputUnitType = + | 'stdout' // Standard output text + | 'stderr' // Standard error text + | 'thought' // AI reasoning/thinking + | 'code' // Code block content + | 'file_diff' // File modification diff + | 'progress' // Progress updates + | 'metadata' // Session/execution metadata + | 'system'; // System events/messages + +/** + * Intermediate Representation unit + * Common structure for all CLI output chunks + */ +export interface CliOutputUnit { + type: CliOutputUnitType; + content: T; + timestamp: string; // ISO 8601 format +} + +// ========== Parser Interface ========== + +/** + * Parser interface for converting raw output into IR + */ +export interface IOutputParser { + /** + * Parse a chunk of data from stdout/stderr stream + * @param chunk - Raw buffer from stream + * @param streamType - Source stream (stdout or stderr) + * @returns Array of parsed output units + */ + parse(chunk: Buffer, streamType: 'stdout' | 'stderr'): CliOutputUnit[]; + + /** + * Flush any remaining buffered data + * Called when stream ends to ensure no data is lost + * @returns Array of remaining output units + */ + flush(): CliOutputUnit[]; +} + +// ========== Plain Text Parser ========== + +/** + * PlainTextParser - Converts plain text output to IR + * Simply wraps text in appropriate type envelope + */ +export class PlainTextParser implements IOutputParser { + parse(chunk: Buffer, streamType: 'stdout' | 'stderr'): CliOutputUnit[] { + const text = chunk.toString('utf8'); + + if (!text) { + return []; + } + + return [{ + type: streamType, + content: text, + timestamp: new Date().toISOString() + }]; + } + + /** + * Flush any remaining buffered data + * Called when stream ends to ensure no data is lost + * + * Note: PlainTextParser does not buffer data internally, so this method + * always returns an empty array. Other parsers (e.g., JsonLinesParser) + * may have buffered incomplete lines that need to be flushed. + * + * @returns Array of remaining output units (always empty for PlainTextParser) + */ + flush(): CliOutputUnit[] { + // Plain text parser has no internal buffer + return []; + } +} + +// ========== JSON Lines Parser ========== + +/** + * JsonLinesParser - Parses newline-delimited JSON output + * + * Features: + * - Handles incomplete lines across chunks + * - Maps JSON events to appropriate IR types + * - Falls back to stdout for unparseable lines + * - Robust error handling for malformed JSON + */ +export class JsonLinesParser implements IOutputParser { + private buffer: string = ''; + + parse(chunk: Buffer, streamType: 'stdout' | 'stderr'): CliOutputUnit[] { + const text = chunk.toString('utf8'); + this.buffer += text; + + const units: CliOutputUnit[] = []; + const lines = this.buffer.split('\n'); + + // Keep the last incomplete line in buffer + this.buffer = lines.pop() || ''; + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + + // Try to parse as JSON + let parsed: any; + try { + parsed = JSON.parse(trimmed); + } catch { + // Not valid JSON, treat as plain text + units.push({ + type: streamType, + content: line, + timestamp: new Date().toISOString() + }); + continue; + } + + // Map JSON structure to IR type + const unit = this.mapJsonToIR(parsed, streamType); + if (unit) { + units.push(unit); + } + } + + return units; + } + + flush(): CliOutputUnit[] { + const units: CliOutputUnit[] = []; + + if (this.buffer.trim()) { + // Try to parse remaining buffer + try { + const parsed = JSON.parse(this.buffer.trim()); + const unit = this.mapJsonToIR(parsed, 'stdout'); + if (unit) { + units.push(unit); + } + } catch { + // Not valid JSON, return as plain text + units.push({ + type: 'stdout', + content: this.buffer, + timestamp: new Date().toISOString() + }); + } + } + + this.buffer = ''; + return units; + } + + /** + * Map parsed JSON object to appropriate IR type + * Handles various JSON event formats from different CLI tools + */ + private mapJsonToIR(json: any, fallbackStreamType: 'stdout' | 'stderr'): CliOutputUnit | null { + const timestamp = json.timestamp || new Date().toISOString(); + + // Detect type from JSON structure + if (json.type) { + switch (json.type) { + case 'thought': + case 'thinking': + case 'reasoning': + return { + type: 'thought', + content: json.content || json.text || json.message, + timestamp + }; + + case 'code': + case 'code_block': + return { + type: 'code', + content: json.content || json.code, + timestamp + }; + + case 'diff': + case 'file_diff': + case 'file_change': + return { + type: 'file_diff', + content: { + path: json.path || json.file, + diff: json.diff || json.content, + action: json.action || 'modify' + }, + timestamp + }; + + case 'progress': + case 'status': + return { + type: 'progress', + content: { + message: json.message || json.content, + progress: json.progress, + total: json.total + }, + timestamp + }; + + case 'metadata': + case 'session_meta': + return { + type: 'metadata', + content: json.payload || json.data || json, + timestamp + }; + + case 'system': + case 'event': + return { + type: 'system', + content: json.message || json.content || json, + timestamp + }; + } + } + + // Check for Codex JSONL format + if (json.type === 'response_item' && json.payload) { + const payloadType = json.payload.type; + + if (payloadType === 'message') { + // User or assistant message + const content = json.payload.content + ?.map((c: any) => c.text || '') + .filter((t: string) => t) + .join('\n') || ''; + + return { + type: 'stdout', + content, + timestamp + }; + } + + if (payloadType === 'reasoning') { + return { + type: 'thought', + content: json.payload.summary || json.payload.content, + timestamp + }; + } + + if (payloadType === 'function_call' || payloadType === 'function_call_output') { + return { + type: 'system', + content: json.payload, + timestamp + }; + } + } + + // Check for Gemini/Qwen message format + if (json.role === 'user' || json.role === 'assistant') { + return { + type: 'stdout', + content: json.content || json.text || '', + timestamp + }; + } + + if (json.thoughts && Array.isArray(json.thoughts)) { + return { + type: 'thought', + content: json.thoughts.map((t: any) => + typeof t === 'string' ? t : `${t.subject}: ${t.description}` + ).join('\n'), + timestamp + }; + } + + // Default: treat as stdout/stderr based on fallback + if (json.content || json.message || json.text) { + return { + type: fallbackStreamType, + content: json.content || json.message || json.text, + timestamp + }; + } + + // Unrecognized structure, return as metadata + return { + type: 'metadata', + content: json, + timestamp + }; + } +} + +// ========== Factory Function ========== + +/** + * Create an output parser instance based on format + * @param format - Output format type + * @returns Parser instance + */ +export function createOutputParser(format: 'text' | 'json-lines'): IOutputParser { + switch (format) { + case 'json-lines': + return new JsonLinesParser(); + case 'text': + default: + return new PlainTextParser(); + } +} + +// ========== Utility Functions ========== + +/** + * Flatten output units into plain text string + * Useful for Resume scenario where we need concatenated context + * + * @param units - Array of output units to flatten + * @param options - Filtering and formatting options + * @returns Concatenated text content + */ +export function flattenOutputUnits( + units: CliOutputUnit[], + options?: { + includeTypes?: CliOutputUnitType[]; + excludeTypes?: CliOutputUnitType[]; + includeTimestamps?: boolean; + separator?: string; + } +): string { + const { + includeTypes, + excludeTypes, + includeTimestamps = false, + separator = '\n' + } = options || {}; + + // Filter units by type + let filtered = units; + if (includeTypes && includeTypes.length > 0) { + filtered = filtered.filter(u => includeTypes.includes(u.type)); + } + if (excludeTypes && excludeTypes.length > 0) { + filtered = filtered.filter(u => !excludeTypes.includes(u.type)); + } + + // Convert to text + const lines = filtered.map(unit => { + let text = ''; + + if (includeTimestamps) { + text += `[${unit.timestamp}] `; + } + + // Extract text content based on type + if (typeof unit.content === 'string') { + text += unit.content; + } else if (typeof unit.content === 'object' && unit.content !== null) { + // Handle structured content with type-specific formatting + switch (unit.type) { + case 'file_diff': + // Format file diff with path and diff content + text += `File: ${unit.content.path}\n\`\`\`diff\n${unit.content.diff}\n\`\`\``; + break; + + case 'code': + // Format code block with language + const lang = unit.content.language || ''; + const code = unit.content.code || unit.content; + text += `\`\`\`${lang}\n${typeof code === 'string' ? code : JSON.stringify(code)}\n\`\`\``; + break; + + case 'thought': + // Format thought/reasoning content + text += `[Thought] ${typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content)}`; + break; + + case 'progress': + // Format progress updates + if (unit.content.message) { + text += unit.content.message; + if (unit.content.progress !== undefined && unit.content.total !== undefined) { + text += ` (${unit.content.progress}/${unit.content.total})`; + } + } else { + text += JSON.stringify(unit.content); + } + break; + + case 'metadata': + case 'system': + // Metadata and system events are typically excluded from prompt context + // Include minimal representation if they passed filtering + text += JSON.stringify(unit.content); + break; + + default: + // Fallback for unknown structured types + text += JSON.stringify(unit.content); + } + } else { + text += String(unit.content); + } + + return text; + }); + + return lines.join(separator); +} + +/** + * Extract specific content type from units + * Convenience helper for common extraction patterns + */ +export function extractContent( + units: CliOutputUnit[], + type: CliOutputUnitType +): string[] { + return units + .filter(u => u.type === type) + .map(u => typeof u.content === 'string' ? u.content : JSON.stringify(u.content)); +} + +/** + * Get statistics about output units + * Useful for debugging and analytics + */ +export function getOutputStats(units: CliOutputUnit[]): { + total: number; + byType: Record; + firstTimestamp?: string; + lastTimestamp?: string; +} { + const byType: Record = {}; + let firstTimestamp: string | undefined; + let lastTimestamp: string | undefined; + + for (const unit of units) { + byType[unit.type] = (byType[unit.type] || 0) + 1; + + if (!firstTimestamp || unit.timestamp < firstTimestamp) { + firstTimestamp = unit.timestamp; + } + if (!lastTimestamp || unit.timestamp > lastTimestamp) { + lastTimestamp = unit.timestamp; + } + } + + return { + total: units.length, + byType: byType as Record, + firstTimestamp, + lastTimestamp + }; +} diff --git a/ccw/src/tools/cli-prompt-builder.ts b/ccw/src/tools/cli-prompt-builder.ts index 8ba90a5d..122d01dc 100644 --- a/ccw/src/tools/cli-prompt-builder.ts +++ b/ccw/src/tools/cli-prompt-builder.ts @@ -4,10 +4,59 @@ */ import type { ConversationRecord, ConversationTurn } from './cli-executor-state.js'; +import { flattenOutputUnits, type CliOutputUnit, type CliOutputUnitType } from './cli-output-converter.js'; // Prompt concatenation format types export type PromptFormat = 'plain' | 'yaml' | 'json'; +/** + * Extract clean AI output content from ConversationTurn + * Prioritizes structured IR data, falls back to raw stdout + * + * This function performs noise filtering to extract only meaningful content + * for use in prompt context, excluding progress updates, metadata, and system events. + * + * @param turn - Conversation turn containing output + * @param options - Extraction options + * @param options.includeThoughts - Whether to include AI reasoning/thinking in output (default: false) + * @returns Clean output string suitable for prompt context + * + * @example + * ```typescript + * const cleanOutput = extractCleanOutput(turn, { includeThoughts: true }); + * // Returns: stdout + code + file_diff + thought (excludes: progress, metadata, system) + * ``` + */ +function extractCleanOutput( + turn: ConversationTurn, + options?: { + includeThoughts?: boolean; + } +): string { + // Priority 1: Use structured IR if available (clean, noise-filtered) + if (turn.output?.structured && turn.output.structured.length > 0) { + const includeTypes: CliOutputUnitType[] = ['stdout', 'code', 'file_diff']; + + // Optionally include thought processes + if (options?.includeThoughts) { + includeTypes.push('thought'); + } + + return flattenOutputUnits(turn.output.structured, { + includeTypes, + excludeTypes: ['progress', 'metadata', 'system'] + }); + } + + // Priority 2: Use full output if available + if (turn.output?.stdout_full) { + return turn.output.stdout_full; + } + + // Priority 3: Fall back to truncated stdout + return turn.output?.stdout || '[No output]'; +} + /** * Merge multiple conversations into a unified context * Returns merged turns sorted by timestamp with source tracking @@ -166,7 +215,11 @@ export class PromptConcatenator { timestamp: turn.timestamp, source_id: sourceId }); - this.addAssistantTurn(turn.output.stdout || '[No output]', { + + // Use extractCleanOutput to get noise-filtered content + const cleanOutput = extractCleanOutput(turn); + + this.addAssistantTurn(cleanOutput, { turn: turn.turn * 2, timestamp: turn.timestamp, status: turn.status, diff --git a/ccw/src/tools/litellm-executor.ts b/ccw/src/tools/litellm-executor.ts index 6dc03cf4..b2046979 100644 --- a/ccw/src/tools/litellm-executor.ts +++ b/ccw/src/tools/litellm-executor.ts @@ -10,6 +10,7 @@ import { getProviderWithResolvedEnvVars, } from '../config/litellm-api-config-manager.js'; import type { CustomEndpoint, ProviderCredential } from '../types/litellm-api-config.js'; +import type { CliOutputUnit } from './cli-output-converter.js'; export interface LiteLLMExecutionOptions { prompt: string; @@ -18,7 +19,7 @@ export interface LiteLLMExecutionOptions { cwd?: string; // Working directory for file resolution includeDirs?: string[]; // Additional directories for @patterns enableCache?: boolean; // Override endpoint cache setting - onOutput?: (data: { type: string; data: string }) => void; + onOutput?: (unit: CliOutputUnit) => void; /** Number of retries after the initial attempt (default: 0) */ maxRetries?: number; /** Base delay for exponential backoff in milliseconds (default: 1000) */ @@ -105,7 +106,11 @@ export async function executeLiteLLMEndpoint( const patterns = extractPatterns(prompt); if (patterns.length > 0) { if (onOutput) { - onOutput({ type: 'stderr', data: `[Context cache: Found ${patterns.length} @patterns]\n` }); + onOutput({ + type: 'stderr', + content: `[Context cache: Found ${patterns.length} @patterns]\n`, + timestamp: new Date().toISOString() + }); } // Pack files into cache @@ -124,7 +129,8 @@ export async function executeLiteLLMEndpoint( if (onOutput) { onOutput({ type: 'stderr', - data: `[Context cache: Packed ${pack.files_packed} files, ${pack.total_bytes} bytes]\n`, + content: `[Context cache: Packed ${pack.files_packed} files, ${pack.total_bytes} bytes]\n`, + timestamp: new Date().toISOString() }); } @@ -143,12 +149,20 @@ export async function executeLiteLLMEndpoint( cachedFiles = pack.files_packed ? Array(pack.files_packed).fill('...') : []; if (onOutput) { - onOutput({ type: 'stderr', data: `[Context cache: Applied to prompt]\n` }); + onOutput({ + type: 'stderr', + content: `[Context cache: Applied to prompt]\n`, + timestamp: new Date().toISOString() + }); } } } else if (packResult.error) { if (onOutput) { - onOutput({ type: 'stderr', data: `[Context cache warning: ${packResult.error}]\n` }); + onOutput({ + type: 'stderr', + content: `[Context cache warning: ${packResult.error}]\n`, + timestamp: new Date().toISOString() + }); } } } @@ -159,7 +173,8 @@ export async function executeLiteLLMEndpoint( if (onOutput) { onOutput({ type: 'stderr', - data: `[LiteLLM: Calling ${provider.type}/${endpoint.model}]\n`, + content: `[LiteLLM: Calling ${provider.type}/${endpoint.model}]\n`, + timestamp: new Date().toISOString() }); } @@ -195,7 +210,11 @@ export async function executeLiteLLMEndpoint( ); if (onOutput) { - onOutput({ type: 'stdout', data: response }); + onOutput({ + type: 'stdout', + content: response, + timestamp: new Date().toISOString() + }); } return { @@ -209,7 +228,11 @@ export async function executeLiteLLMEndpoint( } catch (error) { const errorMsg = (error as Error).message; if (onOutput) { - onOutput({ type: 'stderr', data: `[LiteLLM error: ${errorMsg}]\n` }); + onOutput({ + type: 'stderr', + content: `[LiteLLM error: ${errorMsg}]\n`, + timestamp: new Date().toISOString() + }); } return { @@ -279,7 +302,7 @@ async function callWithRetries( options: { maxRetries: number; baseDelayMs: number; - onOutput?: (data: { type: string; data: string }) => void; + onOutput?: (unit: CliOutputUnit) => void; rateLimitKey: string; }, ): Promise { @@ -301,7 +324,8 @@ async function callWithRetries( if (onOutput) { onOutput({ type: 'stderr', - data: `[LiteLLM retry ${attempt + 1}/${maxRetries}: waiting ${delayMs}ms] ${errorMessage}\n`, + content: `[LiteLLM retry ${attempt + 1}/${maxRetries}: waiting ${delayMs}ms] ${errorMessage}\n`, + timestamp: new Date().toISOString() }); }