feat: Enhance CLI output handling with structured Intermediate Representation (IR)

- Introduced `CliOutputUnit` and `IOutputParser` interfaces for unified output processing.
- Implemented `PlainTextParser` and `JsonLinesParser` for parsing raw CLI output into structured units.
- Updated `executeCliTool` to utilize output parsers and handle structured output.
- Added `flattenOutputUnits` utility for extracting clean output from structured data.
- Enhanced `ConversationTurn` and `ExecutionRecord` interfaces to include structured output.
- Created comprehensive documentation for CLI Output Converter usage and integration.
- Improved error handling and type mapping for various output formats.
This commit is contained in:
catlog22
2026-01-08 17:26:40 +08:00
parent b86cdd6644
commit d0523684e5
22 changed files with 1618 additions and 111 deletions

View File

@@ -0,0 +1,338 @@
# CLI Output Converter Usage Guide
## Overview
The CLI Output Converter provides a unified Intermediate Representation (IR) layer for CLI tool output, enabling clean separation between output parsing and consumption scenarios.
## Architecture
```
┌─────────────────┐
│ CLI Tool │
│ (stdout/stderr)│
└────────┬────────┘
│ Buffer chunks
┌─────────────────┐
│ Output Parser │
│ (text/json-lines)│
└────────┬────────┘
│ CliOutputUnit[]
┌─────────────────────────────────────┐
│ Intermediate Representation (IR) │
│ - type: stdout|stderr|thought|... │
│ - content: string | object │
│ - timestamp: ISO 8601 │
└────┬────────────────────────────┬───┘
│ │
▼ ▼
┌─────────────┐ ┌─────────────┐
│ Storage │ │ View │
│ (SQLite) │ │ (Dashboard)│
└─────────────┘ └─────────────┘
┌─────────────┐
│ Resume │
│ (Flatten) │
└─────────────┘
```
## Basic Usage
### 1. Creating Parsers
```typescript
import { createOutputParser } from './cli-output-converter.js';
// For plain text output (e.g., Gemini/Qwen plain mode)
const textParser = createOutputParser('text');
// For JSON Lines output (e.g., Codex JSONL format)
const jsonParser = createOutputParser('json-lines');
```
### 2. Parsing Stream Chunks
```typescript
import { spawn } from 'child_process';
import { createOutputParser } from './cli-output-converter.js';
const parser = createOutputParser('json-lines');
const allUnits: CliOutputUnit[] = [];
const child = spawn('codex', ['run', 'task']);
child.stdout.on('data', (chunk: Buffer) => {
const units = parser.parse(chunk, 'stdout');
allUnits.push(...units);
// Real-time processing
for (const unit of units) {
console.log(`[${unit.type}] ${unit.content}`);
}
});
child.stderr.on('data', (chunk: Buffer) => {
const units = parser.parse(chunk, 'stderr');
allUnits.push(...units);
});
child.on('close', () => {
// Flush remaining buffer
const remaining = parser.flush();
allUnits.push(...remaining);
// Save to storage
saveToDatabase(allUnits);
});
```
### 3. Integration with CLI Executor
```typescript
// In cli-executor-core.ts
import { createOutputParser, type CliOutputUnit } from './cli-output-converter.js';
async function executeCliTool(params: CliParams) {
const parser = createOutputParser(params.format === 'json-lines' ? 'json-lines' : 'text');
const structuredOutput: CliOutputUnit[] = [];
child.stdout.on('data', (data) => {
const text = data.toString();
stdout += text;
// Parse into IR
const units = parser.parse(data, 'stdout');
structuredOutput.push(...units);
// Existing streaming logic
if (onOutput) {
onOutput({ type: 'stdout', data: text });
}
});
child.on('close', () => {
// Flush parser
structuredOutput.push(...parser.flush());
// Create turn with structured output
const newTurn: ConversationTurn = {
turn: turnNumber,
timestamp: new Date().toISOString(),
prompt,
duration_ms,
status,
exit_code: code,
output: {
stdout: stdout.substring(0, 10240),
stderr: stderr.substring(0, 2048),
truncated: stdout.length > 10240 || stderr.length > 2048,
cached: shouldCache,
stdout_full: shouldCache ? stdout : undefined,
stderr_full: shouldCache ? stderr : undefined,
structured: structuredOutput.length > 0 ? structuredOutput : undefined
}
};
});
}
```
## Scenario-Specific Usage
### Scenario 1: View (Dashboard Display)
```typescript
// In dashboard rendering logic
import { type CliOutputUnit } from '../tools/cli-output-converter.js';
function renderOutputUnits(units: CliOutputUnit[]) {
return units.map(unit => {
switch (unit.type) {
case 'thought':
return `<div class="thought">${unit.content}</div>`;
case 'code':
return `<pre><code>${unit.content}</code></pre>`;
case 'file_diff':
return renderDiff(unit.content);
case 'progress':
return renderProgress(unit.content);
default:
return `<div class="output">${unit.content}</div>`;
}
}).join('\n');
}
```
### Scenario 2: Storage (Backward Compatible)
```typescript
// In cli-history-store.ts
function saveConversation(conversation: ConversationRecord) {
for (const turn of conversation.turns) {
// Save traditional fields
db.run(`
INSERT INTO turns (stdout, stderr, truncated, ...)
VALUES (?, ?, ?, ...)
`, turn.output.stdout, turn.output.stderr, turn.output.truncated);
// Optionally save structured output
if (turn.output.structured) {
db.run(`
INSERT INTO turn_structured_output (turn_id, units)
VALUES (?, ?)
`, turnId, JSON.stringify(turn.output.structured));
}
}
}
```
### Scenario 3: Resume (Context Concatenation)
```typescript
// In resume-strategy.ts or cli-prompt-builder.ts
import { flattenOutputUnits } from './cli-output-converter.js';
function buildContextFromTurns(turns: ConversationTurn[]): string {
const lines: string[] = [];
for (const turn of turns) {
lines.push(`USER: ${turn.prompt}`);
// Use structured output if available
if (turn.output.structured) {
const assistantText = flattenOutputUnits(turn.output.structured, {
excludeTypes: ['metadata', 'system'], // Skip noise
includeTypes: ['stdout', 'thought', 'code'] // Keep meaningful content
});
lines.push(`ASSISTANT: ${assistantText}`);
} else {
// Fallback to plain stdout
lines.push(`ASSISTANT: ${turn.output.stdout}`);
}
}
return lines.join('\n\n');
}
```
## Advanced Features
### Filtering by Type
```typescript
import { flattenOutputUnits, extractContent } from './cli-output-converter.js';
// Extract only AI thoughts for analysis
const thoughts = extractContent(units, 'thought');
// Get only code blocks
const codeBlocks = extractContent(units, 'code');
// Create clean context (exclude system noise)
const cleanContext = flattenOutputUnits(units, {
excludeTypes: ['metadata', 'system', 'progress']
});
```
### Analytics
```typescript
import { getOutputStats } from './cli-output-converter.js';
const stats = getOutputStats(units);
console.log(`Total units: ${stats.total}`);
console.log(`Thoughts: ${stats.byType.thought || 0}`);
console.log(`Code blocks: ${stats.byType.code || 0}`);
console.log(`Duration: ${stats.firstTimestamp} - ${stats.lastTimestamp}`);
```
### Custom Processing
```typescript
function processUnits(units: CliOutputUnit[]) {
for (const unit of units) {
switch (unit.type) {
case 'file_diff':
// Apply diff to file system
applyDiff(unit.content.path, unit.content.diff);
break;
case 'progress':
// Update progress bar
updateProgress(unit.content.progress, unit.content.total);
break;
case 'thought':
// Log reasoning
logThought(unit.content);
break;
}
}
}
```
## Type Reference
### CliOutputUnitType
```typescript
type CliOutputUnitType =
| 'stdout' // Standard output text
| 'stderr' // Standard error text
| 'thought' // AI reasoning/thinking
| 'code' // Code block content
| 'file_diff' // File modification diff
| 'progress' // Progress updates
| 'metadata' // Session/execution metadata
| 'system'; // System events/messages
```
### CliOutputUnit
```typescript
interface CliOutputUnit<T = any> {
type: CliOutputUnitType;
content: T; // string for text types, object for structured types
timestamp: string; // ISO 8601 format
}
```
## Migration Path
### Phase 1: Optional Enhancement (Current)
- Add `structured` field to `ConversationTurn.output`
- Populate during parsing (optional)
- Existing code ignores it (backward compatible)
### Phase 2: View Integration
- Dashboard uses `structured` when available
- Falls back to plain `stdout` when not present
- Better rendering for thoughts, code, diffs
### Phase 3: Resume Optimization
- Resume logic prefers `structured` for cleaner context
- Filters out noise (metadata, system events)
- Reduces token usage while preserving semantics
### Phase 4: Full Adoption
- All CLI tools use converters
- Storage optimized for structured data
- Analytics and insights from IR layer
## Best Practices
1. **Always flush the parser** when stream ends to capture incomplete lines
2. **Filter by type** for Resume scenarios to reduce token usage
3. **Use structured content** when available for better display
4. **Keep backward compatibility** by making `structured` optional
5. **Handle missing fields** gracefully (not all tools output JSON)
6. **Validate JSON** before parsing to avoid crashes
## Future Enhancements
1. **Streaming transformers** - Apply transformations during parsing
2. **Custom type mappers** - Register custom JSON-to-IR mappings
3. **Compression** - Store structured output more efficiently
4. **Semantic search** - Index thoughts and code separately
5. **Diff viewer** - Interactive file_diff rendering
6. **Progress tracking** - Aggregate progress events across tools

View File

@@ -6,6 +6,7 @@
import chalk from 'chalk';
import http from 'http';
import inquirer from 'inquirer';
import type { CliOutputUnit } from '../tools/cli-output-converter.js';
import {
cliExecutorTool,
getCliToolsStatus,
@@ -725,7 +726,8 @@ async function execAction(positionalPrompt: string | undefined, options: CliExec
// Generate execution ID for streaming (use custom ID or timestamp-based)
const executionId = id || `${Date.now()}-${tool}`;
const startTime = Date.now();
const spinnerBaseText = `Executing ${tool} (${mode} mode${resumeInfo}${nativeMode})${idInfo}...`;
const modelInfo = model ? ` @${model}` : '';
const spinnerBaseText = `Executing ${tool}${modelInfo} (${mode} mode${resumeInfo}${nativeMode})${idInfo}...`;
console.log();
const spinner = stream ? null : createSpinner(` ${spinnerBaseText}`).start();
@@ -787,20 +789,49 @@ async function execAction(positionalPrompt: string | undefined, options: CliExec
});
// Streaming output handler - broadcasts to dashboard AND writes to stdout
const onOutput = (chunk: any) => {
const onOutput = (unit: CliOutputUnit) => {
// Always broadcast to dashboard for real-time viewing
// Note: /api/hook wraps extraData into payload, so send fields directly
// Maintain backward compatibility with frontend expecting { chunkType, data }
const content = typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content);
broadcastStreamEvent('CLI_OUTPUT', {
executionId,
chunkType: chunk.type,
data: chunk.data
chunkType: unit.type, // For backward compatibility
data: content, // For backward compatibility
unit // New structured format
});
// Write to terminal only when --stream flag is passed
if (stream) {
process.stdout.write(chunk.data);
switch (unit.type) {
case 'stdout':
case 'code':
process.stdout.write(typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content));
break;
case 'stderr':
process.stderr.write(typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content));
break;
case 'thought':
// Optional: display thinking process with different color
// For now, skip to reduce noise
break;
case 'progress':
// Optional: update progress bar
// For now, skip
break;
default:
// Other types: output content if available
if (unit.content) {
process.stdout.write(typeof unit.content === 'string' ? unit.content : '');
}
}
}
};
// Use JSON-lines parsing by default to enable type badges (thought, code, file_diff, etc.)
// All CLI tools may output structured JSON that can be parsed for richer UI
const outputFormat = 'json-lines';
try {
const result = await cliExecutorTool.execute({
tool,
@@ -813,7 +844,8 @@ async function execAction(positionalPrompt: string | undefined, options: CliExec
resume,
id, // custom execution ID
noNative,
stream: !!stream // stream=true → streaming enabled (no cache), stream=false → cache output (default)
stream: !!stream, // stream=true → streaming enabled (no cache), stream=false → cache output (default)
outputFormat // Enable JSONL parsing for tools that support it
}, onOutput); // Always pass onOutput for real-time dashboard streaming
if (elapsedInterval) clearInterval(elapsedInterval);

View File

@@ -118,8 +118,9 @@ export async function csrfValidation(ctx: CsrfMiddlewareContext): Promise<boolea
const method = (req.method || 'GET').toUpperCase();
if (!['POST', 'PUT', 'PATCH', 'DELETE'].includes(method)) return true;
// Always allow token acquisition routes.
// Always allow token acquisition routes and webhook endpoints.
if (pathname === '/api/auth/token') return true;
if (pathname === '/api/hook') return true;
// Requests authenticated via Authorization header do not require CSRF protection.
const authorization = getHeaderValue(req.headers.authorization);

View File

@@ -627,7 +627,18 @@ export async function handleClaudeRoutes(ctx: RouteContext): Promise<boolean> {
stream: false,
category: 'internal',
id: syncId
}, onOutput);
}, (unit) => {
// CliOutputUnit handler: convert to string content for broadcast
const content = typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content);
broadcastToClients({
type: 'CLI_OUTPUT',
payload: {
executionId: syncId,
chunkType: unit.type,
data: content
}
});
});
// Broadcast CLI_EXECUTION_COMPLETED event
broadcastToClients({

View File

@@ -195,7 +195,7 @@ export async function handleCliRoutes(ctx: RouteContext): Promise<boolean> {
}
// API: Get/Update Tool Config
const configMatch = pathname.match(/^\/api\/cli\/config\/(gemini|qwen|codex)$/);
const configMatch = pathname.match(/^\/api\/cli\/config\/(gemini|qwen|codex|claude|opencode)$/);
if (configMatch) {
const tool = configMatch[1];
@@ -216,7 +216,7 @@ export async function handleCliRoutes(ctx: RouteContext): Promise<boolean> {
if (req.method === 'PUT') {
handlePostRequest(req, res, async (body: unknown) => {
try {
const updates = body as { enabled?: boolean; primaryModel?: string; secondaryModel?: string };
const updates = body as { enabled?: boolean; primaryModel?: string; secondaryModel?: string; tags?: string[] };
const updated = updateToolConfig(initialPath, tool, updates);
// Broadcast config updated event
@@ -559,19 +559,22 @@ export async function handleCliRoutes(ctx: RouteContext): Promise<boolean> {
category: category || 'user',
parentExecutionId,
stream: true
}, (chunk) => {
// Append chunk to active execution buffer
}, (unit) => {
// CliOutputUnit handler: convert to string content
const content = typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content);
// Append to active execution buffer
const activeExec = activeExecutions.get(executionId);
if (activeExec) {
activeExec.output += chunk.data || '';
activeExec.output += content || '';
}
broadcastToClients({
type: 'CLI_OUTPUT',
payload: {
executionId,
chunkType: chunk.type,
data: chunk.data
chunkType: unit.type,
data: content
}
});
});

View File

@@ -1007,7 +1007,18 @@ RULES: Be concise. Focus on practical understanding. Include function signatures
stream: false,
category: 'internal',
id: syncId
}, onOutput);
}, (unit) => {
// CliOutputUnit handler: convert to string content for broadcast
const content = typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content);
broadcastToClients({
type: 'CLI_OUTPUT',
payload: {
executionId: syncId,
chunkType: unit.type,
data: content
}
});
});
// Broadcast CLI_EXECUTION_COMPLETED event
broadcastToClients({

View File

@@ -661,13 +661,15 @@ FILE NAME: ${fileName}`;
// Create onOutput callback for real-time streaming
const onOutput = broadcastToClients
? (chunk: { type: string; data: string }) => {
? (unit: import('../../tools/cli-output-converter.js').CliOutputUnit) => {
// CliOutputUnit handler: convert to string content for broadcast
const content = typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content);
broadcastToClients({
type: 'CLI_OUTPUT',
payload: {
executionId,
chunkType: chunk.type,
data: chunk.data
chunkType: unit.type,
data: content
}
});
}
@@ -746,13 +748,15 @@ FILE NAME: ${fileName}`;
// Create onOutput callback for review step
const reviewOnOutput = broadcastToClients
? (chunk: { type: string; data: string }) => {
? (unit: import('../../tools/cli-output-converter.js').CliOutputUnit) => {
// CliOutputUnit handler: convert to string content for broadcast
const content = typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content);
broadcastToClients({
type: 'CLI_OUTPUT',
payload: {
executionId: reviewExecutionId,
chunkType: chunk.type,
data: chunk.data
chunkType: unit.type,
data: content
}
});
}

View File

@@ -579,13 +579,15 @@ Create a new Claude Code skill with the following specifications:
// Create onOutput callback for real-time streaming
const onOutput = broadcastToClients
? (chunk: { type: string; data: string }) => {
? (unit: import('../../tools/cli-output-converter.js').CliOutputUnit) => {
// CliOutputUnit handler: convert to string content for broadcast
const content = typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content);
broadcastToClients({
type: 'CLI_OUTPUT',
payload: {
executionId,
chunkType: chunk.type,
data: chunk.data
chunkType: unit.type,
data: content
}
});
}

View File

@@ -357,7 +357,7 @@ export async function startServer(options: ServerOptions = {}): Promise<http.Ser
const tokenManager = getTokenManager();
const secretKey = tokenManager.getSecretKey();
tokenManager.getOrCreateAuthToken();
const unauthenticatedPaths = new Set<string>(['/api/auth/token', '/api/csrf-token']);
const unauthenticatedPaths = new Set<string>(['/api/auth/token', '/api/csrf-token', '/api/hook']);
const server = http.createServer(async (req, res) => {
const url = new URL(req.url ?? '/', `http://localhost:${serverPort}`);

View File

@@ -203,6 +203,168 @@
color: hsl(142 76% 36%);
}
/* Tool Tags - displayed in tool cards */
.tool-tags {
display: flex;
flex-wrap: wrap;
gap: 0.25rem;
margin-top: 0.25rem;
}
.tool-tag {
font-size: 0.5625rem;
font-weight: 500;
padding: 0.125rem 0.375rem;
background: hsl(var(--primary) / 0.1);
color: hsl(var(--primary));
border-radius: 0.25rem;
}
/* Tags Input - used in config modal */
.tags-input-group {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
/* Unified tag input - tags and input in one container */
.tags-unified-input {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 0.375rem;
min-height: 2.5rem;
padding: 0.375rem 0.5rem;
background: hsl(var(--background));
border: 1px solid hsl(var(--border));
border-radius: 0.5rem;
cursor: text;
transition: border-color 0.15s ease;
}
.tags-unified-input:focus-within {
border-color: hsl(var(--primary));
box-shadow: 0 0 0 2px hsl(var(--primary) / 0.1);
}
.tag-inline-input {
flex: 1;
min-width: 120px;
border: none;
background: transparent;
outline: none;
font-size: 0.8125rem;
color: hsl(var(--foreground));
padding: 0.25rem 0;
}
.tag-inline-input::placeholder {
color: hsl(var(--muted-foreground) / 0.6);
}
.tags-container {
display: flex;
flex-wrap: wrap;
gap: 0.375rem;
min-height: 1.75rem;
padding: 0.25rem;
background: hsl(var(--muted) / 0.3);
border-radius: 0.375rem;
}
.tag-item {
display: inline-flex;
align-items: center;
gap: 0.25rem;
font-size: 0.75rem;
font-weight: 500;
padding: 0.25rem 0.5rem;
background: hsl(var(--primary) / 0.15);
color: hsl(var(--primary));
border-radius: 0.25rem;
}
.tag-remove {
display: flex;
align-items: center;
justify-content: center;
width: 1rem;
height: 1rem;
padding: 0;
border: none;
background: transparent;
color: hsl(var(--primary) / 0.6);
cursor: pointer;
font-size: 0.875rem;
line-height: 1;
border-radius: 0.125rem;
}
.tag-remove:hover {
background: hsl(var(--destructive) / 0.2);
color: hsl(var(--destructive));
}
/* Predefined Tags Row - prominent quick add buttons */
.predefined-tags-row {
display: flex;
flex-wrap: wrap;
gap: 0.5rem;
margin-top: 0.75rem;
}
.predefined-tag-btn {
display: inline-flex;
align-items: center;
gap: 0.375rem;
font-size: 0.75rem;
font-weight: 500;
padding: 0.375rem 0.625rem;
background: hsl(var(--muted));
color: hsl(var(--foreground));
border: 1px solid hsl(var(--border));
border-radius: 0.375rem;
cursor: pointer;
transition: all 0.15s ease;
}
.predefined-tag-btn:hover {
background: hsl(var(--primary) / 0.15);
color: hsl(var(--primary));
border-color: hsl(var(--primary) / 0.3);
}
.predefined-tag-btn i {
opacity: 0.7;
}
/* Legacy predefined tags */
.predefined-tags {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 0.375rem;
margin-top: 0.5rem;
}
.predefined-tag {
font-size: 0.6875rem;
font-weight: 500;
padding: 0.25rem 0.5rem;
background: hsl(var(--muted) / 0.5);
color: hsl(var(--muted-foreground));
border: 1px dashed hsl(var(--border));
border-radius: 0.25rem;
cursor: pointer;
transition: all 0.15s ease;
}
.predefined-tag:hover {
background: hsl(var(--primary) / 0.1);
color: hsl(var(--primary));
border-color: hsl(var(--primary) / 0.3);
}
.tool-item-right {
display: flex;
align-items: center;

View File

@@ -549,6 +549,77 @@
color: hsl(200 70% 70%);
}
/* ===== Backend ChunkType Badges (CliOutputUnit.type) ===== */
/* Thought/Thinking Message (from JSONL parser) */
.cli-stream-line.formatted.thought {
background: hsl(280 50% 20% / 0.3);
border-left: 3px solid hsl(280 70% 65%);
font-style: italic;
}
.cli-msg-badge.cli-msg-thought {
background: hsl(280 70% 65% / 0.2);
color: hsl(280 70% 75%);
}
/* Code Block Message */
.cli-stream-line.formatted.code {
background: hsl(220 40% 18% / 0.4);
border-left: 3px solid hsl(220 60% 55%);
font-family: var(--font-mono, 'Consolas', 'Monaco', 'Courier New', monospace);
}
.cli-msg-badge.cli-msg-code {
background: hsl(220 60% 55% / 0.25);
color: hsl(220 60% 70%);
}
/* File Diff Message */
.cli-stream-line.formatted.file_diff {
background: hsl(35 50% 18% / 0.4);
border-left: 3px solid hsl(35 80% 55%);
}
.cli-msg-badge.cli-msg-file_diff {
background: hsl(35 80% 55% / 0.25);
color: hsl(35 80% 65%);
}
/* Progress Message */
.cli-stream-line.formatted.progress {
background: hsl(190 40% 18% / 0.3);
border-left: 3px solid hsl(190 70% 50%);
}
.cli-msg-badge.cli-msg-progress {
background: hsl(190 70% 50% / 0.2);
color: hsl(190 70% 65%);
}
/* Metadata Message */
.cli-stream-line.formatted.metadata {
background: hsl(250 30% 18% / 0.3);
border-left: 3px solid hsl(250 50% 60%);
font-size: 0.85em;
}
.cli-msg-badge.cli-msg-metadata {
background: hsl(250 50% 60% / 0.2);
color: hsl(250 50% 75%);
}
/* Stderr Message (Error) */
.cli-stream-line.formatted.stderr {
background: hsl(0 50% 20% / 0.4);
border-left: 3px solid hsl(0 70% 55%);
}
.cli-msg-badge.cli-msg-stderr {
background: hsl(0 70% 55% / 0.25);
color: hsl(0 70% 70%);
}
/* Inline Code */
.cli-inline-code {
padding: 1px 5px;

View File

@@ -346,16 +346,50 @@ function renderFormattedLine(line, searchFilter) {
// Format inline code
content = content.replace(/`([^`]+)`/g, '<code class="cli-inline-code">$1</code>');
// Build type badge if has prefix
const typeBadge = parsed.hasPrefix ?
`<span class="cli-msg-badge cli-msg-${parsed.type}">
// Type badge icons for backend chunkType (CliOutputUnit.type)
const CHUNK_TYPE_ICONS = {
thought: 'brain',
code: 'code',
file_diff: 'git-compare',
progress: 'loader',
system: 'settings',
stderr: 'alert-circle',
metadata: 'info'
};
// Type badge labels for backend chunkType
const CHUNK_TYPE_LABELS = {
thought: 'Thinking',
code: 'Code',
file_diff: 'Diff',
progress: 'Progress',
system: 'System',
stderr: 'Error',
metadata: 'Info'
};
// Build type badge - prioritize content prefix, then fall back to chunkType
let typeBadge = '';
let lineClass = '';
if (parsed.hasPrefix) {
// Content has Chinese prefix like [系统], [思考], etc.
typeBadge = `<span class="cli-msg-badge cli-msg-${parsed.type}">
<i data-lucide="${MESSAGE_TYPE_ICONS[parsed.type] || 'circle'}"></i>
<span>${parsed.label}</span>
</span>` : '';
// Determine line class based on original type and parsed type
const lineClass = parsed.hasPrefix ? `cli-stream-line formatted ${parsed.type}` :
`cli-stream-line ${line.type}`;
</span>`;
lineClass = `cli-stream-line formatted ${parsed.type}`;
} else if (line.type && line.type !== 'stdout' && CHUNK_TYPE_LABELS[line.type]) {
// No content prefix, but backend sent a meaningful chunkType
typeBadge = `<span class="cli-msg-badge cli-msg-${line.type}">
<i data-lucide="${CHUNK_TYPE_ICONS[line.type] || 'circle'}"></i>
<span>${CHUNK_TYPE_LABELS[line.type]}</span>
</span>`;
lineClass = `cli-stream-line formatted ${line.type}`;
} else {
// Plain stdout, no badge
lineClass = `cli-stream-line ${line.type || 'stdout'}`;
}
return `<div class="${lineClass}">${typeBadge}<span class="cli-msg-content">${content}</span></div>`;
}

View File

@@ -1721,6 +1721,28 @@ const i18n = {
'apiSettings.modelIdExists': 'Model ID already exists',
'apiSettings.useModelTreeToManage': 'Use the model tree to manage individual models',
// CLI Settings
'apiSettings.cliSettings': 'CLI Settings',
'apiSettings.addCliSettings': 'Add CLI Settings',
'apiSettings.editCliSettings': 'Edit CLI Settings',
'apiSettings.noCliSettings': 'No CLI settings configured',
'apiSettings.noCliSettingsSelected': 'No CLI Settings Selected',
'apiSettings.cliSettingsHint': 'Select a CLI settings endpoint or create a new one',
'apiSettings.cliProviderHint': 'Select an Anthropic provider to use its API key and base URL',
'apiSettings.noAnthropicProviders': 'No Anthropic providers configured. Please add one in the Providers tab first.',
'apiSettings.selectProviderFirst': 'Select a provider first',
'apiSettings.providerRequired': 'Provider is required',
'apiSettings.modelRequired': 'Model is required',
'apiSettings.providerNotFound': 'Provider not found',
'apiSettings.settingsSaved': 'Settings saved successfully',
'apiSettings.settingsDeleted': 'Settings deleted successfully',
'apiSettings.confirmDeleteSettings': 'Are you sure you want to delete this CLI settings?',
'apiSettings.endpointName': 'Endpoint Name',
'apiSettings.envSettings': 'Environment Settings',
'apiSettings.settingsFilePath': 'Settings File Path',
'apiSettings.nameRequired': 'Name is required',
'apiSettings.status': 'Status',
// Common
'common.cancel': 'Cancel',
'common.optional': '(Optional)',
@@ -3777,6 +3799,29 @@ const i18n = {
'apiSettings.modelIdExists': '模型 ID 已存在',
'apiSettings.useModelTreeToManage': '使用模型树管理各个模型',
// CLI Settings
'apiSettings.cliSettings': 'CLI 配置',
'apiSettings.addCliSettings': '添加 CLI 配置',
'apiSettings.editCliSettings': '编辑 CLI 配置',
'apiSettings.noCliSettings': '未配置 CLI 设置',
'apiSettings.noCliSettingsSelected': '未选择 CLI 配置',
'apiSettings.cliSettingsHint': '选择一个 CLI 配置端点或创建新的',
'apiSettings.cliProviderHint': '选择一个 Anthropic 供应商以使用其 API 密钥和基础 URL',
'apiSettings.noAnthropicProviders': '未配置 Anthropic 供应商。请先在供应商标签页中添加。',
'apiSettings.selectProviderFirst': '请先选择供应商',
'apiSettings.providerRequired': '供应商为必填项',
'apiSettings.modelRequired': '模型为必填项',
'apiSettings.providerNotFound': '未找到供应商',
'apiSettings.settingsSaved': '设置保存成功',
'apiSettings.settingsDeleted': '设置删除成功',
'apiSettings.confirmDeleteSettings': '确定要删除此 CLI 配置吗?',
'apiSettings.endpointName': '端点名称',
'apiSettings.envSettings': '环境变量设置',
'apiSettings.settingsFilePath': '配置文件路径',
'apiSettings.nameRequired': '名称为必填项',
'apiSettings.tokenRequired': 'API 令牌为必填项',
'apiSettings.status': '状态',
// Common
'common.cancel': '取消',
'common.optional': '(可选)',

View File

@@ -3737,23 +3737,94 @@ function renderCliSettingsEmptyState() {
if (window.lucide) lucide.createIcons();
}
/**
* Get available Anthropic providers
*/
function getAvailableAnthropicProviders() {
if (!apiSettingsData || !apiSettingsData.providers) {
return [];
}
return apiSettingsData.providers.filter(function(p) {
return p.type === 'anthropic' && p.enabled;
});
}
/**
* Build provider options HTML for CLI Settings
*/
function buildCliProviderOptions(selectedProviderId) {
var providers = getAvailableAnthropicProviders();
var optionsHtml = '<option value="">' + t('apiSettings.selectProvider') + '</option>';
providers.forEach(function(provider) {
var isSelected = provider.id === selectedProviderId ? ' selected' : '';
optionsHtml += '<option value="' + escapeHtml(provider.id) + '"' + isSelected + '>' + escapeHtml(provider.name) + '</option>';
});
return optionsHtml;
}
/**
* Build model options HTML for CLI Settings based on selected provider
*/
function buildCliModelOptions(providerId, selectedModel) {
var providers = getAvailableAnthropicProviders();
var provider = providers.find(function(p) { return p.id === providerId; });
if (!provider || !provider.llmModels || provider.llmModels.length === 0) {
return '<option value="">' + t('apiSettings.selectProviderFirst') + '</option>';
}
var optionsHtml = '';
provider.llmModels.forEach(function(model) {
var isSelected = model.id === selectedModel ? ' selected' : '';
optionsHtml += '<option value="' + escapeHtml(model.id) + '"' + isSelected + '>' + escapeHtml(model.name || model.id) + '</option>';
});
return optionsHtml;
}
/**
* Update CLI Settings model dropdown when provider changes
*/
function onCliProviderChange() {
var providerId = document.getElementById('cli-settings-provider').value;
var modelSelect = document.getElementById('cli-settings-model');
if (modelSelect) {
modelSelect.innerHTML = buildCliModelOptions(providerId, '');
}
}
/**
* Show Add CLI Settings Modal
*/
function showAddCliSettingsModal(existingEndpoint) {
var isEdit = !!existingEndpoint;
var settings = existingEndpoint ? existingEndpoint.settings : { env: {}, model: 'sonnet' };
var env = settings.env || {};
var settings = existingEndpoint ? existingEndpoint.settings : { env: {}, model: '' };
var selectedProviderId = settings.providerId || '';
var providerOptionsHtml = buildCliProviderOptions(selectedProviderId);
var modelOptionsHtml = buildCliModelOptions(selectedProviderId, settings.model);
// Check if any Anthropic providers are configured
var hasProviders = getAvailableAnthropicProviders().length > 0;
var noProvidersWarning = !hasProviders ?
'<div class="info-message" style="margin-bottom: 1rem;">' +
'<i data-lucide="alert-circle"></i>' +
'<span>' + t('apiSettings.noAnthropicProviders') + '</span>' +
'</div>' : '';
var modalHtml =
'<div class="modal-overlay" onclick="closeModal(event)">' +
'<div class="modal" onclick="event.stopPropagation()">' +
'<div class="modal-header">' +
'<h2>' + (isEdit ? t('apiSettings.editCliSettings') : t('apiSettings.addCliSettings')) + '</h2>' +
'<button class="modal-close" onclick="closeCliSettingsModal()">&times;</button>' +
'<div class="generic-modal-overlay active" id="cliSettingsModal">' +
'<div class="generic-modal">' +
'<div class="generic-modal-header">' +
'<h3 class="generic-modal-title">' + (isEdit ? t('apiSettings.editCliSettings') : t('apiSettings.addCliSettings')) + '</h3>' +
'<button class="generic-modal-close" onclick="closeCliSettingsModal()">&times;</button>' +
'</div>' +
'<div class="modal-body">' +
'<form id="cli-settings-form">' +
'<div class="generic-modal-body">' +
noProvidersWarning +
'<form id="cli-settings-form" class="api-settings-form">' +
(isEdit ? '<input type="hidden" id="cli-settings-id" value="' + existingEndpoint.id + '">' : '') +
'<div class="form-group">' +
'<label for="cli-settings-name">' + t('apiSettings.endpointName') + ' *</label>' +
@@ -3764,20 +3835,17 @@ function showAddCliSettingsModal(existingEndpoint) {
'<input type="text" id="cli-settings-description" class="cli-input" value="' + escapeHtml(existingEndpoint ? (existingEndpoint.description || '') : '') + '" />' +
'</div>' +
'<div class="form-group">' +
'<label for="cli-settings-model">' + t('apiSettings.model') + '</label>' +
'<select id="cli-settings-model" class="cli-select">' +
'<option value="opus"' + (settings.model === 'opus' ? ' selected' : '') + '>Claude Opus</option>' +
'<option value="sonnet"' + (settings.model === 'sonnet' ? ' selected' : '') + '>Claude Sonnet</option>' +
'<option value="haiku"' + (settings.model === 'haiku' ? ' selected' : '') + '>Claude Haiku</option>' +
'<label for="cli-settings-provider">' + t('apiSettings.provider') + ' *</label>' +
'<select id="cli-settings-provider" class="cli-input" onchange="onCliProviderChange()" required>' +
providerOptionsHtml +
'</select>' +
'<small class="form-hint">' + t('apiSettings.cliProviderHint') + '</small>' +
'</div>' +
'<div class="form-group">' +
'<label for="cli-settings-token">ANTHROPIC_AUTH_TOKEN *</label>' +
'<input type="password" id="cli-settings-token" class="cli-input" value="' + escapeHtml(env.ANTHROPIC_AUTH_TOKEN || '') + '" placeholder="sk-..." required />' +
'</div>' +
'<div class="form-group">' +
'<label for="cli-settings-base-url">ANTHROPIC_BASE_URL</label>' +
'<input type="text" id="cli-settings-base-url" class="cli-input" value="' + escapeHtml(env.ANTHROPIC_BASE_URL || '') + '" placeholder="https://api.anthropic.com/v1" />' +
'<label for="cli-settings-model">' + t('apiSettings.model') + ' *</label>' +
'<select id="cli-settings-model" class="cli-input" required>' +
modelOptionsHtml +
'</select>' +
'</div>' +
'<div class="form-group">' +
'<label class="checkbox-label">' +
@@ -3786,22 +3854,17 @@ function showAddCliSettingsModal(existingEndpoint) {
'</label>' +
'</div>' +
'</form>' +
'<div class="modal-actions">' +
'<button class="btn btn-secondary" onclick="closeCliSettingsModal()">' + t('common.cancel') + '</button>' +
'<button class="btn btn-primary" onclick="submitCliSettings()"' + (!hasProviders ? ' disabled' : '') + '>' + (isEdit ? t('common.save') : t('common.create')) + '</button>' +
'</div>' +
'<div class="modal-footer">' +
'<button class="btn btn-ghost" onclick="closeCliSettingsModal()">' + t('common.cancel') + '</button>' +
'<button class="btn btn-primary" onclick="submitCliSettings()">' + (isEdit ? t('common.save') : t('common.create')) + '</button>' +
'</div>' +
'</div>' +
'</div>';
// Append modal to body
var modalsContainer = document.getElementById('modals');
if (!modalsContainer) {
modalsContainer = document.createElement('div');
modalsContainer.id = 'modals';
document.body.appendChild(modalsContainer);
}
modalsContainer.innerHTML = modalHtml;
document.body.insertAdjacentHTML('beforeend', modalHtml);
if (window.lucide) lucide.createIcons();
}
/**
@@ -3821,10 +3884,8 @@ function editCliSettings(endpointId) {
* Close CLI Settings Modal
*/
function closeCliSettingsModal() {
var modalsContainer = document.getElementById('modals');
if (modalsContainer) {
modalsContainer.innerHTML = '';
}
var modal = document.getElementById('cliSettingsModal');
if (modal) modal.remove();
}
/**
@@ -3833,9 +3894,8 @@ function closeCliSettingsModal() {
async function submitCliSettings() {
var name = document.getElementById('cli-settings-name').value.trim();
var description = document.getElementById('cli-settings-description').value.trim();
var providerId = document.getElementById('cli-settings-provider').value;
var model = document.getElementById('cli-settings-model').value;
var token = document.getElementById('cli-settings-token').value.trim();
var baseUrl = document.getElementById('cli-settings-base-url').value.trim();
var enabled = document.getElementById('cli-settings-enabled').checked;
var idInput = document.getElementById('cli-settings-id');
var id = idInput ? idInput.value : null;
@@ -3845,26 +3905,42 @@ async function submitCliSettings() {
return;
}
if (!token) {
showRefreshToast(t('apiSettings.tokenRequired'), 'error');
if (!providerId) {
showRefreshToast(t('apiSettings.providerRequired'), 'error');
return;
}
if (!model) {
showRefreshToast(t('apiSettings.modelRequired'), 'error');
return;
}
// Get provider configuration
var providers = getAvailableAnthropicProviders();
var provider = providers.find(function(p) { return p.id === providerId; });
if (!provider) {
showRefreshToast(t('apiSettings.providerNotFound'), 'error');
return;
}
// Build settings from provider
var data = {
name: name,
description: description,
enabled: enabled,
settings: {
env: {
ANTHROPIC_AUTH_TOKEN: token,
ANTHROPIC_AUTH_TOKEN: provider.apiKey || '',
DISABLE_AUTOUPDATER: '1'
},
model: model
model: model,
providerId: providerId // Store for editing
}
};
if (baseUrl) {
data.settings.env.ANTHROPIC_BASE_URL = baseUrl;
if (provider.apiBase) {
data.settings.env.ANTHROPIC_BASE_URL = provider.apiBase;
}
if (id) {
@@ -3889,6 +3965,7 @@ window.showAddCliSettingsModal = showAddCliSettingsModal;
window.editCliSettings = editCliSettings;
window.closeCliSettingsModal = closeCliSettingsModal;
window.submitCliSettings = submitCliSettings;
window.onCliProviderChange = onCliProviderChange;
// ========== Utility Functions ==========

View File

@@ -330,6 +330,25 @@ function buildToolConfigModalContent(tool, config, models, status) {
'</div>' +
'</div>' +
// Tags Section - Unified input with inline tags
'<div class="tool-config-section">' +
'<h4>Tags <span class="text-muted">(optional labels)</span></h4>' +
'<div class="tags-unified-input" id="tagsUnifiedInput">' +
(config.tags || []).map(function(tag) {
return '<span class="tag-item">' + escapeHtml(tag) + '<button type="button" class="tag-remove" data-tag="' + escapeHtml(tag) + '">&times;</button></span>';
}).join('') +
'<input type="text" id="tagInput" class="tag-inline-input" placeholder="输入标签按 Enter 添加" />' +
'</div>' +
'<div class="predefined-tags-row">' +
'<button type="button" class="predefined-tag-btn" data-tag="分析"><i data-lucide="search" class="w-3 h-3"></i> 分析</button>' +
'<button type="button" class="predefined-tag-btn" data-tag="编码"><i data-lucide="code" class="w-3 h-3"></i> 编码</button>' +
'<button type="button" class="predefined-tag-btn" data-tag="Debug"><i data-lucide="bug" class="w-3 h-3"></i> Debug</button>' +
'<button type="button" class="predefined-tag-btn" data-tag="重构"><i data-lucide="refresh-cw" class="w-3 h-3"></i> 重构</button>' +
'<button type="button" class="predefined-tag-btn" data-tag="测试"><i data-lucide="check-square" class="w-3 h-3"></i> 测试</button>' +
'<button type="button" class="predefined-tag-btn" data-tag="文档"><i data-lucide="file-text" class="w-3 h-3"></i> 文档</button>' +
'</div>' +
'</div>' +
// Footer
'<div class="tool-config-footer">' +
'<button class="btn btn-outline" onclick="closeModal()">' + t('common.cancel') + '</button>' +
@@ -341,6 +360,79 @@ function buildToolConfigModalContent(tool, config, models, status) {
}
function initToolConfigModalEvents(tool, currentConfig, models) {
// Local tags state (copy from config)
var currentTags = (currentConfig.tags || []).slice();
// Helper to render tags inline with input
function renderTags() {
var container = document.getElementById('tagsUnifiedInput');
var input = document.getElementById('tagInput');
if (!container) return;
// Remove existing tag items but keep the input
container.querySelectorAll('.tag-item').forEach(function(el) { el.remove(); });
// Insert tags before the input
currentTags.forEach(function(tag) {
var tagEl = document.createElement('span');
tagEl.className = 'tag-item';
tagEl.innerHTML = escapeHtml(tag) + '<button type="button" class="tag-remove" data-tag="' + escapeHtml(tag) + '">&times;</button>';
container.insertBefore(tagEl, input);
});
// Re-attach remove handlers
container.querySelectorAll('.tag-remove').forEach(function(btn) {
btn.onclick = function(e) {
e.stopPropagation();
var tagToRemove = this.getAttribute('data-tag');
currentTags = currentTags.filter(function(t) { return t !== tagToRemove; });
renderTags();
};
});
}
// Click on unified input container focuses the input
var unifiedInput = document.getElementById('tagsUnifiedInput');
if (unifiedInput) {
unifiedInput.onclick = function(e) {
if (e.target === this) {
document.getElementById('tagInput').focus();
}
};
}
// Tag input handler
var tagInput = document.getElementById('tagInput');
if (tagInput) {
tagInput.onkeydown = function(e) {
if (e.key === 'Enter') {
e.preventDefault();
var newTag = this.value.trim();
if (newTag && currentTags.indexOf(newTag) === -1) {
currentTags.push(newTag);
renderTags();
}
this.value = '';
}
};
}
// Predefined tag click handlers
document.querySelectorAll('.predefined-tag-btn').forEach(function(btn) {
btn.onclick = function() {
var tag = this.getAttribute('data-tag');
if (tag && currentTags.indexOf(tag) === -1) {
currentTags.push(tag);
renderTags();
}
};
});
// Initialize tags display
renderTags();
// Initialize lucide icons for predefined buttons
if (window.lucide) lucide.createIcons();
// Toggle Enable/Disable
var toggleBtn = document.getElementById('toggleEnableBtn');
if (toggleBtn) {
@@ -426,10 +518,15 @@ function initToolConfigModalEvents(tool, currentConfig, models) {
try {
await updateCliToolConfig(tool, {
primaryModel: primaryModel,
secondaryModel: secondaryModel
secondaryModel: secondaryModel,
tags: currentTags
});
// Reload config to reflect changes
await loadCliToolConfig();
showRefreshToast('Configuration saved', 'success');
closeModal();
renderToolsSection();
if (window.lucide) lucide.createIcons();
} catch (err) {
showRefreshToast('Failed to save: ' + err.message, 'error');
}
@@ -554,35 +651,42 @@ function renderToolsSection() {
var toolDescriptions = {
gemini: t('cli.geminiDesc'),
qwen: t('cli.qwenDesc'),
codex: t('cli.codexDesc')
codex: t('cli.codexDesc'),
claude: t('cli.claudeDesc') || 'Anthropic Claude Code CLI for AI-assisted development',
opencode: t('cli.opencodeDesc') || 'OpenCode CLI - Multi-provider AI coding assistant'
};
var tools = ['gemini', 'qwen', 'codex'];
var tools = ['gemini', 'qwen', 'codex', 'claude', 'opencode'];
var available = Object.values(cliToolStatus).filter(function(t) { return t.available; }).length;
var toolsHtml = tools.map(function(tool) {
var status = cliToolStatus[tool] || {};
var isAvailable = status.available;
var isDefault = defaultCliTool === tool;
var toolConfig = cliToolConfig && cliToolConfig.tools ? cliToolConfig.tools[tool] : null;
var tags = toolConfig && toolConfig.tags ? toolConfig.tags : [];
// Build tags HTML
var tagsHtml = tags.length > 0
? '<div class="tool-tags">' + tags.map(function(tag) {
return '<span class="tool-tag">' + escapeHtml(tag) + '</span>';
}).join('') + '</div>'
: '';
return '<div class="tool-item clickable ' + (isAvailable ? 'available' : 'unavailable') + '" onclick="showToolConfigModal(\'' + tool + '\')">' +
'<div class="tool-item-left">' +
'<span class="tool-status-dot ' + (isAvailable ? 'status-available' : 'status-unavailable') + '"></span>' +
'<div class="tool-item-info">' +
'<div class="tool-item-name">' + tool.charAt(0).toUpperCase() + tool.slice(1) +
(isDefault ? '<span class="tool-default-badge">' + t('cli.default') + '</span>' : '') +
'<i data-lucide="settings" class="w-3 h-3 tool-config-icon"></i>' +
'</div>' +
'<div class="tool-item-desc">' + toolDescriptions[tool] + '</div>' +
tagsHtml +
'</div>' +
'</div>' +
'<div class="tool-item-right">' +
(isAvailable
? '<span class="tool-status-text success"><i data-lucide="check-circle" class="w-3.5 h-3.5"></i> ' + t('cli.ready') + '</span>'
: '<span class="tool-status-text muted"><i data-lucide="circle-dashed" class="w-3.5 h-3.5"></i> ' + t('cli.notInstalled') + '</span>') +
(isAvailable && !isDefault
? '<button class="btn-sm btn-outline" onclick="event.stopPropagation(); setDefaultCliTool(\'' + tool + '\')"><i data-lucide="star" class="w-3 h-3"></i> ' + t('cli.setDefault') + '</button>'
: '') +
'</div>' +
'</div>';
}).join('');

View File

@@ -13,6 +13,7 @@ export interface CliToolConfig {
enabled: boolean;
primaryModel: string; // For CLI endpoint calls (ccw cli -p)
secondaryModel: string; // For internal calls (llm_enhancer, generate_module_docs)
tags?: string[]; // User-defined tags/labels for the tool
}
export interface CliConfig {
@@ -204,7 +205,8 @@ export function updateToolConfig(
const updatedToolConfig: CliToolConfig = {
enabled: updates.enabled !== undefined ? updates.enabled : currentToolConfig.enabled,
primaryModel: updates.primaryModel || currentToolConfig.primaryModel,
secondaryModel: updates.secondaryModel || currentToolConfig.secondaryModel
secondaryModel: updates.secondaryModel || currentToolConfig.secondaryModel,
tags: updates.tags !== undefined ? updates.tags : currentToolConfig.tags
};
// Save updated config

View File

@@ -10,6 +10,12 @@ import { validatePath } from '../utils/path-resolver.js';
import { escapeWindowsArg } from '../utils/shell-escape.js';
import { buildCommand, checkToolAvailability, clearToolCache, debugLog, errorLog, type NativeResumeConfig, type ToolAvailability } from './cli-executor-utils.js';
import type { ConversationRecord, ConversationTurn, ExecutionOutput, ExecutionRecord } from './cli-executor-state.js';
import {
createOutputParser,
type CliOutputUnit,
type IOutputParser,
flattenOutputUnits
} from './cli-output-converter.js';
import {
buildMergedPrompt,
buildMultiTurnPrompt,
@@ -110,6 +116,7 @@ const ParamsSchema = z.object({
category: z.enum(['user', 'internal', 'insight']).default('user'), // Execution category for tracking
parentExecutionId: z.string().optional(), // Parent execution ID for fork/retry scenarios
stream: z.boolean().default(false), // false = cache full output (default), true = stream output via callback
outputFormat: z.enum(['text', 'json-lines']).optional().default('json-lines'), // Output parsing format (default: json-lines for type badges)
});
type Params = z.infer<typeof ParamsSchema>;
@@ -127,14 +134,14 @@ function assertNonEmptyArray<T>(items: T[], message: string): asserts items is N
*/
async function executeCliTool(
params: Record<string, unknown>,
onOutput?: ((data: { type: string; data: string }) => void) | null
onOutput?: ((unit: CliOutputUnit) => void) | null
): Promise<ExecutionOutput> {
const parsed = ParamsSchema.safeParse(params);
if (!parsed.success) {
throw new Error(`Invalid params: ${parsed.error.message}`);
}
const { tool, prompt, mode, format, model, cd, includeDirs, timeout, resume, id: customId, noNative, category, parentExecutionId } = parsed.data;
const { tool, prompt, mode, format, model, cd, includeDirs, timeout, resume, id: customId, noNative, category, parentExecutionId, outputFormat } = parsed.data;
// Validate and determine working directory early (needed for conversation lookup)
let workingDir: string;
@@ -155,7 +162,11 @@ async function executeCliTool(
if (endpoint) {
// Route to LiteLLM executor
if (onOutput) {
onOutput({ type: 'stderr', data: `[Routing to LiteLLM endpoint: ${model}]\n` });
onOutput({
type: 'stderr',
content: `[Routing to LiteLLM endpoint: ${model}]\n`,
timestamp: new Date().toISOString()
});
}
const result = await executeLiteLLMEndpoint({
@@ -363,7 +374,11 @@ async function executeCliTool(
if (resumeDecision) {
const modeDesc = getResumeModeDescription(resumeDecision);
if (onOutput) {
onOutput({ type: 'stderr', data: `[Resume mode: ${modeDesc}]\n` });
onOutput({
type: 'stderr',
content: `[Resume mode: ${modeDesc}]\n`,
timestamp: new Date().toISOString()
});
}
}
@@ -381,6 +396,10 @@ async function executeCliTool(
nativeResume: nativeResumeConfig
});
// Create output parser and IR storage
const parser = createOutputParser(outputFormat);
const allOutputUnits: CliOutputUnit[] = [];
const startTime = Date.now();
debugLog('EXEC', `Starting CLI execution`, {
@@ -390,7 +409,8 @@ async function executeCliTool(
conversationId,
promptLength: finalPrompt.length,
hasResume: !!resume,
hasCustomId: !!customId
hasCustomId: !!customId,
outputFormat
});
return new Promise((resolve, reject) => {
@@ -436,20 +456,36 @@ async function executeCliTool(
let timedOut = false;
// Handle stdout
child.stdout!.on('data', (data) => {
child.stdout!.on('data', (data: Buffer) => {
const text = data.toString();
stdout += text;
// Parse into IR units
const units = parser.parse(data, 'stdout');
allOutputUnits.push(...units);
if (onOutput) {
onOutput({ type: 'stdout', data: text });
// Send each IR unit to callback
for (const unit of units) {
onOutput(unit);
}
}
});
// Handle stderr
child.stderr!.on('data', (data) => {
child.stderr!.on('data', (data: Buffer) => {
const text = data.toString();
stderr += text;
// Parse into IR units
const units = parser.parse(data, 'stderr');
allOutputUnits.push(...units);
if (onOutput) {
onOutput({ type: 'stderr', data: text });
// Send each IR unit to callback
for (const unit of units) {
onOutput(unit);
}
}
});
@@ -464,6 +500,15 @@ async function executeCliTool(
// Clear current child process reference
currentChildProcess = null;
// Flush remaining buffer from parser
const remainingUnits = parser.flush();
allOutputUnits.push(...remainingUnits);
if (onOutput) {
for (const unit of remainingUnits) {
onOutput(unit);
}
}
const endTime = Date.now();
const duration = endTime - startTime;
@@ -472,7 +517,8 @@ async function executeCliTool(
duration: `${duration}ms`,
timedOut,
stdoutLength: stdout.length,
stderrLength: stderr.length
stderrLength: stderr.length,
outputUnitsCount: allOutputUnits.length
});
// Determine status - prioritize output content over exit code
@@ -524,7 +570,8 @@ async function executeCliTool(
truncated: stdout.length > 10240 || stderr.length > 2048,
cached: shouldCache,
stdout_full: shouldCache ? stdout : undefined,
stderr_full: shouldCache ? stderr : undefined
stderr_full: shouldCache ? stderr : undefined,
structured: allOutputUnits // Save structured IR units
};
// Determine base turn number for merge scenarios
@@ -677,13 +724,16 @@ async function executeCliTool(
id: conversationId,
timestamp: new Date(startTime).toISOString(),
tool,
model: model || 'default',
model: effectiveModel || 'default',
mode,
prompt,
status,
exit_code: code,
duration_ms: duration,
output: newTurnOutput
output: newTurnOutput,
parsedOutput: flattenOutputUnits(allOutputUnits, {
excludeTypes: ['stderr', 'progress', 'metadata', 'system']
})
};
resolve({
@@ -691,7 +741,8 @@ async function executeCliTool(
execution,
conversation,
stdout,
stderr
stderr,
parsedOutput: execution.parsedOutput
});
});

View File

@@ -5,6 +5,7 @@
import type { HistoryIndexEntry } from './cli-history-store.js';
import { StoragePaths, ensureStorageDir } from '../config/storage-paths.js';
import type { CliOutputUnit } from './cli-output-converter.js';
// Lazy-loaded SQLite store module
let sqliteStoreModule: typeof import('./cli-history-store.js') | null = null;
@@ -44,6 +45,10 @@ export interface ConversationTurn {
stdout: string;
stderr: string;
truncated: boolean;
cached?: boolean;
stdout_full?: string;
stderr_full?: string;
structured?: CliOutputUnit[]; // Structured IR sequence for advanced parsing
};
}
@@ -79,6 +84,7 @@ export interface ExecutionRecord {
stderr: string;
truncated: boolean;
};
parsedOutput?: string; // Extracted clean text from structured output units
}
interface HistoryIndex {

View File

@@ -8,6 +8,7 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, unlinkSync,
import { join } from 'path';
import { parseSessionFile, formatConversation, extractConversationPairs, type ParsedSession, type ParsedTurn } from './session-content-parser.js';
import { StoragePaths, ensureStorageDir, getProjectId } from '../config/storage-paths.js';
import type { CliOutputUnit } from './cli-output-converter.js';
// Types
export interface ConversationTurn {
@@ -26,6 +27,7 @@ export interface ConversationTurn {
cached?: boolean;
stdout_full?: string;
stderr_full?: string;
structured?: CliOutputUnit[]; // Structured IR sequence for advanced parsing
};
}

View File

@@ -0,0 +1,474 @@
/**
* CLI Output Converter
* Converts raw CLI tool output into structured Intermediate Representation (IR)
*
* Purpose: Decouple output parsing from consumption scenarios (View, Storage, Resume)
* Supports: Plain text, JSON Lines, and other structured formats
*/
// ========== Type Definitions ==========
/**
* Unified output unit types for the intermediate representation layer
*/
export type CliOutputUnitType =
| 'stdout' // Standard output text
| 'stderr' // Standard error text
| 'thought' // AI reasoning/thinking
| 'code' // Code block content
| 'file_diff' // File modification diff
| 'progress' // Progress updates
| 'metadata' // Session/execution metadata
| 'system'; // System events/messages
/**
* Intermediate Representation unit
* Common structure for all CLI output chunks
*/
export interface CliOutputUnit<T = any> {
type: CliOutputUnitType;
content: T;
timestamp: string; // ISO 8601 format
}
// ========== Parser Interface ==========
/**
* Parser interface for converting raw output into IR
*/
export interface IOutputParser {
/**
* Parse a chunk of data from stdout/stderr stream
* @param chunk - Raw buffer from stream
* @param streamType - Source stream (stdout or stderr)
* @returns Array of parsed output units
*/
parse(chunk: Buffer, streamType: 'stdout' | 'stderr'): CliOutputUnit[];
/**
* Flush any remaining buffered data
* Called when stream ends to ensure no data is lost
* @returns Array of remaining output units
*/
flush(): CliOutputUnit[];
}
// ========== Plain Text Parser ==========
/**
* PlainTextParser - Converts plain text output to IR
* Simply wraps text in appropriate type envelope
*/
export class PlainTextParser implements IOutputParser {
parse(chunk: Buffer, streamType: 'stdout' | 'stderr'): CliOutputUnit[] {
const text = chunk.toString('utf8');
if (!text) {
return [];
}
return [{
type: streamType,
content: text,
timestamp: new Date().toISOString()
}];
}
/**
* Flush any remaining buffered data
* Called when stream ends to ensure no data is lost
*
* Note: PlainTextParser does not buffer data internally, so this method
* always returns an empty array. Other parsers (e.g., JsonLinesParser)
* may have buffered incomplete lines that need to be flushed.
*
* @returns Array of remaining output units (always empty for PlainTextParser)
*/
flush(): CliOutputUnit[] {
// Plain text parser has no internal buffer
return [];
}
}
// ========== JSON Lines Parser ==========
/**
* JsonLinesParser - Parses newline-delimited JSON output
*
* Features:
* - Handles incomplete lines across chunks
* - Maps JSON events to appropriate IR types
* - Falls back to stdout for unparseable lines
* - Robust error handling for malformed JSON
*/
export class JsonLinesParser implements IOutputParser {
private buffer: string = '';
parse(chunk: Buffer, streamType: 'stdout' | 'stderr'): CliOutputUnit[] {
const text = chunk.toString('utf8');
this.buffer += text;
const units: CliOutputUnit[] = [];
const lines = this.buffer.split('\n');
// Keep the last incomplete line in buffer
this.buffer = lines.pop() || '';
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed) {
continue;
}
// Try to parse as JSON
let parsed: any;
try {
parsed = JSON.parse(trimmed);
} catch {
// Not valid JSON, treat as plain text
units.push({
type: streamType,
content: line,
timestamp: new Date().toISOString()
});
continue;
}
// Map JSON structure to IR type
const unit = this.mapJsonToIR(parsed, streamType);
if (unit) {
units.push(unit);
}
}
return units;
}
flush(): CliOutputUnit[] {
const units: CliOutputUnit[] = [];
if (this.buffer.trim()) {
// Try to parse remaining buffer
try {
const parsed = JSON.parse(this.buffer.trim());
const unit = this.mapJsonToIR(parsed, 'stdout');
if (unit) {
units.push(unit);
}
} catch {
// Not valid JSON, return as plain text
units.push({
type: 'stdout',
content: this.buffer,
timestamp: new Date().toISOString()
});
}
}
this.buffer = '';
return units;
}
/**
* Map parsed JSON object to appropriate IR type
* Handles various JSON event formats from different CLI tools
*/
private mapJsonToIR(json: any, fallbackStreamType: 'stdout' | 'stderr'): CliOutputUnit | null {
const timestamp = json.timestamp || new Date().toISOString();
// Detect type from JSON structure
if (json.type) {
switch (json.type) {
case 'thought':
case 'thinking':
case 'reasoning':
return {
type: 'thought',
content: json.content || json.text || json.message,
timestamp
};
case 'code':
case 'code_block':
return {
type: 'code',
content: json.content || json.code,
timestamp
};
case 'diff':
case 'file_diff':
case 'file_change':
return {
type: 'file_diff',
content: {
path: json.path || json.file,
diff: json.diff || json.content,
action: json.action || 'modify'
},
timestamp
};
case 'progress':
case 'status':
return {
type: 'progress',
content: {
message: json.message || json.content,
progress: json.progress,
total: json.total
},
timestamp
};
case 'metadata':
case 'session_meta':
return {
type: 'metadata',
content: json.payload || json.data || json,
timestamp
};
case 'system':
case 'event':
return {
type: 'system',
content: json.message || json.content || json,
timestamp
};
}
}
// Check for Codex JSONL format
if (json.type === 'response_item' && json.payload) {
const payloadType = json.payload.type;
if (payloadType === 'message') {
// User or assistant message
const content = json.payload.content
?.map((c: any) => c.text || '')
.filter((t: string) => t)
.join('\n') || '';
return {
type: 'stdout',
content,
timestamp
};
}
if (payloadType === 'reasoning') {
return {
type: 'thought',
content: json.payload.summary || json.payload.content,
timestamp
};
}
if (payloadType === 'function_call' || payloadType === 'function_call_output') {
return {
type: 'system',
content: json.payload,
timestamp
};
}
}
// Check for Gemini/Qwen message format
if (json.role === 'user' || json.role === 'assistant') {
return {
type: 'stdout',
content: json.content || json.text || '',
timestamp
};
}
if (json.thoughts && Array.isArray(json.thoughts)) {
return {
type: 'thought',
content: json.thoughts.map((t: any) =>
typeof t === 'string' ? t : `${t.subject}: ${t.description}`
).join('\n'),
timestamp
};
}
// Default: treat as stdout/stderr based on fallback
if (json.content || json.message || json.text) {
return {
type: fallbackStreamType,
content: json.content || json.message || json.text,
timestamp
};
}
// Unrecognized structure, return as metadata
return {
type: 'metadata',
content: json,
timestamp
};
}
}
// ========== Factory Function ==========
/**
* Create an output parser instance based on format
* @param format - Output format type
* @returns Parser instance
*/
export function createOutputParser(format: 'text' | 'json-lines'): IOutputParser {
switch (format) {
case 'json-lines':
return new JsonLinesParser();
case 'text':
default:
return new PlainTextParser();
}
}
// ========== Utility Functions ==========
/**
* Flatten output units into plain text string
* Useful for Resume scenario where we need concatenated context
*
* @param units - Array of output units to flatten
* @param options - Filtering and formatting options
* @returns Concatenated text content
*/
export function flattenOutputUnits(
units: CliOutputUnit[],
options?: {
includeTypes?: CliOutputUnitType[];
excludeTypes?: CliOutputUnitType[];
includeTimestamps?: boolean;
separator?: string;
}
): string {
const {
includeTypes,
excludeTypes,
includeTimestamps = false,
separator = '\n'
} = options || {};
// Filter units by type
let filtered = units;
if (includeTypes && includeTypes.length > 0) {
filtered = filtered.filter(u => includeTypes.includes(u.type));
}
if (excludeTypes && excludeTypes.length > 0) {
filtered = filtered.filter(u => !excludeTypes.includes(u.type));
}
// Convert to text
const lines = filtered.map(unit => {
let text = '';
if (includeTimestamps) {
text += `[${unit.timestamp}] `;
}
// Extract text content based on type
if (typeof unit.content === 'string') {
text += unit.content;
} else if (typeof unit.content === 'object' && unit.content !== null) {
// Handle structured content with type-specific formatting
switch (unit.type) {
case 'file_diff':
// Format file diff with path and diff content
text += `File: ${unit.content.path}\n\`\`\`diff\n${unit.content.diff}\n\`\`\``;
break;
case 'code':
// Format code block with language
const lang = unit.content.language || '';
const code = unit.content.code || unit.content;
text += `\`\`\`${lang}\n${typeof code === 'string' ? code : JSON.stringify(code)}\n\`\`\``;
break;
case 'thought':
// Format thought/reasoning content
text += `[Thought] ${typeof unit.content === 'string' ? unit.content : JSON.stringify(unit.content)}`;
break;
case 'progress':
// Format progress updates
if (unit.content.message) {
text += unit.content.message;
if (unit.content.progress !== undefined && unit.content.total !== undefined) {
text += ` (${unit.content.progress}/${unit.content.total})`;
}
} else {
text += JSON.stringify(unit.content);
}
break;
case 'metadata':
case 'system':
// Metadata and system events are typically excluded from prompt context
// Include minimal representation if they passed filtering
text += JSON.stringify(unit.content);
break;
default:
// Fallback for unknown structured types
text += JSON.stringify(unit.content);
}
} else {
text += String(unit.content);
}
return text;
});
return lines.join(separator);
}
/**
* Extract specific content type from units
* Convenience helper for common extraction patterns
*/
export function extractContent(
units: CliOutputUnit[],
type: CliOutputUnitType
): string[] {
return units
.filter(u => u.type === type)
.map(u => typeof u.content === 'string' ? u.content : JSON.stringify(u.content));
}
/**
* Get statistics about output units
* Useful for debugging and analytics
*/
export function getOutputStats(units: CliOutputUnit[]): {
total: number;
byType: Record<CliOutputUnitType, number>;
firstTimestamp?: string;
lastTimestamp?: string;
} {
const byType: Record<string, number> = {};
let firstTimestamp: string | undefined;
let lastTimestamp: string | undefined;
for (const unit of units) {
byType[unit.type] = (byType[unit.type] || 0) + 1;
if (!firstTimestamp || unit.timestamp < firstTimestamp) {
firstTimestamp = unit.timestamp;
}
if (!lastTimestamp || unit.timestamp > lastTimestamp) {
lastTimestamp = unit.timestamp;
}
}
return {
total: units.length,
byType: byType as Record<CliOutputUnitType, number>,
firstTimestamp,
lastTimestamp
};
}

View File

@@ -4,10 +4,59 @@
*/
import type { ConversationRecord, ConversationTurn } from './cli-executor-state.js';
import { flattenOutputUnits, type CliOutputUnit, type CliOutputUnitType } from './cli-output-converter.js';
// Prompt concatenation format types
export type PromptFormat = 'plain' | 'yaml' | 'json';
/**
* Extract clean AI output content from ConversationTurn
* Prioritizes structured IR data, falls back to raw stdout
*
* This function performs noise filtering to extract only meaningful content
* for use in prompt context, excluding progress updates, metadata, and system events.
*
* @param turn - Conversation turn containing output
* @param options - Extraction options
* @param options.includeThoughts - Whether to include AI reasoning/thinking in output (default: false)
* @returns Clean output string suitable for prompt context
*
* @example
* ```typescript
* const cleanOutput = extractCleanOutput(turn, { includeThoughts: true });
* // Returns: stdout + code + file_diff + thought (excludes: progress, metadata, system)
* ```
*/
function extractCleanOutput(
turn: ConversationTurn,
options?: {
includeThoughts?: boolean;
}
): string {
// Priority 1: Use structured IR if available (clean, noise-filtered)
if (turn.output?.structured && turn.output.structured.length > 0) {
const includeTypes: CliOutputUnitType[] = ['stdout', 'code', 'file_diff'];
// Optionally include thought processes
if (options?.includeThoughts) {
includeTypes.push('thought');
}
return flattenOutputUnits(turn.output.structured, {
includeTypes,
excludeTypes: ['progress', 'metadata', 'system']
});
}
// Priority 2: Use full output if available
if (turn.output?.stdout_full) {
return turn.output.stdout_full;
}
// Priority 3: Fall back to truncated stdout
return turn.output?.stdout || '[No output]';
}
/**
* Merge multiple conversations into a unified context
* Returns merged turns sorted by timestamp with source tracking
@@ -166,7 +215,11 @@ export class PromptConcatenator {
timestamp: turn.timestamp,
source_id: sourceId
});
this.addAssistantTurn(turn.output.stdout || '[No output]', {
// Use extractCleanOutput to get noise-filtered content
const cleanOutput = extractCleanOutput(turn);
this.addAssistantTurn(cleanOutput, {
turn: turn.turn * 2,
timestamp: turn.timestamp,
status: turn.status,

View File

@@ -10,6 +10,7 @@ import {
getProviderWithResolvedEnvVars,
} from '../config/litellm-api-config-manager.js';
import type { CustomEndpoint, ProviderCredential } from '../types/litellm-api-config.js';
import type { CliOutputUnit } from './cli-output-converter.js';
export interface LiteLLMExecutionOptions {
prompt: string;
@@ -18,7 +19,7 @@ export interface LiteLLMExecutionOptions {
cwd?: string; // Working directory for file resolution
includeDirs?: string[]; // Additional directories for @patterns
enableCache?: boolean; // Override endpoint cache setting
onOutput?: (data: { type: string; data: string }) => void;
onOutput?: (unit: CliOutputUnit) => void;
/** Number of retries after the initial attempt (default: 0) */
maxRetries?: number;
/** Base delay for exponential backoff in milliseconds (default: 1000) */
@@ -105,7 +106,11 @@ export async function executeLiteLLMEndpoint(
const patterns = extractPatterns(prompt);
if (patterns.length > 0) {
if (onOutput) {
onOutput({ type: 'stderr', data: `[Context cache: Found ${patterns.length} @patterns]\n` });
onOutput({
type: 'stderr',
content: `[Context cache: Found ${patterns.length} @patterns]\n`,
timestamp: new Date().toISOString()
});
}
// Pack files into cache
@@ -124,7 +129,8 @@ export async function executeLiteLLMEndpoint(
if (onOutput) {
onOutput({
type: 'stderr',
data: `[Context cache: Packed ${pack.files_packed} files, ${pack.total_bytes} bytes]\n`,
content: `[Context cache: Packed ${pack.files_packed} files, ${pack.total_bytes} bytes]\n`,
timestamp: new Date().toISOString()
});
}
@@ -143,12 +149,20 @@ export async function executeLiteLLMEndpoint(
cachedFiles = pack.files_packed ? Array(pack.files_packed).fill('...') : [];
if (onOutput) {
onOutput({ type: 'stderr', data: `[Context cache: Applied to prompt]\n` });
onOutput({
type: 'stderr',
content: `[Context cache: Applied to prompt]\n`,
timestamp: new Date().toISOString()
});
}
}
} else if (packResult.error) {
if (onOutput) {
onOutput({ type: 'stderr', data: `[Context cache warning: ${packResult.error}]\n` });
onOutput({
type: 'stderr',
content: `[Context cache warning: ${packResult.error}]\n`,
timestamp: new Date().toISOString()
});
}
}
}
@@ -159,7 +173,8 @@ export async function executeLiteLLMEndpoint(
if (onOutput) {
onOutput({
type: 'stderr',
data: `[LiteLLM: Calling ${provider.type}/${endpoint.model}]\n`,
content: `[LiteLLM: Calling ${provider.type}/${endpoint.model}]\n`,
timestamp: new Date().toISOString()
});
}
@@ -195,7 +210,11 @@ export async function executeLiteLLMEndpoint(
);
if (onOutput) {
onOutput({ type: 'stdout', data: response });
onOutput({
type: 'stdout',
content: response,
timestamp: new Date().toISOString()
});
}
return {
@@ -209,7 +228,11 @@ export async function executeLiteLLMEndpoint(
} catch (error) {
const errorMsg = (error as Error).message;
if (onOutput) {
onOutput({ type: 'stderr', data: `[LiteLLM error: ${errorMsg}]\n` });
onOutput({
type: 'stderr',
content: `[LiteLLM error: ${errorMsg}]\n`,
timestamp: new Date().toISOString()
});
}
return {
@@ -279,7 +302,7 @@ async function callWithRetries(
options: {
maxRetries: number;
baseDelayMs: number;
onOutput?: (data: { type: string; data: string }) => void;
onOutput?: (unit: CliOutputUnit) => void;
rateLimitKey: string;
},
): Promise<string> {
@@ -301,7 +324,8 @@ async function callWithRetries(
if (onOutput) {
onOutput({
type: 'stderr',
data: `[LiteLLM retry ${attempt + 1}/${maxRetries}: waiting ${delayMs}ms] ${errorMessage}\n`,
content: `[LiteLLM retry ${attempt + 1}/${maxRetries}: waiting ${delayMs}ms] ${errorMessage}\n`,
timestamp: new Date().toISOString()
});
}