From a98db07731dcbcdc3ea3c154b6f324e821e2e1d0 Mon Sep 17 00:00:00 2001 From: catlog22 Date: Mon, 12 Jan 2026 23:51:16 +0800 Subject: [PATCH] chore(release): v6.3.19 - Dense Reranker, CLI Tools & Issue Workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Documentation Updates - Update all version references to v6.3.19 - Add Dense + Reranker search documentation - Add OpenCode AI CLI tool integration docs - Add Issue workflow (plan → queue → execute) with Codex recommendation - Update CHANGELOG with complete v6.3.19 release notes ## Features - Cross-Encoder reranking for improved search relevance - OpenCode CLI tool support - Issue multi-queue parallel execution - Service architecture improvements (cache-manager, preload-service) Co-Authored-By: Claude Opus 4.5 --- .claude/commands/issue/execute.md | 2 + CHANGELOG.md | 61 ++++ README.md | 20 +- README_CN.md | 20 +- WORKFLOW_DECISION_GUIDE.md | 45 ++- WORKFLOW_DECISION_GUIDE_EN.md | 45 ++- ccw/README.md | 14 +- ccw/src/config/litellm-api-config-manager.ts | 161 ++++++--- ccw/src/core/routes/litellm-api-routes.ts | 196 ++++++++++ ccw/src/core/server.ts | 12 + ccw/src/core/services/api-key-tester.ts | 137 +++++++ ccw/src/core/services/health-check-service.ts | 340 ++++++++++++++++++ .../dashboard-js/views/api-settings.js | 251 +++++++++++-- ccw/src/types/litellm-api-config.ts | 3 + 14 files changed, 1195 insertions(+), 112 deletions(-) create mode 100644 ccw/src/core/services/api-key-tester.ts create mode 100644 ccw/src/core/services/health-check-service.ts diff --git a/.claude/commands/issue/execute.md b/.claude/commands/issue/execute.md index f0b9f16e..5d8537e3 100644 --- a/.claude/commands/issue/execute.md +++ b/.claude/commands/issue/execute.md @@ -33,6 +33,8 @@ Minimal orchestrator that dispatches **solution IDs** to executors. Each executo **Executor & Dry-run**: Selected via interactive prompt (AskUserQuestion) **Worktree**: Creates isolated git worktrees for each parallel executor +**⭐ Recommended Executor**: **Codex** - Best for long-running autonomous work (2hr timeout), supports background execution and full write access + **Worktree Options**: - `--worktree` - Create a new worktree with timestamp-based name - `--worktree ` - Resume in an existing worktree (for recovery/continuation) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13b478f8..701856f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,67 @@ All notable changes to Claude Code Workflow (CCW) will be documented in this fil The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [6.3.19] - 2026-01-12 + +### 🚀 Major New Features | 主要新功能 + +#### SPLADE & Dense Reranker Search System | SPLADE 与密集重排序搜索系统 +- **Added**: SPLADE sparse encoder implementation for precise semantic search (currently hidden, dense mode primary) +- **Added**: Cross-Encoder reranking with FastEmbed integration for improved result relevance +- **Added**: Unified reranker architecture with file watcher support +- **Added**: Centralized vector storage and metadata management for embeddings +- **Added**: Dynamic batch size calculation for embedding generation +- **Added**: Multiple embedding backends for cascade retrieval + +#### CLI Tools System Overhaul | CLI 工具系统全面升级 +- **Added**: OpenCode AI assistant support with full CLI integration +- **Added**: CLI Wrapper endpoints management with Dashboard UI +- **Added**: Smart Content Formatter for intelligent output processing +- **Added**: Structured Intermediate Representation (IR) for CLI output +- **Added**: High-availability model pool with path resolution +- **Added**: Custom API header support and tool type descriptions + +#### Service Architecture | 服务架构 +- **Added**: Core service modules: cache-manager, event-manager, preload-service +- **Added**: CLI state caching with preload optimization +- **Added**: UV package manager support for optimized installation +- **Added**: ccw-litellm installation improvements with venv prioritization + +#### Issue Management | Issue 管理 +- **Added**: Multi-queue parallel execution support +- **Added**: Worktree auto-detection with user choice (merge/PR/keep) +- **Added**: Enhanced worktree management with recovery support + +### 🎨 Dashboard & UI Improvements | Dashboard 与 UI 改进 + +- **Added**: Workspace index status interface with real-time monitoring +- **Added**: Watcher status handling and control modal +- **Added**: CLI stream viewer with active execution synchronization +- **Added**: Danger protection hooks with i18n confirmation dialogs +- **Added**: Navigation status routes with badge aggregation + +### 🛠️ Skills & Templates | 技能与模板 + +- **Added**: CCW orchestrator skill for workflow automation +- **Added**: Code analysis and LLM action templates +- **Added**: Autonomous actions and sequential phase templates +- **Added**: Swagger docs command for RESTful API documentation +- **Added**: Debug explore agent with 5-phase workflow and NDJSON logging + +### 🔒 Security & Quality | 安全与质量 + +- **Fixed**: Command injection prevention with strengthened input validation +- **Fixed**: Path validation for CLI executor --cd parameter +- **Added**: E2E tests for MCP tool execution and session lifecycle +- **Added**: Integration tests for CodexLens UV installation + +### 🌐 Internationalization | 国际化 + +- **Added**: Index management, incremental update translations +- **Added**: Environment variables and dynamic batch size i18n support + +--- + ## [6.3.11] - 2025-12-28 ### 🔧 Issue System Enhancements | Issue系统增强 diff --git a/README.md b/README.md index 5194391f..796012f6 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@
-[![Version](https://img.shields.io/badge/version-v6.3.18-blue.svg)](https://github.com/catlog22/Claude-Code-Workflow/releases) +[![Version](https://img.shields.io/badge/version-v6.3.19-blue.svg)](https://github.com/catlog22/Claude-Code-Workflow/releases) [![npm](https://img.shields.io/npm/v/claude-code-workflow.svg)](https://www.npmjs.com/package/claude-code-workflow) [![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) [![Platform](https://img.shields.io/badge/platform-Windows%20%7C%20Linux%20%7C%20macOS-lightgrey.svg)]() @@ -19,18 +19,16 @@ **Claude Code Workflow (CCW)** is a JSON-driven multi-agent development framework with intelligent CLI orchestration (Gemini/Qwen/Codex), context-first architecture, and automated workflow execution. It transforms AI development from simple prompt chaining into a powerful orchestration system. -> **🎉 Version 6.3.18: Native CodexLens & Dashboard Revolution** +> **🎉 Version 6.3.19: Search Enhancement & CLI Tools Upgrade** > -> **Breaking Changes**: -> - ⚠️ CLI command refactored: `ccw cli exec` → `ccw cli -p` -> - ⚠️ Code Index MCP replaced with native CodexLens -> - ⚠️ Knowledge Graph replaced with Session Clustering +> **New Features**: +> - 🔍 **Dense + Reranker Search**: Cross-Encoder reranking for improved result relevance +> - 💻 **OpenCode AI Support**: New OpenCode CLI tool integration +> - 🛠️ **Service Architecture**: Preload service, cache management, UV package manager support +> - 📊 **Issue Multi-Queue Execution**: Supports Codex for long-running autonomous work > -> **Core Features**: -> - 🔍 **Native CodexLens**: Full-Text Search + Semantic Search + HNSW vector index -> - 🖥️ **New Dashboard Views**: CLAUDE.md Manager, Skills Manager, Graph Explorer, Core Memory -> - 📘 **TypeScript Migration**: Full backend modernization -> - 🧠 **Session Clustering**: Intelligent memory management with cluster visualization +> **Recommended Workflow**: +> - 🚀 **Issue Workflow** (`/issue:plan` → `/issue:queue` → `/issue:execute`): Recommend **Codex** executor for long-running autonomous coding > > See [CHANGELOG.md](CHANGELOG.md) for complete details and migration guide. diff --git a/README_CN.md b/README_CN.md index 63e19f42..439c8a15 100644 --- a/README_CN.md +++ b/README_CN.md @@ -5,7 +5,7 @@
-[![Version](https://img.shields.io/badge/version-v6.3.18-blue.svg)](https://github.com/catlog22/Claude-Code-Workflow/releases) +[![Version](https://img.shields.io/badge/version-v6.3.19-blue.svg)](https://github.com/catlog22/Claude-Code-Workflow/releases) [![npm](https://img.shields.io/npm/v/claude-code-workflow.svg)](https://www.npmjs.com/package/claude-code-workflow) [![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) [![Platform](https://img.shields.io/badge/platform-Windows%20%7C%20Linux%20%7C%20macOS-lightgrey.svg)]() @@ -18,18 +18,16 @@ **Claude Code Workflow (CCW)** 是一个 JSON 驱动的多智能体开发框架，具有智能 CLI 编排（Gemini/Qwen/Codex）、上下文优先架构和自动化工作流执行。它将 AI 开发从简单的提示词链接转变为一个强大的编排系统。 -> **🎉 版本 6.3.18: 原生 CodexLens 与 Dashboard 革新** +> **🎉 版本 6.3.19: 搜索增强与 CLI 工具升级** > -> **破坏性变更**: -> - ⚠️ CLI 命令重构: `ccw cli exec` → `ccw cli -p` -> - ⚠️ Code Index MCP 替换为原生 CodexLens -> - ⚠️ 知识图谱替换为会话聚类系统 +> **新功能亮点**: +> - 🔍 **Dense + Reranker 搜索**: Cross-Encoder 重排序提升结果相关性 +> - 💻 **OpenCode AI 支持**: 新增 OpenCode CLI 工具集成 +> - 🛠️ **服务架构优化**: 预加载服务、缓存管理、UV 包管理器支持 +> - 📊 **Issue 多队列执行**: 支持 Codex 长时间自主工作 > -> **核心功能**: -> - 🔍 **原生 CodexLens**: 全文搜索 + 语义搜索 + HNSW 向量索引 -> - 🖥️ **新 Dashboard 视图**: CLAUDE.md 管理器、技能管理器、图浏览器、核心记忆 -> - 📘 **TypeScript 迁移**: 后端全面现代化 -> - 🧠 **会话聚类**: 智能记忆管理与聚类可视化 +> **推荐工作流**: +> - 🚀 **Issue 工作流** (`/issue:plan` → `/issue:queue` → `/issue:execute`): 推荐使用 **Codex** 执行器，支持长时间自主编码工作 > > 详见 [CHANGELOG.md](CHANGELOG.md) 获取完整详情和迁移指南。 diff --git a/WORKFLOW_DECISION_GUIDE.md b/WORKFLOW_DECISION_GUIDE.md index d8b68cc8..8a550662 100644 --- a/WORKFLOW_DECISION_GUIDE.md +++ b/WORKFLOW_DECISION_GUIDE.md @@ -778,6 +778,46 @@ Phase 1: Gemini 分析 ──┐ ## 💡 专家建议 +### 🔧 Issue 批量执行工作流 (v6.3.19 新增) + +**适用场景**: 多个相关 Issue 需要批量规划和执行，支持长时间自主工作 + +| 阶段 | 命令 | 说明 | +|------|------|------| +| **规划** | `/issue:plan` | 为 Issue 生成解决方案和任务分解 | +| **排队** | `/issue:queue` | 将解决方案形成执行队列，分析依赖关系 | +| **执行** | `/issue:execute` | DAG 驱动的并行执行，每个解决方案一次提交 | + +**执行器选择**: +| 执行器 | 推荐场景 | 超时 | +|--------|----------|------| +| **Codex (推荐)** | 长时间自主编码，复杂多任务解决方案 | 2小时 | +| Gemini | 需要大上下文分析的实现 | 1小时 | +| Agent | Claude Code 子代理执行复杂任务 | 同步 | + +**为什么推荐 Codex**: +- ✅ **长时间自主工作**: 2小时超时，适合复杂解决方案 +- ✅ **完整写权限**: 自主创建、修改、删除文件 +- ✅ **后台执行**: 支持 `run_in_background: true`，不阻塞其他工作 +- ✅ **工作树隔离**: 配合 `--worktree` 实现真正并行执行 + +**示例工作流**: +```bash +# 1. 规划多个 Issue +/issue:plan ISS-001 ISS-002 ISS-003 + +# 2. 形成执行队列 +/issue:queue + +# 3. 使用 Codex 执行（推荐长时间任务） +/issue:execute --worktree +# → 选择 Codex 执行器 +# → 启用工作树隔离 +# → 并行批次自动执行 +``` + +--- + ### ✅ 最佳实践 1. **不确定时用头脑风暴**：宁可多花10分钟探索方案，也不要盲目实现后推翻重来 @@ -786,6 +826,7 @@ Phase 1: Gemini 分析 ──┐ 4. **小任务用Lite工作流**：快速完成，减少overhead 5. **关键模块用TDD**：测试驱动开发保证质量 6. **定期更新内存**：`/memory:update-related` 保持上下文准确 +7. **批量 Issue 用 issue 工作流**：`/issue:plan` → `/issue:queue` → `/issue:execute`，推荐 Codex 执行长时间任务 ### ❌ 常见陷阱 @@ -808,5 +849,5 @@ Phase 1: Gemini 分析 ──┐ --- -**最后更新**: 2025-01-23 -**版本**: 5.9.2 +**最后更新**: 2026-01-12 +**版本**: 6.3.19 diff --git a/WORKFLOW_DECISION_GUIDE_EN.md b/WORKFLOW_DECISION_GUIDE_EN.md index 78321f1d..09a5ca80 100644 --- a/WORKFLOW_DECISION_GUIDE_EN.md +++ b/WORKFLOW_DECISION_GUIDE_EN.md @@ -671,6 +671,46 @@ Use gemini to review code quality ## 💡 Expert Advice +### 🔧 Issue Batch Execution Workflow (v6.3.19 New) + +**Use Case**: Multiple related issues requiring batch planning and execution, supports long-running autonomous work + +| Phase | Command | Description | +|-------|---------|-------------| +| **Plan** | `/issue:plan` | Generate solutions and task breakdown for issues | +| **Queue** | `/issue:queue` | Form execution queue, analyze dependencies | +| **Execute** | `/issue:execute` | DAG-driven parallel execution, one commit per solution | + +**Executor Selection**: +| Executor | Recommended For | Timeout | +|----------|-----------------|---------| +| **Codex (Recommended)** | Long-running autonomous coding, complex multi-task solutions | 2 hours | +| Gemini | Large context analysis and implementation | 1 hour | +| Agent | Claude Code sub-agent for complex tasks | Sync | + +**Why Codex is Recommended**: +- ✅ **Long-running autonomous work**: 2-hour timeout, suitable for complex solutions +- ✅ **Full write access**: Autonomously create, modify, delete files +- ✅ **Background execution**: Supports `run_in_background: true`, non-blocking +- ✅ **Worktree isolation**: Combined with `--worktree` for true parallel execution + +**Example Workflow**: +```bash +# 1. Plan multiple issues +/issue:plan ISS-001 ISS-002 ISS-003 + +# 2. Form execution queue +/issue:queue + +# 3. Execute with Codex (recommended for long tasks) +/issue:execute --worktree +# → Select Codex executor +# → Enable worktree isolation +# → Parallel batches auto-execute +``` + +--- + ### ✅ Best Practices 1. **Use brainstorming when uncertain**: Better to spend 10 minutes exploring solutions than blindly implementing and rewriting @@ -678,6 +718,7 @@ Use gemini to review code quality 3. **Use Lite workflow for small tasks**: Complete quickly, reduce overhead 4. **Use TDD for critical modules**: Test-driven development ensures quality 5. **Regularly update memory**: `/memory:update-related` keeps context accurate +6. **Use issue workflow for batch issues**: `/issue:plan` → `/issue:queue` → `/issue:execute`, recommend Codex for long-running tasks ### ❌ Common Pitfalls @@ -699,5 +740,5 @@ Use gemini to review code quality --- -**Last Updated**: 2025-11-20 -**Version**: 5.8.1 +**Last Updated**: 2026-01-12 +**Version**: 6.3.19 diff --git a/ccw/README.md b/ccw/README.md index 5dfc4a32..fe370a61 100644 --- a/ccw/README.md +++ b/ccw/README.md @@ -1,6 +1,6 @@ # CCW - Claude Code Workflow CLI -[![Version](https://img.shields.io/badge/version-v6.2.0-blue.svg)](https://github.com/catlog22/Claude-Code-Workflow/releases) +[![Version](https://img.shields.io/badge/version-v6.3.19-blue.svg)](https://github.com/catlog22/Claude-Code-Workflow/releases) A powerful command-line tool for managing Claude Code Workflow with native CodexLens code intelligence, multi-model CLI orchestration, and interactive dashboard. @@ -36,17 +36,23 @@ ccw view -o report.html ## Features -### 🔍 Native CodexLens (v6.2) +### 🔍 Native CodexLens (v6.3) - **Full-Text Search (FTS)**: SQLite-based fast keyword search with symbol extraction -- **Semantic Search**: Embedding-based similarity search with vector store +- **Semantic Search**: Dense embedding-based similarity search with vector store - **Hybrid Search**: RRF (Reciprocal Rank Fusion) combining FTS and semantic results +- **Cross-Encoder Reranking**: Second-stage reranker for improved result relevance - **HNSW Index**: Approximate Nearest Neighbor index for significantly faster vector search +- **Dynamic Batch Processing**: Intelligent batch size calculation for embedding generation +- **Workspace Index Status**: Real-time index status monitoring and management ### 💻 CLI Tools Integration -- **Multi-Model Support**: Execute prompts with Gemini, Qwen, or Codex +- **Multi-Model Support**: Execute prompts with Gemini, Qwen, Codex, Claude, or OpenCode +- **CLI Wrapper Endpoints**: Custom API endpoints with tool calling support +- **Smart Content Formatter**: Intelligent output formatting with structured IR - **Session Resume**: Resume from last session or merge multiple sessions - **SQLite History**: Persistent execution history with conversation tracking - **Custom IDs**: Support for custom execution IDs and multi-turn conversations +- **Preload Service**: Optimized data fetching with caching for faster responses ### 🧠 Core Memory & Clustering - **Session Clustering**: Intelligent grouping of related sessions diff --git a/ccw/src/config/litellm-api-config-manager.ts b/ccw/src/config/litellm-api-config-manager.ts index 5fbbc7d7..ae9535ae 100644 --- a/ccw/src/config/litellm-api-config-manager.ts +++ b/ccw/src/config/litellm-api-config-manager.ts @@ -545,18 +545,34 @@ export function getEmbeddingProvidersForRotation(baseDir: string): Array<{ } /** - * Generate rotation endpoints for ccw_litellm - * Creates endpoint list from rotation config for parallel embedding - * Supports both legacy codexlensEmbeddingRotation and new embeddingPoolConfig + * Extended rotation endpoint with routing and health check info */ -export function generateRotationEndpoints(baseDir: string): Array<{ +export interface RotationEndpointConfig { name: string; api_key: string; api_base: string; model: string; weight: number; max_concurrent: number; -}> { + /** Routing strategy for load balancing */ + routing_strategy?: string; + /** Health check configuration */ + health_check?: { + enabled: boolean; + interval_seconds: number; + cooldown_seconds: number; + failure_threshold: number; + }; + /** Last recorded latency in milliseconds */ + last_latency_ms?: number; +} + +/** + * Generate rotation endpoints for ccw_litellm + * Creates endpoint list from rotation config for parallel embedding + * Supports both legacy codexlensEmbeddingRotation and new embeddingPoolConfig + */ +export function generateRotationEndpoints(baseDir: string): RotationEndpointConfig[] { const config = loadLiteLLMApiConfig(baseDir); // Prefer embeddingPoolConfig, fallback to codexlensEmbeddingRotation for backward compatibility @@ -583,22 +599,8 @@ function generateEndpointsFromPool( baseDir: string, poolConfig: EmbeddingPoolConfig, config: LiteLLMApiConfig -): Array<{ - name: string; - api_key: string; - api_base: string; - model: string; - weight: number; - max_concurrent: number; -}> { - const endpoints: Array<{ - name: string; - api_key: string; - api_base: string; - model: string; - weight: number; - max_concurrent: number; - }> = []; +): RotationEndpointConfig[] { + const endpoints: RotationEndpointConfig[] = []; if (poolConfig.autoDiscover) { // Auto-discover all providers offering targetModel @@ -626,10 +628,20 @@ function generateEndpointsFromPool( let keysToUse: Array<{ id: string; key: string; label: string }> = []; if (provider.apiKeys && provider.apiKeys.length > 0) { - // Use all enabled keys - keysToUse = provider.apiKeys - .filter(k => k.enabled) - .map(k => ({ id: k.id, key: k.key, label: k.label || k.id })); + // Use all enabled and healthy keys (filter out unhealthy) + const healthyKeys = provider.apiKeys.filter(k => + k.enabled && k.healthStatus !== 'unhealthy' + ); + + // Log filtered keys for debugging + const unhealthyCount = provider.apiKeys.filter(k => + k.enabled && k.healthStatus === 'unhealthy' + ).length; + if (unhealthyCount > 0) { + console.log(`[RotationEndpoints] Filtered ${unhealthyCount} unhealthy key(s) from provider ${provider.name}`); + } + + keysToUse = healthyKeys.map(k => ({ id: k.id, key: k.key, label: k.label || k.id })); } else if (provider.apiKey) { // Single key fallback keysToUse = [{ id: 'default', key: provider.apiKey, label: 'Default' }]; @@ -637,14 +649,37 @@ function generateEndpointsFromPool( // Create endpoint for each key for (const keyInfo of keysToUse) { - endpoints.push({ + const endpoint: RotationEndpointConfig = { name: `${provider.name}-${keyInfo.label}`, api_key: resolveEnvVar(keyInfo.key), api_base: apiBase, model: embeddingModel.name, weight: 1.0, // Default weight for auto-discovered providers max_concurrent: poolConfig.defaultMaxConcurrentPerKey, - }); + }; + + // Add routing strategy from provider config + if (provider.routingStrategy) { + endpoint.routing_strategy = provider.routingStrategy; + } + + // Add health check config from provider + if (provider.healthCheck) { + endpoint.health_check = { + enabled: provider.healthCheck.enabled, + interval_seconds: provider.healthCheck.intervalSeconds, + cooldown_seconds: provider.healthCheck.cooldownSeconds, + failure_threshold: provider.healthCheck.failureThreshold, + }; + } + + // Add last latency if available from key entry + const keyEntry = provider.apiKeys?.find(k => k.id === keyInfo.id); + if (keyEntry && keyEntry.lastLatencyMs !== undefined) { + endpoint.last_latency_ms = keyEntry.lastLatencyMs; + } + + endpoints.push(endpoint); } } } @@ -659,22 +694,8 @@ function generateEndpointsFromLegacyRotation( baseDir: string, rotationConfig: CodexLensEmbeddingRotation, config: LiteLLMApiConfig -): Array<{ - name: string; - api_key: string; - api_base: string; - model: string; - weight: number; - max_concurrent: number; -}> { - const endpoints: Array<{ - name: string; - api_key: string; - api_base: string; - model: string; - weight: number; - max_concurrent: number; - }> = []; +): RotationEndpointConfig[] { + const endpoints: RotationEndpointConfig[] = []; for (const rotationProvider of rotationConfig.providers) { if (!rotationProvider.enabled) continue; @@ -696,15 +717,26 @@ function generateEndpointsFromLegacyRotation( let keysToUse: Array<{ id: string; key: string; label: string }> = []; if (provider.apiKeys && provider.apiKeys.length > 0) { + // Filter out unhealthy keys first + const healthyKeys = provider.apiKeys.filter(k => + k.enabled && k.healthStatus !== 'unhealthy' + ); + + // Log filtered keys for debugging + const unhealthyCount = provider.apiKeys.filter(k => + k.enabled && k.healthStatus === 'unhealthy' + ).length; + if (unhealthyCount > 0) { + console.log(`[RotationEndpoints] Filtered ${unhealthyCount} unhealthy key(s) from provider ${provider.name}`); + } + if (rotationProvider.useAllKeys) { - // Use all enabled keys - keysToUse = provider.apiKeys - .filter(k => k.enabled) - .map(k => ({ id: k.id, key: k.key, label: k.label || k.id })); + // Use all enabled and healthy keys + keysToUse = healthyKeys.map(k => ({ id: k.id, key: k.key, label: k.label || k.id })); } else if (rotationProvider.selectedKeyIds && rotationProvider.selectedKeyIds.length > 0) { - // Use only selected keys - keysToUse = provider.apiKeys - .filter(k => k.enabled && rotationProvider.selectedKeyIds!.includes(k.id)) + // Use only selected healthy keys + keysToUse = healthyKeys + .filter(k => rotationProvider.selectedKeyIds!.includes(k.id)) .map(k => ({ id: k.id, key: k.key, label: k.label || k.id })); } } else if (provider.apiKey) { @@ -714,14 +746,37 @@ function generateEndpointsFromLegacyRotation( // Create endpoint for each key for (const keyInfo of keysToUse) { - endpoints.push({ + const endpoint: RotationEndpointConfig = { name: `${provider.name}-${keyInfo.label}`, api_key: resolveEnvVar(keyInfo.key), api_base: apiBase, model: embeddingModel.name, weight: rotationProvider.weight, max_concurrent: rotationProvider.maxConcurrentPerKey, - }); + }; + + // Add routing strategy from provider config + if (provider.routingStrategy) { + endpoint.routing_strategy = provider.routingStrategy; + } + + // Add health check config from provider + if (provider.healthCheck) { + endpoint.health_check = { + enabled: provider.healthCheck.enabled, + interval_seconds: provider.healthCheck.intervalSeconds, + cooldown_seconds: provider.healthCheck.cooldownSeconds, + failure_threshold: provider.healthCheck.failureThreshold, + }; + } + + // Add last latency if available from key entry + const keyEntry = provider.apiKeys?.find(k => k.id === keyInfo.id); + if (keyEntry && keyEntry.lastLatencyMs !== undefined) { + endpoint.last_latency_ms = keyEntry.lastLatencyMs; + } + + endpoints.push(endpoint); } } @@ -1289,4 +1344,4 @@ export function getAvailableModelsForType( } // Re-export types -export type { ProviderCredential, CustomEndpoint, ProviderType, CacheStrategy, CodexLensEmbeddingRotation, CodexLensEmbeddingProvider, EmbeddingPoolConfig }; +export type { ProviderCredential, CustomEndpoint, ProviderType, CacheStrategy, CodexLensEmbeddingRotation, CodexLensEmbeddingProvider, EmbeddingPoolConfig, RotationEndpointConfig }; diff --git a/ccw/src/core/routes/litellm-api-routes.ts b/ccw/src/core/routes/litellm-api-routes.ts index 6e78c92e..d79fbc94 100644 --- a/ccw/src/core/routes/litellm-api-routes.ts +++ b/ccw/src/core/routes/litellm-api-routes.ts @@ -86,6 +86,7 @@ import { } from '../../config/litellm-api-config-manager.js'; import { getContextCacheStore } from '../../tools/context-cache-store.js'; import { getLiteLLMClient } from '../../tools/litellm-client.js'; +import { testApiKeyConnection, getDefaultApiBase } from '../services/api-key-tester.js'; // Cache for ccw-litellm status check let ccwLitellmStatusCache: { @@ -338,6 +339,201 @@ export async function handleLiteLLMApiRoutes(ctx: RouteContext): Promise { + const { keyId } = body as { keyId?: string }; + + if (!keyId) { + return { valid: false, error: 'keyId is required', status: 400 }; + } + + try { + const provider = getProvider(initialPath, providerId); + + if (!provider) { + return { valid: false, error: 'Provider not found', status: 404 }; + } + + // Find the specific API key + let apiKeyValue: string | null = null; + let keyLabel = 'Default'; + + if (keyId === 'default' && provider.apiKey) { + // Use the single default apiKey + apiKeyValue = provider.apiKey; + } else if (provider.apiKeys && provider.apiKeys.length > 0) { + const keyEntry = provider.apiKeys.find(k => k.id === keyId); + if (keyEntry) { + apiKeyValue = keyEntry.key; + keyLabel = keyEntry.label || keyEntry.id; + } + } + + if (!apiKeyValue) { + return { valid: false, error: 'API key not found' }; + } + + // Resolve environment variables + const { resolveEnvVar } = await import('../../config/litellm-api-config-manager.js'); + const resolvedKey = resolveEnvVar(apiKeyValue); + + if (!resolvedKey) { + return { valid: false, error: 'API key is empty or environment variable not set' }; + } + + // Determine API base URL + const apiBase = provider.apiBase || getDefaultApiBase(provider.type); + + // Test the API key with appropriate endpoint based on provider type + const startTime = Date.now(); + const testResult = await testApiKeyConnection(provider.type, apiBase, resolvedKey); + const latencyMs = Date.now() - startTime; + + // Update key health status in provider config + if (provider.apiKeys && provider.apiKeys.length > 0) { + const keyEntry = provider.apiKeys.find(k => k.id === keyId); + if (keyEntry) { + keyEntry.healthStatus = testResult.valid ? 'healthy' : 'unhealthy'; + keyEntry.lastHealthCheck = new Date().toISOString(); + if (!testResult.valid) { + keyEntry.lastError = testResult.error; + } else { + delete keyEntry.lastError; + } + + // Save updated provider + try { + updateProvider(initialPath, providerId, { apiKeys: provider.apiKeys }); + } catch (updateErr) { + console.warn('[test-key] Failed to update key health status:', updateErr); + } + } + } + + return { + valid: testResult.valid, + error: testResult.error, + latencyMs: testResult.valid ? latencyMs : undefined, + keyLabel, + }; + } catch (err) { + return { valid: false, error: (err as Error).message }; + } + }); + return true; + } + + // GET /api/litellm-api/providers/:id/health-status - Get health status for all keys + const providerHealthStatusMatch = pathname.match(/^\/api\/litellm-api\/providers\/([^/]+)\/health-status$/); + if (providerHealthStatusMatch && req.method === 'GET') { + const providerId = providerHealthStatusMatch[1]; + + try { + const provider = getProvider(initialPath, providerId); + + if (!provider) { + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Provider not found' })); + return true; + } + + // Import health check service to get runtime state + const { getHealthCheckService } = await import('../services/health-check-service.js'); + const healthService = getHealthCheckService(); + const healthStatus = healthService.getProviderHealthStatus(providerId); + + // Merge persisted key data with runtime health status + const keys = (provider.apiKeys || []).map(key => { + const runtimeStatus = healthStatus.find(s => s.keyId === key.id); + return { + keyId: key.id, + label: key.label || key.id, + status: runtimeStatus?.status || key.healthStatus || 'unknown', + lastCheck: runtimeStatus?.lastCheck || key.lastHealthCheck, + lastLatencyMs: key.lastLatencyMs, + consecutiveFailures: runtimeStatus?.consecutiveFailures || 0, + inCooldown: runtimeStatus?.inCooldown || false, + lastError: runtimeStatus?.lastError || key.lastError, + }; + }); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + providerId, + providerName: provider.name, + keys, + })); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + return true; + } + + // POST /api/litellm-api/providers/:id/health-check-now - Trigger immediate health check + const providerHealthCheckNowMatch = pathname.match(/^\/api\/litellm-api\/providers\/([^/]+)\/health-check-now$/); + if (providerHealthCheckNowMatch && req.method === 'POST') { + const providerId = providerHealthCheckNowMatch[1]; + + try { + const provider = getProvider(initialPath, providerId); + + if (!provider) { + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Provider not found' })); + return true; + } + + // Import health check service and trigger check + const { getHealthCheckService } = await import('../services/health-check-service.js'); + const healthService = getHealthCheckService(); + + // Trigger immediate check (async, but we wait for completion) + await healthService.checkProviderNow(providerId); + + // Get updated status + const healthStatus = healthService.getProviderHealthStatus(providerId); + + // Reload provider to get updated persisted data + const updatedProvider = getProvider(initialPath, providerId); + const keys = (updatedProvider?.apiKeys || []).map(key => { + const runtimeStatus = healthStatus.find(s => s.keyId === key.id); + return { + keyId: key.id, + label: key.label || key.id, + status: runtimeStatus?.status || key.healthStatus || 'unknown', + lastCheck: runtimeStatus?.lastCheck || key.lastHealthCheck, + lastLatencyMs: key.lastLatencyMs, + consecutiveFailures: runtimeStatus?.consecutiveFailures || 0, + inCooldown: runtimeStatus?.inCooldown || false, + lastError: runtimeStatus?.lastError || key.lastError, + }; + }); + + broadcastToClients({ + type: 'PROVIDER_HEALTH_CHECKED', + payload: { providerId, keys, timestamp: new Date().toISOString() } + }); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + success: true, + providerId, + providerName: updatedProvider?.name, + keys, + checkedAt: new Date().toISOString(), + })); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: false, error: (err as Error).message })); + } + return true; + } + // =========================== // Endpoint Management Routes // =========================== diff --git a/ccw/src/core/server.ts b/ccw/src/core/server.ts index c5a370f0..1aa55f4b 100644 --- a/ccw/src/core/server.ts +++ b/ccw/src/core/server.ts @@ -39,6 +39,9 @@ import { csrfValidation } from './auth/csrf-middleware.js'; import { getCsrfTokenManager } from './auth/csrf-manager.js'; import { randomBytes } from 'crypto'; +// Import health check service +import { getHealthCheckService } from './services/health-check-service.js'; + import type { ServerConfig } from '../types/config.js'; import type { PostRequestHandler } from './routes/types.js'; @@ -632,6 +635,15 @@ export async function startServer(options: ServerOptions = {}): Promise = { + openai: 'https://api.openai.com/v1', + anthropic: 'https://api.anthropic.com/v1', + custom: 'https://api.openai.com/v1', // Assume OpenAI-compatible by default + }; + return defaults[providerType] || defaults.openai; +} + +/** + * Test API key connection by making a minimal API request + * @param providerType - The type of provider (openai, anthropic, custom) + * @param apiBase - The base URL for the API + * @param apiKey - The API key to test + * @param timeout - Timeout in milliseconds (default: 10000) + * @returns TestResult indicating if the key is valid + */ +export async function testApiKeyConnection( + providerType: ProviderType, + apiBase: string, + apiKey: string, + timeout: number = 10000 +): Promise { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeout); + const startTime = Date.now(); + + try { + if (providerType === 'anthropic') { + // Anthropic format: POST /v1/messages with minimal payload + const response = await fetch(`${apiBase}/messages`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-api-key': apiKey, + 'anthropic-version': '2023-06-01', + }, + body: JSON.stringify({ + model: 'claude-3-haiku-20240307', + max_tokens: 1, + messages: [{ role: 'user', content: 'Hi' }], + }), + signal: controller.signal, + }); + + clearTimeout(timeoutId); + const latencyMs = Date.now() - startTime; + + if (response.ok) { + return { valid: true, latencyMs }; + } + + // Parse error response + const errorBody = await response.json().catch(() => ({})); + const errorMessage = (errorBody as any)?.error?.message || response.statusText; + + // 401 = invalid API key, other 4xx might be valid key with other issues + if (response.status === 401) { + return { valid: false, error: 'Invalid API key' }; + } + if (response.status === 403) { + return { valid: false, error: 'Access denied - check API key permissions' }; + } + if (response.status === 429) { + // Rate limited means the key is valid but being throttled + return { valid: true, latencyMs }; + } + + return { valid: false, error: errorMessage }; + } else { + // OpenAI-compatible format: GET /v1/models + const modelsUrl = apiBase.endsWith('/v1') ? `${apiBase}/models` : `${apiBase}/v1/models`; + const response = await fetch(modelsUrl, { + method: 'GET', + headers: { + 'Authorization': `Bearer ${apiKey}`, + }, + signal: controller.signal, + }); + + clearTimeout(timeoutId); + const latencyMs = Date.now() - startTime; + + if (response.ok) { + return { valid: true, latencyMs }; + } + + // Parse error response + const errorBody = await response.json().catch(() => ({})); + const errorMessage = (errorBody as any)?.error?.message || response.statusText; + + if (response.status === 401) { + return { valid: false, error: 'Invalid API key' }; + } + if (response.status === 403) { + return { valid: false, error: 'Access denied - check API key permissions' }; + } + if (response.status === 429) { + // Rate limited means the key is valid but being throttled + return { valid: true, latencyMs }; + } + + return { valid: false, error: errorMessage }; + } + } catch (err) { + clearTimeout(timeoutId); + + if ((err as Error).name === 'AbortError') { + return { valid: false, error: 'Connection timed out' }; + } + + return { valid: false, error: `Connection failed: ${(err as Error).message}` }; + } +} diff --git a/ccw/src/core/services/health-check-service.ts b/ccw/src/core/services/health-check-service.ts new file mode 100644 index 00000000..801dbbc3 --- /dev/null +++ b/ccw/src/core/services/health-check-service.ts @@ -0,0 +1,340 @@ +/** + * Health Check Service + * Singleton service that periodically checks API key health for providers + * with health check enabled. Updates key health status and filters unhealthy + * keys from rotation endpoints. + */ + +import { + getAllProviders, + getProvider, + updateProvider, + resolveEnvVar, +} from '../../config/litellm-api-config-manager.js'; +import { testApiKeyConnection, getDefaultApiBase } from './api-key-tester.js'; +import type { ProviderCredential, ApiKeyEntry, HealthCheckConfig } from '../../types/litellm-api-config.js'; + +/** + * Internal state for tracking consecutive failures per key + */ +interface KeyHealthState { + consecutiveFailures: number; + cooldownUntil?: Date; +} + +/** + * Health Check Service - Singleton + * Manages periodic health checks for API keys across all providers + */ +export class HealthCheckService { + private static instance: HealthCheckService; + + /** Timer handles for each provider's health check interval */ + private timers: Map = new Map(); + + /** Track consecutive failures and cooldown per key (providerId:keyId -> state) */ + private keyStates: Map = new Map(); + + /** Base directory for config operations */ + private baseDir: string = ''; + + /** Lock to prevent concurrent checks on same provider */ + private checkingProviders: Set = new Set(); + + private constructor() {} + + /** + * Get singleton instance + */ + static getInstance(): HealthCheckService { + if (!HealthCheckService.instance) { + HealthCheckService.instance = new HealthCheckService(); + } + return HealthCheckService.instance; + } + + /** + * Start health check for a specific provider + * @param providerId - The provider ID to start health checks for + */ + startHealthCheck(providerId: string): void { + // Stop existing timer if any + this.stopHealthCheck(providerId); + + const provider = getProvider(this.baseDir, providerId); + if (!provider) { + console.warn(`[HealthCheck] Provider not found: ${providerId}`); + return; + } + + if (!provider.enabled) { + console.log(`[HealthCheck] Provider ${providerId} is disabled, skipping`); + return; + } + + const healthConfig = provider.healthCheck; + if (!healthConfig || !healthConfig.enabled) { + console.log(`[HealthCheck] Health check not enabled for provider ${providerId}`); + return; + } + + const intervalMs = (healthConfig.intervalSeconds || 300) * 1000; + + console.log(`[HealthCheck] Starting health check for ${provider.name} (${providerId}), interval: ${healthConfig.intervalSeconds}s`); + + // Run initial check immediately + void this.checkProviderNow(providerId); + + // Set up interval + const timer = setInterval(() => { + void this.checkProviderNow(providerId); + }, intervalMs); + + this.timers.set(providerId, timer); + } + + /** + * Stop health check for a specific provider + * @param providerId - The provider ID to stop health checks for + */ + stopHealthCheck(providerId: string): void { + const timer = this.timers.get(providerId); + if (timer) { + clearInterval(timer); + this.timers.delete(providerId); + console.log(`[HealthCheck] Stopped health check for provider ${providerId}`); + } + } + + /** + * Start health checks for all providers that have health check enabled + * @param baseDir - Base directory for config operations + */ + startAllHealthChecks(baseDir: string): void { + this.baseDir = baseDir; + + const providers = getAllProviders(baseDir); + let startedCount = 0; + + for (const provider of providers) { + if (provider.enabled && provider.healthCheck?.enabled) { + this.startHealthCheck(provider.id); + startedCount++; + } + } + + if (startedCount > 0) { + console.log(`[HealthCheck] Started health checks for ${startedCount} provider(s)`); + } else { + console.log('[HealthCheck] No providers with health check enabled'); + } + } + + /** + * Stop all active health checks + */ + stopAllHealthChecks(): void { + const providerIds = Array.from(this.timers.keys()); + for (const providerId of providerIds) { + this.stopHealthCheck(providerId); + } + this.keyStates.clear(); + console.log('[HealthCheck] Stopped all health checks'); + } + + /** + * Manually trigger a health check for a provider + * @param providerId - The provider ID to check + */ + async checkProviderNow(providerId: string): Promise { + // Prevent concurrent checks on same provider + if (this.checkingProviders.has(providerId)) { + console.log(`[HealthCheck] Already checking provider ${providerId}, skipping`); + return; + } + + this.checkingProviders.add(providerId); + + try { + const provider = getProvider(this.baseDir, providerId); + if (!provider) { + console.warn(`[HealthCheck] Provider not found: ${providerId}`); + return; + } + + if (!provider.enabled) { + return; + } + + const healthConfig = provider.healthCheck; + if (!healthConfig) { + return; + } + + const apiBase = provider.apiBase || getDefaultApiBase(provider.type); + const apiKeys = provider.apiKeys || []; + + if (apiKeys.length === 0 && provider.apiKey) { + // Single key mode - create virtual key entry + await this.checkSingleKey(provider, 'default', provider.apiKey, apiBase, healthConfig); + } else { + // Multi-key mode + for (const keyEntry of apiKeys) { + if (!keyEntry.enabled) continue; + await this.checkSingleKey(provider, keyEntry.id, keyEntry.key, apiBase, healthConfig); + } + } + + // Persist updated health statuses + const updatedApiKeys = provider.apiKeys; + if (updatedApiKeys && updatedApiKeys.length > 0) { + try { + updateProvider(this.baseDir, providerId, { apiKeys: updatedApiKeys }); + } catch (err) { + console.error(`[HealthCheck] Failed to persist health status for ${providerId}:`, err); + } + } + } finally { + this.checkingProviders.delete(providerId); + } + } + + /** + * Check a single API key's health + */ + private async checkSingleKey( + provider: ProviderCredential, + keyId: string, + keyValue: string, + apiBase: string, + healthConfig: HealthCheckConfig + ): Promise { + const stateKey = `${provider.id}:${keyId}`; + let state = this.keyStates.get(stateKey); + + if (!state) { + state = { consecutiveFailures: 0 }; + this.keyStates.set(stateKey, state); + } + + // Check if in cooldown + if (state.cooldownUntil && new Date() < state.cooldownUntil) { + console.log(`[HealthCheck] Key ${keyId} for ${provider.name} is in cooldown until ${state.cooldownUntil.toISOString()}`); + return; + } + + // Resolve environment variables + const resolvedKey = resolveEnvVar(keyValue); + if (!resolvedKey) { + console.warn(`[HealthCheck] Key ${keyId} for ${provider.name} has empty value (env var not set?)`); + this.updateKeyHealth(provider, keyId, 'unhealthy', 'API key is empty or environment variable not set'); + return; + } + + // Test the key + const result = await testApiKeyConnection(provider.type, apiBase, resolvedKey); + const now = new Date().toISOString(); + + if (result.valid) { + // Reset failure count on success + state.consecutiveFailures = 0; + state.cooldownUntil = undefined; + this.updateKeyHealth(provider, keyId, 'healthy', undefined, now, result.latencyMs); + console.log(`[HealthCheck] Key ${keyId} for ${provider.name}: healthy (${result.latencyMs}ms)`); + } else { + // Increment failure count + state.consecutiveFailures++; + + if (state.consecutiveFailures >= healthConfig.failureThreshold) { + // Mark as unhealthy and enter cooldown + const cooldownMs = (healthConfig.cooldownSeconds || 60) * 1000; + state.cooldownUntil = new Date(Date.now() + cooldownMs); + this.updateKeyHealth(provider, keyId, 'unhealthy', result.error, now); + console.warn(`[HealthCheck] Key ${keyId} for ${provider.name}: UNHEALTHY after ${state.consecutiveFailures} failures. Cooldown until ${state.cooldownUntil.toISOString()}`); + } else { + // Still unknown/degraded, not yet unhealthy + this.updateKeyHealth(provider, keyId, 'unknown', result.error, now); + console.log(`[HealthCheck] Key ${keyId} for ${provider.name}: failed (${state.consecutiveFailures}/${healthConfig.failureThreshold}): ${result.error}`); + } + } + } + + /** + * Update the health status of a key in the provider's apiKeys array + */ + private updateKeyHealth( + provider: ProviderCredential, + keyId: string, + status: 'healthy' | 'unhealthy' | 'unknown', + error?: string, + timestamp?: string, + latencyMs?: number + ): void { + if (!provider.apiKeys) return; + + const keyEntry = provider.apiKeys.find(k => k.id === keyId); + if (keyEntry) { + keyEntry.healthStatus = status; + keyEntry.lastHealthCheck = timestamp || new Date().toISOString(); + if (error) { + keyEntry.lastError = error; + } else { + delete keyEntry.lastError; + } + // Save latency if provided (only on successful checks) + if (latencyMs !== undefined) { + keyEntry.lastLatencyMs = latencyMs; + } + } + } + + /** + * Get the current health status of all keys for a provider + */ + getProviderHealthStatus(providerId: string): Array<{ + keyId: string; + status: 'healthy' | 'unhealthy' | 'unknown'; + lastCheck?: string; + lastError?: string; + consecutiveFailures: number; + inCooldown: boolean; + }> { + const provider = getProvider(this.baseDir, providerId); + if (!provider || !provider.apiKeys) return []; + + return provider.apiKeys.map(key => { + const stateKey = `${providerId}:${key.id}`; + const state = this.keyStates.get(stateKey) || { consecutiveFailures: 0 }; + + return { + keyId: key.id, + status: key.healthStatus || 'unknown', + lastCheck: key.lastHealthCheck, + lastError: key.lastError, + consecutiveFailures: state.consecutiveFailures, + inCooldown: state.cooldownUntil ? new Date() < state.cooldownUntil : false, + }; + }); + } + + /** + * Check if the service is running health checks for any provider + */ + isRunning(): boolean { + return this.timers.size > 0; + } + + /** + * Get list of provider IDs currently being monitored + */ + getMonitoredProviders(): string[] { + return Array.from(this.timers.keys()); + } +} + +/** + * Get the singleton health check service instance + */ +export function getHealthCheckService(): HealthCheckService { + return HealthCheckService.getInstance(); +} diff --git a/ccw/src/templates/dashboard-js/views/api-settings.js b/ccw/src/templates/dashboard-js/views/api-settings.js index fb4fd6ed..2585d898 100644 --- a/ccw/src/templates/dashboard-js/views/api-settings.js +++ b/ccw/src/templates/dashboard-js/views/api-settings.js @@ -3172,16 +3172,75 @@ function renderDiscoveredProviders() { if (window.lucide) lucide.createIcons(); } +/** + * Format relative time for display (e.g., "2 minutes ago") + */ +function formatKeyRelativeTime(dateString) { + if (!dateString) return ''; + try { + var date = new Date(dateString); + var now = new Date(); + var diffMs = now - date; + var diffSecs = Math.floor(diffMs / 1000); + var diffMins = Math.floor(diffSecs / 60); + var diffHours = Math.floor(diffMins / 60); + var diffDays = Math.floor(diffHours / 24); + + if (diffSecs < 60) return t('apiSettings.justNow') || 'just now'; + if (diffMins < 60) return diffMins + (t('apiSettings.minutesAgo') || 'm ago'); + if (diffHours < 24) return diffHours + (t('apiSettings.hoursAgo') || 'h ago'); + if (diffDays < 7) return diffDays + (t('apiSettings.daysAgo') || 'd ago'); + return date.toLocaleDateString(); + } catch (e) { + return dateString; + } +} + +/** + * Get status indicator color class + */ +function getKeyStatusColorClass(status) { + switch (status) { + case 'healthy': return 'text-success'; + case 'unhealthy': return 'text-danger'; + default: return 'text-muted'; + } +} + /** * Render API keys section */ function renderApiKeysSection(provider) { - const keys = provider.apiKeys || []; - const hasMultipleKeys = keys.length > 0; + var keys = provider.apiKeys || []; + var hasMultipleKeys = keys.length > 0; - let keysHtml = ''; + var keysHtml = ''; if (hasMultipleKeys) { keysHtml = keys.map(function(key, index) { + var healthStatus = key.healthStatus || 'unknown'; + var statusColorClass = getKeyStatusColorClass(healthStatus); + var lastCheckTime = key.lastHealthCheck ? formatKeyRelativeTime(key.lastHealthCheck) : ''; + var latencyInfo = (healthStatus === 'healthy' && key.lastLatencyMs !== undefined) + ? ' (' + key.lastLatencyMs + 'ms)' + : ''; + + // Status display with latency and time + var statusDisplay = '
' + + '
' + + '' + + '' + + t('apiSettings.' + healthStatus) + latencyInfo + + '' + + '
'; + + // Show last check time if available + if (lastCheckTime) { + statusDisplay += '
' + + (t('apiSettings.lastCheck') || 'Last check') + ': ' + lastCheckTime + + '
'; + } + statusDisplay += '
'; + return '
' + '' + - '
' + - '' + - '' + t('apiSettings.' + (key.healthStatus || 'unknown')) + '' + - '
' + + statusDisplay + '
' + - '' + '' + + '
' + + nextCheckInfo + + '
' + '
' + '' + ' 0) { + statusMsg += ', ' + unhealthyCount + ' ' + (t('apiSettings.unhealthy') || 'unhealthy'); + } + showToast(statusMsg, unhealthyCount > 0 ? 'warning' : 'success'); + + // Reload data and refresh modal + loadApiSettings(true).then(function() { + refreshMultiKeyModal(providerId); + }); + } else { + showToast((t('apiSettings.healthCheckFailed') || 'Health check failed') + ': ' + (result.error || 'Unknown error'), 'error'); + } + }) + .catch(function(err) { + // Reset button state on error + if (btn) { + btn.disabled = false; + btn.innerHTML = originalHtml; + if (window.lucide) lucide.createIcons(); + } + console.error('Failed to trigger health check:', err); + showToast((t('apiSettings.healthCheckFailed') || 'Health check failed') + ': ' + err.message, 'error'); + }); +} + +/** + * Test API key + * @param {string} providerId - Provider ID + * @param {string} keyId - Key ID to test + * @param {Event} event - Click event (optional, for button reference) + */ +function testApiKey(providerId, keyId, event) { + // Get button element - either from event or find by data attributes + var btn; + if (event && event.target) { + btn = event.target.closest('button'); + } + if (!btn) { + // Fallback: find button by data attributes + btn = document.querySelector('button[onclick*="testApiKey"][onclick*="' + keyId + '"]'); + } + + if (!btn) { + showToast('Test failed: Could not find test button', 'error'); + return; + } + + // Store original button text + var originalText = btn.textContent; + + // Set loading state btn.disabled = true; btn.classList.add('testing'); - btn.textContent = t('apiSettings.testingKey'); + btn.textContent = t('apiSettings.testingKey') || 'Testing...'; csrfFetch('/api/litellm-api/providers/' + providerId + '/test-key', { method: 'POST', @@ -3496,29 +3657,61 @@ function testApiKey(providerId, keyId) { }) .then(function(res) { return res.json(); }) .then(function(result) { + // Reset button state btn.disabled = false; btn.classList.remove('testing'); - btn.textContent = t('apiSettings.testKey'); + btn.textContent = originalText; - const keyItem = btn.closest('.api-key-item'); - const statusIndicator = keyItem.querySelector('.key-status-indicator'); - const statusText = keyItem.querySelector('.key-status-text'); + // Find status elements within the key item + var keyItem = btn.closest('.api-key-item'); + if (!keyItem) { + // Try parent elements + keyItem = btn.parentElement; + while (keyItem && !keyItem.classList.contains('api-key-item')) { + keyItem = keyItem.parentElement; + } + } + + var statusIndicator = keyItem ? keyItem.querySelector('.key-status-indicator') : null; + var statusText = keyItem ? keyItem.querySelector('.key-status-text') : null; if (result.valid) { - statusIndicator.className = 'key-status-indicator healthy'; - statusText.textContent = t('apiSettings.healthy'); - showToast(t('apiSettings.keyValid'), 'success'); + // Update status indicators if found + if (statusIndicator) { + statusIndicator.className = 'key-status-indicator healthy'; + } + if (statusText) { + var latencyInfo = result.latencyMs ? ' (' + result.latencyMs + 'ms)' : ''; + statusText.textContent = (t('apiSettings.healthy') || 'Healthy') + latencyInfo; + } + // Show success toast with latency info + var successMsg = (t('apiSettings.keyValid') || 'API key is valid'); + if (result.latencyMs) { + successMsg += ' (' + result.latencyMs + 'ms)'; + } + showToast(successMsg, 'success'); } else { - statusIndicator.className = 'key-status-indicator unhealthy'; - statusText.textContent = t('apiSettings.unhealthy'); - showToast(t('apiSettings.keyInvalid') + ': ' + (result.error || ''), 'error'); + // Update status indicators if found + if (statusIndicator) { + statusIndicator.className = 'key-status-indicator unhealthy'; + } + if (statusText) { + statusText.textContent = t('apiSettings.unhealthy') || 'Unhealthy'; + } + // Show error toast + var errorMsg = (t('apiSettings.keyInvalid') || 'API key is invalid'); + if (result.error) { + errorMsg += ': ' + result.error; + } + showToast(errorMsg, 'error'); } }) .catch(function(err) { + // Reset button state on error btn.disabled = false; btn.classList.remove('testing'); - btn.textContent = t('apiSettings.testKey'); - showToast('Test failed: ' + err.message, 'error'); + btn.textContent = originalText; + showToast('Test failed: ' + (err.message || 'Unknown error'), 'error'); }); } diff --git a/ccw/src/types/litellm-api-config.ts b/ccw/src/types/litellm-api-config.ts index 5d77baa7..cbe4fdf5 100644 --- a/ccw/src/types/litellm-api-config.ts +++ b/ccw/src/types/litellm-api-config.ts @@ -109,6 +109,9 @@ export interface ApiKeyEntry { /** Error message if unhealthy */ lastError?: string; + + /** Last recorded latency in milliseconds */ + lastLatencyMs?: number; } /**