mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-02-09 02:24:11 +08:00
chore(release): v6.3.19 - Dense Reranker, CLI Tools & Issue Workflow
## Documentation Updates - Update all version references to v6.3.19 - Add Dense + Reranker search documentation - Add OpenCode AI CLI tool integration docs - Add Issue workflow (plan → queue → execute) with Codex recommendation - Update CHANGELOG with complete v6.3.19 release notes ## Features - Cross-Encoder reranking for improved search relevance - OpenCode CLI tool support - Issue multi-queue parallel execution - Service architecture improvements (cache-manager, preload-service) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
catlog22
@@ -86,6 +86,7 @@ import {
|
||||
} from '../../config/litellm-api-config-manager.js';
|
||||
import { getContextCacheStore } from '../../tools/context-cache-store.js';
|
||||
import { getLiteLLMClient } from '../../tools/litellm-client.js';
|
||||
import { testApiKeyConnection, getDefaultApiBase } from '../services/api-key-tester.js';
|
||||
|
||||
// Cache for ccw-litellm status check
|
||||
let ccwLitellmStatusCache: {
|
||||
@@ -338,6 +339,201 @@ export async function handleLiteLLMApiRoutes(ctx: RouteContext): Promise<boolean
|
||||
return true;
|
||||
}
|
||||
|
||||
// POST /api/litellm-api/providers/:id/test-key - Test specific API key
|
||||
const providerTestKeyMatch = pathname.match(/^\/api\/litellm-api\/providers\/([^/]+)\/test-key$/);
|
||||
if (providerTestKeyMatch && req.method === 'POST') {
|
||||
const providerId = providerTestKeyMatch[1];
|
||||
|
||||
handlePostRequest(req, res, async (body: unknown) => {
|
||||
const { keyId } = body as { keyId?: string };
|
||||
|
||||
if (!keyId) {
|
||||
return { valid: false, error: 'keyId is required', status: 400 };
|
||||
}
|
||||
|
||||
try {
|
||||
const provider = getProvider(initialPath, providerId);
|
||||
|
||||
if (!provider) {
|
||||
return { valid: false, error: 'Provider not found', status: 404 };
|
||||
}
|
||||
|
||||
// Find the specific API key
|
||||
let apiKeyValue: string | null = null;
|
||||
let keyLabel = 'Default';
|
||||
|
||||
if (keyId === 'default' && provider.apiKey) {
|
||||
// Use the single default apiKey
|
||||
apiKeyValue = provider.apiKey;
|
||||
} else if (provider.apiKeys && provider.apiKeys.length > 0) {
|
||||
const keyEntry = provider.apiKeys.find(k => k.id === keyId);
|
||||
if (keyEntry) {
|
||||
apiKeyValue = keyEntry.key;
|
||||
keyLabel = keyEntry.label || keyEntry.id;
|
||||
}
|
||||
}
|
||||
|
||||
if (!apiKeyValue) {
|
||||
return { valid: false, error: 'API key not found' };
|
||||
}
|
||||
|
||||
// Resolve environment variables
|
||||
const { resolveEnvVar } = await import('../../config/litellm-api-config-manager.js');
|
||||
const resolvedKey = resolveEnvVar(apiKeyValue);
|
||||
|
||||
if (!resolvedKey) {
|
||||
return { valid: false, error: 'API key is empty or environment variable not set' };
|
||||
}
|
||||
|
||||
// Determine API base URL
|
||||
const apiBase = provider.apiBase || getDefaultApiBase(provider.type);
|
||||
|
||||
// Test the API key with appropriate endpoint based on provider type
|
||||
const startTime = Date.now();
|
||||
const testResult = await testApiKeyConnection(provider.type, apiBase, resolvedKey);
|
||||
const latencyMs = Date.now() - startTime;
|
||||
|
||||
// Update key health status in provider config
|
||||
if (provider.apiKeys && provider.apiKeys.length > 0) {
|
||||
const keyEntry = provider.apiKeys.find(k => k.id === keyId);
|
||||
if (keyEntry) {
|
||||
keyEntry.healthStatus = testResult.valid ? 'healthy' : 'unhealthy';
|
||||
keyEntry.lastHealthCheck = new Date().toISOString();
|
||||
if (!testResult.valid) {
|
||||
keyEntry.lastError = testResult.error;
|
||||
} else {
|
||||
delete keyEntry.lastError;
|
||||
}
|
||||
|
||||
// Save updated provider
|
||||
try {
|
||||
updateProvider(initialPath, providerId, { apiKeys: provider.apiKeys });
|
||||
} catch (updateErr) {
|
||||
console.warn('[test-key] Failed to update key health status:', updateErr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
valid: testResult.valid,
|
||||
error: testResult.error,
|
||||
latencyMs: testResult.valid ? latencyMs : undefined,
|
||||
keyLabel,
|
||||
};
|
||||
} catch (err) {
|
||||
return { valid: false, error: (err as Error).message };
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
// GET /api/litellm-api/providers/:id/health-status - Get health status for all keys
|
||||
const providerHealthStatusMatch = pathname.match(/^\/api\/litellm-api\/providers\/([^/]+)\/health-status$/);
|
||||
if (providerHealthStatusMatch && req.method === 'GET') {
|
||||
const providerId = providerHealthStatusMatch[1];
|
||||
|
||||
try {
|
||||
const provider = getProvider(initialPath, providerId);
|
||||
|
||||
if (!provider) {
|
||||
res.writeHead(404, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Provider not found' }));
|
||||
return true;
|
||||
}
|
||||
|
||||
// Import health check service to get runtime state
|
||||
const { getHealthCheckService } = await import('../services/health-check-service.js');
|
||||
const healthService = getHealthCheckService();
|
||||
const healthStatus = healthService.getProviderHealthStatus(providerId);
|
||||
|
||||
// Merge persisted key data with runtime health status
|
||||
const keys = (provider.apiKeys || []).map(key => {
|
||||
const runtimeStatus = healthStatus.find(s => s.keyId === key.id);
|
||||
return {
|
||||
keyId: key.id,
|
||||
label: key.label || key.id,
|
||||
status: runtimeStatus?.status || key.healthStatus || 'unknown',
|
||||
lastCheck: runtimeStatus?.lastCheck || key.lastHealthCheck,
|
||||
lastLatencyMs: key.lastLatencyMs,
|
||||
consecutiveFailures: runtimeStatus?.consecutiveFailures || 0,
|
||||
inCooldown: runtimeStatus?.inCooldown || false,
|
||||
lastError: runtimeStatus?.lastError || key.lastError,
|
||||
};
|
||||
});
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
providerId,
|
||||
providerName: provider.name,
|
||||
keys,
|
||||
}));
|
||||
} catch (err) {
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: (err as Error).message }));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// POST /api/litellm-api/providers/:id/health-check-now - Trigger immediate health check
|
||||
const providerHealthCheckNowMatch = pathname.match(/^\/api\/litellm-api\/providers\/([^/]+)\/health-check-now$/);
|
||||
if (providerHealthCheckNowMatch && req.method === 'POST') {
|
||||
const providerId = providerHealthCheckNowMatch[1];
|
||||
|
||||
try {
|
||||
const provider = getProvider(initialPath, providerId);
|
||||
|
||||
if (!provider) {
|
||||
res.writeHead(404, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Provider not found' }));
|
||||
return true;
|
||||
}
|
||||
|
||||
// Import health check service and trigger check
|
||||
const { getHealthCheckService } = await import('../services/health-check-service.js');
|
||||
const healthService = getHealthCheckService();
|
||||
|
||||
// Trigger immediate check (async, but we wait for completion)
|
||||
await healthService.checkProviderNow(providerId);
|
||||
|
||||
// Get updated status
|
||||
const healthStatus = healthService.getProviderHealthStatus(providerId);
|
||||
|
||||
// Reload provider to get updated persisted data
|
||||
const updatedProvider = getProvider(initialPath, providerId);
|
||||
const keys = (updatedProvider?.apiKeys || []).map(key => {
|
||||
const runtimeStatus = healthStatus.find(s => s.keyId === key.id);
|
||||
return {
|
||||
keyId: key.id,
|
||||
label: key.label || key.id,
|
||||
status: runtimeStatus?.status || key.healthStatus || 'unknown',
|
||||
lastCheck: runtimeStatus?.lastCheck || key.lastHealthCheck,
|
||||
lastLatencyMs: key.lastLatencyMs,
|
||||
consecutiveFailures: runtimeStatus?.consecutiveFailures || 0,
|
||||
inCooldown: runtimeStatus?.inCooldown || false,
|
||||
lastError: runtimeStatus?.lastError || key.lastError,
|
||||
};
|
||||
});
|
||||
|
||||
broadcastToClients({
|
||||
type: 'PROVIDER_HEALTH_CHECKED',
|
||||
payload: { providerId, keys, timestamp: new Date().toISOString() }
|
||||
});
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
success: true,
|
||||
providerId,
|
||||
providerName: updatedProvider?.name,
|
||||
keys,
|
||||
checkedAt: new Date().toISOString(),
|
||||
}));
|
||||
} catch (err) {
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ success: false, error: (err as Error).message }));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// ===========================
|
||||
// Endpoint Management Routes
|
||||
// ===========================
|
||||
|
||||
@@ -39,6 +39,9 @@ import { csrfValidation } from './auth/csrf-middleware.js';
|
||||
import { getCsrfTokenManager } from './auth/csrf-manager.js';
|
||||
import { randomBytes } from 'crypto';
|
||||
|
||||
// Import health check service
|
||||
import { getHealthCheckService } from './services/health-check-service.js';
|
||||
|
||||
import type { ServerConfig } from '../types/config.js';
|
||||
import type { PostRequestHandler } from './routes/types.js';
|
||||
|
||||
@@ -632,6 +635,15 @@ export async function startServer(options: ServerOptions = {}): Promise<http.Ser
|
||||
console.log(`Dashboard server running at http://${host}:${serverPort}`);
|
||||
console.log(`WebSocket endpoint available at ws://${host}:${serverPort}/ws`);
|
||||
console.log(`Hook endpoint available at POST http://${host}:${serverPort}/api/hook`);
|
||||
|
||||
// Start health check service for all enabled providers
|
||||
try {
|
||||
const healthCheckService = getHealthCheckService();
|
||||
healthCheckService.startAllHealthChecks(initialPath);
|
||||
} catch (err) {
|
||||
console.warn('[Server] Failed to start health check service:', err);
|
||||
}
|
||||
|
||||
resolve(server);
|
||||
});
|
||||
server.on('error', reject);
|
||||
|
||||
137
ccw/src/core/services/api-key-tester.ts
Normal file
137
ccw/src/core/services/api-key-tester.ts
Normal file
@@ -0,0 +1,137 @@
|
||||
/**
|
||||
* API Key Tester Service
|
||||
* Shared module for testing API key connectivity across different provider types.
|
||||
* Used by both manual testing (litellm-api-routes.ts) and health check service.
|
||||
*/
|
||||
|
||||
import type { ProviderType } from '../../types/litellm-api-config.js';
|
||||
|
||||
/**
|
||||
* Result of an API key connection test
|
||||
*/
|
||||
export interface TestResult {
|
||||
/** Whether the API key is valid */
|
||||
valid: boolean;
|
||||
/** Error message if validation failed */
|
||||
error?: string;
|
||||
/** Latency in milliseconds (only if valid) */
|
||||
latencyMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get default API base URL for a provider type
|
||||
*/
|
||||
export function getDefaultApiBase(providerType: ProviderType): string {
|
||||
const defaults: Record<string, string> = {
|
||||
openai: 'https://api.openai.com/v1',
|
||||
anthropic: 'https://api.anthropic.com/v1',
|
||||
custom: 'https://api.openai.com/v1', // Assume OpenAI-compatible by default
|
||||
};
|
||||
return defaults[providerType] || defaults.openai;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test API key connection by making a minimal API request
|
||||
* @param providerType - The type of provider (openai, anthropic, custom)
|
||||
* @param apiBase - The base URL for the API
|
||||
* @param apiKey - The API key to test
|
||||
* @param timeout - Timeout in milliseconds (default: 10000)
|
||||
* @returns TestResult indicating if the key is valid
|
||||
*/
|
||||
export async function testApiKeyConnection(
|
||||
providerType: ProviderType,
|
||||
apiBase: string,
|
||||
apiKey: string,
|
||||
timeout: number = 10000
|
||||
): Promise<TestResult> {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeout);
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
if (providerType === 'anthropic') {
|
||||
// Anthropic format: POST /v1/messages with minimal payload
|
||||
const response = await fetch(`${apiBase}/messages`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': apiKey,
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'claude-3-haiku-20240307',
|
||||
max_tokens: 1,
|
||||
messages: [{ role: 'user', content: 'Hi' }],
|
||||
}),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
const latencyMs = Date.now() - startTime;
|
||||
|
||||
if (response.ok) {
|
||||
return { valid: true, latencyMs };
|
||||
}
|
||||
|
||||
// Parse error response
|
||||
const errorBody = await response.json().catch(() => ({}));
|
||||
const errorMessage = (errorBody as any)?.error?.message || response.statusText;
|
||||
|
||||
// 401 = invalid API key, other 4xx might be valid key with other issues
|
||||
if (response.status === 401) {
|
||||
return { valid: false, error: 'Invalid API key' };
|
||||
}
|
||||
if (response.status === 403) {
|
||||
return { valid: false, error: 'Access denied - check API key permissions' };
|
||||
}
|
||||
if (response.status === 429) {
|
||||
// Rate limited means the key is valid but being throttled
|
||||
return { valid: true, latencyMs };
|
||||
}
|
||||
|
||||
return { valid: false, error: errorMessage };
|
||||
} else {
|
||||
// OpenAI-compatible format: GET /v1/models
|
||||
const modelsUrl = apiBase.endsWith('/v1') ? `${apiBase}/models` : `${apiBase}/v1/models`;
|
||||
const response = await fetch(modelsUrl, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${apiKey}`,
|
||||
},
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
const latencyMs = Date.now() - startTime;
|
||||
|
||||
if (response.ok) {
|
||||
return { valid: true, latencyMs };
|
||||
}
|
||||
|
||||
// Parse error response
|
||||
const errorBody = await response.json().catch(() => ({}));
|
||||
const errorMessage = (errorBody as any)?.error?.message || response.statusText;
|
||||
|
||||
if (response.status === 401) {
|
||||
return { valid: false, error: 'Invalid API key' };
|
||||
}
|
||||
if (response.status === 403) {
|
||||
return { valid: false, error: 'Access denied - check API key permissions' };
|
||||
}
|
||||
if (response.status === 429) {
|
||||
// Rate limited means the key is valid but being throttled
|
||||
return { valid: true, latencyMs };
|
||||
}
|
||||
|
||||
return { valid: false, error: errorMessage };
|
||||
}
|
||||
} catch (err) {
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if ((err as Error).name === 'AbortError') {
|
||||
return { valid: false, error: 'Connection timed out' };
|
||||
}
|
||||
|
||||
return { valid: false, error: `Connection failed: ${(err as Error).message}` };
|
||||
}
|
||||
}
|
||||
340
ccw/src/core/services/health-check-service.ts
Normal file
340
ccw/src/core/services/health-check-service.ts
Normal file
@@ -0,0 +1,340 @@
|
||||
/**
|
||||
* Health Check Service
|
||||
* Singleton service that periodically checks API key health for providers
|
||||
* with health check enabled. Updates key health status and filters unhealthy
|
||||
* keys from rotation endpoints.
|
||||
*/
|
||||
|
||||
import {
|
||||
getAllProviders,
|
||||
getProvider,
|
||||
updateProvider,
|
||||
resolveEnvVar,
|
||||
} from '../../config/litellm-api-config-manager.js';
|
||||
import { testApiKeyConnection, getDefaultApiBase } from './api-key-tester.js';
|
||||
import type { ProviderCredential, ApiKeyEntry, HealthCheckConfig } from '../../types/litellm-api-config.js';
|
||||
|
||||
/**
|
||||
* Internal state for tracking consecutive failures per key
|
||||
*/
|
||||
interface KeyHealthState {
|
||||
consecutiveFailures: number;
|
||||
cooldownUntil?: Date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Health Check Service - Singleton
|
||||
* Manages periodic health checks for API keys across all providers
|
||||
*/
|
||||
export class HealthCheckService {
|
||||
private static instance: HealthCheckService;
|
||||
|
||||
/** Timer handles for each provider's health check interval */
|
||||
private timers: Map<string, NodeJS.Timeout> = new Map();
|
||||
|
||||
/** Track consecutive failures and cooldown per key (providerId:keyId -> state) */
|
||||
private keyStates: Map<string, KeyHealthState> = new Map();
|
||||
|
||||
/** Base directory for config operations */
|
||||
private baseDir: string = '';
|
||||
|
||||
/** Lock to prevent concurrent checks on same provider */
|
||||
private checkingProviders: Set<string> = new Set();
|
||||
|
||||
private constructor() {}
|
||||
|
||||
/**
|
||||
* Get singleton instance
|
||||
*/
|
||||
static getInstance(): HealthCheckService {
|
||||
if (!HealthCheckService.instance) {
|
||||
HealthCheckService.instance = new HealthCheckService();
|
||||
}
|
||||
return HealthCheckService.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start health check for a specific provider
|
||||
* @param providerId - The provider ID to start health checks for
|
||||
*/
|
||||
startHealthCheck(providerId: string): void {
|
||||
// Stop existing timer if any
|
||||
this.stopHealthCheck(providerId);
|
||||
|
||||
const provider = getProvider(this.baseDir, providerId);
|
||||
if (!provider) {
|
||||
console.warn(`[HealthCheck] Provider not found: ${providerId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!provider.enabled) {
|
||||
console.log(`[HealthCheck] Provider ${providerId} is disabled, skipping`);
|
||||
return;
|
||||
}
|
||||
|
||||
const healthConfig = provider.healthCheck;
|
||||
if (!healthConfig || !healthConfig.enabled) {
|
||||
console.log(`[HealthCheck] Health check not enabled for provider ${providerId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const intervalMs = (healthConfig.intervalSeconds || 300) * 1000;
|
||||
|
||||
console.log(`[HealthCheck] Starting health check for ${provider.name} (${providerId}), interval: ${healthConfig.intervalSeconds}s`);
|
||||
|
||||
// Run initial check immediately
|
||||
void this.checkProviderNow(providerId);
|
||||
|
||||
// Set up interval
|
||||
const timer = setInterval(() => {
|
||||
void this.checkProviderNow(providerId);
|
||||
}, intervalMs);
|
||||
|
||||
this.timers.set(providerId, timer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop health check for a specific provider
|
||||
* @param providerId - The provider ID to stop health checks for
|
||||
*/
|
||||
stopHealthCheck(providerId: string): void {
|
||||
const timer = this.timers.get(providerId);
|
||||
if (timer) {
|
||||
clearInterval(timer);
|
||||
this.timers.delete(providerId);
|
||||
console.log(`[HealthCheck] Stopped health check for provider ${providerId}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start health checks for all providers that have health check enabled
|
||||
* @param baseDir - Base directory for config operations
|
||||
*/
|
||||
startAllHealthChecks(baseDir: string): void {
|
||||
this.baseDir = baseDir;
|
||||
|
||||
const providers = getAllProviders(baseDir);
|
||||
let startedCount = 0;
|
||||
|
||||
for (const provider of providers) {
|
||||
if (provider.enabled && provider.healthCheck?.enabled) {
|
||||
this.startHealthCheck(provider.id);
|
||||
startedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if (startedCount > 0) {
|
||||
console.log(`[HealthCheck] Started health checks for ${startedCount} provider(s)`);
|
||||
} else {
|
||||
console.log('[HealthCheck] No providers with health check enabled');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop all active health checks
|
||||
*/
|
||||
stopAllHealthChecks(): void {
|
||||
const providerIds = Array.from(this.timers.keys());
|
||||
for (const providerId of providerIds) {
|
||||
this.stopHealthCheck(providerId);
|
||||
}
|
||||
this.keyStates.clear();
|
||||
console.log('[HealthCheck] Stopped all health checks');
|
||||
}
|
||||
|
||||
/**
|
||||
* Manually trigger a health check for a provider
|
||||
* @param providerId - The provider ID to check
|
||||
*/
|
||||
async checkProviderNow(providerId: string): Promise<void> {
|
||||
// Prevent concurrent checks on same provider
|
||||
if (this.checkingProviders.has(providerId)) {
|
||||
console.log(`[HealthCheck] Already checking provider ${providerId}, skipping`);
|
||||
return;
|
||||
}
|
||||
|
||||
this.checkingProviders.add(providerId);
|
||||
|
||||
try {
|
||||
const provider = getProvider(this.baseDir, providerId);
|
||||
if (!provider) {
|
||||
console.warn(`[HealthCheck] Provider not found: ${providerId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!provider.enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const healthConfig = provider.healthCheck;
|
||||
if (!healthConfig) {
|
||||
return;
|
||||
}
|
||||
|
||||
const apiBase = provider.apiBase || getDefaultApiBase(provider.type);
|
||||
const apiKeys = provider.apiKeys || [];
|
||||
|
||||
if (apiKeys.length === 0 && provider.apiKey) {
|
||||
// Single key mode - create virtual key entry
|
||||
await this.checkSingleKey(provider, 'default', provider.apiKey, apiBase, healthConfig);
|
||||
} else {
|
||||
// Multi-key mode
|
||||
for (const keyEntry of apiKeys) {
|
||||
if (!keyEntry.enabled) continue;
|
||||
await this.checkSingleKey(provider, keyEntry.id, keyEntry.key, apiBase, healthConfig);
|
||||
}
|
||||
}
|
||||
|
||||
// Persist updated health statuses
|
||||
const updatedApiKeys = provider.apiKeys;
|
||||
if (updatedApiKeys && updatedApiKeys.length > 0) {
|
||||
try {
|
||||
updateProvider(this.baseDir, providerId, { apiKeys: updatedApiKeys });
|
||||
} catch (err) {
|
||||
console.error(`[HealthCheck] Failed to persist health status for ${providerId}:`, err);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
this.checkingProviders.delete(providerId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check a single API key's health
|
||||
*/
|
||||
private async checkSingleKey(
|
||||
provider: ProviderCredential,
|
||||
keyId: string,
|
||||
keyValue: string,
|
||||
apiBase: string,
|
||||
healthConfig: HealthCheckConfig
|
||||
): Promise<void> {
|
||||
const stateKey = `${provider.id}:${keyId}`;
|
||||
let state = this.keyStates.get(stateKey);
|
||||
|
||||
if (!state) {
|
||||
state = { consecutiveFailures: 0 };
|
||||
this.keyStates.set(stateKey, state);
|
||||
}
|
||||
|
||||
// Check if in cooldown
|
||||
if (state.cooldownUntil && new Date() < state.cooldownUntil) {
|
||||
console.log(`[HealthCheck] Key ${keyId} for ${provider.name} is in cooldown until ${state.cooldownUntil.toISOString()}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Resolve environment variables
|
||||
const resolvedKey = resolveEnvVar(keyValue);
|
||||
if (!resolvedKey) {
|
||||
console.warn(`[HealthCheck] Key ${keyId} for ${provider.name} has empty value (env var not set?)`);
|
||||
this.updateKeyHealth(provider, keyId, 'unhealthy', 'API key is empty or environment variable not set');
|
||||
return;
|
||||
}
|
||||
|
||||
// Test the key
|
||||
const result = await testApiKeyConnection(provider.type, apiBase, resolvedKey);
|
||||
const now = new Date().toISOString();
|
||||
|
||||
if (result.valid) {
|
||||
// Reset failure count on success
|
||||
state.consecutiveFailures = 0;
|
||||
state.cooldownUntil = undefined;
|
||||
this.updateKeyHealth(provider, keyId, 'healthy', undefined, now, result.latencyMs);
|
||||
console.log(`[HealthCheck] Key ${keyId} for ${provider.name}: healthy (${result.latencyMs}ms)`);
|
||||
} else {
|
||||
// Increment failure count
|
||||
state.consecutiveFailures++;
|
||||
|
||||
if (state.consecutiveFailures >= healthConfig.failureThreshold) {
|
||||
// Mark as unhealthy and enter cooldown
|
||||
const cooldownMs = (healthConfig.cooldownSeconds || 60) * 1000;
|
||||
state.cooldownUntil = new Date(Date.now() + cooldownMs);
|
||||
this.updateKeyHealth(provider, keyId, 'unhealthy', result.error, now);
|
||||
console.warn(`[HealthCheck] Key ${keyId} for ${provider.name}: UNHEALTHY after ${state.consecutiveFailures} failures. Cooldown until ${state.cooldownUntil.toISOString()}`);
|
||||
} else {
|
||||
// Still unknown/degraded, not yet unhealthy
|
||||
this.updateKeyHealth(provider, keyId, 'unknown', result.error, now);
|
||||
console.log(`[HealthCheck] Key ${keyId} for ${provider.name}: failed (${state.consecutiveFailures}/${healthConfig.failureThreshold}): ${result.error}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the health status of a key in the provider's apiKeys array
|
||||
*/
|
||||
private updateKeyHealth(
|
||||
provider: ProviderCredential,
|
||||
keyId: string,
|
||||
status: 'healthy' | 'unhealthy' | 'unknown',
|
||||
error?: string,
|
||||
timestamp?: string,
|
||||
latencyMs?: number
|
||||
): void {
|
||||
if (!provider.apiKeys) return;
|
||||
|
||||
const keyEntry = provider.apiKeys.find(k => k.id === keyId);
|
||||
if (keyEntry) {
|
||||
keyEntry.healthStatus = status;
|
||||
keyEntry.lastHealthCheck = timestamp || new Date().toISOString();
|
||||
if (error) {
|
||||
keyEntry.lastError = error;
|
||||
} else {
|
||||
delete keyEntry.lastError;
|
||||
}
|
||||
// Save latency if provided (only on successful checks)
|
||||
if (latencyMs !== undefined) {
|
||||
keyEntry.lastLatencyMs = latencyMs;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current health status of all keys for a provider
|
||||
*/
|
||||
getProviderHealthStatus(providerId: string): Array<{
|
||||
keyId: string;
|
||||
status: 'healthy' | 'unhealthy' | 'unknown';
|
||||
lastCheck?: string;
|
||||
lastError?: string;
|
||||
consecutiveFailures: number;
|
||||
inCooldown: boolean;
|
||||
}> {
|
||||
const provider = getProvider(this.baseDir, providerId);
|
||||
if (!provider || !provider.apiKeys) return [];
|
||||
|
||||
return provider.apiKeys.map(key => {
|
||||
const stateKey = `${providerId}:${key.id}`;
|
||||
const state = this.keyStates.get(stateKey) || { consecutiveFailures: 0 };
|
||||
|
||||
return {
|
||||
keyId: key.id,
|
||||
status: key.healthStatus || 'unknown',
|
||||
lastCheck: key.lastHealthCheck,
|
||||
lastError: key.lastError,
|
||||
consecutiveFailures: state.consecutiveFailures,
|
||||
inCooldown: state.cooldownUntil ? new Date() < state.cooldownUntil : false,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the service is running health checks for any provider
|
||||
*/
|
||||
isRunning(): boolean {
|
||||
return this.timers.size > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of provider IDs currently being monitored
|
||||
*/
|
||||
getMonitoredProviders(): string[] {
|
||||
return Array.from(this.timers.keys());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the singleton health check service instance
|
||||
*/
|
||||
export function getHealthCheckService(): HealthCheckService {
|
||||
return HealthCheckService.getInstance();
|
||||
}
|
||||
Reference in New Issue
Block a user