mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-02-05 01:50:27 +08:00
test(litellm): add comprehensive error scenario tests for client
Solution-ID: SOL-1735410001 Issue-ID: ISS-1766921318981-21 Task-ID: T1
This commit is contained in:
catlog22
@@ -1,96 +1,348 @@
|
||||
/**
|
||||
* LiteLLM Client Tests
|
||||
* Tests for the LiteLLM TypeScript bridge
|
||||
* Unit tests for LiteLLM client bridge (ccw/dist/tools/litellm-client.js).
|
||||
*
|
||||
* Notes:
|
||||
* - Uses Node's built-in test runner (node:test) (no Jest in this repo).
|
||||
* - Stubs `child_process.spawn` to avoid depending on local Python/ccw_litellm installation.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach } from '@jest/globals';
|
||||
import { LiteLLMClient, getLiteLLMClient, checkLiteLLMAvailable, getLiteLLMStatus } from '../src/tools/litellm-client';
|
||||
import { after, beforeEach, describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { createRequire } from 'node:module';
|
||||
|
||||
describe('LiteLLMClient', () => {
|
||||
let client: LiteLLMClient;
|
||||
const require = createRequire(import.meta.url);
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const childProcess = require('child_process') as typeof import('child_process');
|
||||
|
||||
beforeEach(() => {
|
||||
client = new LiteLLMClient({ timeout: 5000 });
|
||||
type SpawnBehavior =
|
||||
| { type: 'close'; code?: number; stdout?: string; stderr?: string }
|
||||
| { type: 'error'; error: Error }
|
||||
| { type: 'hang' };
|
||||
|
||||
class FakeChildProcess extends EventEmitter {
|
||||
stdout = new EventEmitter();
|
||||
stderr = new EventEmitter();
|
||||
killCalls: string[] = [];
|
||||
|
||||
kill(signal?: NodeJS.Signals | number | string): boolean {
|
||||
this.killCalls.push(signal === undefined ? 'undefined' : String(signal));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
type SpawnCall = {
|
||||
command: string;
|
||||
args: string[];
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
options: any;
|
||||
proc: FakeChildProcess;
|
||||
};
|
||||
|
||||
const spawnCalls: SpawnCall[] = [];
|
||||
const spawnPlan: SpawnBehavior[] = [];
|
||||
|
||||
const originalSpawn = childProcess.spawn;
|
||||
|
||||
childProcess.spawn = ((command: string, args: string[] = [], options: any = {}) => {
|
||||
const normalizedArgs = (args ?? []).map(String);
|
||||
const shouldIntercept = normalizedArgs[0] === '-m' && normalizedArgs[1] === 'ccw_litellm.cli';
|
||||
if (!shouldIntercept) {
|
||||
return originalSpawn(command as any, args as any, options as any);
|
||||
}
|
||||
|
||||
const proc = new FakeChildProcess();
|
||||
spawnCalls.push({ command: String(command), args: normalizedArgs, options, proc });
|
||||
|
||||
const next = spawnPlan.shift() ?? { type: 'close', code: 0, stdout: '' };
|
||||
|
||||
queueMicrotask(() => {
|
||||
if (next.type === 'error') {
|
||||
proc.emit('error', next.error);
|
||||
return;
|
||||
}
|
||||
|
||||
if (next.type === 'close') {
|
||||
if (next.stdout !== undefined) proc.stdout.emit('data', next.stdout);
|
||||
if (next.stderr !== undefined) proc.stderr.emit('data', next.stderr);
|
||||
proc.emit('close', next.code ?? 0);
|
||||
return;
|
||||
}
|
||||
|
||||
// hang: intentionally do nothing
|
||||
});
|
||||
|
||||
describe('Constructor', () => {
|
||||
it('should create client with default config', () => {
|
||||
const defaultClient = new LiteLLMClient();
|
||||
expect(defaultClient).toBeDefined();
|
||||
});
|
||||
return proc as any;
|
||||
}) as any;
|
||||
|
||||
it('should create client with custom config', () => {
|
||||
const customClient = new LiteLLMClient({
|
||||
pythonPath: 'python3',
|
||||
timeout: 10000
|
||||
});
|
||||
expect(customClient).toBeDefined();
|
||||
});
|
||||
});
|
||||
function getClientModuleUrl(): URL {
|
||||
const url = new URL('../dist/tools/litellm-client.js', import.meta.url);
|
||||
url.searchParams.set('t', `${Date.now()}-${Math.random()}`);
|
||||
return url;
|
||||
}
|
||||
|
||||
describe('isAvailable', () => {
|
||||
it('should check if ccw-litellm is available', async () => {
|
||||
const available = await client.isAvailable();
|
||||
expect(typeof available).toBe('boolean');
|
||||
});
|
||||
});
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
let mod: any;
|
||||
|
||||
describe('getStatus', () => {
|
||||
it('should return status object', async () => {
|
||||
const status = await client.getStatus();
|
||||
expect(status).toHaveProperty('available');
|
||||
expect(typeof status.available).toBe('boolean');
|
||||
});
|
||||
});
|
||||
|
||||
describe('embed', () => {
|
||||
it('should throw error for empty texts array', async () => {
|
||||
await expect(client.embed([])).rejects.toThrow('texts array cannot be empty');
|
||||
});
|
||||
|
||||
it('should throw error for null texts', async () => {
|
||||
await expect(client.embed(null as any)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('chat', () => {
|
||||
it('should throw error for empty message', async () => {
|
||||
await expect(client.chat('')).rejects.toThrow('message cannot be empty');
|
||||
});
|
||||
});
|
||||
|
||||
describe('chatMessages', () => {
|
||||
it('should throw error for empty messages array', async () => {
|
||||
await expect(client.chatMessages([])).rejects.toThrow('messages array cannot be empty');
|
||||
});
|
||||
|
||||
it('should throw error for null messages', async () => {
|
||||
await expect(client.chatMessages(null as any)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
beforeEach(async () => {
|
||||
spawnCalls.length = 0;
|
||||
spawnPlan.length = 0;
|
||||
mod = await import(getClientModuleUrl().href);
|
||||
});
|
||||
|
||||
describe('Singleton Functions', () => {
|
||||
describe('getLiteLLMClient', () => {
|
||||
it('should return singleton instance', () => {
|
||||
const client1 = getLiteLLMClient();
|
||||
const client2 = getLiteLLMClient();
|
||||
expect(client1).toBe(client2);
|
||||
});
|
||||
after(() => {
|
||||
childProcess.spawn = originalSpawn;
|
||||
});
|
||||
|
||||
describe('LiteLLM client bridge', () => {
|
||||
it('uses default pythonPath and version check arguments', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 0, stdout: '1.2.3\n' });
|
||||
|
||||
const client = new mod.LiteLLMClient();
|
||||
const available = await client.isAvailable();
|
||||
|
||||
assert.equal(available, true);
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
assert.equal(spawnCalls[0].command, 'python');
|
||||
assert.deepEqual(spawnCalls[0].args, ['-m', 'ccw_litellm.cli', 'version']);
|
||||
});
|
||||
|
||||
describe('checkLiteLLMAvailable', () => {
|
||||
it('should return boolean', async () => {
|
||||
const available = await checkLiteLLMAvailable();
|
||||
expect(typeof available).toBe('boolean');
|
||||
});
|
||||
it('uses custom pythonPath when provided', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 0, stdout: 'ok' });
|
||||
|
||||
const client = new mod.LiteLLMClient({ pythonPath: 'python3', timeout: 10 });
|
||||
await client.chat('hello', 'default');
|
||||
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
assert.equal(spawnCalls[0].command, 'python3');
|
||||
});
|
||||
|
||||
describe('getLiteLLMStatus', () => {
|
||||
it('should return status object', async () => {
|
||||
const status = await getLiteLLMStatus();
|
||||
expect(status).toHaveProperty('available');
|
||||
expect(typeof status.available).toBe('boolean');
|
||||
});
|
||||
it('isAvailable returns false on spawn error', async () => {
|
||||
spawnPlan.push({ type: 'error', error: new Error('ENOENT') });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
const available = await client.isAvailable();
|
||||
|
||||
assert.equal(available, false);
|
||||
});
|
||||
|
||||
it('getStatus returns version on success', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 0, stdout: 'v9.9.9\n' });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
const status = await client.getStatus();
|
||||
|
||||
assert.equal(status.available, true);
|
||||
assert.equal(status.version, 'v9.9.9');
|
||||
});
|
||||
|
||||
it('getStatus returns error details on non-zero exit', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 1, stderr: 'HTTP 500 Internal Server Error' });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
const status = await client.getStatus();
|
||||
|
||||
assert.equal(status.available, false);
|
||||
assert.ok(String(status.error).includes('HTTP 500'));
|
||||
});
|
||||
|
||||
it('getConfig parses JSON output', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 0, stdout: JSON.stringify({ ok: true }) });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
const cfg = await client.getConfig();
|
||||
|
||||
assert.deepEqual(cfg, { ok: true });
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
assert.deepEqual(spawnCalls[0].args, ['-m', 'ccw_litellm.cli', 'config', '--json']);
|
||||
});
|
||||
|
||||
it('getConfig throws on malformed JSON', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 0, stdout: '{not-json' });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
await assert.rejects(() => client.getConfig());
|
||||
});
|
||||
|
||||
it('embed rejects empty texts input and does not spawn', async () => {
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
await assert.rejects(() => client.embed([]), /texts array cannot be empty/);
|
||||
assert.equal(spawnCalls.length, 0);
|
||||
});
|
||||
|
||||
it('embed rejects null/undefined input', async () => {
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
await assert.rejects(() => client.embed(null as any), /texts array cannot be empty/);
|
||||
await assert.rejects(() => client.embed(undefined as any), /texts array cannot be empty/);
|
||||
assert.equal(spawnCalls.length, 0);
|
||||
});
|
||||
|
||||
it('embed returns vectors with derived dimensions', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 0, stdout: JSON.stringify([[1, 2, 3], [4, 5, 6]]) });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
const res = await client.embed(['a', 'b'], 'embed-model');
|
||||
|
||||
assert.equal(res.model, 'embed-model');
|
||||
assert.equal(res.dimensions, 3);
|
||||
assert.deepEqual(res.vectors, [
|
||||
[1, 2, 3],
|
||||
[4, 5, 6],
|
||||
]);
|
||||
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
assert.deepEqual(spawnCalls[0].args, [
|
||||
'-m',
|
||||
'ccw_litellm.cli',
|
||||
'embed',
|
||||
'--model',
|
||||
'embed-model',
|
||||
'--output',
|
||||
'json',
|
||||
'a',
|
||||
'b',
|
||||
]);
|
||||
});
|
||||
|
||||
it('embed throws on malformed JSON output', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 0, stdout: 'not-json' });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
await assert.rejects(() => client.embed(['a'], 'embed-model'));
|
||||
});
|
||||
|
||||
it('chat rejects empty message and does not spawn', async () => {
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
await assert.rejects(() => client.chat(''), /message cannot be empty/);
|
||||
assert.equal(spawnCalls.length, 0);
|
||||
});
|
||||
|
||||
it('chat returns trimmed stdout on success', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 0, stdout: 'Hello\n' });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
const out = await client.chat('hi', 'chat-model');
|
||||
|
||||
assert.equal(out, 'Hello');
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
assert.deepEqual(spawnCalls[0].args, ['-m', 'ccw_litellm.cli', 'chat', '--model', 'chat-model', 'hi']);
|
||||
});
|
||||
|
||||
it('chat propagates auth errors (401)', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 1, stderr: 'HTTP 401 Unauthorized' });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
await assert.rejects(() => client.chat('hi', 'chat-model'), /401/);
|
||||
});
|
||||
|
||||
it('chat propagates auth errors (403)', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 1, stderr: 'HTTP 403 Forbidden' });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
await assert.rejects(() => client.chat('hi', 'chat-model'), /403/);
|
||||
});
|
||||
|
||||
it('chat propagates rate limit errors (429)', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 1, stderr: 'HTTP 429 Too Many Requests' });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
await assert.rejects(() => client.chat('hi', 'chat-model'), /429/);
|
||||
});
|
||||
|
||||
it('chat propagates server errors (500)', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 1, stderr: 'HTTP 500 Internal Server Error' });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
await assert.rejects(() => client.chat('hi', 'chat-model'), /500/);
|
||||
});
|
||||
|
||||
it('chat propagates server errors (503)', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 1, stderr: 'HTTP 503 Service Unavailable' });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
await assert.rejects(() => client.chat('hi', 'chat-model'), /503/);
|
||||
});
|
||||
|
||||
it('chat falls back to exit code when stderr is empty', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 2, stdout: '' });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
await assert.rejects(() => client.chat('hi', 'chat-model'), /Process exited with code 2/);
|
||||
});
|
||||
|
||||
it('chat surfaces spawn failures with descriptive message', async () => {
|
||||
spawnPlan.push({ type: 'error', error: new Error('spawn ENOENT') });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
await assert.rejects(() => client.chat('hi', 'chat-model'), /Failed to spawn Python process: spawn ENOENT/);
|
||||
});
|
||||
|
||||
it('chat enforces timeout and terminates process', async () => {
|
||||
const originalSetTimeout = global.setTimeout;
|
||||
let observedDelay: number | null = null;
|
||||
|
||||
(global as any).setTimeout = ((fn: any, delay: number, ...args: any[]) => {
|
||||
observedDelay = delay;
|
||||
return originalSetTimeout(fn, 0, ...args);
|
||||
}) as any;
|
||||
|
||||
try {
|
||||
spawnPlan.push({ type: 'hang' });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 11 });
|
||||
await assert.rejects(() => client.chat('hi', 'chat-model'), /Command timed out after 22ms/);
|
||||
|
||||
assert.equal(observedDelay, 22);
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
assert.ok(spawnCalls[0].proc.killCalls.includes('SIGTERM'));
|
||||
} finally {
|
||||
(global as any).setTimeout = originalSetTimeout;
|
||||
}
|
||||
});
|
||||
|
||||
it('chatMessages rejects empty inputs', async () => {
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
await assert.rejects(() => client.chatMessages([]), /messages array cannot be empty/);
|
||||
await assert.rejects(() => client.chatMessages(null as any), /messages array cannot be empty/);
|
||||
assert.equal(spawnCalls.length, 0);
|
||||
});
|
||||
|
||||
it('chatMessages uses the last message content', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 0, stdout: 'OK' });
|
||||
|
||||
const client = new mod.LiteLLMClient({ timeout: 10 });
|
||||
const res = await client.chatMessages(
|
||||
[
|
||||
{ role: 'user', content: 'first' },
|
||||
{ role: 'user', content: 'last' },
|
||||
],
|
||||
'chat-model',
|
||||
);
|
||||
|
||||
assert.equal(res.content, 'OK');
|
||||
assert.equal(res.model, 'chat-model');
|
||||
assert.equal(spawnCalls.length, 1);
|
||||
assert.equal(spawnCalls[0].args.at(-1), 'last');
|
||||
});
|
||||
|
||||
it('getLiteLLMClient returns a singleton instance', () => {
|
||||
const c1 = mod.getLiteLLMClient();
|
||||
const c2 = mod.getLiteLLMClient();
|
||||
assert.equal(c1, c2);
|
||||
});
|
||||
|
||||
it('checkLiteLLMAvailable returns false when version check fails', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 1, stderr: 'ccw_litellm not installed' });
|
||||
|
||||
const available = await mod.checkLiteLLMAvailable();
|
||||
assert.equal(available, false);
|
||||
});
|
||||
|
||||
it('getLiteLLMStatus includes error message when unavailable', async () => {
|
||||
spawnPlan.push({ type: 'close', code: 1, stderr: 'ccw_litellm not installed' });
|
||||
|
||||
const status = await mod.getLiteLLMStatus();
|
||||
assert.equal(status.available, false);
|
||||
assert.ok(String(status.error).includes('ccw_litellm not installed'));
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user