diff --git a/ccw-litellm/README.md b/ccw-litellm/README.md new file mode 100644 index 00000000..70bac9c0 --- /dev/null +++ b/ccw-litellm/README.md @@ -0,0 +1,180 @@ +# ccw-litellm + +Unified LiteLLM interface layer shared by ccw and codex-lens projects. + +## Features + +- **Unified LLM Interface**: Abstract interface for LLM operations (chat, completion) +- **Unified Embedding Interface**: Abstract interface for text embeddings +- **Multi-Provider Support**: OpenAI, Anthropic, Azure, and more via LiteLLM +- **Configuration Management**: YAML-based configuration with environment variable substitution +- **Type Safety**: Full type annotations with Pydantic models + +## Installation + +```bash +pip install -e . +``` + +## Quick Start + +### Configuration + +Create a configuration file at `~/.ccw/config/litellm-config.yaml`: + +```yaml +version: 1 +default_provider: openai + +providers: + openai: + api_key: ${OPENAI_API_KEY} + api_base: https://api.openai.com/v1 + +llm_models: + default: + provider: openai + model: gpt-4 + +embedding_models: + default: + provider: openai + model: text-embedding-3-small + dimensions: 1536 +``` + +### Usage + +#### LLM Client + +```python +from ccw_litellm import LiteLLMClient, ChatMessage + +# Initialize client with default model +client = LiteLLMClient(model="default") + +# Chat completion +messages = [ + ChatMessage(role="user", content="Hello, how are you?") +] +response = client.chat(messages) +print(response.content) + +# Text completion +response = client.complete("Once upon a time") +print(response.content) +``` + +#### Embedder + +```python +from ccw_litellm import LiteLLMEmbedder + +# Initialize embedder with default model +embedder = LiteLLMEmbedder(model="default") + +# Embed single text +vector = embedder.embed("Hello world") +print(vector.shape) # (1, 1536) + +# Embed multiple texts +vectors = embedder.embed(["Text 1", "Text 2", "Text 3"]) +print(vectors.shape) # (3, 1536) +``` + +#### Custom Configuration + +```python +from ccw_litellm import LiteLLMClient, load_config + +# Load custom configuration +config = load_config("/path/to/custom-config.yaml") + +# Use custom configuration +client = LiteLLMClient(model="fast", config=config) +``` + +## Configuration Reference + +### Provider Configuration + +```yaml +providers: + : + api_key: + api_base: +``` + +Supported providers: `openai`, `anthropic`, `azure`, `vertex_ai`, `bedrock`, etc. + +### LLM Model Configuration + +```yaml +llm_models: + : + provider: + model: +``` + +### Embedding Model Configuration + +```yaml +embedding_models: + : + provider: + model: + dimensions: +``` + +## Environment Variables + +The configuration supports environment variable substitution using the `${VAR}` or `${VAR:-default}` syntax: + +```yaml +providers: + openai: + api_key: ${OPENAI_API_KEY} # Required + api_base: ${OPENAI_API_BASE:-https://api.openai.com/v1} # With default +``` + +## API Reference + +### Interfaces + +- `AbstractLLMClient`: Abstract base class for LLM clients +- `AbstractEmbedder`: Abstract base class for embedders +- `ChatMessage`: Message data class (role, content) +- `LLMResponse`: Response data class (content, raw) + +### Implementations + +- `LiteLLMClient`: LiteLLM implementation of AbstractLLMClient +- `LiteLLMEmbedder`: LiteLLM implementation of AbstractEmbedder + +### Configuration + +- `LiteLLMConfig`: Root configuration model +- `ProviderConfig`: Provider configuration model +- `LLMModelConfig`: LLM model configuration model +- `EmbeddingModelConfig`: Embedding model configuration model +- `load_config(path)`: Load configuration from YAML file +- `get_config(path, reload)`: Get global configuration singleton +- `reset_config()`: Reset global configuration (for testing) + +## Development + +### Running Tests + +```bash +pytest tests/ -v +``` + +### Type Checking + +```bash +mypy src/ccw_litellm +``` + +## License + +MIT diff --git a/ccw-litellm/litellm-config.yaml.example b/ccw-litellm/litellm-config.yaml.example new file mode 100644 index 00000000..752305de --- /dev/null +++ b/ccw-litellm/litellm-config.yaml.example @@ -0,0 +1,53 @@ +# LiteLLM Unified Configuration +# Copy to ~/.ccw/config/litellm-config.yaml + +version: 1 + +# Default provider for LLM calls +default_provider: openai + +# Provider configurations +providers: + openai: + api_key: ${OPENAI_API_KEY} + api_base: https://api.openai.com/v1 + + anthropic: + api_key: ${ANTHROPIC_API_KEY} + + ollama: + api_base: http://localhost:11434 + + azure: + api_key: ${AZURE_API_KEY} + api_base: ${AZURE_API_BASE} + +# LLM model configurations +llm_models: + default: + provider: openai + model: gpt-4o + fast: + provider: openai + model: gpt-4o-mini + claude: + provider: anthropic + model: claude-sonnet-4-20250514 + local: + provider: ollama + model: llama3.2 + +# Embedding model configurations +embedding_models: + default: + provider: openai + model: text-embedding-3-small + dimensions: 1536 + large: + provider: openai + model: text-embedding-3-large + dimensions: 3072 + ada: + provider: openai + model: text-embedding-ada-002 + dimensions: 1536 diff --git a/ccw-litellm/pyproject.toml b/ccw-litellm/pyproject.toml new file mode 100644 index 00000000..08d53e6d --- /dev/null +++ b/ccw-litellm/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "ccw-litellm" +version = "0.1.0" +description = "Unified LiteLLM interface layer shared by ccw and codex-lens" +requires-python = ">=3.10" +authors = [{ name = "ccw-litellm contributors" }] +dependencies = [ + "litellm>=1.0.0", + "pyyaml", + "numpy", + "pydantic>=2.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0", +] + +[project.scripts] +ccw-litellm = "ccw_litellm.cli:main" + +[tool.setuptools] +package-dir = { "" = "src" } + +[tool.setuptools.packages.find] +where = ["src"] +include = ["ccw_litellm*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" diff --git a/ccw-litellm/src/ccw_litellm.egg-info/PKG-INFO b/ccw-litellm/src/ccw_litellm.egg-info/PKG-INFO new file mode 100644 index 00000000..a4ef174e --- /dev/null +++ b/ccw-litellm/src/ccw_litellm.egg-info/PKG-INFO @@ -0,0 +1,12 @@ +Metadata-Version: 2.4 +Name: ccw-litellm +Version: 0.1.0 +Summary: Unified LiteLLM interface layer shared by ccw and codex-lens +Author: ccw-litellm contributors +Requires-Python: >=3.10 +Requires-Dist: litellm>=1.0.0 +Requires-Dist: pyyaml +Requires-Dist: numpy +Requires-Dist: pydantic>=2.0 +Provides-Extra: dev +Requires-Dist: pytest>=7.0; extra == "dev" diff --git a/ccw-litellm/src/ccw_litellm.egg-info/SOURCES.txt b/ccw-litellm/src/ccw_litellm.egg-info/SOURCES.txt new file mode 100644 index 00000000..479776fe --- /dev/null +++ b/ccw-litellm/src/ccw_litellm.egg-info/SOURCES.txt @@ -0,0 +1,17 @@ +pyproject.toml +src/ccw_litellm/__init__.py +src/ccw_litellm.egg-info/PKG-INFO +src/ccw_litellm.egg-info/SOURCES.txt +src/ccw_litellm.egg-info/dependency_links.txt +src/ccw_litellm.egg-info/requires.txt +src/ccw_litellm.egg-info/top_level.txt +src/ccw_litellm/clients/__init__.py +src/ccw_litellm/clients/litellm_embedder.py +src/ccw_litellm/clients/litellm_llm.py +src/ccw_litellm/config/__init__.py +src/ccw_litellm/config/loader.py +src/ccw_litellm/config/models.py +src/ccw_litellm/interfaces/__init__.py +src/ccw_litellm/interfaces/embedder.py +src/ccw_litellm/interfaces/llm.py +tests/test_interfaces.py \ No newline at end of file diff --git a/ccw-litellm/src/ccw_litellm.egg-info/dependency_links.txt b/ccw-litellm/src/ccw_litellm.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/ccw-litellm/src/ccw_litellm.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/ccw-litellm/src/ccw_litellm.egg-info/requires.txt b/ccw-litellm/src/ccw_litellm.egg-info/requires.txt new file mode 100644 index 00000000..c36eab78 --- /dev/null +++ b/ccw-litellm/src/ccw_litellm.egg-info/requires.txt @@ -0,0 +1,7 @@ +litellm>=1.0.0 +pyyaml +numpy +pydantic>=2.0 + +[dev] +pytest>=7.0 diff --git a/ccw-litellm/src/ccw_litellm.egg-info/top_level.txt b/ccw-litellm/src/ccw_litellm.egg-info/top_level.txt new file mode 100644 index 00000000..3489caa4 --- /dev/null +++ b/ccw-litellm/src/ccw_litellm.egg-info/top_level.txt @@ -0,0 +1 @@ +ccw_litellm diff --git a/ccw-litellm/src/ccw_litellm/__init__.py b/ccw-litellm/src/ccw_litellm/__init__.py new file mode 100644 index 00000000..8d49979b --- /dev/null +++ b/ccw-litellm/src/ccw_litellm/__init__.py @@ -0,0 +1,47 @@ +"""ccw-litellm package. + +This package provides a small, stable interface layer around LiteLLM to share +between the ccw and codex-lens projects. +""" + +from __future__ import annotations + +from .clients import LiteLLMClient, LiteLLMEmbedder +from .config import ( + EmbeddingModelConfig, + LiteLLMConfig, + LLMModelConfig, + ProviderConfig, + get_config, + load_config, + reset_config, +) +from .interfaces import ( + AbstractEmbedder, + AbstractLLMClient, + ChatMessage, + LLMResponse, +) + +__version__ = "0.1.0" + +__all__ = [ + "__version__", + # Abstract interfaces + "AbstractEmbedder", + "AbstractLLMClient", + "ChatMessage", + "LLMResponse", + # Client implementations + "LiteLLMClient", + "LiteLLMEmbedder", + # Configuration + "LiteLLMConfig", + "ProviderConfig", + "LLMModelConfig", + "EmbeddingModelConfig", + "load_config", + "get_config", + "reset_config", +] + diff --git a/ccw-litellm/src/ccw_litellm/cli.py b/ccw-litellm/src/ccw_litellm/cli.py new file mode 100644 index 00000000..75e664f2 --- /dev/null +++ b/ccw-litellm/src/ccw_litellm/cli.py @@ -0,0 +1,108 @@ +"""CLI entry point for ccw-litellm.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def main() -> int: + """Main CLI entry point.""" + parser = argparse.ArgumentParser( + prog="ccw-litellm", + description="Unified LiteLLM interface for ccw and codex-lens", + ) + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + # config command + config_parser = subparsers.add_parser("config", help="Show configuration") + config_parser.add_argument( + "--path", + type=Path, + help="Configuration file path", + ) + + # embed command + embed_parser = subparsers.add_parser("embed", help="Generate embeddings") + embed_parser.add_argument("texts", nargs="+", help="Texts to embed") + embed_parser.add_argument( + "--model", + default="default", + help="Embedding model name (default: default)", + ) + embed_parser.add_argument( + "--output", + choices=["json", "shape"], + default="shape", + help="Output format (default: shape)", + ) + + # chat command + chat_parser = subparsers.add_parser("chat", help="Chat with LLM") + chat_parser.add_argument("message", help="Message to send") + chat_parser.add_argument( + "--model", + default="default", + help="LLM model name (default: default)", + ) + + # version command + subparsers.add_parser("version", help="Show version") + + args = parser.parse_args() + + if args.command == "version": + from . import __version__ + + print(f"ccw-litellm {__version__}") + return 0 + + if args.command == "config": + from .config import get_config + + try: + config = get_config(config_path=args.path if hasattr(args, "path") else None) + print(config.model_dump_json(indent=2)) + except Exception as e: + print(f"Error loading config: {e}", file=sys.stderr) + return 1 + return 0 + + if args.command == "embed": + from .clients import LiteLLMEmbedder + + try: + embedder = LiteLLMEmbedder(model=args.model) + vectors = embedder.embed(args.texts) + + if args.output == "json": + print(json.dumps(vectors.tolist())) + else: + print(f"Shape: {vectors.shape}") + print(f"Dimensions: {embedder.dimensions}") + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + return 0 + + if args.command == "chat": + from .clients import LiteLLMClient + from .interfaces import ChatMessage + + try: + client = LiteLLMClient(model=args.model) + response = client.chat([ChatMessage(role="user", content=args.message)]) + print(response.content) + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + return 0 + + parser.print_help() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ccw-litellm/src/ccw_litellm/clients/__init__.py b/ccw-litellm/src/ccw_litellm/clients/__init__.py new file mode 100644 index 00000000..d5a66d48 --- /dev/null +++ b/ccw-litellm/src/ccw_litellm/clients/__init__.py @@ -0,0 +1,12 @@ +"""Client implementations for ccw-litellm.""" + +from __future__ import annotations + +from .litellm_embedder import LiteLLMEmbedder +from .litellm_llm import LiteLLMClient + +__all__ = [ + "LiteLLMClient", + "LiteLLMEmbedder", +] + diff --git a/ccw-litellm/src/ccw_litellm/clients/litellm_embedder.py b/ccw-litellm/src/ccw_litellm/clients/litellm_embedder.py new file mode 100644 index 00000000..b71cc7d5 --- /dev/null +++ b/ccw-litellm/src/ccw_litellm/clients/litellm_embedder.py @@ -0,0 +1,170 @@ +"""LiteLLM embedder implementation for text embeddings.""" + +from __future__ import annotations + +import logging +from typing import Any, Sequence + +import litellm +import numpy as np +from numpy.typing import NDArray + +from ..config import LiteLLMConfig, get_config +from ..interfaces.embedder import AbstractEmbedder + +logger = logging.getLogger(__name__) + + +class LiteLLMEmbedder(AbstractEmbedder): + """LiteLLM embedder implementation. + + Supports multiple embedding providers (OpenAI, etc.) through LiteLLM's unified interface. + + Example: + embedder = LiteLLMEmbedder(model="default") + vectors = embedder.embed(["Hello world", "Another text"]) + print(vectors.shape) # (2, 1536) + """ + + def __init__( + self, + model: str = "default", + config: LiteLLMConfig | None = None, + **litellm_kwargs: Any, + ) -> None: + """Initialize LiteLLM embedder. + + Args: + model: Model name from configuration (default: "default") + config: Configuration instance (default: use global config) + **litellm_kwargs: Additional arguments to pass to litellm.embedding() + """ + self._config = config or get_config() + self._model_name = model + self._litellm_kwargs = litellm_kwargs + + # Get embedding model configuration + try: + self._model_config = self._config.get_embedding_model(model) + except ValueError as e: + logger.error(f"Failed to get embedding model configuration: {e}") + raise + + # Get provider configuration + try: + self._provider_config = self._config.get_provider(self._model_config.provider) + except ValueError as e: + logger.error(f"Failed to get provider configuration: {e}") + raise + + # Set up LiteLLM environment + self._setup_litellm() + + def _setup_litellm(self) -> None: + """Configure LiteLLM with provider settings.""" + provider = self._model_config.provider + + # Set API key + if self._provider_config.api_key: + litellm.api_key = self._provider_config.api_key + # Also set environment-specific keys + if provider == "openai": + litellm.openai_key = self._provider_config.api_key + elif provider == "anthropic": + litellm.anthropic_key = self._provider_config.api_key + + # Set API base + if self._provider_config.api_base: + litellm.api_base = self._provider_config.api_base + + def _format_model_name(self) -> str: + """Format model name for LiteLLM. + + Returns: + Formatted model name (e.g., "text-embedding-3-small") + """ + provider = self._model_config.provider + model = self._model_config.model + + # For some providers, LiteLLM expects explicit prefix + if provider in ["azure", "vertex_ai", "bedrock"]: + return f"{provider}/{model}" + + return model + + @property + def dimensions(self) -> int: + """Embedding vector size.""" + return self._model_config.dimensions + + def embed( + self, + texts: str | Sequence[str], + *, + batch_size: int | None = None, + **kwargs: Any, + ) -> NDArray[np.floating]: + """Embed one or more texts. + + Args: + texts: Single text or sequence of texts + batch_size: Batch size for processing (currently unused, LiteLLM handles batching) + **kwargs: Additional arguments for litellm.embedding() + + Returns: + A numpy array of shape (n_texts, dimensions). + + Raises: + Exception: If LiteLLM embedding fails + """ + # Normalize input to list + if isinstance(texts, str): + text_list = [texts] + single_input = True + else: + text_list = list(texts) + single_input = False + + if not text_list: + # Return empty array with correct shape + return np.empty((0, self.dimensions), dtype=np.float32) + + # Merge kwargs + embedding_kwargs = {**self._litellm_kwargs, **kwargs} + + try: + # Call LiteLLM embedding + response = litellm.embedding( + model=self._format_model_name(), + input=text_list, + **embedding_kwargs, + ) + + # Extract embeddings + embeddings = [item["embedding"] for item in response.data] + + # Convert to numpy array + result = np.array(embeddings, dtype=np.float32) + + # Validate dimensions + if result.shape[1] != self.dimensions: + logger.warning( + f"Expected {self.dimensions} dimensions, got {result.shape[1]}. " + f"Configuration may be incorrect." + ) + + return result + + except Exception as e: + logger.error(f"LiteLLM embedding failed: {e}") + raise + + @property + def model_name(self) -> str: + """Get configured model name.""" + return self._model_name + + @property + def provider(self) -> str: + """Get configured provider name.""" + return self._model_config.provider diff --git a/ccw-litellm/src/ccw_litellm/clients/litellm_llm.py b/ccw-litellm/src/ccw_litellm/clients/litellm_llm.py new file mode 100644 index 00000000..54a7b450 --- /dev/null +++ b/ccw-litellm/src/ccw_litellm/clients/litellm_llm.py @@ -0,0 +1,165 @@ +"""LiteLLM client implementation for LLM operations.""" + +from __future__ import annotations + +import logging +from typing import Any, Sequence + +import litellm + +from ..config import LiteLLMConfig, get_config +from ..interfaces.llm import AbstractLLMClient, ChatMessage, LLMResponse + +logger = logging.getLogger(__name__) + + +class LiteLLMClient(AbstractLLMClient): + """LiteLLM client implementation. + + Supports multiple providers (OpenAI, Anthropic, etc.) through LiteLLM's unified interface. + + Example: + client = LiteLLMClient(model="default") + response = client.chat([ + ChatMessage(role="user", content="Hello!") + ]) + print(response.content) + """ + + def __init__( + self, + model: str = "default", + config: LiteLLMConfig | None = None, + **litellm_kwargs: Any, + ) -> None: + """Initialize LiteLLM client. + + Args: + model: Model name from configuration (default: "default") + config: Configuration instance (default: use global config) + **litellm_kwargs: Additional arguments to pass to litellm.completion() + """ + self._config = config or get_config() + self._model_name = model + self._litellm_kwargs = litellm_kwargs + + # Get model configuration + try: + self._model_config = self._config.get_llm_model(model) + except ValueError as e: + logger.error(f"Failed to get model configuration: {e}") + raise + + # Get provider configuration + try: + self._provider_config = self._config.get_provider(self._model_config.provider) + except ValueError as e: + logger.error(f"Failed to get provider configuration: {e}") + raise + + # Set up LiteLLM environment + self._setup_litellm() + + def _setup_litellm(self) -> None: + """Configure LiteLLM with provider settings.""" + provider = self._model_config.provider + + # Set API key + if self._provider_config.api_key: + env_var = f"{provider.upper()}_API_KEY" + litellm.api_key = self._provider_config.api_key + # Also set environment-specific keys + if provider == "openai": + litellm.openai_key = self._provider_config.api_key + elif provider == "anthropic": + litellm.anthropic_key = self._provider_config.api_key + + # Set API base + if self._provider_config.api_base: + litellm.api_base = self._provider_config.api_base + + def _format_model_name(self) -> str: + """Format model name for LiteLLM. + + Returns: + Formatted model name (e.g., "gpt-4", "claude-3-opus-20240229") + """ + # LiteLLM expects model names in format: "provider/model" or just "model" + # If provider is explicit, use provider/model format + provider = self._model_config.provider + model = self._model_config.model + + # For some providers, LiteLLM expects explicit prefix + if provider in ["anthropic", "azure", "vertex_ai", "bedrock"]: + return f"{provider}/{model}" + + return model + + def chat( + self, + messages: Sequence[ChatMessage], + **kwargs: Any, + ) -> LLMResponse: + """Chat completion for a sequence of messages. + + Args: + messages: Sequence of chat messages + **kwargs: Additional arguments for litellm.completion() + + Returns: + LLM response with content and raw response + + Raises: + Exception: If LiteLLM completion fails + """ + # Convert messages to LiteLLM format + litellm_messages = [ + {"role": msg.role, "content": msg.content} for msg in messages + ] + + # Merge kwargs + completion_kwargs = {**self._litellm_kwargs, **kwargs} + + try: + # Call LiteLLM + response = litellm.completion( + model=self._format_model_name(), + messages=litellm_messages, + **completion_kwargs, + ) + + # Extract content + content = response.choices[0].message.content or "" + + return LLMResponse(content=content, raw=response) + + except Exception as e: + logger.error(f"LiteLLM completion failed: {e}") + raise + + def complete(self, prompt: str, **kwargs: Any) -> LLMResponse: + """Text completion for a prompt. + + Args: + prompt: Input prompt + **kwargs: Additional arguments for litellm.completion() + + Returns: + LLM response with content and raw response + + Raises: + Exception: If LiteLLM completion fails + """ + # Convert to chat format (most modern models use chat interface) + messages = [ChatMessage(role="user", content=prompt)] + return self.chat(messages, **kwargs) + + @property + def model_name(self) -> str: + """Get configured model name.""" + return self._model_name + + @property + def provider(self) -> str: + """Get configured provider name.""" + return self._model_config.provider diff --git a/ccw-litellm/src/ccw_litellm/config/__init__.py b/ccw-litellm/src/ccw_litellm/config/__init__.py new file mode 100644 index 00000000..027c95e6 --- /dev/null +++ b/ccw-litellm/src/ccw_litellm/config/__init__.py @@ -0,0 +1,22 @@ +"""Configuration management for LiteLLM integration.""" + +from __future__ import annotations + +from .loader import get_config, load_config, reset_config +from .models import ( + EmbeddingModelConfig, + LiteLLMConfig, + LLMModelConfig, + ProviderConfig, +) + +__all__ = [ + "LiteLLMConfig", + "ProviderConfig", + "LLMModelConfig", + "EmbeddingModelConfig", + "load_config", + "get_config", + "reset_config", +] + diff --git a/ccw-litellm/src/ccw_litellm/config/loader.py b/ccw-litellm/src/ccw_litellm/config/loader.py new file mode 100644 index 00000000..d738d09f --- /dev/null +++ b/ccw-litellm/src/ccw_litellm/config/loader.py @@ -0,0 +1,150 @@ +"""Configuration loader with environment variable substitution.""" + +from __future__ import annotations + +import os +import re +from pathlib import Path +from typing import Any + +import yaml + +from .models import LiteLLMConfig + +# Default configuration path +DEFAULT_CONFIG_PATH = Path.home() / ".ccw" / "config" / "litellm-config.yaml" + +# Global configuration singleton +_config_instance: LiteLLMConfig | None = None + + +def _substitute_env_vars(value: Any) -> Any: + """Recursively substitute environment variables in configuration values. + + Supports ${ENV_VAR} and ${ENV_VAR:-default} syntax. + + Args: + value: Configuration value (str, dict, list, or primitive) + + Returns: + Value with environment variables substituted + """ + if isinstance(value, str): + # Pattern: ${VAR} or ${VAR:-default} + pattern = r"\$\{([^:}]+)(?::-(.*?))?\}" + + def replace_var(match: re.Match) -> str: + var_name = match.group(1) + default_value = match.group(2) if match.group(2) is not None else "" + return os.environ.get(var_name, default_value) + + return re.sub(pattern, replace_var, value) + + if isinstance(value, dict): + return {k: _substitute_env_vars(v) for k, v in value.items()} + + if isinstance(value, list): + return [_substitute_env_vars(item) for item in value] + + return value + + +def _get_default_config() -> dict[str, Any]: + """Get default configuration when no config file exists. + + Returns: + Default configuration dictionary + """ + return { + "version": 1, + "default_provider": "openai", + "providers": { + "openai": { + "api_key": "${OPENAI_API_KEY}", + "api_base": "https://api.openai.com/v1", + }, + }, + "llm_models": { + "default": { + "provider": "openai", + "model": "gpt-4", + }, + "fast": { + "provider": "openai", + "model": "gpt-3.5-turbo", + }, + }, + "embedding_models": { + "default": { + "provider": "openai", + "model": "text-embedding-3-small", + "dimensions": 1536, + }, + }, + } + + +def load_config(config_path: Path | str | None = None) -> LiteLLMConfig: + """Load LiteLLM configuration from YAML file. + + Args: + config_path: Path to configuration file (default: ~/.ccw/config/litellm-config.yaml) + + Returns: + Parsed and validated configuration + + Raises: + FileNotFoundError: If config file not found and no default available + ValueError: If configuration is invalid + """ + if config_path is None: + config_path = DEFAULT_CONFIG_PATH + else: + config_path = Path(config_path) + + # Load configuration + if config_path.exists(): + try: + with open(config_path, "r", encoding="utf-8") as f: + raw_config = yaml.safe_load(f) + except Exception as e: + raise ValueError(f"Failed to load configuration from {config_path}: {e}") from e + else: + # Use default configuration + raw_config = _get_default_config() + + # Substitute environment variables + config_data = _substitute_env_vars(raw_config) + + # Validate and parse with Pydantic + try: + return LiteLLMConfig.model_validate(config_data) + except Exception as e: + raise ValueError(f"Invalid configuration: {e}") from e + + +def get_config(config_path: Path | str | None = None, reload: bool = False) -> LiteLLMConfig: + """Get global configuration singleton. + + Args: + config_path: Path to configuration file (default: ~/.ccw/config/litellm-config.yaml) + reload: Force reload configuration from disk + + Returns: + Global configuration instance + """ + global _config_instance + + if _config_instance is None or reload: + _config_instance = load_config(config_path) + + return _config_instance + + +def reset_config() -> None: + """Reset global configuration singleton. + + Useful for testing. + """ + global _config_instance + _config_instance = None diff --git a/ccw-litellm/src/ccw_litellm/config/models.py b/ccw-litellm/src/ccw_litellm/config/models.py new file mode 100644 index 00000000..ee76cdc8 --- /dev/null +++ b/ccw-litellm/src/ccw_litellm/config/models.py @@ -0,0 +1,130 @@ +"""Pydantic configuration models for LiteLLM integration.""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field + + +class ProviderConfig(BaseModel): + """Provider API configuration. + + Supports environment variable substitution in the format ${ENV_VAR}. + """ + + api_key: str | None = None + api_base: str | None = None + + model_config = {"extra": "allow"} + + +class LLMModelConfig(BaseModel): + """LLM model configuration.""" + + provider: str + model: str + + model_config = {"extra": "allow"} + + +class EmbeddingModelConfig(BaseModel): + """Embedding model configuration.""" + + provider: str # "openai", "fastembed", "ollama", etc. + model: str + dimensions: int + + model_config = {"extra": "allow"} + + +class LiteLLMConfig(BaseModel): + """Root configuration for LiteLLM integration. + + Example YAML: + version: 1 + default_provider: openai + providers: + openai: + api_key: ${OPENAI_API_KEY} + api_base: https://api.openai.com/v1 + anthropic: + api_key: ${ANTHROPIC_API_KEY} + llm_models: + default: + provider: openai + model: gpt-4 + fast: + provider: openai + model: gpt-3.5-turbo + embedding_models: + default: + provider: openai + model: text-embedding-3-small + dimensions: 1536 + """ + + version: int = 1 + default_provider: str = "openai" + providers: dict[str, ProviderConfig] = Field(default_factory=dict) + llm_models: dict[str, LLMModelConfig] = Field(default_factory=dict) + embedding_models: dict[str, EmbeddingModelConfig] = Field(default_factory=dict) + + model_config = {"extra": "allow"} + + def get_llm_model(self, model: str = "default") -> LLMModelConfig: + """Get LLM model configuration by name. + + Args: + model: Model name or "default" + + Returns: + LLM model configuration + + Raises: + ValueError: If model not found + """ + if model not in self.llm_models: + raise ValueError( + f"LLM model '{model}' not found in configuration. " + f"Available models: {list(self.llm_models.keys())}" + ) + return self.llm_models[model] + + def get_embedding_model(self, model: str = "default") -> EmbeddingModelConfig: + """Get embedding model configuration by name. + + Args: + model: Model name or "default" + + Returns: + Embedding model configuration + + Raises: + ValueError: If model not found + """ + if model not in self.embedding_models: + raise ValueError( + f"Embedding model '{model}' not found in configuration. " + f"Available models: {list(self.embedding_models.keys())}" + ) + return self.embedding_models[model] + + def get_provider(self, provider: str) -> ProviderConfig: + """Get provider configuration by name. + + Args: + provider: Provider name + + Returns: + Provider configuration + + Raises: + ValueError: If provider not found + """ + if provider not in self.providers: + raise ValueError( + f"Provider '{provider}' not found in configuration. " + f"Available providers: {list(self.providers.keys())}" + ) + return self.providers[provider] diff --git a/ccw-litellm/src/ccw_litellm/interfaces/__init__.py b/ccw-litellm/src/ccw_litellm/interfaces/__init__.py new file mode 100644 index 00000000..97a615c1 --- /dev/null +++ b/ccw-litellm/src/ccw_litellm/interfaces/__init__.py @@ -0,0 +1,14 @@ +"""Abstract interfaces for ccw-litellm.""" + +from __future__ import annotations + +from .embedder import AbstractEmbedder +from .llm import AbstractLLMClient, ChatMessage, LLMResponse + +__all__ = [ + "AbstractEmbedder", + "AbstractLLMClient", + "ChatMessage", + "LLMResponse", +] + diff --git a/ccw-litellm/src/ccw_litellm/interfaces/embedder.py b/ccw-litellm/src/ccw_litellm/interfaces/embedder.py new file mode 100644 index 00000000..3816d436 --- /dev/null +++ b/ccw-litellm/src/ccw_litellm/interfaces/embedder.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import asyncio +from abc import ABC, abstractmethod +from typing import Any, Sequence + +import numpy as np +from numpy.typing import NDArray + + +class AbstractEmbedder(ABC): + """Embedding interface compatible with fastembed-style embedders. + + Implementers only need to provide the synchronous `embed` method; an + asynchronous `aembed` wrapper is provided for convenience. + """ + + @property + @abstractmethod + def dimensions(self) -> int: + """Embedding vector size.""" + + @abstractmethod + def embed( + self, + texts: str | Sequence[str], + *, + batch_size: int | None = None, + **kwargs: Any, + ) -> NDArray[np.floating]: + """Embed one or more texts. + + Returns: + A numpy array of shape (n_texts, dimensions). + """ + + async def aembed( + self, + texts: str | Sequence[str], + *, + batch_size: int | None = None, + **kwargs: Any, + ) -> NDArray[np.floating]: + """Async wrapper around `embed` using a worker thread by default.""" + + return await asyncio.to_thread( + self.embed, + texts, + batch_size=batch_size, + **kwargs, + ) + diff --git a/ccw-litellm/src/ccw_litellm/interfaces/llm.py b/ccw-litellm/src/ccw_litellm/interfaces/llm.py new file mode 100644 index 00000000..6d2e17a4 --- /dev/null +++ b/ccw-litellm/src/ccw_litellm/interfaces/llm.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import asyncio +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, Literal, Sequence + + +@dataclass(frozen=True, slots=True) +class ChatMessage: + role: Literal["system", "user", "assistant", "tool"] + content: str + + +@dataclass(frozen=True, slots=True) +class LLMResponse: + content: str + raw: Any | None = None + + +class AbstractLLMClient(ABC): + """LiteLLM-like client interface. + + Implementers only need to provide synchronous methods; async wrappers are + provided via `asyncio.to_thread`. + """ + + @abstractmethod + def chat(self, messages: Sequence[ChatMessage], **kwargs: Any) -> LLMResponse: + """Chat completion for a sequence of messages.""" + + @abstractmethod + def complete(self, prompt: str, **kwargs: Any) -> LLMResponse: + """Text completion for a prompt.""" + + async def achat(self, messages: Sequence[ChatMessage], **kwargs: Any) -> LLMResponse: + """Async wrapper around `chat` using a worker thread by default.""" + + return await asyncio.to_thread(self.chat, messages, **kwargs) + + async def acomplete(self, prompt: str, **kwargs: Any) -> LLMResponse: + """Async wrapper around `complete` using a worker thread by default.""" + + return await asyncio.to_thread(self.complete, prompt, **kwargs) + diff --git a/ccw-litellm/tests/conftest.py b/ccw-litellm/tests/conftest.py new file mode 100644 index 00000000..6ccd7722 --- /dev/null +++ b/ccw-litellm/tests/conftest.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +import sys +from pathlib import Path + + +def pytest_configure() -> None: + project_root = Path(__file__).resolve().parents[1] + src_dir = project_root / "src" + sys.path.insert(0, str(src_dir)) + diff --git a/ccw-litellm/tests/test_interfaces.py b/ccw-litellm/tests/test_interfaces.py new file mode 100644 index 00000000..558ab5fd --- /dev/null +++ b/ccw-litellm/tests/test_interfaces.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import asyncio +from typing import Any, Sequence + +import numpy as np + +from ccw_litellm.interfaces import AbstractEmbedder, AbstractLLMClient, ChatMessage, LLMResponse + + +class _DummyEmbedder(AbstractEmbedder): + @property + def dimensions(self) -> int: + return 3 + + def embed( + self, + texts: str | Sequence[str], + *, + batch_size: int | None = None, + **kwargs: Any, + ) -> np.ndarray: + if isinstance(texts, str): + texts = [texts] + _ = batch_size + _ = kwargs + return np.zeros((len(texts), self.dimensions), dtype=np.float32) + + +class _DummyLLM(AbstractLLMClient): + def chat(self, messages: Sequence[ChatMessage], **kwargs: Any) -> LLMResponse: + _ = kwargs + return LLMResponse(content="".join(m.content for m in messages)) + + def complete(self, prompt: str, **kwargs: Any) -> LLMResponse: + _ = kwargs + return LLMResponse(content=prompt) + + +def test_embed_sync_shape_and_dtype() -> None: + emb = _DummyEmbedder() + out = emb.embed(["a", "b"]) + assert out.shape == (2, 3) + assert out.dtype == np.float32 + + +def test_embed_async_wrapper() -> None: + emb = _DummyEmbedder() + out = asyncio.run(emb.aembed("x")) + assert out.shape == (1, 3) + + +def test_llm_sync() -> None: + llm = _DummyLLM() + out = llm.chat([ChatMessage(role="user", content="hi")]) + assert out == LLMResponse(content="hi") + + +def test_llm_async_wrappers() -> None: + llm = _DummyLLM() + out1 = asyncio.run(llm.achat([ChatMessage(role="user", content="a")])) + out2 = asyncio.run(llm.acomplete("b")) + assert out1.content == "a" + assert out2.content == "b" diff --git a/ccw/src/config/litellm-api-config-manager.ts b/ccw/src/config/litellm-api-config-manager.ts new file mode 100644 index 00000000..81bd0e27 --- /dev/null +++ b/ccw/src/config/litellm-api-config-manager.ts @@ -0,0 +1,360 @@ +/** + * LiteLLM API Configuration Manager + * Manages provider credentials, custom endpoints, and cache settings + */ + +import { existsSync, readFileSync, writeFileSync } from 'fs'; +import { join } from 'path'; +import { StoragePaths, ensureStorageDir } from './storage-paths.js'; +import type { + LiteLLMApiConfig, + ProviderCredential, + CustomEndpoint, + GlobalCacheSettings, + ProviderType, + CacheStrategy, +} from '../types/litellm-api-config.js'; + +/** + * Default configuration + */ +function getDefaultConfig(): LiteLLMApiConfig { + return { + version: 1, + providers: [], + endpoints: [], + globalCacheSettings: { + enabled: true, + cacheDir: '~/.ccw/cache/context', + maxTotalSizeMB: 100, + }, + }; +} + +/** + * Get config file path for a project + */ +function getConfigPath(baseDir: string): string { + const paths = StoragePaths.project(baseDir); + ensureStorageDir(paths.config); + return join(paths.config, 'litellm-api-config.json'); +} + +/** + * Load configuration from file + */ +export function loadLiteLLMApiConfig(baseDir: string): LiteLLMApiConfig { + const configPath = getConfigPath(baseDir); + + if (!existsSync(configPath)) { + return getDefaultConfig(); + } + + try { + const content = readFileSync(configPath, 'utf-8'); + return JSON.parse(content) as LiteLLMApiConfig; + } catch (error) { + console.error('[LiteLLM Config] Failed to load config:', error); + return getDefaultConfig(); + } +} + +/** + * Save configuration to file + */ +function saveConfig(baseDir: string, config: LiteLLMApiConfig): void { + const configPath = getConfigPath(baseDir); + writeFileSync(configPath, JSON.stringify(config, null, 2), 'utf-8'); +} + +/** + * Resolve environment variables in API key + * Supports ${ENV_VAR} syntax + */ +export function resolveEnvVar(value: string): string { + if (!value) return value; + + const envVarMatch = value.match(/^\$\{(.+)\}$/); + if (envVarMatch) { + const envVarName = envVarMatch[1]; + return process.env[envVarName] || ''; + } + + return value; +} + +// =========================== +// Provider Management +// =========================== + +/** + * Get all providers + */ +export function getAllProviders(baseDir: string): ProviderCredential[] { + const config = loadLiteLLMApiConfig(baseDir); + return config.providers; +} + +/** + * Get provider by ID + */ +export function getProvider(baseDir: string, providerId: string): ProviderCredential | null { + const config = loadLiteLLMApiConfig(baseDir); + return config.providers.find((p) => p.id === providerId) || null; +} + +/** + * Get provider with resolved environment variables + */ +export function getProviderWithResolvedEnvVars( + baseDir: string, + providerId: string +): (ProviderCredential & { resolvedApiKey: string }) | null { + const provider = getProvider(baseDir, providerId); + if (!provider) return null; + + return { + ...provider, + resolvedApiKey: resolveEnvVar(provider.apiKey), + }; +} + +/** + * Add new provider + */ +export function addProvider( + baseDir: string, + providerData: Omit +): ProviderCredential { + const config = loadLiteLLMApiConfig(baseDir); + + const provider: ProviderCredential = { + ...providerData, + id: `${providerData.type}-${Date.now()}`, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }; + + config.providers.push(provider); + saveConfig(baseDir, config); + + return provider; +} + +/** + * Update provider + */ +export function updateProvider( + baseDir: string, + providerId: string, + updates: Partial> +): ProviderCredential { + const config = loadLiteLLMApiConfig(baseDir); + const providerIndex = config.providers.findIndex((p) => p.id === providerId); + + if (providerIndex === -1) { + throw new Error(`Provider not found: ${providerId}`); + } + + config.providers[providerIndex] = { + ...config.providers[providerIndex], + ...updates, + updatedAt: new Date().toISOString(), + }; + + saveConfig(baseDir, config); + return config.providers[providerIndex]; +} + +/** + * Delete provider + */ +export function deleteProvider(baseDir: string, providerId: string): boolean { + const config = loadLiteLLMApiConfig(baseDir); + const initialLength = config.providers.length; + + config.providers = config.providers.filter((p) => p.id !== providerId); + + if (config.providers.length === initialLength) { + return false; + } + + // Also remove endpoints using this provider + config.endpoints = config.endpoints.filter((e) => e.providerId !== providerId); + + saveConfig(baseDir, config); + return true; +} + +// =========================== +// Endpoint Management +// =========================== + +/** + * Get all endpoints + */ +export function getAllEndpoints(baseDir: string): CustomEndpoint[] { + const config = loadLiteLLMApiConfig(baseDir); + return config.endpoints; +} + +/** + * Get endpoint by ID + */ +export function getEndpoint(baseDir: string, endpointId: string): CustomEndpoint | null { + const config = loadLiteLLMApiConfig(baseDir); + return config.endpoints.find((e) => e.id === endpointId) || null; +} + +/** + * Find endpoint by ID (alias for getEndpoint) + */ +export function findEndpointById(baseDir: string, endpointId: string): CustomEndpoint | null { + return getEndpoint(baseDir, endpointId); +} + +/** + * Add new endpoint + */ +export function addEndpoint( + baseDir: string, + endpointData: Omit +): CustomEndpoint { + const config = loadLiteLLMApiConfig(baseDir); + + // Check if ID already exists + if (config.endpoints.some((e) => e.id === endpointData.id)) { + throw new Error(`Endpoint ID already exists: ${endpointData.id}`); + } + + // Verify provider exists + if (!config.providers.find((p) => p.id === endpointData.providerId)) { + throw new Error(`Provider not found: ${endpointData.providerId}`); + } + + const endpoint: CustomEndpoint = { + ...endpointData, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }; + + config.endpoints.push(endpoint); + saveConfig(baseDir, config); + + return endpoint; +} + +/** + * Update endpoint + */ +export function updateEndpoint( + baseDir: string, + endpointId: string, + updates: Partial> +): CustomEndpoint { + const config = loadLiteLLMApiConfig(baseDir); + const endpointIndex = config.endpoints.findIndex((e) => e.id === endpointId); + + if (endpointIndex === -1) { + throw new Error(`Endpoint not found: ${endpointId}`); + } + + // Verify provider exists if updating providerId + if (updates.providerId && !config.providers.find((p) => p.id === updates.providerId)) { + throw new Error(`Provider not found: ${updates.providerId}`); + } + + config.endpoints[endpointIndex] = { + ...config.endpoints[endpointIndex], + ...updates, + updatedAt: new Date().toISOString(), + }; + + saveConfig(baseDir, config); + return config.endpoints[endpointIndex]; +} + +/** + * Delete endpoint + */ +export function deleteEndpoint(baseDir: string, endpointId: string): boolean { + const config = loadLiteLLMApiConfig(baseDir); + const initialLength = config.endpoints.length; + + config.endpoints = config.endpoints.filter((e) => e.id !== endpointId); + + if (config.endpoints.length === initialLength) { + return false; + } + + // Clear default endpoint if deleted + if (config.defaultEndpoint === endpointId) { + delete config.defaultEndpoint; + } + + saveConfig(baseDir, config); + return true; +} + +// =========================== +// Default Endpoint Management +// =========================== + +/** + * Get default endpoint + */ +export function getDefaultEndpoint(baseDir: string): string | undefined { + const config = loadLiteLLMApiConfig(baseDir); + return config.defaultEndpoint; +} + +/** + * Set default endpoint + */ +export function setDefaultEndpoint(baseDir: string, endpointId?: string): void { + const config = loadLiteLLMApiConfig(baseDir); + + if (endpointId) { + // Verify endpoint exists + if (!config.endpoints.find((e) => e.id === endpointId)) { + throw new Error(`Endpoint not found: ${endpointId}`); + } + config.defaultEndpoint = endpointId; + } else { + delete config.defaultEndpoint; + } + + saveConfig(baseDir, config); +} + +// =========================== +// Cache Settings Management +// =========================== + +/** + * Get global cache settings + */ +export function getGlobalCacheSettings(baseDir: string): GlobalCacheSettings { + const config = loadLiteLLMApiConfig(baseDir); + return config.globalCacheSettings; +} + +/** + * Update global cache settings + */ +export function updateGlobalCacheSettings( + baseDir: string, + settings: Partial +): void { + const config = loadLiteLLMApiConfig(baseDir); + + config.globalCacheSettings = { + ...config.globalCacheSettings, + ...settings, + }; + + saveConfig(baseDir, config); +} + +// Re-export types +export type { ProviderCredential, CustomEndpoint, ProviderType, CacheStrategy }; diff --git a/ccw/src/config/provider-models.ts b/ccw/src/config/provider-models.ts new file mode 100644 index 00000000..86c1e4c2 --- /dev/null +++ b/ccw/src/config/provider-models.ts @@ -0,0 +1,259 @@ +/** + * Provider Model Presets + * + * Predefined model information for each supported LLM provider. + * Used for UI dropdowns and validation. + */ + +import type { ProviderType } from '../types/litellm-api-config.js'; + +/** + * Model information metadata + */ +export interface ModelInfo { + /** Model identifier (used in API calls) */ + id: string; + + /** Human-readable display name */ + name: string; + + /** Context window size in tokens */ + contextWindow: number; + + /** Whether this model supports prompt caching */ + supportsCaching: boolean; +} + +/** + * Predefined models for each provider + * Used for UI selection and validation + */ +export const PROVIDER_MODELS: Record = { + openai: [ + { + id: 'gpt-4o', + name: 'GPT-4o', + contextWindow: 128000, + supportsCaching: true + }, + { + id: 'gpt-4o-mini', + name: 'GPT-4o Mini', + contextWindow: 128000, + supportsCaching: true + }, + { + id: 'o1', + name: 'O1', + contextWindow: 200000, + supportsCaching: true + }, + { + id: 'o1-mini', + name: 'O1 Mini', + contextWindow: 128000, + supportsCaching: true + }, + { + id: 'gpt-4-turbo', + name: 'GPT-4 Turbo', + contextWindow: 128000, + supportsCaching: false + } + ], + + anthropic: [ + { + id: 'claude-sonnet-4-20250514', + name: 'Claude Sonnet 4', + contextWindow: 200000, + supportsCaching: true + }, + { + id: 'claude-3-5-sonnet-20241022', + name: 'Claude 3.5 Sonnet', + contextWindow: 200000, + supportsCaching: true + }, + { + id: 'claude-3-5-haiku-20241022', + name: 'Claude 3.5 Haiku', + contextWindow: 200000, + supportsCaching: true + }, + { + id: 'claude-3-opus-20240229', + name: 'Claude 3 Opus', + contextWindow: 200000, + supportsCaching: false + } + ], + + ollama: [ + { + id: 'llama3.2', + name: 'Llama 3.2', + contextWindow: 128000, + supportsCaching: false + }, + { + id: 'llama3.1', + name: 'Llama 3.1', + contextWindow: 128000, + supportsCaching: false + }, + { + id: 'qwen2.5-coder', + name: 'Qwen 2.5 Coder', + contextWindow: 32000, + supportsCaching: false + }, + { + id: 'codellama', + name: 'Code Llama', + contextWindow: 16000, + supportsCaching: false + }, + { + id: 'mistral', + name: 'Mistral', + contextWindow: 32000, + supportsCaching: false + } + ], + + azure: [ + { + id: 'gpt-4o', + name: 'GPT-4o (Azure)', + contextWindow: 128000, + supportsCaching: true + }, + { + id: 'gpt-4o-mini', + name: 'GPT-4o Mini (Azure)', + contextWindow: 128000, + supportsCaching: true + }, + { + id: 'gpt-4-turbo', + name: 'GPT-4 Turbo (Azure)', + contextWindow: 128000, + supportsCaching: false + }, + { + id: 'gpt-35-turbo', + name: 'GPT-3.5 Turbo (Azure)', + contextWindow: 16000, + supportsCaching: false + } + ], + + google: [ + { + id: 'gemini-2.0-flash-exp', + name: 'Gemini 2.0 Flash Experimental', + contextWindow: 1048576, + supportsCaching: true + }, + { + id: 'gemini-1.5-pro', + name: 'Gemini 1.5 Pro', + contextWindow: 2097152, + supportsCaching: true + }, + { + id: 'gemini-1.5-flash', + name: 'Gemini 1.5 Flash', + contextWindow: 1048576, + supportsCaching: true + }, + { + id: 'gemini-1.0-pro', + name: 'Gemini 1.0 Pro', + contextWindow: 32000, + supportsCaching: false + } + ], + + mistral: [ + { + id: 'mistral-large-latest', + name: 'Mistral Large', + contextWindow: 128000, + supportsCaching: false + }, + { + id: 'mistral-medium-latest', + name: 'Mistral Medium', + contextWindow: 32000, + supportsCaching: false + }, + { + id: 'mistral-small-latest', + name: 'Mistral Small', + contextWindow: 32000, + supportsCaching: false + }, + { + id: 'codestral-latest', + name: 'Codestral', + contextWindow: 32000, + supportsCaching: false + } + ], + + deepseek: [ + { + id: 'deepseek-chat', + name: 'DeepSeek Chat', + contextWindow: 64000, + supportsCaching: false + }, + { + id: 'deepseek-coder', + name: 'DeepSeek Coder', + contextWindow: 64000, + supportsCaching: false + } + ], + + custom: [ + { + id: 'custom-model', + name: 'Custom Model', + contextWindow: 128000, + supportsCaching: false + } + ] +}; + +/** + * Get models for a specific provider + * @param providerType - Provider type to get models for + * @returns Array of model information + */ +export function getModelsForProvider(providerType: ProviderType): ModelInfo[] { + return PROVIDER_MODELS[providerType] || []; +} + +/** + * Get model information by ID within a provider + * @param providerType - Provider type + * @param modelId - Model identifier + * @returns Model information or undefined if not found + */ +export function getModelInfo(providerType: ProviderType, modelId: string): ModelInfo | undefined { + const models = PROVIDER_MODELS[providerType] || []; + return models.find(m => m.id === modelId); +} + +/** + * Validate if a model ID is supported by a provider + * @param providerType - Provider type + * @param modelId - Model identifier to validate + * @returns true if model is valid for provider + */ +export function isValidModel(providerType: ProviderType, modelId: string): boolean { + return getModelInfo(providerType, modelId) !== undefined; +} diff --git a/ccw/src/core/dashboard-generator.ts b/ccw/src/core/dashboard-generator.ts index cbe798c3..d1dfc358 100644 --- a/ccw/src/core/dashboard-generator.ts +++ b/ccw/src/core/dashboard-generator.ts @@ -46,7 +46,8 @@ const MODULE_CSS_FILES = [ '27-graph-explorer.css', '28-mcp-manager.css', '29-help.css', - '30-core-memory.css' + '30-core-memory.css', + '31-api-settings.css' ]; const MODULE_FILES = [ @@ -95,6 +96,7 @@ const MODULE_FILES = [ 'views/skills-manager.js', 'views/rules-manager.js', 'views/claude-manager.js', + 'views/api-settings.js', 'views/help.js', 'main.js' ]; diff --git a/ccw/src/core/routes/litellm-api-routes.ts b/ccw/src/core/routes/litellm-api-routes.ts new file mode 100644 index 00000000..ae5127b2 --- /dev/null +++ b/ccw/src/core/routes/litellm-api-routes.ts @@ -0,0 +1,485 @@ +// @ts-nocheck +/** + * LiteLLM API Routes Module + * Handles LiteLLM provider management, endpoint configuration, and cache management + */ +import type { IncomingMessage, ServerResponse } from 'http'; +import { + getAllProviders, + getProvider, + addProvider, + updateProvider, + deleteProvider, + getAllEndpoints, + getEndpoint, + addEndpoint, + updateEndpoint, + deleteEndpoint, + getDefaultEndpoint, + setDefaultEndpoint, + getGlobalCacheSettings, + updateGlobalCacheSettings, + loadLiteLLMApiConfig, + type ProviderCredential, + type CustomEndpoint, + type ProviderType, +} from '../../config/litellm-api-config-manager.js'; +import { getContextCacheStore } from '../../tools/context-cache-store.js'; +import { getLiteLLMClient } from '../../tools/litellm-client.js'; + +export interface RouteContext { + pathname: string; + url: URL; + req: IncomingMessage; + res: ServerResponse; + initialPath: string; + handlePostRequest: (req: IncomingMessage, res: ServerResponse, handler: (body: unknown) => Promise) => void; + broadcastToClients: (data: unknown) => void; +} + +// =========================== +// Model Information +// =========================== + +interface ModelInfo { + id: string; + name: string; + provider: ProviderType; + description?: string; +} + +const PROVIDER_MODELS: Record = { + openai: [ + { id: 'gpt-4-turbo', name: 'GPT-4 Turbo', provider: 'openai', description: '128K context' }, + { id: 'gpt-4', name: 'GPT-4', provider: 'openai', description: '8K context' }, + { id: 'gpt-3.5-turbo', name: 'GPT-3.5 Turbo', provider: 'openai', description: '16K context' }, + ], + anthropic: [ + { id: 'claude-3-opus-20240229', name: 'Claude 3 Opus', provider: 'anthropic', description: '200K context' }, + { id: 'claude-3-sonnet-20240229', name: 'Claude 3 Sonnet', provider: 'anthropic', description: '200K context' }, + { id: 'claude-3-haiku-20240307', name: 'Claude 3 Haiku', provider: 'anthropic', description: '200K context' }, + ], + google: [ + { id: 'gemini-pro', name: 'Gemini Pro', provider: 'google', description: '32K context' }, + { id: 'gemini-pro-vision', name: 'Gemini Pro Vision', provider: 'google', description: '16K context' }, + ], + ollama: [ + { id: 'llama2', name: 'Llama 2', provider: 'ollama', description: 'Local model' }, + { id: 'mistral', name: 'Mistral', provider: 'ollama', description: 'Local model' }, + ], + azure: [], + mistral: [ + { id: 'mistral-large-latest', name: 'Mistral Large', provider: 'mistral', description: '32K context' }, + { id: 'mistral-medium-latest', name: 'Mistral Medium', provider: 'mistral', description: '32K context' }, + ], + deepseek: [ + { id: 'deepseek-chat', name: 'DeepSeek Chat', provider: 'deepseek', description: '64K context' }, + { id: 'deepseek-coder', name: 'DeepSeek Coder', provider: 'deepseek', description: '64K context' }, + ], + custom: [], +}; + +/** + * Handle LiteLLM API routes + * @returns true if route was handled, false otherwise + */ +export async function handleLiteLLMApiRoutes(ctx: RouteContext): Promise { + const { pathname, url, req, res, initialPath, handlePostRequest, broadcastToClients } = ctx; + + // =========================== + // Provider Management Routes + // =========================== + + // GET /api/litellm-api/providers - List all providers + if (pathname === '/api/litellm-api/providers' && req.method === 'GET') { + try { + const providers = getAllProviders(initialPath); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ providers, count: providers.length })); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + return true; + } + + // POST /api/litellm-api/providers - Create provider + if (pathname === '/api/litellm-api/providers' && req.method === 'POST') { + handlePostRequest(req, res, async (body: unknown) => { + const providerData = body as Omit; + + if (!providerData.name || !providerData.type || !providerData.apiKey) { + return { error: 'Provider name, type, and apiKey are required', status: 400 }; + } + + try { + const provider = addProvider(initialPath, providerData); + + broadcastToClients({ + type: 'LITELLM_PROVIDER_CREATED', + payload: { provider, timestamp: new Date().toISOString() } + }); + + return { success: true, provider }; + } catch (err) { + return { error: (err as Error).message, status: 500 }; + } + }); + return true; + } + + // GET /api/litellm-api/providers/:id - Get provider by ID + const providerGetMatch = pathname.match(/^\/api\/litellm-api\/providers\/([^/]+)$/); + if (providerGetMatch && req.method === 'GET') { + const providerId = providerGetMatch[1]; + + try { + const provider = getProvider(initialPath, providerId); + if (!provider) { + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Provider not found' })); + return true; + } + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(provider)); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + return true; + } + + // PUT /api/litellm-api/providers/:id - Update provider + const providerUpdateMatch = pathname.match(/^\/api\/litellm-api\/providers\/([^/]+)$/); + if (providerUpdateMatch && req.method === 'PUT') { + const providerId = providerUpdateMatch[1]; + + handlePostRequest(req, res, async (body: unknown) => { + const updates = body as Partial>; + + try { + const provider = updateProvider(initialPath, providerId, updates); + + broadcastToClients({ + type: 'LITELLM_PROVIDER_UPDATED', + payload: { provider, timestamp: new Date().toISOString() } + }); + + return { success: true, provider }; + } catch (err) { + return { error: (err as Error).message, status: 404 }; + } + }); + return true; + } + + // DELETE /api/litellm-api/providers/:id - Delete provider + const providerDeleteMatch = pathname.match(/^\/api\/litellm-api\/providers\/([^/]+)$/); + if (providerDeleteMatch && req.method === 'DELETE') { + const providerId = providerDeleteMatch[1]; + + try { + const success = deleteProvider(initialPath, providerId); + + if (!success) { + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Provider not found' })); + return true; + } + + broadcastToClients({ + type: 'LITELLM_PROVIDER_DELETED', + payload: { providerId, timestamp: new Date().toISOString() } + }); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: true, message: 'Provider deleted' })); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + return true; + } + + // POST /api/litellm-api/providers/:id/test - Test provider connection + const providerTestMatch = pathname.match(/^\/api\/litellm-api\/providers\/([^/]+)\/test$/); + if (providerTestMatch && req.method === 'POST') { + const providerId = providerTestMatch[1]; + + try { + const provider = getProvider(initialPath, providerId); + + if (!provider) { + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: false, error: 'Provider not found' })); + return true; + } + + if (!provider.enabled) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: false, error: 'Provider is disabled' })); + return true; + } + + // Test connection using litellm client + const client = getLiteLLMClient(); + const available = await client.isAvailable(); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: available, provider: provider.type })); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: false, error: (err as Error).message })); + } + return true; + } + + // =========================== + // Endpoint Management Routes + // =========================== + + // GET /api/litellm-api/endpoints - List all endpoints + if (pathname === '/api/litellm-api/endpoints' && req.method === 'GET') { + try { + const endpoints = getAllEndpoints(initialPath); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ endpoints, count: endpoints.length })); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + return true; + } + + // POST /api/litellm-api/endpoints - Create endpoint + if (pathname === '/api/litellm-api/endpoints' && req.method === 'POST') { + handlePostRequest(req, res, async (body: unknown) => { + const endpointData = body as Omit; + + if (!endpointData.id || !endpointData.name || !endpointData.providerId || !endpointData.model) { + return { error: 'Endpoint id, name, providerId, and model are required', status: 400 }; + } + + try { + const endpoint = addEndpoint(initialPath, endpointData); + + broadcastToClients({ + type: 'LITELLM_ENDPOINT_CREATED', + payload: { endpoint, timestamp: new Date().toISOString() } + }); + + return { success: true, endpoint }; + } catch (err) { + return { error: (err as Error).message, status: 500 }; + } + }); + return true; + } + + // GET /api/litellm-api/endpoints/:id - Get endpoint by ID + const endpointGetMatch = pathname.match(/^\/api\/litellm-api\/endpoints\/([^/]+)$/); + if (endpointGetMatch && req.method === 'GET') { + const endpointId = endpointGetMatch[1]; + + try { + const endpoint = getEndpoint(initialPath, endpointId); + if (!endpoint) { + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Endpoint not found' })); + return true; + } + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(endpoint)); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + return true; + } + + // PUT /api/litellm-api/endpoints/:id - Update endpoint + const endpointUpdateMatch = pathname.match(/^\/api\/litellm-api\/endpoints\/([^/]+)$/); + if (endpointUpdateMatch && req.method === 'PUT') { + const endpointId = endpointUpdateMatch[1]; + + handlePostRequest(req, res, async (body: unknown) => { + const updates = body as Partial>; + + try { + const endpoint = updateEndpoint(initialPath, endpointId, updates); + + broadcastToClients({ + type: 'LITELLM_ENDPOINT_UPDATED', + payload: { endpoint, timestamp: new Date().toISOString() } + }); + + return { success: true, endpoint }; + } catch (err) { + return { error: (err as Error).message, status: 404 }; + } + }); + return true; + } + + // DELETE /api/litellm-api/endpoints/:id - Delete endpoint + const endpointDeleteMatch = pathname.match(/^\/api\/litellm-api\/endpoints\/([^/]+)$/); + if (endpointDeleteMatch && req.method === 'DELETE') { + const endpointId = endpointDeleteMatch[1]; + + try { + const success = deleteEndpoint(initialPath, endpointId); + + if (!success) { + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Endpoint not found' })); + return true; + } + + broadcastToClients({ + type: 'LITELLM_ENDPOINT_DELETED', + payload: { endpointId, timestamp: new Date().toISOString() } + }); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: true, message: 'Endpoint deleted' })); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + return true; + } + + // =========================== + // Model Discovery Routes + // =========================== + + // GET /api/litellm-api/models/:providerType - Get available models for provider type + const modelsMatch = pathname.match(/^\/api\/litellm-api\/models\/([^/]+)$/); + if (modelsMatch && req.method === 'GET') { + const providerType = modelsMatch[1] as ProviderType; + + try { + const models = PROVIDER_MODELS[providerType]; + + if (!models) { + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Provider type not found' })); + return true; + } + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ providerType, models, count: models.length })); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + return true; + } + + // =========================== + // Cache Management Routes + // =========================== + + // GET /api/litellm-api/cache/stats - Get cache statistics + if (pathname === '/api/litellm-api/cache/stats' && req.method === 'GET') { + try { + const cacheStore = getContextCacheStore(); + const stats = cacheStore.getStatus(); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(stats)); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + return true; + } + + // POST /api/litellm-api/cache/clear - Clear cache + if (pathname === '/api/litellm-api/cache/clear' && req.method === 'POST') { + try { + const cacheStore = getContextCacheStore(); + const result = cacheStore.clear(); + + broadcastToClients({ + type: 'LITELLM_CACHE_CLEARED', + payload: { removed: result.removed, timestamp: new Date().toISOString() } + }); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: true, removed: result.removed })); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + return true; + } + + // =========================== + // Config Management Routes + // =========================== + + // GET /api/litellm-api/config - Get full config + if (pathname === '/api/litellm-api/config' && req.method === 'GET') { + try { + const config = loadLiteLLMApiConfig(initialPath); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(config)); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + return true; + } + + // PUT /api/litellm-api/config/cache - Update global cache settings + if (pathname === '/api/litellm-api/config/cache' && req.method === 'PUT') { + handlePostRequest(req, res, async (body: unknown) => { + const settings = body as Partial<{ enabled: boolean; cacheDir: string; maxTotalSizeMB: number }>; + + try { + updateGlobalCacheSettings(initialPath, settings); + + const updatedSettings = getGlobalCacheSettings(initialPath); + + broadcastToClients({ + type: 'LITELLM_CACHE_SETTINGS_UPDATED', + payload: { settings: updatedSettings, timestamp: new Date().toISOString() } + }); + + return { success: true, settings: updatedSettings }; + } catch (err) { + return { error: (err as Error).message, status: 500 }; + } + }); + return true; + } + + // PUT /api/litellm-api/config/default-endpoint - Set default endpoint + if (pathname === '/api/litellm-api/config/default-endpoint' && req.method === 'PUT') { + handlePostRequest(req, res, async (body: unknown) => { + const { endpointId } = body as { endpointId?: string }; + + try { + setDefaultEndpoint(initialPath, endpointId); + + const defaultEndpoint = getDefaultEndpoint(initialPath); + + broadcastToClients({ + type: 'LITELLM_DEFAULT_ENDPOINT_UPDATED', + payload: { endpointId, defaultEndpoint, timestamp: new Date().toISOString() } + }); + + return { success: true, defaultEndpoint }; + } catch (err) { + return { error: (err as Error).message, status: 500 }; + } + }); + return true; + } + + return false; +} diff --git a/ccw/src/core/routes/litellm-routes.ts b/ccw/src/core/routes/litellm-routes.ts new file mode 100644 index 00000000..1e453113 --- /dev/null +++ b/ccw/src/core/routes/litellm-routes.ts @@ -0,0 +1,107 @@ +// @ts-nocheck +/** + * LiteLLM Routes Module + * Handles all LiteLLM-related API endpoints + */ +import type { IncomingMessage, ServerResponse } from 'http'; +import { getLiteLLMClient, getLiteLLMStatus, checkLiteLLMAvailable } from '../../tools/litellm-client.js'; + +export interface RouteContext { + pathname: string; + url: URL; + req: IncomingMessage; + res: ServerResponse; + initialPath: string; + handlePostRequest: (req: IncomingMessage, res: ServerResponse, handler: (body: unknown) => Promise) => void; + broadcastToClients: (data: unknown) => void; +} + +/** + * Handle LiteLLM routes + * @returns true if route was handled, false otherwise + */ +export async function handleLiteLLMRoutes(ctx: RouteContext): Promise { + const { pathname, url, req, res, initialPath, handlePostRequest } = ctx; + + // API: LiteLLM Status - Check availability and version + if (pathname === '/api/litellm/status') { + try { + const status = await getLiteLLMStatus(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(status)); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ available: false, error: err.message })); + } + return true; + } + + // API: LiteLLM Config - Get configuration + if (pathname === '/api/litellm/config' && req.method === 'GET') { + try { + const client = getLiteLLMClient(); + const config = await client.getConfig(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(config)); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: err.message })); + } + return true; + } + + // API: LiteLLM Embed - Generate embeddings + if (pathname === '/api/litellm/embed' && req.method === 'POST') { + handlePostRequest(req, res, async (body) => { + const { texts, model = 'default' } = body; + + if (!texts || !Array.isArray(texts)) { + return { error: 'texts array is required', status: 400 }; + } + + if (texts.length === 0) { + return { error: 'texts array cannot be empty', status: 400 }; + } + + try { + const client = getLiteLLMClient(); + const result = await client.embed(texts, model); + return { success: true, ...result }; + } catch (err) { + return { error: err.message, status: 500 }; + } + }); + return true; + } + + // API: LiteLLM Chat - Chat with LLM + if (pathname === '/api/litellm/chat' && req.method === 'POST') { + handlePostRequest(req, res, async (body) => { + const { message, messages, model = 'default' } = body; + + // Support both single message and messages array + if (!message && (!messages || !Array.isArray(messages))) { + return { error: 'message or messages array is required', status: 400 }; + } + + try { + const client = getLiteLLMClient(); + + if (messages && Array.isArray(messages)) { + // Multi-turn chat + const result = await client.chatMessages(messages, model); + return { success: true, ...result }; + } else { + // Single message chat + const content = await client.chat(message, model); + return { success: true, content, model }; + } + } catch (err) { + return { error: err.message, status: 500 }; + } + }); + return true; + } + + return false; +} diff --git a/ccw/src/core/server.ts b/ccw/src/core/server.ts index 1ac79ae0..6b7934f5 100644 --- a/ccw/src/core/server.ts +++ b/ccw/src/core/server.ts @@ -22,6 +22,8 @@ import { handleSessionRoutes } from './routes/session-routes.js'; import { handleCcwRoutes } from './routes/ccw-routes.js'; import { handleClaudeRoutes } from './routes/claude-routes.js'; import { handleHelpRoutes } from './routes/help-routes.js'; +import { handleLiteLLMRoutes } from './routes/litellm-routes.js'; +import { handleLiteLLMApiRoutes } from './routes/litellm-api-routes.js'; // Import WebSocket handling import { handleWebSocketUpgrade, broadcastToClients } from './websocket.js'; @@ -311,6 +313,16 @@ export async function startServer(options: ServerOptions = {}): Promise', + 'apiSettings.provider': 'Provider', + 'apiSettings.model': 'Model', + 'apiSettings.selectModel': 'Select model', + 'apiSettings.cacheStrategy': 'Cache Strategy', + 'apiSettings.enableContextCaching': 'Enable Context Caching', + 'apiSettings.cacheTTL': 'TTL (minutes)', + 'apiSettings.cacheMaxSize': 'Max Size (KB)', + 'apiSettings.autoCachePatterns': 'Auto-cache file patterns', + 'apiSettings.enableGlobalCaching': 'Enable Global Caching', + 'apiSettings.cacheUsed': 'Used', + 'apiSettings.cacheEntries': 'Entries', + 'apiSettings.clearCache': 'Clear Cache', + 'apiSettings.noProviders': 'No providers configured', + 'apiSettings.noEndpoints': 'No endpoints configured', + 'apiSettings.enabled': 'Enabled', + 'apiSettings.disabled': 'Disabled', + 'apiSettings.cacheEnabled': 'Cache Enabled', + 'apiSettings.cacheDisabled': 'Cache Disabled', + 'apiSettings.providerSaved': 'Provider saved successfully', + 'apiSettings.providerDeleted': 'Provider deleted successfully', + 'apiSettings.endpointSaved': 'Endpoint saved successfully', + 'apiSettings.endpointDeleted': 'Endpoint deleted successfully', + 'apiSettings.cacheCleared': 'Cache cleared successfully', + 'apiSettings.cacheSettingsUpdated': 'Cache settings updated', + 'apiSettings.confirmDeleteProvider': 'Are you sure you want to delete this provider?', + 'apiSettings.confirmDeleteEndpoint': 'Are you sure you want to delete this endpoint?', + 'apiSettings.confirmClearCache': 'Are you sure you want to clear the cache?', + 'apiSettings.connectionSuccess': 'Connection successful', + 'apiSettings.connectionFailed': 'Connection failed', + 'apiSettings.saveProviderFirst': 'Please save the provider first', + 'apiSettings.addProviderFirst': 'Please add a provider first', + 'apiSettings.failedToLoad': 'Failed to load API settings', + 'apiSettings.toggleVisibility': 'Toggle visibility', + // Common 'common.cancel': 'Cancel', 'common.optional': '(Optional)', @@ -2799,6 +2855,62 @@ const i18n = { 'claudeManager.saved': 'File saved successfully', 'claudeManager.saveError': 'Failed to save file', + + // API Settings + 'nav.apiSettings': 'API 设置', + 'title.apiSettings': 'API 设置', + 'apiSettings.providers': '提供商', + 'apiSettings.customEndpoints': '自定义端点', + 'apiSettings.cacheSettings': '缓存设置', + 'apiSettings.addProvider': '添加提供商', + 'apiSettings.editProvider': '编辑提供商', + 'apiSettings.deleteProvider': '删除提供商', + 'apiSettings.addEndpoint': '添加端点', + 'apiSettings.editEndpoint': '编辑端点', + 'apiSettings.deleteEndpoint': '删除端点', + 'apiSettings.providerType': '提供商类型', + 'apiSettings.displayName': '显示名称', + 'apiSettings.apiKey': 'API 密钥', + 'apiSettings.apiBaseUrl': 'API 基础 URL', + 'apiSettings.useEnvVar': '使用环境变量', + 'apiSettings.enableProvider': '启用提供商', + 'apiSettings.testConnection': '测试连接', + 'apiSettings.endpointId': '端点 ID', + 'apiSettings.endpointIdHint': '用法: ccw cli -p "..." --model <端点ID>', + 'apiSettings.provider': '提供商', + 'apiSettings.model': '模型', + 'apiSettings.selectModel': '选择模型', + 'apiSettings.cacheStrategy': '缓存策略', + 'apiSettings.enableContextCaching': '启用上下文缓存', + 'apiSettings.cacheTTL': 'TTL (分钟)', + 'apiSettings.cacheMaxSize': '最大大小 (KB)', + 'apiSettings.autoCachePatterns': '自动缓存文件模式', + 'apiSettings.enableGlobalCaching': '启用全局缓存', + 'apiSettings.cacheUsed': '已使用', + 'apiSettings.cacheEntries': '条目数', + 'apiSettings.clearCache': '清除缓存', + 'apiSettings.noProviders': '未配置提供商', + 'apiSettings.noEndpoints': '未配置端点', + 'apiSettings.enabled': '已启用', + 'apiSettings.disabled': '已禁用', + 'apiSettings.cacheEnabled': '缓存已启用', + 'apiSettings.cacheDisabled': '缓存已禁用', + 'apiSettings.providerSaved': '提供商保存成功', + 'apiSettings.providerDeleted': '提供商删除成功', + 'apiSettings.endpointSaved': '端点保存成功', + 'apiSettings.endpointDeleted': '端点删除成功', + 'apiSettings.cacheCleared': '缓存清除成功', + 'apiSettings.cacheSettingsUpdated': '缓存设置已更新', + 'apiSettings.confirmDeleteProvider': '确定要删除此提供商吗？', + 'apiSettings.confirmDeleteEndpoint': '确定要删除此端点吗？', + 'apiSettings.confirmClearCache': '确定要清除缓存吗？', + 'apiSettings.connectionSuccess': '连接成功', + 'apiSettings.connectionFailed': '连接失败', + 'apiSettings.saveProviderFirst': '请先保存提供商', + 'apiSettings.addProviderFirst': '请先添加提供商', + 'apiSettings.failedToLoad': '加载 API 设置失败', + 'apiSettings.toggleVisibility': '切换可见性', + // Common 'common.cancel': '取消', 'common.optional': '(可选)', diff --git a/ccw/src/templates/dashboard-js/views/api-settings.js b/ccw/src/templates/dashboard-js/views/api-settings.js new file mode 100644 index 00000000..8ca07389 --- /dev/null +++ b/ccw/src/templates/dashboard-js/views/api-settings.js @@ -0,0 +1,815 @@ +// API Settings View +// Manages LiteLLM API providers, custom endpoints, and cache settings + +// ========== State Management ========== +var apiSettingsData = null; +var providerModels = {}; +var currentModal = null; + +// ========== Data Loading ========== + +/** + * Load API configuration + */ +async function loadApiSettings() { + try { + var response = await fetch('/api/litellm-api/config'); + if (!response.ok) throw new Error('Failed to load API settings'); + apiSettingsData = await response.json(); + return apiSettingsData; + } catch (err) { + console.error('Failed to load API settings:', err); + showRefreshToast(t('common.error') + ': ' + err.message, 'error'); + return null; + } +} + +/** + * Load available models for a provider type + */ +async function loadProviderModels(providerType) { + try { + var response = await fetch('/api/litellm-api/models/' + providerType); + if (!response.ok) throw new Error('Failed to load models'); + var data = await response.json(); + providerModels[providerType] = data.models || []; + return data.models; + } catch (err) { + console.error('Failed to load provider models:', err); + return []; + } +} + +/** + * Load cache statistics + */ +async function loadCacheStats() { + try { + var response = await fetch('/api/litellm-api/cache/stats'); + if (!response.ok) throw new Error('Failed to load cache stats'); + return await response.json(); + } catch (err) { + console.error('Failed to load cache stats:', err); + return { enabled: false, totalSize: 0, maxSize: 104857600, entries: 0 }; + } +} + +// ========== Provider Management ========== + +/** + * Show add provider modal + */ +async function showAddProviderModal() { + var modalHtml = '
' + + '
' + + '
' + + '

' + t('apiSettings.addProvider') + '

' + + '' + + '
' + + '
' + + '
' + + '
' + + '' + + '' + + '
' + + '
' + + '' + + '' + + '
' + + '
' + + '' + + '
' + + '' + + '' + + '
' + + '' + + '' + + '
' + + '
' + + '' + + '' + + '
' + + '
' + + '' + + '
' + + '
' + + '' + + '' + + '' + + '
' + + '
' + + '
' + + '
' + + '
'; + + document.body.insertAdjacentHTML('beforeend', modalHtml); + + document.getElementById('providerForm').addEventListener('submit', async function(e) { + e.preventDefault(); + await saveProvider(); + }); + + if (window.lucide) lucide.createIcons(); +} + +/** + * Show edit provider modal + */ +async function showEditProviderModal(providerId) { + if (!apiSettingsData) return; + + var provider = apiSettingsData.providers?.find(function(p) { return p.id === providerId; }); + if (!provider) return; + + await showAddProviderModal(); + + // Update modal title + document.querySelector('#providerModal .generic-modal-title').textContent = t('apiSettings.editProvider'); + + // Populate form + document.getElementById('provider-type').value = provider.type; + document.getElementById('provider-name').value = provider.name; + document.getElementById('provider-apikey').value = provider.apiKey; + if (provider.apiBase) { + document.getElementById('provider-apibase').value = provider.apiBase; + } + document.getElementById('provider-enabled').checked = provider.enabled !== false; + + // Store provider ID for update + document.getElementById('providerForm').dataset.providerId = providerId; +} + +/** + * Save provider (create or update) + */ +async function saveProvider() { + var form = document.getElementById('providerForm'); + var providerId = form.dataset.providerId; + + var useEnvVar = document.getElementById('use-env-var').checked; + var apiKey = useEnvVar + ? '${' + document.getElementById('env-var-name').value + '}' + : document.getElementById('provider-apikey').value; + + var providerData = { + type: document.getElementById('provider-type').value, + name: document.getElementById('provider-name').value, + apiKey: apiKey, + apiBase: document.getElementById('provider-apibase').value || undefined, + enabled: document.getElementById('provider-enabled').checked + }; + + try { + var url = providerId + ? '/api/litellm-api/providers/' + providerId + : '/api/litellm-api/providers'; + var method = providerId ? 'PUT' : 'POST'; + + var response = await fetch(url, { + method: method, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(providerData) + }); + + if (!response.ok) throw new Error('Failed to save provider'); + + var result = await response.json(); + showRefreshToast(t('apiSettings.providerSaved'), 'success'); + + closeProviderModal(); + await renderApiSettings(); + } catch (err) { + console.error('Failed to save provider:', err); + showRefreshToast(t('common.error') + ': ' + err.message, 'error'); + } +} + +/** + * Delete provider + */ +async function deleteProvider(providerId) { + if (!confirm(t('apiSettings.confirmDeleteProvider'))) return; + + try { + var response = await fetch('/api/litellm-api/providers/' + providerId, { + method: 'DELETE' + }); + + if (!response.ok) throw new Error('Failed to delete provider'); + + showRefreshToast(t('apiSettings.providerDeleted'), 'success'); + await renderApiSettings(); + } catch (err) { + console.error('Failed to delete provider:', err); + showRefreshToast(t('common.error') + ': ' + err.message, 'error'); + } +} + +/** + * Test provider connection + */ +async function testProviderConnection() { + var form = document.getElementById('providerForm'); + var providerId = form.dataset.providerId; + + if (!providerId) { + showRefreshToast(t('apiSettings.saveProviderFirst'), 'warning'); + return; + } + + try { + var response = await fetch('/api/litellm-api/providers/' + providerId + '/test', { + method: 'POST' + }); + + if (!response.ok) throw new Error('Failed to test provider'); + + var result = await response.json(); + + if (result.success) { + showRefreshToast(t('apiSettings.connectionSuccess'), 'success'); + } else { + showRefreshToast(t('apiSettings.connectionFailed') + ': ' + (result.error || 'Unknown error'), 'error'); + } + } catch (err) { + console.error('Failed to test provider:', err); + showRefreshToast(t('common.error') + ': ' + err.message, 'error'); + } +} + +/** + * Close provider modal + */ +function closeProviderModal() { + var modal = document.getElementById('providerModal'); + if (modal) modal.remove(); +} + +/** + * Toggle API key visibility + */ +function toggleApiKeyVisibility(inputId) { + var input = document.getElementById(inputId); + var icon = event.target.closest('button').querySelector('i'); + + if (input.type === 'password') { + input.type = 'text'; + icon.setAttribute('data-lucide', 'eye-off'); + } else { + input.type = 'password'; + icon.setAttribute('data-lucide', 'eye'); + } + + if (window.lucide) lucide.createIcons(); +} + +/** + * Toggle environment variable input + */ +function toggleEnvVarInput() { + var useEnvVar = document.getElementById('use-env-var').checked; + var apiKeyInput = document.getElementById('provider-apikey'); + var envVarInput = document.getElementById('env-var-name'); + + if (useEnvVar) { + apiKeyInput.style.display = 'none'; + apiKeyInput.required = false; + envVarInput.style.display = 'block'; + envVarInput.required = true; + } else { + apiKeyInput.style.display = 'block'; + apiKeyInput.required = true; + envVarInput.style.display = 'none'; + envVarInput.required = false; + } +} + +// ========== Endpoint Management ========== + +/** + * Show add endpoint modal + */ +async function showAddEndpointModal() { + if (!apiSettingsData || !apiSettingsData.providers || apiSettingsData.providers.length === 0) { + showRefreshToast(t('apiSettings.addProviderFirst'), 'warning'); + return; + } + + var providerOptions = apiSettingsData.providers + .filter(function(p) { return p.enabled !== false; }) + .map(function(p) { + return ''; + }) + .join(''); + + var modalHtml = '
' + + '
' + + '
' + + '

' + t('apiSettings.addEndpoint') + '

' + + '' + + '
' + + '
' + + '
' + + '
' + + '' + + '' + + '' + t('apiSettings.endpointIdHint') + '' + + '
' + + '
' + + '' + + '' + + '
' + + '
' + + '' + + '' + + '
' + + '
' + + '' + + '' + + '
' + + '
' + + '' + t('apiSettings.cacheStrategy') + '' + + '' + + '' + + '
' + + '
' + + '' + + '' + + '
' + + '
' + + '
' + + '
' + + '
'; + + document.body.insertAdjacentHTML('beforeend', modalHtml); + + document.getElementById('endpointForm').addEventListener('submit', async function(e) { + e.preventDefault(); + await saveEndpoint(); + }); + + // Load models for first provider + await loadModelsForProvider(); + + if (window.lucide) lucide.createIcons(); +} + +/** + * Show edit endpoint modal + */ +async function showEditEndpointModal(endpointId) { + if (!apiSettingsData) return; + + var endpoint = apiSettingsData.endpoints?.find(function(e) { return e.id === endpointId; }); + if (!endpoint) return; + + await showAddEndpointModal(); + + // Update modal title + document.querySelector('#endpointModal .generic-modal-title').textContent = t('apiSettings.editEndpoint'); + + // Populate form + document.getElementById('endpoint-id').value = endpoint.id; + document.getElementById('endpoint-id').disabled = true; + document.getElementById('endpoint-name').value = endpoint.name; + document.getElementById('endpoint-provider').value = endpoint.providerId; + + await loadModelsForProvider(); + document.getElementById('endpoint-model').value = endpoint.model; + + if (endpoint.cacheStrategy) { + document.getElementById('cache-enabled').checked = endpoint.cacheStrategy.enabled; + if (endpoint.cacheStrategy.enabled) { + toggleCacheSettings(); + document.getElementById('cache-ttl').value = endpoint.cacheStrategy.ttlMinutes || 60; + document.getElementById('cache-maxsize').value = endpoint.cacheStrategy.maxSizeKB || 512; + document.getElementById('cache-patterns').value = endpoint.cacheStrategy.autoCachePatterns?.join(', ') || ''; + } + } + + // Store endpoint ID for update + document.getElementById('endpointForm').dataset.endpointId = endpointId; +} + +/** + * Save endpoint (create or update) + */ +async function saveEndpoint() { + var form = document.getElementById('endpointForm'); + var endpointId = form.dataset.endpointId || document.getElementById('endpoint-id').value; + + var cacheEnabled = document.getElementById('cache-enabled').checked; + var cacheStrategy = cacheEnabled ? { + enabled: true, + ttlMinutes: parseInt(document.getElementById('cache-ttl').value) || 60, + maxSizeKB: parseInt(document.getElementById('cache-maxsize').value) || 512, + autoCachePatterns: document.getElementById('cache-patterns').value + .split(',') + .map(function(p) { return p.trim(); }) + .filter(function(p) { return p; }) + } : { enabled: false }; + + var endpointData = { + id: endpointId, + name: document.getElementById('endpoint-name').value, + providerId: document.getElementById('endpoint-provider').value, + model: document.getElementById('endpoint-model').value, + cacheStrategy: cacheStrategy + }; + + try { + var url = form.dataset.endpointId + ? '/api/litellm-api/endpoints/' + form.dataset.endpointId + : '/api/litellm-api/endpoints'; + var method = form.dataset.endpointId ? 'PUT' : 'POST'; + + var response = await fetch(url, { + method: method, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(endpointData) + }); + + if (!response.ok) throw new Error('Failed to save endpoint'); + + var result = await response.json(); + showRefreshToast(t('apiSettings.endpointSaved'), 'success'); + + closeEndpointModal(); + await renderApiSettings(); + } catch (err) { + console.error('Failed to save endpoint:', err); + showRefreshToast(t('common.error') + ': ' + err.message, 'error'); + } +} + +/** + * Delete endpoint + */ +async function deleteEndpoint(endpointId) { + if (!confirm(t('apiSettings.confirmDeleteEndpoint'))) return; + + try { + var response = await fetch('/api/litellm-api/endpoints/' + endpointId, { + method: 'DELETE' + }); + + if (!response.ok) throw new Error('Failed to delete endpoint'); + + showRefreshToast(t('apiSettings.endpointDeleted'), 'success'); + await renderApiSettings(); + } catch (err) { + console.error('Failed to delete endpoint:', err); + showRefreshToast(t('common.error') + ': ' + err.message, 'error'); + } +} + +/** + * Close endpoint modal + */ +function closeEndpointModal() { + var modal = document.getElementById('endpointModal'); + if (modal) modal.remove(); +} + +/** + * Load models for selected provider + */ +async function loadModelsForProvider() { + var providerSelect = document.getElementById('endpoint-provider'); + var modelSelect = document.getElementById('endpoint-model'); + + if (!providerSelect || !modelSelect) return; + + var providerId = providerSelect.value; + var provider = apiSettingsData.providers.find(function(p) { return p.id === providerId; }); + + if (!provider) return; + + // Load models for provider type + var models = await loadProviderModels(provider.type); + + modelSelect.innerHTML = '' + + models.map(function(m) { + var desc = m.description ? ' - ' + m.description : ''; + return ''; + }).join(''); +} + +/** + * Toggle cache settings visibility + */ +function toggleCacheSettings() { + var enabled = document.getElementById('cache-enabled').checked; + var settings = document.getElementById('cache-settings'); + settings.style.display = enabled ? 'block' : 'none'; +} + +// ========== Cache Management ========== + +/** + * Clear cache + */ +async function clearCache() { + if (!confirm(t('apiSettings.confirmClearCache'))) return; + + try { + var response = await fetch('/api/litellm-api/cache/clear', { + method: 'POST' + }); + + if (!response.ok) throw new Error('Failed to clear cache'); + + var result = await response.json(); + showRefreshToast(t('apiSettings.cacheCleared') + ' (' + result.removed + ' entries)', 'success'); + + await renderApiSettings(); + } catch (err) { + console.error('Failed to clear cache:', err); + showRefreshToast(t('common.error') + ': ' + err.message, 'error'); + } +} + +/** + * Toggle global cache + */ +async function toggleGlobalCache() { + var enabled = document.getElementById('global-cache-enabled').checked; + + try { + var response = await fetch('/api/litellm-api/config/cache', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ enabled: enabled }) + }); + + if (!response.ok) throw new Error('Failed to update cache settings'); + + showRefreshToast(t('apiSettings.cacheSettingsUpdated'), 'success'); + } catch (err) { + console.error('Failed to update cache settings:', err); + showRefreshToast(t('common.error') + ': ' + err.message, 'error'); + // Revert checkbox + document.getElementById('global-cache-enabled').checked = !enabled; + } +} + +// ========== Rendering ========== + +/** + * Render API Settings page + */ +async function renderApiSettings() { + var container = document.getElementById('mainContent'); + if (!container) return; + + // Hide stats grid and search + var statsGrid = document.getElementById('statsGrid'); + var searchInput = document.getElementById('searchInput'); + if (statsGrid) statsGrid.style.display = 'none'; + if (searchInput) searchInput.parentElement.style.display = 'none'; + + // Load data + await loadApiSettings(); + var cacheStats = await loadCacheStats(); + + if (!apiSettingsData) { + container.innerHTML = '
' + + '
' + t('apiSettings.failedToLoad') + '
' + + '
'; + return; + } + + container.innerHTML = '
' + + '
' + + '
' + + '

' + t('apiSettings.providers') + '

' + + '' + + '
' + + '
' + + '
' + + '
' + + '
' + + '

' + t('apiSettings.customEndpoints') + '

' + + '' + + '
' + + '
' + + '
' + + '
' + + '
' + + '

' + t('apiSettings.cacheSettings') + '

' + + '
' + + '
' + + '
' + + '
'; + + renderProvidersList(); + renderEndpointsList(); + renderCacheSettings(cacheStats); + + if (window.lucide) lucide.createIcons(); +} + +/** + * Render providers list + */ +function renderProvidersList() { + var container = document.getElementById('providers-list'); + if (!container) return; + + var providers = apiSettingsData.providers || []; + + if (providers.length === 0) { + container.innerHTML = '
' + + '' + + '

' + t('apiSettings.noProviders') + '

' + + '
'; + if (window.lucide) lucide.createIcons(); + return; + } + + container.innerHTML = providers.map(function(provider) { + var statusClass = provider.enabled === false ? 'disabled' : 'enabled'; + var statusText = provider.enabled === false ? t('apiSettings.disabled') : t('apiSettings.enabled'); + + return '
' + + '
' + + '
' + + '

' + provider.name + '

' + + '' + provider.type + '' + + '
' + + '
' + + '' + + '' + + '
' + + '
' + + '
' + + '
' + + ' ' + maskApiKey(provider.apiKey) + '' + + (provider.apiBase ? ' ' + provider.apiBase + '' : '') + + '' + statusText + '' + + '
' + + '
' + + '
'; + }).join(''); + + if (window.lucide) lucide.createIcons(); +} + +/** + * Render endpoints list + */ +function renderEndpointsList() { + var container = document.getElementById('endpoints-list'); + if (!container) return; + + var endpoints = apiSettingsData.endpoints || []; + + if (endpoints.length === 0) { + container.innerHTML = '
' + + '' + + '

' + t('apiSettings.noEndpoints') + '

' + + '
'; + if (window.lucide) lucide.createIcons(); + return; + } + + container.innerHTML = endpoints.map(function(endpoint) { + var provider = apiSettingsData.providers.find(function(p) { return p.id === endpoint.providerId; }); + var providerName = provider ? provider.name : endpoint.providerId; + + var cacheStatus = endpoint.cacheStrategy?.enabled + ? t('apiSettings.cacheEnabled') + ' (' + endpoint.cacheStrategy.ttlMinutes + ' min)' + : t('apiSettings.cacheDisabled'); + + return '
' + + '
' + + '
' + + '

' + endpoint.name + '

' + + '' + endpoint.id + '' + + '
' + + '
' + + '' + + '' + + '
' + + '
' + + '
' + + '
' + + ' ' + providerName + '' + + ' ' + endpoint.model + '' + + ' ' + cacheStatus + '' + + '
' + + '
' + + ' ' + + 'ccw cli -p "..." --model ' + endpoint.id + '' + + '
' + + '
' + + '
'; + }).join(''); + + if (window.lucide) lucide.createIcons(); +} + +/** + * Render cache settings panel + */ +function renderCacheSettings(stats) { + var container = document.getElementById('cache-settings-panel'); + if (!container) return; + + var globalSettings = apiSettingsData.globalCache || { enabled: false }; + var usedMB = (stats.totalSize / 1024 / 1024).toFixed(2); + var maxMB = (stats.maxSize / 1024 / 1024).toFixed(0); + var usagePercent = stats.maxSize > 0 ? ((stats.totalSize / stats.maxSize) * 100).toFixed(1) : 0; + + container.innerHTML = '
' + + '' + + '
' + + '
' + + '' + t('apiSettings.cacheUsed') + '' + + '' + usedMB + ' MB / ' + maxMB + ' MB (' + usagePercent + '%)' + + '
' + + '
' + + '' + t('apiSettings.cacheEntries') + '' + + '' + stats.entries + '' + + '
' + + '
' + + '
' + + '
' + + '
' + + '' + + '
'; + + if (window.lucide) lucide.createIcons(); +} + +// ========== Utility Functions ========== + +/** + * Mask API key for display + */ +function maskApiKey(apiKey) { + if (!apiKey) return ''; + if (apiKey.startsWith('${')) return apiKey; // Environment variable + if (apiKey.length <= 8) return '***'; + return apiKey.substring(0, 4) + '...' + apiKey.substring(apiKey.length - 4); +} diff --git a/ccw/src/templates/dashboard.html b/ccw/src/templates/dashboard.html index ec8a41c9..428b0459 100644 --- a/ccw/src/templates/dashboard.html +++ b/ccw/src/templates/dashboard.html @@ -336,6 +336,10 @@ CodexLens - +
  • + + API Settings +