feat: Implement CodexLens multi-provider embedding rotation management

- Added functions to get and update CodexLens embedding rotation configuration.
- Introduced functionality to retrieve enabled embedding providers for rotation.
- Created endpoints for managing rotation configuration via API.
- Enhanced dashboard UI to support multi-provider rotation configuration.
- Updated internationalization strings for new rotation features.
- Adjusted CLI commands and embedding manager to support increased concurrency limits.
- Modified hybrid search weights for improved ranking behavior.
This commit is contained in:
catlog22
2025-12-25 14:13:27 +08:00
parent dfa8b541b4
commit 8e744597d1
12 changed files with 713 additions and 20 deletions

View File

@@ -103,12 +103,12 @@ def init(
"-l",
help="Limit indexing to specific languages (repeat or comma-separated).",
),
workers: Optional[int] = typer.Option(None, "--workers", "-w", min=1, max=16, help="Parallel worker processes (default: auto-detect based on CPU count, max 16)."),
workers: Optional[int] = typer.Option(None, "--workers", "-w", min=1, max=32, help="Parallel worker processes (default: auto-detect based on CPU count, max 32)."),
force: bool = typer.Option(False, "--force", "-f", help="Force full reindex (skip incremental mode)."),
no_embeddings: bool = typer.Option(False, "--no-embeddings", help="Skip automatic embedding generation (if semantic deps installed)."),
embedding_backend: str = typer.Option("fastembed", "--embedding-backend", help="Embedding backend: fastembed (local) or litellm (remote API)."),
embedding_model: str = typer.Option("code", "--embedding-model", help="Embedding model: profile name for fastembed (fast/code/multilingual/balanced) or model name for litellm (e.g. text-embedding-3-small)."),
max_workers: int = typer.Option(1, "--max-workers", min=1, max=16, help="Max concurrent API calls for embedding generation. Recommended: 4-8 for litellm backend."),
max_workers: int = typer.Option(1, "--max-workers", min=1, max=32, help="Max concurrent API calls for embedding generation. Recommended: 4-8 for litellm backend."),
json_mode: bool = typer.Option(False, "--json", help="Output JSON response."),
verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."),
) -> None:
@@ -351,7 +351,7 @@ def search(
Use 'codexlens embeddings-generate' to create embeddings first.
Hybrid Mode:
Default weights: exact=0.4, fuzzy=0.3, vector=0.3
Default weights: exact=0.3, fuzzy=0.1, vector=0.6
Use --weights to customize (e.g., --weights 0.5,0.3,0.2)
Examples:
@@ -1852,7 +1852,7 @@ def embeddings_generate(
"--max-workers",
"-w",
min=1,
max=16,
max=32,
help="Max concurrent API calls. Recommended: 4-8 for litellm backend. Default: 1 (sequential).",
),
json_mode: bool = typer.Option(False, "--json", help="Output JSON response."),

View File

@@ -331,7 +331,7 @@ def generate_embeddings(
if max_workers is None:
if embedding_backend == "litellm":
if endpoint_count > 1:
max_workers = min(endpoint_count * 2, 16) # Cap at 16 workers
max_workers = min(endpoint_count * 2, 32) # Cap at 32 workers
else:
max_workers = 4
else:
@@ -806,7 +806,7 @@ def generate_embeddings_recursive(
if max_workers is None:
if embedding_backend == "litellm":
if endpoint_count > 1:
max_workers = min(endpoint_count * 2, 16)
max_workers = min(endpoint_count * 2, 32)
else:
max_workers = 4
else: