catlog22/Claude-Code-Workflow
1
0
Fork 0
mirror of https://github.com/catlog22/Claude-Code-Workflow.git synced 2026-03-18 18:48:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add APIEmbedder for remote embedding with multi-endpoint support

Browse Source 
- Introduced APIEmbedder class to handle embeddings via a remote HTTP API.
- Implemented token packing to optimize batch sizes based on token limits.
- Added support for multiple API endpoints with round-robin dispatching.
- Included retry logic for API calls with exponential backoff on failures.
- Enhanced indexing pipeline with file exclusion checks and smart chunking strategies.
- Updated tests to cover new APIEmbedder functionality and ensure robustness.
This commit is contained in:
catlog22
2026-03-17 17:17:24 +08:00
parent 34749d2fad
commit f37189dc64
18 changed files with 1633 additions and 476 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ccw/frontend/src/components/codexlens/OverviewTab.tsx
View File

@@ -16,7 +16,6 @@ import { cn } from '@/lib/utils';
import type { CodexLensVenvStatus, CodexLensConfig } from '@/lib/api';
import { IndexOperations } from './IndexOperations';
import { FileWatcherCard } from './FileWatcherCard';
import { LspServerCard } from './LspServerCard';
interface OverviewTabProps {
installed: boolean;
@@ -145,9 +144,8 @@ export function OverviewTab({ installed, status, config, isLoading, onRefresh }:
</div>
{/* Service Management */}
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
<div className="grid grid-cols-1 gap-4">
<FileWatcherCard disabled={!isReady} />
<LspServerCard disabled={!isReady} />
</div>
{/* Index Operations */}

27
ccw/frontend/src/components/codexlens/SchemaFormRenderer.tsx
View File

@@ -11,7 +11,6 @@ import {
ArrowUpDown,
Cpu,
GitBranch,
Scissors,
type LucideIcon,
} from 'lucide-react';
import { Label } from '@/components/ui/Label';
@@ -41,7 +40,6 @@ const iconMap: Record<string, LucideIcon> = {
'arrow-up-down': ArrowUpDown,
cpu: Cpu,
'git-branch': GitBranch,
scissors: Scissors,
};
interface SchemaFormRendererProps {
@@ -214,12 +212,12 @@ function FieldRenderer({
case 'model-select': {
// Determine backend type from related backend env var
const isEmbedding = field.key.includes('EMBEDDING');
const isEmbedding = field.key.includes('EMBED');
const backendKey = isEmbedding
? 'CODEXLENS_EMBEDDING_BACKEND'
: 'CODEXLENS_RERANKER_BACKEND';
const backendValue = allValues[backendKey];
const backendType = (backendValue === 'api' || backendValue === 'litellm') ? 'api' : 'local';
const backendType = backendValue === 'api' ? 'api' : 'local';
return (
<div className="flex items-center gap-2">
@@ -241,6 +239,27 @@ function FieldRenderer({
);
}
case 'password':
return (
<div className="flex items-center gap-2">
<Label
className="text-xs text-muted-foreground w-28 flex-shrink-0"
title={field.key}
>
{label}
</Label>
<Input
type="password"
className="flex-1 h-8 text-xs"
value={value}
onChange={(e) => onChange(e.target.value)}
placeholder={field.placeholder}
disabled={disabled}
autoComplete="off"
/>
</div>
);
case 'text':
default:
return (

564
ccw/frontend/src/components/codexlens/envVarSchema.ts
View File

@@ -1,8 +1,8 @@
// ========================================
// CodexLens Environment Variable Schema
// CodexLens v2 Environment Variable Schema
// ========================================
// TypeScript port of ENV_VAR_GROUPS from codexlens-manager.js
// Defines structured groups for CodexLens configuration
// Defines structured groups for codexlens-search v2 configuration.
// Env var names match what the Python bridge CLI reads.
import type { EnvVarGroupsSchema } from '@/types/codexlens';
@@ -20,20 +20,38 @@ export const envVarGroupsSchema: EnvVarGroupsSchema = {
default: 'local',
settingsPath: 'embedding.backend',
},
CODEXLENS_EMBEDDING_MODEL: {
key: 'CODEXLENS_EMBEDDING_MODEL',
CODEXLENS_EMBED_API_URL: {
key: 'CODEXLENS_EMBED_API_URL',
labelKey: 'codexlens.envField.apiUrl',
type: 'text',
placeholder: 'https://api.siliconflow.cn/v1',
default: '',
settingsPath: 'embedding.api_url',
showWhen: (env) => env['CODEXLENS_EMBEDDING_BACKEND'] === 'api',
},
CODEXLENS_EMBED_API_KEY: {
key: 'CODEXLENS_EMBED_API_KEY',
labelKey: 'codexlens.envField.apiKey',
type: 'password',
placeholder: 'sk-...',
default: '',
settingsPath: 'embedding.api_key',
showWhen: (env) => env['CODEXLENS_EMBEDDING_BACKEND'] === 'api',
},
CODEXLENS_EMBED_API_MODEL: {
key: 'CODEXLENS_EMBED_API_MODEL',
labelKey: 'codexlens.envField.model',
type: 'model-select',
placeholder: 'Select or enter model...',
default: 'fast',
settingsPath: 'embedding.model',
localModels: [
{
group: 'FastEmbed Profiles',
items: ['fast', 'code', 'base', 'minilm', 'multilingual', 'balanced'],
},
],
default: '',
settingsPath: 'embedding.api_model',
showWhen: (env) => env['CODEXLENS_EMBEDDING_BACKEND'] === 'api',
localModels: [],
apiModels: [
{
group: 'SiliconFlow',
items: ['BAAI/bge-m3', 'BAAI/bge-large-zh-v1.5', 'BAAI/bge-large-en-v1.5'],
},
{
group: 'OpenAI',
items: ['text-embedding-3-small', 'text-embedding-3-large', 'text-embedding-ada-002'],
@@ -44,66 +62,90 @@ export const envVarGroupsSchema: EnvVarGroupsSchema = {
},
{
group: 'Voyage',
items: ['voyage-3', 'voyage-3-lite', 'voyage-code-3', 'voyage-multilingual-2'],
},
{
group: 'SiliconFlow',
items: ['BAAI/bge-m3', 'BAAI/bge-large-zh-v1.5', 'BAAI/bge-large-en-v1.5'],
items: ['voyage-3', 'voyage-3-lite', 'voyage-code-3'],
},
{
group: 'Jina',
items: ['jina-embeddings-v3', 'jina-embeddings-v2-base-en', 'jina-embeddings-v2-base-zh'],
items: ['jina-embeddings-v3', 'jina-embeddings-v2-base-en'],
},
],
},
CODEXLENS_AUTO_EMBED_MISSING: {
key: 'CODEXLENS_AUTO_EMBED_MISSING',
labelKey: 'codexlens.envField.autoEmbedMissing',
type: 'checkbox',
default: 'true',
settingsPath: 'embedding.auto_embed_missing',
CODEXLENS_EMBED_API_ENDPOINTS: {
key: 'CODEXLENS_EMBED_API_ENDPOINTS',
labelKey: 'codexlens.envField.multiEndpoints',
type: 'text',
placeholder: 'url1|key1|model1,url2|key2|model2',
default: '',
settingsPath: 'embedding.api_endpoints',
showWhen: (env) => env['CODEXLENS_EMBEDDING_BACKEND'] === 'api',
},
CODEXLENS_EMBED_DIM: {
key: 'CODEXLENS_EMBED_DIM',
labelKey: 'codexlens.envField.embedDim',
type: 'number',
placeholder: '384',
default: '384',
settingsPath: 'embedding.dim',
min: 64,
max: 4096,
showWhen: (env) => env['CODEXLENS_EMBEDDING_BACKEND'] === 'api',
},
CODEXLENS_EMBED_API_CONCURRENCY: {
key: 'CODEXLENS_EMBED_API_CONCURRENCY',
labelKey: 'codexlens.envField.apiConcurrency',
type: 'number',
placeholder: '4',
default: '4',
settingsPath: 'embedding.api_concurrency',
min: 1,
max: 32,
showWhen: (env) => env['CODEXLENS_EMBEDDING_BACKEND'] === 'api',
},
CODEXLENS_EMBED_API_MAX_TOKENS: {
key: 'CODEXLENS_EMBED_API_MAX_TOKENS',
labelKey: 'codexlens.envField.maxTokensPerBatch',
type: 'number',
placeholder: '8192',
default: '8192',
settingsPath: 'embedding.api_max_tokens_per_batch',
min: 512,
max: 65536,
showWhen: (env) => env['CODEXLENS_EMBEDDING_BACKEND'] === 'api',
},
CODEXLENS_EMBEDDING_MODEL: {
key: 'CODEXLENS_EMBEDDING_MODEL',
labelKey: 'codexlens.envField.localModel',
type: 'model-select',
placeholder: 'Select local model...',
default: 'BAAI/bge-small-en-v1.5',
settingsPath: 'embedding.model',
showWhen: (env) => env['CODEXLENS_EMBEDDING_BACKEND'] !== 'api',
localModels: [
{
group: 'FastEmbed Profiles',
items: ['small', 'base', 'large', 'code'],
},
],
apiModels: [],
},
CODEXLENS_USE_GPU: {
key: 'CODEXLENS_USE_GPU',
labelKey: 'codexlens.envField.useGpu',
type: 'select',
options: ['true', 'false'],
default: 'true',
settingsPath: 'embedding.use_gpu',
showWhen: (env) => env['CODEXLENS_EMBEDDING_BACKEND'] === 'local',
options: ['auto', 'cuda', 'cpu'],
default: 'auto',
settingsPath: 'embedding.device',
showWhen: (env) => env['CODEXLENS_EMBEDDING_BACKEND'] !== 'api',
},
CODEXLENS_EMBEDDING_POOL_ENABLED: {
key: 'CODEXLENS_EMBEDDING_POOL_ENABLED',
labelKey: 'codexlens.envField.highAvailability',
type: 'select',
options: ['true', 'false'],
default: 'false',
settingsPath: 'embedding.pool_enabled',
showWhen: (env) => env['CODEXLENS_EMBEDDING_BACKEND'] === 'api',
},
CODEXLENS_EMBEDDING_STRATEGY: {
key: 'CODEXLENS_EMBEDDING_STRATEGY',
labelKey: 'codexlens.envField.loadBalanceStrategy',
type: 'select',
options: ['round_robin', 'latency_aware', 'weighted_random'],
default: 'latency_aware',
settingsPath: 'embedding.strategy',
showWhen: (env) =>
env['CODEXLENS_EMBEDDING_BACKEND'] === 'api' &&
env['CODEXLENS_EMBEDDING_POOL_ENABLED'] === 'true',
},
CODEXLENS_EMBEDDING_COOLDOWN: {
key: 'CODEXLENS_EMBEDDING_COOLDOWN',
labelKey: 'codexlens.envField.rateLimitCooldown',
CODEXLENS_EMBED_BATCH_SIZE: {
key: 'CODEXLENS_EMBED_BATCH_SIZE',
labelKey: 'codexlens.envField.batchSize',
type: 'number',
placeholder: '60',
default: '60',
settingsPath: 'embedding.cooldown',
min: 0,
max: 300,
showWhen: (env) =>
env['CODEXLENS_EMBEDDING_BACKEND'] === 'api' &&
env['CODEXLENS_EMBEDDING_POOL_ENABLED'] === 'true',
placeholder: '64',
default: '64',
settingsPath: 'embedding.batch_size',
min: 1,
max: 512,
},
},
},
@@ -112,29 +154,64 @@ export const envVarGroupsSchema: EnvVarGroupsSchema = {
labelKey: 'codexlens.envGroup.reranker',
icon: 'arrow-up-down',
vars: {
CODEXLENS_RERANKER_ENABLED: {
key: 'CODEXLENS_RERANKER_ENABLED',
labelKey: 'codexlens.envField.enabled',
type: 'select',
options: ['true', 'false'],
default: 'true',
settingsPath: 'reranker.enabled',
},
CODEXLENS_RERANKER_BACKEND: {
key: 'CODEXLENS_RERANKER_BACKEND',
labelKey: 'codexlens.envField.backend',
type: 'select',
options: ['onnx', 'api', 'litellm', 'legacy'],
default: 'onnx',
options: ['local', 'api'],
default: 'local',
settingsPath: 'reranker.backend',
},
CODEXLENS_RERANKER_MODEL: {
key: 'CODEXLENS_RERANKER_MODEL',
CODEXLENS_RERANKER_API_URL: {
key: 'CODEXLENS_RERANKER_API_URL',
labelKey: 'codexlens.envField.apiUrl',
type: 'text',
placeholder: 'https://api.siliconflow.cn/v1',
default: '',
settingsPath: 'reranker.api_url',
showWhen: (env) => env['CODEXLENS_RERANKER_BACKEND'] === 'api',
},
CODEXLENS_RERANKER_API_KEY: {
key: 'CODEXLENS_RERANKER_API_KEY',
labelKey: 'codexlens.envField.apiKey',
type: 'password',
placeholder: 'sk-...',
default: '',
settingsPath: 'reranker.api_key',
showWhen: (env) => env['CODEXLENS_RERANKER_BACKEND'] === 'api',
},
CODEXLENS_RERANKER_API_MODEL: {
key: 'CODEXLENS_RERANKER_API_MODEL',
labelKey: 'codexlens.envField.model',
type: 'model-select',
placeholder: 'Select or enter model...',
default: '',
settingsPath: 'reranker.api_model',
showWhen: (env) => env['CODEXLENS_RERANKER_BACKEND'] === 'api',
localModels: [],
apiModels: [
{
group: 'SiliconFlow',
items: ['BAAI/bge-reranker-v2-m3', 'BAAI/bge-reranker-large', 'BAAI/bge-reranker-base'],
},
{
group: 'Cohere',
items: ['rerank-english-v3.0', 'rerank-multilingual-v3.0'],
},
{
group: 'Jina',
items: ['jina-reranker-v2-base-multilingual'],
},
],
},
CODEXLENS_RERANKER_MODEL: {
key: 'CODEXLENS_RERANKER_MODEL',
labelKey: 'codexlens.envField.localModel',
type: 'model-select',
placeholder: 'Select local model...',
default: 'Xenova/ms-marco-MiniLM-L-6-v2',
settingsPath: 'reranker.model',
showWhen: (env) => env['CODEXLENS_RERANKER_BACKEND'] !== 'api',
localModels: [
{
group: 'FastEmbed/ONNX',
@@ -145,283 +222,128 @@ export const envVarGroupsSchema: EnvVarGroupsSchema = {
],
},
],
apiModels: [
{
group: 'Cohere',
items: ['rerank-english-v3.0', 'rerank-multilingual-v3.0', 'rerank-english-v2.0'],
},
{
group: 'Voyage',
items: ['rerank-2', 'rerank-2-lite', 'rerank-1'],
},
{
group: 'SiliconFlow',
items: ['BAAI/bge-reranker-v2-m3', 'BAAI/bge-reranker-large', 'BAAI/bge-reranker-base'],
},
{
group: 'Jina',
items: ['jina-reranker-v2-base-multilingual', 'jina-reranker-v1-base-en'],
},
],
apiModels: [],
},
CODEXLENS_RERANKER_TOP_K: {
key: 'CODEXLENS_RERANKER_TOP_K',
labelKey: 'codexlens.envField.topKResults',
type: 'number',
placeholder: '50',
default: '50',
placeholder: '20',
default: '20',
settingsPath: 'reranker.top_k',
min: 5,
max: 200,
},
CODEXLENS_RERANKER_POOL_ENABLED: {
key: 'CODEXLENS_RERANKER_POOL_ENABLED',
labelKey: 'codexlens.envField.highAvailability',
type: 'select',
options: ['true', 'false'],
default: 'false',
settingsPath: 'reranker.pool_enabled',
showWhen: (env) => env['CODEXLENS_RERANKER_BACKEND'] === 'api' || env['CODEXLENS_RERANKER_BACKEND'] === 'litellm',
CODEXLENS_RERANKER_BATCH_SIZE: {
key: 'CODEXLENS_RERANKER_BATCH_SIZE',
labelKey: 'codexlens.envField.batchSize',
type: 'number',
placeholder: '32',
default: '32',
settingsPath: 'reranker.batch_size',
min: 1,
max: 128,
},
CODEXLENS_RERANKER_STRATEGY: {
key: 'CODEXLENS_RERANKER_STRATEGY',
labelKey: 'codexlens.envField.loadBalanceStrategy',
type: 'select',
options: ['round_robin', 'latency_aware', 'weighted_random'],
default: 'latency_aware',
settingsPath: 'reranker.strategy',
showWhen: (env) =>
(env['CODEXLENS_RERANKER_BACKEND'] === 'api' || env['CODEXLENS_RERANKER_BACKEND'] === 'litellm') &&
env['CODEXLENS_RERANKER_POOL_ENABLED'] === 'true',
},
},
search: {
id: 'search',
labelKey: 'codexlens.envGroup.search',
icon: 'git-branch',
vars: {
CODEXLENS_BINARY_TOP_K: {
key: 'CODEXLENS_BINARY_TOP_K',
labelKey: 'codexlens.envField.binaryTopK',
type: 'number',
placeholder: '200',
default: '200',
settingsPath: 'search.binary_top_k',
min: 10,
max: 1000,
},
CODEXLENS_RERANKER_COOLDOWN: {
key: 'CODEXLENS_RERANKER_COOLDOWN',
labelKey: 'codexlens.envField.rateLimitCooldown',
CODEXLENS_ANN_TOP_K: {
key: 'CODEXLENS_ANN_TOP_K',
labelKey: 'codexlens.envField.annTopK',
type: 'number',
placeholder: '50',
default: '50',
settingsPath: 'search.ann_top_k',
min: 5,
max: 500,
},
CODEXLENS_FTS_TOP_K: {
key: 'CODEXLENS_FTS_TOP_K',
labelKey: 'codexlens.envField.ftsTopK',
type: 'number',
placeholder: '50',
default: '50',
settingsPath: 'search.fts_top_k',
min: 5,
max: 500,
},
CODEXLENS_FUSION_K: {
key: 'CODEXLENS_FUSION_K',
labelKey: 'codexlens.envField.fusionK',
type: 'number',
placeholder: '60',
default: '60',
settingsPath: 'reranker.cooldown',
min: 0,
max: 300,
showWhen: (env) =>
(env['CODEXLENS_RERANKER_BACKEND'] === 'api' || env['CODEXLENS_RERANKER_BACKEND'] === 'litellm') &&
env['CODEXLENS_RERANKER_POOL_ENABLED'] === 'true',
},
},
},
concurrency: {
id: 'concurrency',
labelKey: 'codexlens.envGroup.concurrency',
icon: 'cpu',
vars: {
CODEXLENS_API_MAX_WORKERS: {
key: 'CODEXLENS_API_MAX_WORKERS',
labelKey: 'codexlens.envField.maxWorkers',
type: 'number',
placeholder: '4',
default: '4',
settingsPath: 'api.max_workers',
settingsPath: 'search.fusion_k',
min: 1,
max: 32,
},
CODEXLENS_API_BATCH_SIZE: {
key: 'CODEXLENS_API_BATCH_SIZE',
labelKey: 'codexlens.envField.batchSize',
type: 'number',
placeholder: '8',
default: '8',
settingsPath: 'api.batch_size',
min: 1,
max: 64,
showWhen: (env) => env['CODEXLENS_API_BATCH_SIZE_DYNAMIC'] !== 'true',
},
CODEXLENS_API_BATCH_SIZE_DYNAMIC: {
key: 'CODEXLENS_API_BATCH_SIZE_DYNAMIC',
labelKey: 'codexlens.envField.dynamicBatchSize',
type: 'checkbox',
default: 'false',
settingsPath: 'api.batch_size_dynamic',
},
CODEXLENS_API_BATCH_SIZE_UTILIZATION: {
key: 'CODEXLENS_API_BATCH_SIZE_UTILIZATION',
labelKey: 'codexlens.envField.batchSizeUtilization',
type: 'number',
placeholder: '0.8',
default: '0.8',
settingsPath: 'api.batch_size_utilization_factor',
min: 0.1,
max: 0.95,
step: 0.05,
showWhen: (env) => env['CODEXLENS_API_BATCH_SIZE_DYNAMIC'] === 'true',
},
CODEXLENS_API_BATCH_SIZE_MAX: {
key: 'CODEXLENS_API_BATCH_SIZE_MAX',
labelKey: 'codexlens.envField.batchSizeMax',
type: 'number',
placeholder: '2048',
default: '2048',
settingsPath: 'api.batch_size_max',
min: 1,
max: 4096,
showWhen: (env) => env['CODEXLENS_API_BATCH_SIZE_DYNAMIC'] === 'true',
},
CODEXLENS_CHARS_PER_TOKEN: {
key: 'CODEXLENS_CHARS_PER_TOKEN',
labelKey: 'codexlens.envField.charsPerToken',
type: 'number',
placeholder: '4',
default: '4',
settingsPath: 'api.chars_per_token_estimate',
min: 1,
max: 10,
showWhen: (env) => env['CODEXLENS_API_BATCH_SIZE_DYNAMIC'] === 'true',
},
},
},
cascade: {
id: 'cascade',
labelKey: 'codexlens.envGroup.cascade',
icon: 'git-branch',
vars: {
CODEXLENS_CASCADE_STRATEGY: {
key: 'CODEXLENS_CASCADE_STRATEGY',
labelKey: 'codexlens.envField.searchStrategy',
type: 'select',
options: ['binary', 'hybrid', 'binary_rerank', 'dense_rerank', 'staged'],
default: 'dense_rerank',
settingsPath: 'cascade.strategy',
},
CODEXLENS_CASCADE_COARSE_K: {
key: 'CODEXLENS_CASCADE_COARSE_K',
labelKey: 'codexlens.envField.coarseK',
type: 'number',
placeholder: '100',
default: '100',
settingsPath: 'cascade.coarse_k',
min: 10,
max: 500,
},
CODEXLENS_CASCADE_FINE_K: {
key: 'CODEXLENS_CASCADE_FINE_K',
labelKey: 'codexlens.envField.fineK',
type: 'number',
placeholder: '10',
default: '10',
settingsPath: 'cascade.fine_k',
min: 1,
max: 100,
},
CODEXLENS_STAGED_STAGE2_MODE: {
key: 'CODEXLENS_STAGED_STAGE2_MODE',
labelKey: 'codexlens.envField.stagedStage2Mode',
type: 'select',
options: ['precomputed', 'realtime', 'static_global_graph'],
default: 'precomputed',
settingsPath: 'staged.stage2_mode',
showWhen: (env) => env['CODEXLENS_CASCADE_STRATEGY'] === 'staged',
},
CODEXLENS_STAGED_CLUSTERING_STRATEGY: {
key: 'CODEXLENS_STAGED_CLUSTERING_STRATEGY',
labelKey: 'codexlens.envField.stagedClusteringStrategy',
type: 'select',
options: ['auto', 'hdbscan', 'dbscan', 'frequency', 'noop', 'score', 'dir_rr', 'path'],
default: 'auto',
settingsPath: 'staged.clustering_strategy',
showWhen: (env) => env['CODEXLENS_CASCADE_STRATEGY'] === 'staged',
},
CODEXLENS_STAGED_CLUSTERING_MIN_SIZE: {
key: 'CODEXLENS_STAGED_CLUSTERING_MIN_SIZE',
labelKey: 'codexlens.envField.stagedClusteringMinSize',
type: 'number',
placeholder: '3',
default: '3',
settingsPath: 'staged.clustering_min_size',
min: 1,
max: 50,
showWhen: (env) => env['CODEXLENS_CASCADE_STRATEGY'] === 'staged',
},
CODEXLENS_ENABLE_STAGED_RERANK: {
key: 'CODEXLENS_ENABLE_STAGED_RERANK',
labelKey: 'codexlens.envField.enableStagedRerank',
type: 'checkbox',
default: 'true',
settingsPath: 'staged.enable_rerank',
showWhen: (env) => env['CODEXLENS_CASCADE_STRATEGY'] === 'staged',
max: 200,
},
},
},
indexing: {
id: 'indexing',
labelKey: 'codexlens.envGroup.indexing',
icon: 'git-branch',
icon: 'cpu',
vars: {
CODEXLENS_USE_ASTGREP: {
key: 'CODEXLENS_USE_ASTGREP',
labelKey: 'codexlens.envField.useAstGrep',
CODEXLENS_CODE_AWARE_CHUNKING: {
key: 'CODEXLENS_CODE_AWARE_CHUNKING',
labelKey: 'codexlens.envField.codeAwareChunking',
type: 'checkbox',
default: 'false',
settingsPath: 'parsing.use_astgrep',
},
CODEXLENS_STATIC_GRAPH_ENABLED: {
key: 'CODEXLENS_STATIC_GRAPH_ENABLED',
labelKey: 'codexlens.envField.staticGraphEnabled',
type: 'checkbox',
default: 'false',
settingsPath: 'indexing.static_graph_enabled',
},
CODEXLENS_STATIC_GRAPH_RELATIONSHIP_TYPES: {
key: 'CODEXLENS_STATIC_GRAPH_RELATIONSHIP_TYPES',
labelKey: 'codexlens.envField.staticGraphRelationshipTypes',
type: 'text',
placeholder: 'imports,inherits,calls',
default: 'imports,inherits',
settingsPath: 'indexing.static_graph_relationship_types',
showWhen: (env) => env['CODEXLENS_STATIC_GRAPH_ENABLED'] === 'true',
},
},
},
chunking: {
id: 'chunking',
labelKey: 'codexlens.envGroup.chunking',
icon: 'scissors',
vars: {
CHUNK_STRIP_COMMENTS: {
key: 'CHUNK_STRIP_COMMENTS',
labelKey: 'codexlens.envField.stripComments',
type: 'select',
options: ['true', 'false'],
default: 'true',
settingsPath: 'chunking.strip_comments',
settingsPath: 'indexing.code_aware_chunking',
},
CHUNK_STRIP_DOCSTRINGS: {
key: 'CHUNK_STRIP_DOCSTRINGS',
labelKey: 'codexlens.envField.stripDocstrings',
type: 'select',
options: ['true', 'false'],
default: 'true',
settingsPath: 'chunking.strip_docstrings',
},
RERANKER_TEST_FILE_PENALTY: {
key: 'RERANKER_TEST_FILE_PENALTY',
labelKey: 'codexlens.envField.testFilePenalty',
CODEXLENS_INDEX_WORKERS: {
key: 'CODEXLENS_INDEX_WORKERS',
labelKey: 'codexlens.envField.indexWorkers',
type: 'number',
placeholder: '0.0',
default: '0.0',
settingsPath: 'reranker.test_file_penalty',
min: 0,
max: 1,
step: 0.1,
placeholder: '2',
default: '2',
settingsPath: 'indexing.workers',
min: 1,
max: 16,
},
RERANKER_DOCSTRING_WEIGHT: {
key: 'RERANKER_DOCSTRING_WEIGHT',
labelKey: 'codexlens.envField.docstringWeight',
CODEXLENS_MAX_FILE_SIZE: {
key: 'CODEXLENS_MAX_FILE_SIZE',
labelKey: 'codexlens.envField.maxFileSize',
type: 'number',
placeholder: '1.0',
default: '1.0',
settingsPath: 'reranker.docstring_weight',
min: 0,
max: 1,
step: 0.1,
placeholder: '1000000',
default: '1000000',
settingsPath: 'indexing.max_file_size_bytes',
min: 10000,
max: 10000000,
},
CODEXLENS_HNSW_EF: {
key: 'CODEXLENS_HNSW_EF',
labelKey: 'codexlens.envField.hnswEf',
type: 'number',
placeholder: '150',
default: '150',
settingsPath: 'indexing.hnsw_ef',
min: 10,
max: 500,
},
CODEXLENS_HNSW_M: {
key: 'CODEXLENS_HNSW_M',
labelKey: 'codexlens.envField.hnswM',
type: 'number',
placeholder: '32',
default: '32',
settingsPath: 'indexing.hnsw_M',
min: 4,
max: 128,
},
},
},

48
ccw/frontend/src/locales/en/codexlens.json
View File

@@ -290,41 +290,31 @@
"envGroup": {
"embedding": "Embedding",
"reranker": "Reranker",
"concurrency": "Concurrency",
"cascade": "Cascade Search",
"indexing": "Indexing",
"chunking": "Chunking"
"search": "Search Pipeline",
"indexing": "Indexing"
},
"envField": {
"backend": "Backend",
"model": "Model",
"autoEmbedMissing": "Auto Build Missing Vectors",
"useGpu": "Use GPU",
"highAvailability": "High Availability",
"loadBalanceStrategy": "Load Balance Strategy",
"rateLimitCooldown": "Rate Limit Cooldown",
"enabled": "Enabled",
"localModel": "Local Model",
"apiUrl": "API URL",
"apiKey": "API Key",
"multiEndpoints": "Multi-Endpoint",
"embedDim": "Embed Dimension",
"apiConcurrency": "Concurrency",
"maxTokensPerBatch": "Max Tokens/Batch",
"useGpu": "Device",
"topKResults": "Top K Results",
"maxWorkers": "Max Workers",
"batchSize": "Batch Size",
"dynamicBatchSize": "Dynamic Batch Size",
"batchSizeUtilization": "Utilization Factor",
"batchSizeMax": "Max Batch Size",
"charsPerToken": "Chars Per Token",
"searchStrategy": "Search Strategy",
"coarseK": "Coarse K",
"fineK": "Fine K",
"stagedStage2Mode": "Stage-2 Mode",
"stagedClusteringStrategy": "Clustering Strategy",
"stagedClusteringMinSize": "Cluster Min Size",
"enableStagedRerank": "Enable Rerank",
"useAstGrep": "Use ast-grep",
"staticGraphEnabled": "Static Graph",
"staticGraphRelationshipTypes": "Relationship Types",
"stripComments": "Strip Comments",
"stripDocstrings": "Strip Docstrings",
"testFilePenalty": "Test File Penalty",
"docstringWeight": "Docstring Weight"
"binaryTopK": "Binary Top K",
"annTopK": "ANN Top K",
"ftsTopK": "FTS Top K",
"fusionK": "Fusion K",
"codeAwareChunking": "Code-Aware Chunking",
"indexWorkers": "Index Workers",
"maxFileSize": "Max File Size (bytes)",
"hnswEf": "HNSW ef",
"hnswM": "HNSW M"
},
"install": {
"title": "Install CodexLens",

48
ccw/frontend/src/locales/zh/codexlens.json
View File

@@ -290,41 +290,31 @@
"envGroup": {
"embedding": "嵌入模型",
"reranker": "重排序",
"concurrency": "并发",
"cascade": "级联搜索",
"indexing": "索引与解析",
"chunking": "分块"
"search": "搜索流水线",
"indexing": "索引"
},
"envField": {
"backend": "后端",
"model": "模型",
"autoEmbedMissing": "缺失向量时自动构建",
"useGpu": "使用 GPU",
"highAvailability": "高可用",
"loadBalanceStrategy": "负载均衡策略",
"rateLimitCooldown": "限流冷却时间",
"enabled": "启用",
"localModel": "本地模型",
"apiUrl": "API 地址",
"apiKey": "API 密钥",
"multiEndpoints": "多端点",
"embedDim": "向量维度",
"apiConcurrency": "并发数",
"maxTokensPerBatch": "每批最大Token数",
"useGpu": "设备",
"topKResults": "Top K 结果数",
"maxWorkers": "最大工作线程",
"batchSize": "批次大小",
"dynamicBatchSize": "动态批次大小",
"batchSizeUtilization": "利用率因子",
"batchSizeMax": "最大批次大小",
"charsPerToken": "每 Token 字符数",
"searchStrategy": "搜索策略",
"coarseK": "粗筛 K 值",
"fineK": "精筛 K 值",
"stagedStage2Mode": "Stage-2 模式",
"stagedClusteringStrategy": "聚类策略",
"stagedClusteringMinSize": "最小聚类大小",
"enableStagedRerank": "启用重排序",
"useAstGrep": "使用 ast-grep",
"staticGraphEnabled": "启用静态图",
"staticGraphRelationshipTypes": "关系类型",
"stripComments": "去除注释",
"stripDocstrings": "去除文档字符串",
"testFilePenalty": "测试文件惩罚",
"docstringWeight": "文档字符串权重"
"binaryTopK": "二值粗筛 K",
"annTopK": "ANN 精筛 K",
"ftsTopK": "全文搜索 K",
"fusionK": "融合 K",
"codeAwareChunking": "代码感知分块",
"indexWorkers": "索引线程数",
"maxFileSize": "最大文件大小(字节)",
"hnswEf": "HNSW ef",
"hnswM": "HNSW M"
},
"install": {
"title": "安装 CodexLens",

2
ccw/frontend/src/types/codexlens.ts
View File

@@ -20,7 +20,7 @@ export interface EnvVarFieldSchema {
/** i18n label key */
labelKey: string;
/** Field type determines which control to render */
type: 'select' | 'model-select' | 'number' | 'checkbox' | 'text';
type: 'select' | 'model-select' | 'number' | 'checkbox' | 'text' | 'password';
/** Options for select type */
options?: string[];
/** Default value */

124
ccw/src/core/routes/codexlens/config-handlers.ts
View File

@@ -941,7 +941,7 @@ export async function handleCodexLensConfigRoutes(ctx: RouteContext): Promise<bo
// Settings file doesn't exist or is invalid, use empty
}
// Map settings to env var format for defaults
// Map settings to env var format for defaults (v2 schema)
const settingsDefaults: Record<string, string> = {};
// Embedding settings
@@ -950,19 +950,34 @@ export async function handleCodexLensConfigRoutes(ctx: RouteContext): Promise<bo
}
if (settings.embedding?.model) {
settingsDefaults['CODEXLENS_EMBEDDING_MODEL'] = settings.embedding.model;
settingsDefaults['LITELLM_EMBEDDING_MODEL'] = settings.embedding.model;
}
if (settings.embedding?.use_gpu !== undefined) {
settingsDefaults['CODEXLENS_USE_GPU'] = String(settings.embedding.use_gpu);
if (settings.embedding?.device) {
settingsDefaults['CODEXLENS_USE_GPU'] = settings.embedding.device;
}
if (settings.embedding?.auto_embed_missing !== undefined) {
settingsDefaults['CODEXLENS_AUTO_EMBED_MISSING'] = String(settings.embedding.auto_embed_missing);
if (settings.embedding?.batch_size !== undefined) {
settingsDefaults['CODEXLENS_EMBED_BATCH_SIZE'] = String(settings.embedding.batch_size);
}
if (settings.embedding?.strategy) {
settingsDefaults['CODEXLENS_EMBEDDING_STRATEGY'] = settings.embedding.strategy;
// Embedding API settings
if (settings.embedding?.api_url) {
settingsDefaults['CODEXLENS_EMBED_API_URL'] = settings.embedding.api_url;
}
if (settings.embedding?.cooldown !== undefined) {
settingsDefaults['CODEXLENS_EMBEDDING_COOLDOWN'] = String(settings.embedding.cooldown);
if (settings.embedding?.api_key) {
settingsDefaults['CODEXLENS_EMBED_API_KEY'] = settings.embedding.api_key;
}
if (settings.embedding?.api_model) {
settingsDefaults['CODEXLENS_EMBED_API_MODEL'] = settings.embedding.api_model;
}
if (settings.embedding?.api_endpoints) {
settingsDefaults['CODEXLENS_EMBED_API_ENDPOINTS'] = settings.embedding.api_endpoints;
}
if (settings.embedding?.dim !== undefined) {
settingsDefaults['CODEXLENS_EMBED_DIM'] = String(settings.embedding.dim);
}
if (settings.embedding?.api_concurrency !== undefined) {
settingsDefaults['CODEXLENS_EMBED_API_CONCURRENCY'] = String(settings.embedding.api_concurrency);
}
if (settings.embedding?.api_max_tokens_per_batch !== undefined) {
settingsDefaults['CODEXLENS_EMBED_API_MAX_TOKENS'] = String(settings.embedding.api_max_tokens_per_batch);
}
// Reranker settings
@@ -971,82 +986,53 @@ export async function handleCodexLensConfigRoutes(ctx: RouteContext): Promise<bo
}
if (settings.reranker?.model) {
settingsDefaults['CODEXLENS_RERANKER_MODEL'] = settings.reranker.model;
settingsDefaults['LITELLM_RERANKER_MODEL'] = settings.reranker.model;
}
if (settings.reranker?.enabled !== undefined) {
settingsDefaults['CODEXLENS_RERANKER_ENABLED'] = String(settings.reranker.enabled);
}
if (settings.reranker?.top_k !== undefined) {
settingsDefaults['CODEXLENS_RERANKER_TOP_K'] = String(settings.reranker.top_k);
}
// API/Concurrency settings
if (settings.api?.max_workers !== undefined) {
settingsDefaults['CODEXLENS_API_MAX_WORKERS'] = String(settings.api.max_workers);
if (settings.reranker?.batch_size !== undefined) {
settingsDefaults['CODEXLENS_RERANKER_BATCH_SIZE'] = String(settings.reranker.batch_size);
}
if (settings.api?.batch_size !== undefined) {
settingsDefaults['CODEXLENS_API_BATCH_SIZE'] = String(settings.api.batch_size);
// Reranker API settings
if (settings.reranker?.api_url) {
settingsDefaults['CODEXLENS_RERANKER_API_URL'] = settings.reranker.api_url;
}
// Dynamic batch size settings
if (settings.api?.batch_size_dynamic !== undefined) {
settingsDefaults['CODEXLENS_API_BATCH_SIZE_DYNAMIC'] = String(settings.api.batch_size_dynamic);
if (settings.reranker?.api_key) {
settingsDefaults['CODEXLENS_RERANKER_API_KEY'] = settings.reranker.api_key;
}
if (settings.api?.batch_size_utilization_factor !== undefined) {
settingsDefaults['CODEXLENS_API_BATCH_SIZE_UTILIZATION'] = String(settings.api.batch_size_utilization_factor);
}
if (settings.api?.batch_size_max !== undefined) {
settingsDefaults['CODEXLENS_API_BATCH_SIZE_MAX'] = String(settings.api.batch_size_max);
}
if (settings.api?.chars_per_token_estimate !== undefined) {
settingsDefaults['CODEXLENS_CHARS_PER_TOKEN'] = String(settings.api.chars_per_token_estimate);
if (settings.reranker?.api_model) {
settingsDefaults['CODEXLENS_RERANKER_API_MODEL'] = settings.reranker.api_model;
}
// Cascade search settings
if (settings.cascade?.strategy) {
settingsDefaults['CODEXLENS_CASCADE_STRATEGY'] = settings.cascade.strategy;
// Search pipeline settings
if (settings.search?.binary_top_k !== undefined) {
settingsDefaults['CODEXLENS_BINARY_TOP_K'] = String(settings.search.binary_top_k);
}
if (settings.cascade?.coarse_k !== undefined) {
settingsDefaults['CODEXLENS_CASCADE_COARSE_K'] = String(settings.cascade.coarse_k);
if (settings.search?.ann_top_k !== undefined) {
settingsDefaults['CODEXLENS_ANN_TOP_K'] = String(settings.search.ann_top_k);
}
if (settings.cascade?.fine_k !== undefined) {
settingsDefaults['CODEXLENS_CASCADE_FINE_K'] = String(settings.cascade.fine_k);
if (settings.search?.fts_top_k !== undefined) {
settingsDefaults['CODEXLENS_FTS_TOP_K'] = String(settings.search.fts_top_k);
}
if (settings.search?.fusion_k !== undefined) {
settingsDefaults['CODEXLENS_FUSION_K'] = String(settings.search.fusion_k);
}
// Staged cascade settings (advanced)
if (settings.staged?.stage2_mode) {
settingsDefaults['CODEXLENS_STAGED_STAGE2_MODE'] = settings.staged.stage2_mode;
// Indexing settings
if (settings.indexing?.code_aware_chunking !== undefined) {
settingsDefaults['CODEXLENS_CODE_AWARE_CHUNKING'] = String(settings.indexing.code_aware_chunking);
}
if (settings.staged?.clustering_strategy) {
settingsDefaults['CODEXLENS_STAGED_CLUSTERING_STRATEGY'] = settings.staged.clustering_strategy;
if (settings.indexing?.workers !== undefined) {
settingsDefaults['CODEXLENS_INDEX_WORKERS'] = String(settings.indexing.workers);
}
if (settings.staged?.clustering_min_size !== undefined) {
settingsDefaults['CODEXLENS_STAGED_CLUSTERING_MIN_SIZE'] = String(settings.staged.clustering_min_size);
if (settings.indexing?.max_file_size_bytes !== undefined) {
settingsDefaults['CODEXLENS_MAX_FILE_SIZE'] = String(settings.indexing.max_file_size_bytes);
}
if (settings.staged?.enable_rerank !== undefined) {
settingsDefaults['CODEXLENS_ENABLE_STAGED_RERANK'] = String(settings.staged.enable_rerank);
if (settings.indexing?.hnsw_ef !== undefined) {
settingsDefaults['CODEXLENS_HNSW_EF'] = String(settings.indexing.hnsw_ef);
}
// LLM settings
if (settings.llm?.enabled !== undefined) {
settingsDefaults['CODEXLENS_LLM_ENABLED'] = String(settings.llm.enabled);
}
if (settings.llm?.batch_size !== undefined) {
settingsDefaults['CODEXLENS_LLM_BATCH_SIZE'] = String(settings.llm.batch_size);
}
// Parsing / indexing settings
if (settings.parsing?.use_astgrep !== undefined) {
settingsDefaults['CODEXLENS_USE_ASTGREP'] = String(settings.parsing.use_astgrep);
}
if (settings.indexing?.static_graph_enabled !== undefined) {
settingsDefaults['CODEXLENS_STATIC_GRAPH_ENABLED'] = String(settings.indexing.static_graph_enabled);
}
if (settings.indexing?.static_graph_relationship_types !== undefined) {
if (Array.isArray(settings.indexing.static_graph_relationship_types)) {
settingsDefaults['CODEXLENS_STATIC_GRAPH_RELATIONSHIP_TYPES'] = settings.indexing.static_graph_relationship_types.join(',');
} else if (typeof settings.indexing.static_graph_relationship_types === 'string') {
settingsDefaults['CODEXLENS_STATIC_GRAPH_RELATIONSHIP_TYPES'] = settings.indexing.static_graph_relationship_types;
}
if (settings.indexing?.hnsw_M !== undefined) {
settingsDefaults['CODEXLENS_HNSW_M'] = String(settings.indexing.hnsw_M);
}
res.writeHead(200, { 'Content-Type': 'application/json' });

32
ccw/src/tools/smart-search.ts
View File

@@ -2286,6 +2286,34 @@ async function executeCodexLensV2Bridge(
});
}
/**
* Load env vars from ~/.codexlens/.env file so they're passed to bridge subprocess.
*/
function loadCodexLensEnvFile(): Record<string, string> {
const envVars: Record<string, string> = {};
try {
const envPath = join(getCodexLensDataDir(), '.env');
const content = readFileSync(envPath, 'utf-8');
for (const line of content.split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const eqIdx = trimmed.indexOf('=');
if (eqIdx <= 0) continue;
const key = trimmed.substring(0, eqIdx).trim();
let value = trimmed.substring(eqIdx + 1).trim();
// Strip surrounding quotes
if ((value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))) {
value = value.slice(1, -1);
}
envVars[key] = value;
}
} catch {
// File doesn't exist — no env overrides
}
return envVars;
}
/**
* Execute a generic codexlens-search v2 bridge subcommand (init, status, sync, watch, etc.).
* Returns parsed JSON output from the bridge CLI.
@@ -2299,11 +2327,13 @@ async function executeV2BridgeCommand(
// --db-path is a global arg and must come BEFORE the subcommand
const globalArgs = options?.dbPath ? ['--db-path', options.dbPath] : [];
const fullArgs = [...globalArgs, subcommand, ...args];
// Merge process.env with .env file settings (file values override process.env)
const codexlensEnv = loadCodexLensEnvFile();
execFile('codexlens-search', fullArgs, {
encoding: 'utf-8',
timeout: options?.timeout ?? EXEC_TIMEOUTS.PROCESS_SPAWN,
windowsHide: true,
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
env: { ...process.env, ...codexlensEnv, PYTHONIOENCODING: 'utf-8' },
}, (error, stdout, stderr) => {
if (error) {
resolve({