feat: enhance search, ranking, reranker and CLI tooling across ccw and codex-lens

Major improvements to smart-search, chain-search cascade, ranking pipeline,
reranker factory, CLI history store, codex-lens integration, and uv-manager.
Simplify command-generator skill by inlining phases. Add comprehensive tests.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

ccw/src/core/memory-embedder-bridge.ts

@@ -16,7 +16,7 @@ import { spawn } from 'child_process';
 import { join, dirname } from 'path';
 import { existsSync } from 'fs';
 import { fileURLToPath } from 'url';
-import { getCodexLensPython } from '../utils/codexlens-path.js';
+import { getCodexLensHiddenPython } from '../utils/codexlens-path.js';
 import { getCoreMemoryStore } from './core-memory-store.js';
 import type { Stage1Output } from './core-memory-store.js';
 import { StoragePaths } from '../config/storage-paths.js';
@@ -26,7 +26,7 @@ const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
 // Venv paths (reuse CodexLens venv)
-const VENV_PYTHON = getCodexLensPython();
+const VENV_PYTHON = getCodexLensHiddenPython();
 
 // Script path
 const EMBEDDER_SCRIPT = join(__dirname, '..', '..', 'scripts', 'memory_embedder.py');
@@ -116,8 +116,11 @@ function runPython(args: string[], timeout: number = 300000): Promise<string> {
 
     // Spawn Python process
     const child = spawn(VENV_PYTHON, [EMBEDDER_SCRIPT, ...args], {
+      shell: false,
       stdio: ['ignore', 'pipe', 'pipe'],
       timeout,
+      windowsHide: true,
+      env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
     });
 
     let stdout = '';

ccw/src/core/routes/codexlens/semantic-handlers.ts

@@ -8,7 +8,7 @@ import {
   executeCodexLens,
   installSemantic,
 } from '../../../tools/codex-lens.js';
-import { getCodexLensPython } from '../../../utils/codexlens-path.js';
+import { getCodexLensHiddenPython } from '../../../utils/codexlens-path.js';
 import { spawn } from 'child_process';
 import type { GpuMode } from '../../../tools/codex-lens.js';
 import { loadLiteLLMApiConfig, getAvailableModelsForType, getProvider, getAllProviders } from '../../../config/litellm-api-config-manager.js';
@@ -59,10 +59,13 @@ except Exception as e:
     sys.exit(1)
 `;
 
-    const pythonPath = getCodexLensPython();
+    const pythonPath = getCodexLensHiddenPython();
     const child = spawn(pythonPath, ['-c', pythonScript], {
+      shell: false,
       stdio: ['ignore', 'pipe', 'pipe'],
       timeout,
+      windowsHide: true,
+      env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
     });
 
     let stdout = '';

ccw/src/core/routes/codexlens/watcher-handlers.ts

@@ -126,8 +126,10 @@ export async function handleCodexLensWatcherRoutes(ctx: RouteContext): Promise<b
         const args = ['-m', 'codexlens', 'watch', targetPath, '--debounce', String(debounceMs)];
         watcherProcess = spawn(pythonPath, args, {
           cwd: targetPath,
+          shell: false,
           stdio: ['ignore', 'pipe', 'pipe'],
-          env: { ...process.env }
+          windowsHide: true,
+          env: { ...process.env, PYTHONIOENCODING: 'utf-8' }
         });
 
         watcherStats = {

ccw/src/core/routes/litellm-api-routes.ts

@@ -4,7 +4,11 @@
  */
 import { z } from 'zod';
 import { spawn } from 'child_process';
-import { getSystemPython } from '../../utils/python-utils.js';
+import {
+  getSystemPythonCommand,
+  parsePythonCommandSpec,
+  type PythonCommandSpec,
+} from '../../utils/python-utils.js';
 import {
   isUvAvailable,
   createCodexLensUvManager
@@ -102,10 +106,11 @@ interface CcwLitellmStatusResponse {
 }
 
 function checkCcwLitellmImport(
-  pythonCmd: string,
-  options: { timeout: number; shell?: boolean }
+  pythonCmd: string | PythonCommandSpec,
+  options: { timeout: number }
 ): Promise<CcwLitellmEnvCheck> {
-  const { timeout, shell = false } = options;
+  const { timeout } = options;
+  const pythonSpec = typeof pythonCmd === 'string' ? parsePythonCommandSpec(pythonCmd) : pythonCmd;
 
   const sanitizePythonError = (stderrText: string): string | undefined => {
     const trimmed = stderrText.trim();
@@ -119,11 +124,12 @@ function checkCcwLitellmImport(
   };
 
   return new Promise((resolve) => {
-    const child = spawn(pythonCmd, ['-c', 'import ccw_litellm; print(ccw_litellm.__version__)'], {
+    const child = spawn(pythonSpec.command, [...pythonSpec.args, '-c', 'import ccw_litellm; print(ccw_litellm.__version__)'], {
       stdio: ['ignore', 'pipe', 'pipe'],
       timeout,
       windowsHide: true,
-      shell,
+      shell: false,
+      env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
     });
 
     let stdout = '';
@@ -142,20 +148,20 @@ function checkCcwLitellmImport(
       const error = sanitizePythonError(stderr);
 
       if (code === 0 && version) {
-        resolve({ python: pythonCmd, installed: true, version });
+        resolve({ python: pythonSpec.display, installed: true, version });
         return;
       }
 
       if (code === null) {
-        resolve({ python: pythonCmd, installed: false, error: `Timed out after ${timeout}ms` });
+        resolve({ python: pythonSpec.display, installed: false, error: `Timed out after ${timeout}ms` });
         return;
       }
 
-      resolve({ python: pythonCmd, installed: false, error: error || undefined });
+      resolve({ python: pythonSpec.display, installed: false, error: error || undefined });
     });
 
     child.on('error', (err) => {
-      resolve({ python: pythonCmd, installed: false, error: err.message });
+      resolve({ python: pythonSpec.display, installed: false, error: err.message });
     });
   });
 }
@@ -940,7 +946,7 @@ export async function handleLiteLLMApiRoutes(ctx: RouteContext): Promise<boolean
       // Diagnostics only: if not installed in venv, also check system python so users understand mismatches.
       // NOTE: `installed` flag remains the CodexLens venv status (we want isolated venv dependencies).
       const systemPython = !codexLensVenv.installed
-        ? await checkCcwLitellmImport(getSystemPython(), { timeout: statusTimeout, shell: true })
+        ? await checkCcwLitellmImport(getSystemPythonCommand(), { timeout: statusTimeout })
         : undefined;
 
       const result: CcwLitellmStatusResponse = {
@@ -1410,10 +1416,19 @@ export async function handleLiteLLMApiRoutes(ctx: RouteContext): Promise<boolean
 
         // Priority 2: Fallback to system pip uninstall
         console.log('[ccw-litellm uninstall] Using pip fallback...');
-        const pythonCmd = getSystemPython();
+        const pythonCmd = getSystemPythonCommand();
 
         return new Promise((resolve) => {
-          const proc = spawn(pythonCmd, ['-m', 'pip', 'uninstall', '-y', 'ccw-litellm'], { shell: true, timeout: 120000 });
+          const proc = spawn(
+            pythonCmd.command,
+            [...pythonCmd.args, '-m', 'pip', 'uninstall', '-y', 'ccw-litellm'],
+            {
+              shell: false,
+              timeout: 120000,
+              windowsHide: true,
+              env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
+            },
+          );
           let output = '';
           let error = '';
           proc.stdout?.on('data', (data) => { output += data.toString(); });

ccw/src/core/unified-vector-index.ts

@@ -16,7 +16,7 @@ import { spawn } from 'child_process';
 import { join, dirname } from 'path';
 import { existsSync } from 'fs';
 import { fileURLToPath } from 'url';
-import { getCodexLensPython } from '../utils/codexlens-path.js';
+import { getCodexLensHiddenPython } from '../utils/codexlens-path.js';
 import { StoragePaths, ensureStorageDir } from '../config/storage-paths.js';
 
 // Get directory of this module
@@ -24,7 +24,7 @@ const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
 // Venv python path (reuse CodexLens venv)
-const VENV_PYTHON = getCodexLensPython();
+const VENV_PYTHON = getCodexLensHiddenPython();
 
 // Script path
 const EMBEDDER_SCRIPT = join(__dirname, '..', '..', 'scripts', 'unified_memory_embedder.py');
@@ -170,8 +170,11 @@ function runPython<T>(request: Record<string, unknown>, timeout: number = 300000
     }
 
     const child = spawn(VENV_PYTHON, [EMBEDDER_SCRIPT], {
+      shell: false,
       stdio: ['pipe', 'pipe', 'pipe'],
       timeout,
+      windowsHide: true,
+      env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
     });
 
     let stdout = '';