Claude-Code-Workflow/ccw/src/tools/codex-lens.ts

/**
 * CodexLens Tool - Bridge between CCW and CodexLens Python package
 * Provides code indexing and semantic search via spawned Python process
 *
 * Features:
 * - Automatic venv bootstrap at ~/.codexlens/venv
 * - JSON protocol communication
 * - Symbol extraction and semantic search
 * - FTS5 full-text search
 */

import { z } from 'zod';
import type { ToolSchema, ToolResult } from '../types/tool.js';
import { spawn, execSync } from 'child_process';
import { existsSync, mkdirSync } from 'fs';
import { join, dirname } from 'path';
import { homedir } from 'os';
import { fileURLToPath } from 'url';

// Get directory of this module
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

// CodexLens configuration
const CODEXLENS_DATA_DIR = join(homedir(), '.codexlens');
const CODEXLENS_VENV = join(CODEXLENS_DATA_DIR, 'venv');
const VENV_PYTHON =
  process.platform === 'win32'
    ? join(CODEXLENS_VENV, 'Scripts', 'python.exe')
    : join(CODEXLENS_VENV, 'bin', 'python');

// Bootstrap status cache
let bootstrapChecked = false;
let bootstrapReady = false;

// Define Zod schema for validation
const ParamsSchema = z.object({
  action: z.enum([
    'init',
    'search',
    'search_files',
    'symbol',
    'status',
    'config_show',
    'config_set',
    'config_migrate',
    'clean',
    'bootstrap',
    'check',
  ]),
  path: z.string().optional(),
  query: z.string().optional(),
  mode: z.enum(['text', 'semantic']).default('text'),
  file: z.string().optional(),
  key: z.string().optional(), // For config_set action
  value: z.string().optional(), // For config_set action
  newPath: z.string().optional(), // For config_migrate action
  all: z.boolean().optional(), // For clean action
  languages: z.array(z.string()).optional(),
  limit: z.number().default(20),
  format: z.enum(['json', 'table', 'plain']).default('json'),
});

type Params = z.infer<typeof ParamsSchema>;

interface ReadyStatus {
  ready: boolean;
  error?: string;
  version?: string;
}

interface SemanticStatus {
  available: boolean;
  backend?: string;
  error?: string;
}

interface BootstrapResult {
  success: boolean;
  error?: string;
  message?: string;
}

interface ExecuteResult {
  success: boolean;
  output?: string;
  error?: string;
  message?: string;
  results?: unknown;
  files?: unknown;
  symbols?: unknown;
  status?: unknown;
  config?: unknown;
  cleanResult?: unknown;
  ready?: boolean;
  version?: string;
}

interface ExecuteOptions {
  timeout?: number;
  cwd?: string;
}

/**
 * Detect available Python 3 executable
 * @returns Python executable command
 */
function getSystemPython(): string {
  const commands = process.platform === 'win32' ? ['python', 'py', 'python3'] : ['python3', 'python'];

  for (const cmd of commands) {
    try {
      const version = execSync(`${cmd} --version 2>&1`, { encoding: 'utf8' });
      if (version.includes('Python 3')) {
        return cmd;
      }
    } catch {
      // Try next command
    }
  }
  throw new Error('Python 3 not found. Please install Python 3 and ensure it is in PATH.');
}

/**
 * Check if CodexLens venv exists and has required packages
 * @returns Ready status
 */
async function checkVenvStatus(): Promise<ReadyStatus> {
  // Check venv exists
  if (!existsSync(CODEXLENS_VENV)) {
    return { ready: false, error: 'Venv not found' };
  }

  // Check python executable exists
  if (!existsSync(VENV_PYTHON)) {
    return { ready: false, error: 'Python executable not found in venv' };
  }

  // Check codexlens is importable
  return new Promise((resolve) => {
    const child = spawn(VENV_PYTHON, ['-c', 'import codexlens; print(codexlens.__version__)'], {
      stdio: ['ignore', 'pipe', 'pipe'],
      timeout: 10000,
    });

    let stdout = '';
    let stderr = '';

    child.stdout.on('data', (data) => {
      stdout += data.toString();
    });
    child.stderr.on('data', (data) => {
      stderr += data.toString();
    });

    child.on('close', (code) => {
      if (code === 0) {
        resolve({ ready: true, version: stdout.trim() });
      } else {
        resolve({ ready: false, error: `CodexLens not installed: ${stderr}` });
      }
    });

    child.on('error', (err) => {
      resolve({ ready: false, error: `Failed to check venv: ${err.message}` });
    });
  });
}

/**
 * Check if semantic search dependencies are installed
 * @returns Semantic status
 */
async function checkSemanticStatus(): Promise<SemanticStatus> {
  // First check if CodexLens is installed
  const venvStatus = await checkVenvStatus();
  if (!venvStatus.ready) {
    return { available: false, error: 'CodexLens not installed' };
  }

  // Check semantic module availability
  return new Promise((resolve) => {
    const checkCode = `
import sys
try:
    from codexlens.semantic import SEMANTIC_AVAILABLE, SEMANTIC_BACKEND
    if SEMANTIC_AVAILABLE:
        print(f"available:{SEMANTIC_BACKEND}")
    else:
        print("unavailable")
except Exception as e:
    print(f"error:{e}")
`;
    const child = spawn(VENV_PYTHON, ['-c', checkCode], {
      stdio: ['ignore', 'pipe', 'pipe'],
      timeout: 15000,
    });

    let stdout = '';
    let stderr = '';

    child.stdout.on('data', (data) => {
      stdout += data.toString();
    });
    child.stderr.on('data', (data) => {
      stderr += data.toString();
    });

    child.on('close', (code) => {
      const output = stdout.trim();
      if (output.startsWith('available:')) {
        const backend = output.split(':')[1];
        resolve({ available: true, backend });
      } else if (output === 'unavailable') {
        resolve({ available: false, error: 'Semantic dependencies not installed' });
      } else {
        resolve({ available: false, error: output || stderr || 'Unknown error' });
      }
    });

    child.on('error', (err) => {
      resolve({ available: false, error: `Check failed: ${err.message}` });
    });
  });
}

/**
 * Install semantic search dependencies (fastembed, ONNX-based, ~200MB)
 * @returns Bootstrap result
 */
async function installSemantic(): Promise<BootstrapResult> {
  // First ensure CodexLens is installed
  const venvStatus = await checkVenvStatus();
  if (!venvStatus.ready) {
    return { success: false, error: 'CodexLens not installed. Install CodexLens first.' };
  }

  const pipPath =
    process.platform === 'win32'
      ? join(CODEXLENS_VENV, 'Scripts', 'pip.exe')
      : join(CODEXLENS_VENV, 'bin', 'pip');

  return new Promise((resolve) => {
    console.log('[CodexLens] Installing semantic search dependencies (fastembed)...');
    console.log('[CodexLens] Using ONNX-based fastembed backend (~200MB)');

    const child = spawn(pipPath, ['install', 'numpy>=1.24', 'fastembed>=0.2'], {
      stdio: ['ignore', 'pipe', 'pipe'],
      timeout: 600000, // 10 minutes for potential model download
    });

    let stdout = '';
    let stderr = '';

    child.stdout.on('data', (data) => {
      stdout += data.toString();
      // Log progress
      const line = data.toString().trim();
      if (line.includes('Downloading') || line.includes('Installing') || line.includes('Collecting')) {
        console.log(`[CodexLens] ${line}`);
      }
    });

    child.stderr.on('data', (data) => {
      stderr += data.toString();
    });

    child.on('close', (code) => {
      if (code === 0) {
        console.log('[CodexLens] Semantic dependencies installed successfully');
        resolve({ success: true });
      } else {
        resolve({ success: false, error: `Installation failed: ${stderr || stdout}` });
      }
    });

    child.on('error', (err) => {
      resolve({ success: false, error: `Failed to run pip: ${err.message}` });
    });
  });
}

/**
 * Bootstrap CodexLens venv with required packages
 * @returns Bootstrap result
 */
async function bootstrapVenv(): Promise<BootstrapResult> {
  // Ensure data directory exists
  if (!existsSync(CODEXLENS_DATA_DIR)) {
    mkdirSync(CODEXLENS_DATA_DIR, { recursive: true });
  }

  // Create venv if not exists
  if (!existsSync(CODEXLENS_VENV)) {
    try {
      console.log('[CodexLens] Creating virtual environment...');
      const pythonCmd = getSystemPython();
      execSync(`${pythonCmd} -m venv "${CODEXLENS_VENV}"`, { stdio: 'inherit' });
    } catch (err) {
      return { success: false, error: `Failed to create venv: ${(err as Error).message}` };
    }
  }

  // Install codexlens with semantic extras
  try {
    console.log('[CodexLens] Installing codexlens package...');
    const pipPath =
      process.platform === 'win32'
        ? join(CODEXLENS_VENV, 'Scripts', 'pip.exe')
        : join(CODEXLENS_VENV, 'bin', 'pip');

    // Try multiple local paths, then fall back to PyPI
    const possiblePaths = [
      join(process.cwd(), 'codex-lens'),
      join(__dirname, '..', '..', '..', 'codex-lens'), // ccw/src/tools -> project root
      join(homedir(), 'codex-lens'),
    ];

    let installed = false;
    for (const localPath of possiblePaths) {
      if (existsSync(join(localPath, 'pyproject.toml'))) {
        console.log(`[CodexLens] Installing from local path: ${localPath}`);
        execSync(`"${pipPath}" install -e "${localPath}"`, { stdio: 'inherit' });
        installed = true;
        break;
      }
    }

    if (!installed) {
      console.log('[CodexLens] Installing from PyPI...');
      execSync(`"${pipPath}" install codexlens`, { stdio: 'inherit' });
    }

    return { success: true };
  } catch (err) {
    return { success: false, error: `Failed to install codexlens: ${(err as Error).message}` };
  }
}

/**
 * Ensure CodexLens is ready to use
 * @returns Ready status
 */
async function ensureReady(): Promise<ReadyStatus> {
  // Use cached result if already checked
  if (bootstrapChecked && bootstrapReady) {
    return { ready: true };
  }

  // Check current status
  const status = await checkVenvStatus();
  if (status.ready) {
    bootstrapChecked = true;
    bootstrapReady = true;
    return { ready: true, version: status.version };
  }

  // Attempt bootstrap
  const bootstrap = await bootstrapVenv();
  if (!bootstrap.success) {
    return { ready: false, error: bootstrap.error };
  }

  // Verify after bootstrap
  const recheck = await checkVenvStatus();
  bootstrapChecked = true;
  bootstrapReady = recheck.ready;

  return recheck;
}

/**
 * Execute CodexLens CLI command
 * @param args - CLI arguments
 * @param options - Execution options
 * @returns Execution result
 */
async function executeCodexLens(args: string[], options: ExecuteOptions = {}): Promise<ExecuteResult> {
  const { timeout = 60000, cwd = process.cwd() } = options;

  // Ensure ready
  const readyStatus = await ensureReady();
  if (!readyStatus.ready) {
    return { success: false, error: readyStatus.error };
  }

  return new Promise((resolve) => {
    const child = spawn(VENV_PYTHON, ['-m', 'codexlens', ...args], {
      cwd,
      stdio: ['ignore', 'pipe', 'pipe'],
    });

    let stdout = '';
    let stderr = '';
    let timedOut = false;

    child.stdout.on('data', (data) => {
      stdout += data.toString();
    });
    child.stderr.on('data', (data) => {
      stderr += data.toString();
    });

    const timeoutId = setTimeout(() => {
      timedOut = true;
      child.kill('SIGTERM');
    }, timeout);

    child.on('close', (code) => {
      clearTimeout(timeoutId);

      if (timedOut) {
        resolve({ success: false, error: 'Command timed out' });
      } else if (code === 0) {
        resolve({ success: true, output: stdout.trim() });
      } else {
        resolve({ success: false, error: stderr || `Exit code: ${code}` });
      }
    });

    child.on('error', (err) => {
      clearTimeout(timeoutId);
      resolve({ success: false, error: `Spawn failed: ${err.message}` });
    });
  });
}

/**
 * Initialize CodexLens index for a directory
 * @param params - Parameters
 * @returns Execution result
 */
async function initIndex(params: Params): Promise<ExecuteResult> {
  const { path = '.', languages } = params;

  const args = ['init', path];
  if (languages && languages.length > 0) {
    args.push('--languages', languages.join(','));
  }

  return executeCodexLens(args, { cwd: path });
}

/**
 * Search code using CodexLens
 * @param params - Search parameters
 * @returns Execution result
 */
async function searchCode(params: Params): Promise<ExecuteResult> {
  const { query, path = '.', limit = 20 } = params;

  if (!query) {
    return { success: false, error: 'Query is required for search action' };
  }

  const args = ['search', query, '--limit', limit.toString(), '--json'];

  const result = await executeCodexLens(args, { cwd: path });

  if (result.success && result.output) {
    try {
      result.results = JSON.parse(result.output);
      delete result.output;
    } catch {
      // Keep raw output if JSON parse fails
    }
  }

  return result;
}

/**
 * Search code and return only file paths
 * @param params - Search parameters
 * @returns Execution result
 */
async function searchFiles(params: Params): Promise<ExecuteResult> {
  const { query, path = '.', limit = 20 } = params;

  if (!query) {
    return { success: false, error: 'Query is required for search_files action' };
  }

  const args = ['search', query, '--files-only', '--limit', limit.toString(), '--json'];

  const result = await executeCodexLens(args, { cwd: path });

  if (result.success && result.output) {
    try {
      result.files = JSON.parse(result.output);
      delete result.output;
    } catch {
      // Keep raw output if JSON parse fails
    }
  }

  return result;
}

/**
 * Extract symbols from a file
 * @param params - Parameters
 * @returns Execution result
 */
async function extractSymbols(params: Params): Promise<ExecuteResult> {
  const { file } = params;

  if (!file) {
    return { success: false, error: 'File is required for symbol action' };
  }

  const args = ['symbol', file, '--json'];

  const result = await executeCodexLens(args);

  if (result.success && result.output) {
    try {
      result.symbols = JSON.parse(result.output);
      delete result.output;
    } catch {
      // Keep raw output if JSON parse fails
    }
  }

  return result;
}

/**
 * Get index status
 * @param params - Parameters
 * @returns Execution result
 */
async function getStatus(params: Params): Promise<ExecuteResult> {
  const { path = '.' } = params;

  const args = ['status', '--json'];

  const result = await executeCodexLens(args, { cwd: path });

  if (result.success && result.output) {
    try {
      result.status = JSON.parse(result.output);
      delete result.output;
    } catch {
      // Keep raw output if JSON parse fails
    }
  }

  return result;
}

/**
 * Show configuration
 * @param params - Parameters
 * @returns Execution result
 */
async function configShow(): Promise<ExecuteResult> {
  const args = ['config', 'show', '--json'];
  const result = await executeCodexLens(args);

  if (result.success && result.output) {
    try {
      result.config = JSON.parse(result.output);
      delete result.output;
    } catch {
      // Keep raw output if JSON parse fails
    }
  }

  return result;
}

/**
 * Set configuration value
 * @param params - Parameters
 * @returns Execution result
 */
async function configSet(params: Params): Promise<ExecuteResult> {
  const { key, value } = params;

  if (!key) {
    return { success: false, error: 'key is required for config_set action' };
  }
  if (!value) {
    return { success: false, error: 'value is required for config_set action' };
  }

  const args = ['config', 'set', key, value, '--json'];
  const result = await executeCodexLens(args);

  if (result.success && result.output) {
    try {
      result.config = JSON.parse(result.output);
      delete result.output;
    } catch {
      // Keep raw output if JSON parse fails
    }
  }

  return result;
}

/**
 * Migrate indexes to new location
 * @param params - Parameters
 * @returns Execution result
 */
async function configMigrate(params: Params): Promise<ExecuteResult> {
  const { newPath } = params;

  if (!newPath) {
    return { success: false, error: 'newPath is required for config_migrate action' };
  }

  const args = ['config', 'migrate', newPath, '--json'];
  const result = await executeCodexLens(args, { timeout: 300000 }); // 5 min for migration

  if (result.success && result.output) {
    try {
      result.config = JSON.parse(result.output);
      delete result.output;
    } catch {
      // Keep raw output if JSON parse fails
    }
  }

  return result;
}

/**
 * Clean indexes
 * @param params - Parameters
 * @returns Execution result
 */
async function cleanIndexes(params: Params): Promise<ExecuteResult> {
  const { path, all } = params;

  const args = ['clean'];

  if (all) {
    args.push('--all');
  } else if (path) {
    args.push(path);
  }

  args.push('--json');
  const result = await executeCodexLens(args);

  if (result.success && result.output) {
    try {
      result.cleanResult = JSON.parse(result.output);
      delete result.output;
    } catch {
      // Keep raw output if JSON parse fails
    }
  }

  return result;
}

// Tool schema for MCP
export const schema: ToolSchema = {
  name: 'codex_lens',
  description: `CodexLens - Code indexing and search.

Usage:
  codex_lens(action="init", path=".")           # Index directory
  codex_lens(action="search", query="func", path=".")  # Search code
  codex_lens(action="search_files", query="x")  # Search, return paths only
  codex_lens(action="symbol", file="f.py")      # Extract symbols
  codex_lens(action="status")                   # Index status
  codex_lens(action="config_show")              # Show configuration
  codex_lens(action="config_set", key="index_dir", value="/path/to/indexes")  # Set config
  codex_lens(action="config_migrate", newPath="/new/path")  # Migrate indexes
  codex_lens(action="clean")                    # Show clean status
  codex_lens(action="clean", path=".")          # Clean specific project
  codex_lens(action="clean", all=true)          # Clean all indexes`,
  inputSchema: {
    type: 'object',
    properties: {
      action: {
        type: 'string',
        enum: [
          'init',
          'search',
          'search_files',
          'symbol',
          'status',
          'config_show',
          'config_set',
          'config_migrate',
          'clean',
          'bootstrap',
          'check',
        ],
        description: 'Action to perform',
      },
      path: {
        type: 'string',
        description: 'Target path (for init, search, search_files, status, clean)',
      },
      query: {
        type: 'string',
        description: 'Search query (for search and search_files actions)',
      },
      mode: {
        type: 'string',
        enum: ['text', 'semantic'],
        description: 'Search mode (default: text)',
        default: 'text',
      },
      file: {
        type: 'string',
        description: 'File path (for symbol action)',
      },
      key: {
        type: 'string',
        description: 'Config key (for config_set action, e.g., "index_dir")',
      },
      value: {
        type: 'string',
        description: 'Config value (for config_set action)',
      },
      newPath: {
        type: 'string',
        description: 'New index path (for config_migrate action)',
      },
      all: {
        type: 'boolean',
        description: 'Clean all indexes (for clean action)',
        default: false,
      },
      languages: {
        type: 'array',
        items: { type: 'string' },
        description: 'Languages to index (for init action)',
      },
      limit: {
        type: 'number',
        description: 'Maximum results (for search and search_files actions)',
        default: 20,
      },
      format: {
        type: 'string',
        enum: ['json', 'table', 'plain'],
        description: 'Output format',
        default: 'json',
      },
    },
    required: ['action'],
  },
};

// Handler function
export async function handler(params: Record<string, unknown>): Promise<ToolResult<ExecuteResult>> {
  const parsed = ParamsSchema.safeParse(params);
  if (!parsed.success) {
    return { success: false, error: `Invalid params: ${parsed.error.message}` };
  }

  const { action } = parsed.data;

  try {
    let result: ExecuteResult;

    switch (action) {
      case 'init':
        result = await initIndex(parsed.data);
        break;

      case 'search':
        result = await searchCode(parsed.data);
        break;

      case 'search_files':
        result = await searchFiles(parsed.data);
        break;

      case 'symbol':
        result = await extractSymbols(parsed.data);
        break;

      case 'status':
        result = await getStatus(parsed.data);
        break;

      case 'config_show':
        result = await configShow();
        break;

      case 'config_set':
        result = await configSet(parsed.data);
        break;

      case 'config_migrate':
        result = await configMigrate(parsed.data);
        break;

      case 'clean':
        result = await cleanIndexes(parsed.data);
        break;

      case 'bootstrap': {
        // Force re-bootstrap
        bootstrapChecked = false;
        bootstrapReady = false;
        const bootstrapResult = await bootstrapVenv();
        result = bootstrapResult.success
          ? { success: true, message: 'CodexLens bootstrapped successfully' }
          : { success: false, error: bootstrapResult.error };
        break;
      }

      case 'check': {
        const checkResult = await checkVenvStatus();
        result = {
          success: checkResult.ready,
          ready: checkResult.ready,
          error: checkResult.error,
          version: checkResult.version,
        };
        break;
      }

      default:
        throw new Error(
          `Unknown action: ${action}. Valid actions: init, search, search_files, symbol, status, config_show, config_set, config_migrate, clean, bootstrap, check`
        );
    }

    return result.success ? { success: true, result } : { success: false, error: result.error };
  } catch (error) {
    return { success: false, error: (error as Error).message };
  }
}

// Export for direct usage
export { ensureReady, executeCodexLens, checkVenvStatus, bootstrapVenv, checkSemanticStatus, installSemantic };

// Backward-compatible export for tests
export const codexLensTool = {
  name: schema.name,
  description: schema.description,
  parameters: schema.inputSchema,
  execute: async (params: Record<string, unknown>) => {
    const result = await handler(params);
    // Return the result directly - tests expect {success: boolean, ...} format
    return result.success ? result.result : { success: false, error: result.error };
  }
};