From ef770ff29bc95953c549f5a3cffd0c9e7c78f7a7 Mon Sep 17 00:00:00 2001 From: catlog22 Date: Tue, 6 Jan 2026 23:11:15 +0800 Subject: [PATCH] Add comprehensive code review specifications and templates - Introduced best practices requirements specification covering code quality, performance, maintainability, error handling, and documentation standards. - Established quality standards with overall quality metrics and mandatory checks for security, code quality, performance, and maintainability. - Created security requirements specification aligned with OWASP Top 10 and CWE Top 25, detailing checks and patterns for common vulnerabilities. - Developed templates for documenting best practice findings, security findings, and generating reports, including structured markdown and JSON formats. - Updated dependencies in the project, ensuring compatibility and stability. - Added test files and README documentation for vector indexing tests. --- .claude/skills/code-reviewer/README.md | 340 ++++++++++++++ .claude/skills/code-reviewer/SKILL.md | 317 +++++++++++++ .../code-reviewer/phases/01-code-discovery.md | 246 ++++++++++ .../phases/02-security-analysis.md | 442 ++++++++++++++++++ .../phases/03-best-practices-review.md | 36 ++ .../phases/04-report-generation.md | 278 +++++++++++ .../specs/best-practices-requirements.md | 346 ++++++++++++++ .../code-reviewer/specs/quality-standards.md | 252 ++++++++++ .../specs/security-requirements.md | 243 ++++++++++ .../templates/best-practice-finding.md | 234 ++++++++++ .../templates/report-template.md | 316 +++++++++++++ .../templates/security-finding.md | 161 +++++++ 1.18.0 | 62 +++ ccw/src/commands/cli.ts | 8 +- ccw/src/core/routes/codexlens-routes.ts | 101 +++- ccw/src/core/routes/skills-routes.ts | 53 ++- .../dashboard-css/30-core-memory.css | 20 + .../dashboard-css/33-cli-stream-viewer.css | 130 ++++++ .../components/cli-stream-viewer.js | 119 ++++- ccw/src/templates/dashboard-js/i18n.js | 4 + .../dashboard-js/views/codexlens-manager.js | 310 +++++++++--- .../dashboard-js/views/core-memory.js | 87 +++- ccw/src/tools/cli-config-manager.ts | 14 +- ccw/src/tools/cli-executor.ts | 353 +++++++++++++- ccw/src/tools/smart-search.ts | 138 ++++-- ccw/tsconfig.tsbuildinfo | 2 +- codex-lens/src/codexlens/cli/commands.py | 1 + codex-lens/src/codexlens/config.py | 68 ++- codex-lens/src/codexlens/env_config.py | 6 + .../src/codexlens/search/chain_search.py | 5 +- test_index_dir/README.md | 1 + test_index_dir/test.js | 1 + 32 files changed, 4530 insertions(+), 164 deletions(-) create mode 100644 .claude/skills/code-reviewer/README.md create mode 100644 .claude/skills/code-reviewer/SKILL.md create mode 100644 .claude/skills/code-reviewer/phases/01-code-discovery.md create mode 100644 .claude/skills/code-reviewer/phases/02-security-analysis.md create mode 100644 .claude/skills/code-reviewer/phases/03-best-practices-review.md create mode 100644 .claude/skills/code-reviewer/phases/04-report-generation.md create mode 100644 .claude/skills/code-reviewer/specs/best-practices-requirements.md create mode 100644 .claude/skills/code-reviewer/specs/quality-standards.md create mode 100644 .claude/skills/code-reviewer/specs/security-requirements.md create mode 100644 .claude/skills/code-reviewer/templates/best-practice-finding.md create mode 100644 .claude/skills/code-reviewer/templates/report-template.md create mode 100644 .claude/skills/code-reviewer/templates/security-finding.md create mode 100644 1.18.0 create mode 100644 test_index_dir/README.md create mode 100644 test_index_dir/test.js diff --git a/.claude/skills/code-reviewer/README.md b/.claude/skills/code-reviewer/README.md new file mode 100644 index 00000000..27810044 --- /dev/null +++ b/.claude/skills/code-reviewer/README.md @@ -0,0 +1,340 @@ +# Code Reviewer Skill + +A comprehensive code review skill for identifying security vulnerabilities and best practices violations. + +## Overview + +The **code-reviewer** skill provides automated code review capabilities covering: +- **Security Analysis**: OWASP Top 10, CWE Top 25, language-specific vulnerabilities +- **Code Quality**: Naming conventions, complexity, duplication, dead code +- **Performance**: N+1 queries, inefficient algorithms, memory leaks +- **Maintainability**: Documentation, test coverage, dependency health + +## Quick Start + +### Basic Usage + +```bash +# Review entire codebase +/code-reviewer + +# Review specific directory +/code-reviewer --scope src/auth + +# Focus on security only +/code-reviewer --focus security + +# Focus on best practices only +/code-reviewer --focus best-practices +``` + +### Advanced Options + +```bash +# Review with custom severity threshold +/code-reviewer --severity critical,high + +# Review specific file types +/code-reviewer --languages typescript,python + +# Generate detailed report +/code-reviewer --report-level detailed + +# Resume from previous session +/code-reviewer --resume +``` + +## Features + +### Security Analysis + +✅ **OWASP Top 10 2021 Coverage** +- Injection vulnerabilities (SQL, Command, XSS) +- Authentication & authorization flaws +- Sensitive data exposure +- Security misconfiguration +- And more... + +✅ **CWE Top 25 Coverage** +- Cross-site scripting (CWE-79) +- SQL injection (CWE-89) +- Command injection (CWE-78) +- Input validation (CWE-20) +- And more... + +✅ **Language-Specific Checks** +- JavaScript/TypeScript: prototype pollution, eval usage +- Python: pickle vulnerabilities, command injection +- Java: deserialization, XXE +- Go: race conditions, memory leaks + +### Best Practices Review + +✅ **Code Quality** +- Naming convention compliance +- Cyclomatic complexity analysis +- Code duplication detection +- Dead code identification + +✅ **Performance** +- N+1 query detection +- Inefficient algorithm patterns +- Memory leak detection +- Resource cleanup verification + +✅ **Maintainability** +- Documentation coverage +- Test coverage analysis +- Dependency health check +- Error handling review + +## Output + +The skill generates comprehensive reports in `.code-review/` directory: + +``` +.code-review/ +├── inventory.json # File inventory with metadata +├── security-findings.json # Security vulnerabilities +├── best-practices-findings.json # Best practices violations +├── summary.json # Summary statistics +├── REPORT.md # Comprehensive markdown report +└── FIX-CHECKLIST.md # Actionable fix checklist +``` + +### Report Contents + +**REPORT.md** includes: +- Executive summary with risk assessment +- Quality scores (Security, Code Quality, Performance, Maintainability) +- Detailed findings organized by severity +- Code examples with fix recommendations +- Action plan prioritized by urgency +- Compliance status (PCI DSS, HIPAA, GDPR, SOC 2) + +**FIX-CHECKLIST.md** provides: +- Checklist format for tracking fixes +- Organized by severity (Critical → Low) +- Effort estimates for each issue +- Priority assignments + +## Configuration + +Create `.code-reviewer.json` in project root: + +```json +{ + "scope": { + "include": ["src/**/*", "lib/**/*"], + "exclude": ["**/*.test.ts", "**/*.spec.ts", "**/node_modules/**"] + }, + "security": { + "enabled": true, + "checks": ["owasp-top-10", "cwe-top-25"], + "severity_threshold": "medium" + }, + "best_practices": { + "enabled": true, + "code_quality": true, + "performance": true, + "maintainability": true + }, + "reporting": { + "format": "markdown", + "output_path": ".code-review/", + "include_snippets": true, + "include_fixes": true + } +} +``` + +## Workflow + +### Phase 1: Code Discovery +- Discover and categorize code files +- Extract metadata (LOC, complexity, framework) +- Prioritize files (Critical, High, Medium, Low) + +### Phase 2: Security Analysis +- Scan for OWASP Top 10 vulnerabilities +- Check CWE Top 25 weaknesses +- Apply language-specific security patterns +- Generate security findings + +### Phase 3: Best Practices Review +- Analyze code quality issues +- Detect performance problems +- Assess maintainability concerns +- Generate best practices findings + +### Phase 4: Report Generation +- Consolidate all findings +- Calculate quality scores +- Generate comprehensive reports +- Create actionable checklists + +## Integration + +### Pre-commit Hook + +Block commits with critical/high issues: + +```bash +#!/bin/bash +# .git/hooks/pre-commit + +staged_files=$(git diff --cached --name-only --diff-filter=ACMR) +ccw run code-reviewer --scope "$staged_files" --severity critical,high + +if [ $? -ne 0 ]; then + echo "❌ Code review found critical/high issues. Commit aborted." + exit 1 +fi +``` + +### CI/CD Integration + +```yaml +# .github/workflows/code-review.yml +name: Code Review +on: [pull_request] + +jobs: + review: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run Code Review + run: | + ccw run code-reviewer --report-level detailed + ccw report upload .code-review/report.md +``` + +## Examples + +### Example 1: Security-Focused Review + +```bash +# Review authentication module for security issues +/code-reviewer --scope src/auth --focus security --severity critical,high +``` + +**Output**: Security findings with OWASP/CWE mappings and fix recommendations + +### Example 2: Performance Review + +```bash +# Review API endpoints for performance issues +/code-reviewer --scope src/api --focus best-practices --check performance +``` + +**Output**: N+1 queries, inefficient algorithms, memory leak detections + +### Example 3: Full Project Audit + +```bash +# Comprehensive review of entire codebase +/code-reviewer --report-level detailed --output .code-review/audit-2024-01.md +``` + +**Output**: Complete audit with all findings, scores, and action plan + +## Compliance Support + +The skill maps findings to compliance requirements: + +- **PCI DSS**: Requirement 6.5 (Common coding vulnerabilities) +- **HIPAA**: Technical safeguards and access controls +- **GDPR**: Article 32 (Security of processing) +- **SOC 2**: Security controls and monitoring + +## Architecture + +### Execution Mode +**Sequential** - Fixed phase order for systematic review: +1. Code Discovery → 2. Security Analysis → 3. Best Practices → 4. Report Generation + +### Tools Used +- `mcp__ace-tool__search_context` - Semantic code search +- `mcp__ccw-tools__smart_search` - Pattern matching +- `Read` - File content access +- `Write` - Report generation + +## Quality Standards + +### Scoring System + +``` +Overall Score = ( + Security Score × 0.4 + + Code Quality Score × 0.25 + + Performance Score × 0.2 + + Maintainability Score × 0.15 +) +``` + +### Score Ranges +- **A (90-100)**: Excellent - Production ready +- **B (80-89)**: Good - Minor improvements needed +- **C (70-79)**: Acceptable - Some issues to address +- **D (60-69)**: Poor - Significant improvements required +- **F (0-59)**: Failing - Major issues, not production ready + +## Troubleshooting + +### Large Codebase + +If review takes too long: +```bash +# Review in batches +/code-reviewer --scope src/module-1 +/code-reviewer --scope src/module-2 --resume + +# Or use parallel execution +/code-reviewer --parallel 4 +``` + +### False Positives + +Configure suppressions in `.code-reviewer.json`: +```json +{ + "suppressions": { + "security": { + "sql-injection": { + "paths": ["src/legacy/**/*"], + "reason": "Legacy code, scheduled for refactor" + } + } + } +} +``` + +## File Structure + +``` +.claude/skills/code-reviewer/ +├── SKILL.md # Main skill documentation +├── README.md # This file +├── phases/ +│ ├── 01-code-discovery.md +│ ├── 02-security-analysis.md +│ ├── 03-best-practices-review.md +│ └── 04-report-generation.md +├── specs/ +│ ├── security-requirements.md +│ ├── best-practices-requirements.md +│ └── quality-standards.md +└── templates/ + ├── security-finding.md + ├── best-practice-finding.md + └── report-template.md +``` + +## Version + +**v1.0.0** - Initial release + +## License + +MIT License diff --git a/.claude/skills/code-reviewer/SKILL.md b/.claude/skills/code-reviewer/SKILL.md new file mode 100644 index 00000000..6051b741 --- /dev/null +++ b/.claude/skills/code-reviewer/SKILL.md @@ -0,0 +1,317 @@ +# Code Reviewer + +Comprehensive code review skill for identifying security vulnerabilities and best practices violations. + +## Metadata + +```yaml +name: code-reviewer +description: 帮助审查代码的安全漏洞和最佳实践 +version: 1.0.0 +execution_mode: sequential +allowed-tools: + - Read + - Glob + - Grep + - mcp__ace-tool__search_context + - mcp__ccw-tools__smart_search +``` + +## Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Code Reviewer Workflow │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ Phase 1: Code Discovery → 发现待审查的代码文件 │ +│ & Scoping - 根据语言/框架识别文件 │ +│ ↓ - 设置审查范围和优先级 │ +│ │ +│ Phase 2: Security → 安全漏洞扫描 │ +│ Analysis - OWASP Top 10 检查 │ +│ ↓ - 常见漏洞模式识别 │ +│ - 敏感数据泄露检查 │ +│ │ +│ Phase 3: Best Practices → 最佳实践审查 │ +│ Review - 代码质量检查 │ +│ ↓ - 性能优化建议 │ +│ - 可维护性评估 │ +│ │ +│ Phase 4: Report → 生成审查报告 │ +│ Generation - 按严重程度分类问题 │ +│ - 提供修复建议和示例 │ +│ - 生成可追踪的修复清单 │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## Features + +### Security Analysis + +- **OWASP Top 10 Coverage** + - Injection vulnerabilities (SQL, Command, LDAP) + - Authentication & authorization bypass + - Sensitive data exposure + - XML External Entities (XXE) + - Broken access control + - Security misconfiguration + - Cross-Site Scripting (XSS) + - Insecure deserialization + - Components with known vulnerabilities + - Insufficient logging & monitoring + +- **Language-Specific Checks** + - JavaScript/TypeScript: prototype pollution, eval usage + - Python: pickle vulnerabilities, command injection + - Java: deserialization, path traversal + - Go: race conditions, memory leaks + +### Best Practices Review + +- **Code Quality** + - Naming conventions + - Function complexity (cyclomatic complexity) + - Code duplication + - Dead code detection + +- **Performance** + - N+1 queries + - Inefficient algorithms + - Memory leaks + - Resource cleanup + +- **Maintainability** + - Documentation quality + - Test coverage + - Error handling patterns + - Dependency management + +## Usage + +### Basic Review + +```bash +# Review entire codebase +/code-reviewer + +# Review specific directory +/code-reviewer --scope src/auth + +# Focus on security only +/code-reviewer --focus security + +# Focus on best practices only +/code-reviewer --focus best-practices +``` + +### Advanced Options + +```bash +# Review with custom severity threshold +/code-reviewer --severity critical,high + +# Review specific file types +/code-reviewer --languages typescript,python + +# Generate detailed report with code snippets +/code-reviewer --report-level detailed + +# Resume from previous session +/code-reviewer --resume +``` + +## Configuration + +Create `.code-reviewer.json` in project root: + +```json +{ + "scope": { + "include": ["src/**/*", "lib/**/*"], + "exclude": ["**/*.test.ts", "**/*.spec.ts", "**/node_modules/**"] + }, + "security": { + "enabled": true, + "checks": ["owasp-top-10", "cwe-top-25"], + "severity_threshold": "medium" + }, + "best_practices": { + "enabled": true, + "code_quality": true, + "performance": true, + "maintainability": true + }, + "reporting": { + "format": "markdown", + "output_path": ".code-review/", + "include_snippets": true, + "include_fixes": true + } +} +``` + +## Output + +### Review Report Structure + +```markdown +# Code Review Report + +## Executive Summary +- Total Issues: 42 +- Critical: 3 +- High: 8 +- Medium: 15 +- Low: 16 + +## Security Findings + +### [CRITICAL] SQL Injection in User Query +**File**: src/auth/user-service.ts:145 +**Issue**: Unsanitized user input in SQL query +**Fix**: Use parameterized queries + +Code Snippet: +\`\`\`typescript +// ❌ Vulnerable +const query = `SELECT * FROM users WHERE username = '${username}'`; + +// ✅ Fixed +const query = 'SELECT * FROM users WHERE username = ?'; +db.execute(query, [username]); +\`\`\` + +## Best Practices Findings + +### [MEDIUM] High Cyclomatic Complexity +**File**: src/utils/validator.ts:78 +**Issue**: Function has complexity score of 15 (threshold: 10) +**Fix**: Break into smaller functions + +... +``` + +## Phase Documentation + +| Phase | Description | Output | +|-------|-------------|--------| +| [01-code-discovery.md](phases/01-code-discovery.md) | Discover and categorize code files | File inventory with metadata | +| [02-security-analysis.md](phases/02-security-analysis.md) | Analyze security vulnerabilities | Security findings list | +| [03-best-practices-review.md](phases/03-best-practices-review.md) | Review code quality and practices | Best practices findings | +| [04-report-generation.md](phases/04-report-generation.md) | Generate comprehensive report | Markdown report | + +## Specifications + +- [specs/security-requirements.md](specs/security-requirements.md) - Security check specifications +- [specs/best-practices-requirements.md](specs/best-practices-requirements.md) - Best practices standards +- [specs/quality-standards.md](specs/quality-standards.md) - Overall quality standards +- [specs/severity-classification.md](specs/severity-classification.md) - Issue severity criteria + +## Templates + +- [templates/security-finding.md](templates/security-finding.md) - Security finding template +- [templates/best-practice-finding.md](templates/best-practice-finding.md) - Best practice finding template +- [templates/report-template.md](templates/report-template.md) - Final report template + +## Integration with Development Workflow + +### Pre-commit Hook + +```bash +#!/bin/bash +# .git/hooks/pre-commit + +# Run code review on staged files +staged_files=$(git diff --cached --name-only --diff-filter=ACMR) +ccw run code-reviewer --scope "$staged_files" --severity critical,high + +if [ $? -ne 0 ]; then + echo "❌ Code review found critical/high issues. Commit aborted." + exit 1 +fi +``` + +### CI/CD Integration + +```yaml +# .github/workflows/code-review.yml +name: Code Review +on: [pull_request] + +jobs: + review: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run Code Review + run: | + ccw run code-reviewer --report-level detailed + ccw report upload .code-review/report.md +``` + +## Examples + +### Example 1: Security-Focused Review + +```bash +# Review authentication module for security issues +/code-reviewer --scope src/auth --focus security --severity critical,high +``` + +### Example 2: Performance Review + +```bash +# Review API endpoints for performance issues +/code-reviewer --scope src/api --focus best-practices --check performance +``` + +### Example 3: Full Project Audit + +```bash +# Comprehensive review of entire codebase +/code-reviewer --report-level detailed --output .code-review/audit-2024-01.md +``` + +## Troubleshooting + +### Large Codebase + +If review takes too long: +```bash +# Review in batches +/code-reviewer --scope src/module-1 +/code-reviewer --scope src/module-2 --resume + +# Or use parallel execution +/code-reviewer --parallel 4 +``` + +### False Positives + +Configure suppressions in `.code-reviewer.json`: +```json +{ + "suppressions": { + "security": { + "sql-injection": { + "paths": ["src/legacy/**/*"], + "reason": "Legacy code, scheduled for refactor" + } + } + } +} +``` + +## Roadmap + +- [ ] AI-powered vulnerability detection +- [ ] Integration with popular security scanners (Snyk, SonarQube) +- [ ] Automated fix suggestions with diffs +- [ ] IDE plugins for real-time feedback +- [ ] Custom rule engine for organization-specific policies + +## License + +MIT License - See LICENSE file for details diff --git a/.claude/skills/code-reviewer/phases/01-code-discovery.md b/.claude/skills/code-reviewer/phases/01-code-discovery.md new file mode 100644 index 00000000..eab727f0 --- /dev/null +++ b/.claude/skills/code-reviewer/phases/01-code-discovery.md @@ -0,0 +1,246 @@ +# Phase 1: Code Discovery & Scoping + +## Objective + +Discover and categorize all code files within the specified scope, preparing them for security analysis and best practices review. + +## Input + +- **User Arguments**: + - `--scope`: Directory or file patterns (default: entire project) + - `--languages`: Specific languages to review (e.g., typescript, python, java) + - `--exclude`: Patterns to exclude (e.g., test files, node_modules) + +- **Configuration**: `.code-reviewer.json` (if exists) + +## Process + +### Step 1: Load Configuration + +```javascript +// Check for project-level configuration +const configPath = path.join(projectRoot, '.code-reviewer.json'); +const config = fileExists(configPath) + ? JSON.parse(readFile(configPath)) + : getDefaultConfig(); + +// Merge user arguments with config +const scope = args.scope || config.scope.include; +const exclude = args.exclude || config.scope.exclude; +const languages = args.languages || config.languages || 'auto'; +``` + +### Step 2: Discover Files + +Use MCP tools for efficient file discovery: + +```javascript +// Use smart_search for file discovery +const files = await mcp__ccw_tools__smart_search({ + action: "find_files", + pattern: scope, + includeHidden: false +}); + +// Apply exclusion patterns +const filteredFiles = files.filter(file => { + return !exclude.some(pattern => minimatch(file, pattern)); +}); +``` + +### Step 3: Categorize Files + +Categorize files by: +- **Language/Framework**: TypeScript, Python, Java, Go, etc. +- **File Type**: Source, config, test, build +- **Priority**: Critical (auth, payment), High (API), Medium (utils), Low (docs) + +```javascript +const inventory = { + critical: { + auth: ['src/auth/login.ts', 'src/auth/jwt.ts'], + payment: ['src/payment/stripe.ts'], + }, + high: { + api: ['src/api/users.ts', 'src/api/orders.ts'], + database: ['src/db/queries.ts'], + }, + medium: { + utils: ['src/utils/validator.ts'], + services: ['src/services/*.ts'], + }, + low: { + types: ['src/types/*.ts'], + } +}; +``` + +### Step 4: Extract Metadata + +For each file, extract: +- **Lines of Code (LOC)** +- **Complexity Indicators**: Function count, class count +- **Dependencies**: Import statements +- **Framework Detection**: Express, React, Django, etc. + +```javascript +const metadata = files.map(file => ({ + path: file, + language: detectLanguage(file), + loc: countLines(file), + complexity: estimateComplexity(file), + framework: detectFramework(file), + priority: categorizePriority(file) +})); +``` + +## Output + +### File Inventory + +Save to `.code-review/inventory.json`: + +```json +{ + "scan_date": "2024-01-15T10:30:00Z", + "total_files": 247, + "by_language": { + "typescript": 185, + "python": 42, + "javascript": 15, + "go": 5 + }, + "by_priority": { + "critical": 12, + "high": 45, + "medium": 120, + "low": 70 + }, + "files": [ + { + "path": "src/auth/login.ts", + "language": "typescript", + "loc": 245, + "functions": 8, + "classes": 2, + "priority": "critical", + "framework": "express", + "dependencies": ["bcrypt", "jsonwebtoken", "express"] + } + ] +} +``` + +### Summary Report + +```markdown +## Code Discovery Summary + +**Scope**: src/**/* +**Total Files**: 247 +**Languages**: TypeScript (75%), Python (17%), JavaScript (6%), Go (2%) + +### Priority Distribution +- Critical: 12 files (authentication, payment processing) +- High: 45 files (API endpoints, database queries) +- Medium: 120 files (utilities, services) +- Low: 70 files (types, configs) + +### Key Areas Identified +1. **Authentication Module** (src/auth/) - 12 files, 2,400 LOC +2. **Payment Processing** (src/payment/) - 5 files, 1,200 LOC +3. **API Layer** (src/api/) - 35 files, 5,600 LOC +4. **Database Layer** (src/db/) - 8 files, 1,800 LOC + +**Next Phase**: Security Analysis on Critical + High priority files +``` + +## State Management + +Save phase state for potential resume: + +```json +{ + "phase": "01-code-discovery", + "status": "completed", + "timestamp": "2024-01-15T10:35:00Z", + "output": { + "inventory_path": ".code-review/inventory.json", + "total_files": 247, + "critical_files": 12, + "high_files": 45 + } +} +``` + +## Agent Instructions + +```markdown +You are in Phase 1 of the Code Review workflow. Your task is to discover and categorize code files. + +**Instructions**: +1. Use mcp__ccw_tools__smart_search with action="find_files" to discover files +2. Apply exclusion patterns from config or arguments +3. Categorize files by language, type, and priority +4. Extract basic metadata (LOC, complexity indicators) +5. Save inventory to .code-review/inventory.json +6. Generate summary report +7. Proceed to Phase 2 with critical + high priority files + +**Tools Available**: +- mcp__ccw_tools__smart_search (file discovery) +- Read (read configuration and sample files) +- Write (save inventory and reports) + +**Output Requirements**: +- inventory.json with complete file list and metadata +- Summary markdown report +- State file for phase tracking +``` + +## Error Handling + +### No Files Found + +```javascript +if (filteredFiles.length === 0) { + throw new Error(`No files found matching scope: ${scope} + + Suggestions: + - Check if scope pattern is correct + - Verify exclude patterns are not too broad + - Ensure project has code files in specified scope + `); +} +``` + +### Large Codebase + +```javascript +if (filteredFiles.length > 1000) { + console.warn(`⚠️ Large codebase detected (${filteredFiles.length} files)`); + console.log(`Consider using --scope to review in batches`); + + // Offer to focus on critical/high priority only + const answer = await askUser("Review critical/high priority files only?"); + if (answer === 'yes') { + filteredFiles = filteredFiles.filter(f => + f.priority === 'critical' || f.priority === 'high' + ); + } +} +``` + +## Validation + +Before proceeding to Phase 2: + +- ✅ Inventory file created +- ✅ At least one file categorized as critical or high priority +- ✅ Metadata extracted for all files +- ✅ Summary report generated +- ✅ State saved for resume capability + +## Next Phase + +**Phase 2: Security Analysis** - Analyze critical and high priority files for security vulnerabilities using OWASP Top 10 and CWE Top 25 checks. diff --git a/.claude/skills/code-reviewer/phases/02-security-analysis.md b/.claude/skills/code-reviewer/phases/02-security-analysis.md new file mode 100644 index 00000000..9fc1a244 --- /dev/null +++ b/.claude/skills/code-reviewer/phases/02-security-analysis.md @@ -0,0 +1,442 @@ +# Phase 2: Security Analysis + +## Objective + +Analyze code files for security vulnerabilities based on OWASP Top 10, CWE Top 25, and language-specific security patterns. + +## Input + +- **File Inventory**: From Phase 1 (`.code-review/inventory.json`) +- **Priority Focus**: Critical and High priority files (unless `--scope all`) +- **User Arguments**: + - `--focus security`: Security-only mode + - `--severity critical,high,medium,low`: Minimum severity to report + - `--checks`: Specific security checks to run (e.g., sql-injection, xss) + +## Process + +### Step 1: Load Security Rules + +```javascript +// Load security check definitions +const securityRules = { + owasp_top_10: [ + 'injection', + 'broken_authentication', + 'sensitive_data_exposure', + 'xxe', + 'broken_access_control', + 'security_misconfiguration', + 'xss', + 'insecure_deserialization', + 'vulnerable_components', + 'insufficient_logging' + ], + cwe_top_25: [ + 'cwe-79', // XSS + 'cwe-89', // SQL Injection + 'cwe-20', // Improper Input Validation + 'cwe-78', // OS Command Injection + 'cwe-190', // Integer Overflow + // ... more CWE checks + ] +}; + +// Load language-specific rules +const languageRules = { + typescript: require('./rules/typescript-security.json'), + python: require('./rules/python-security.json'), + java: require('./rules/java-security.json'), + go: require('./rules/go-security.json'), +}; +``` + +### Step 2: Analyze Files for Vulnerabilities + +For each file in the inventory, perform security analysis: + +```javascript +const findings = []; + +for (const file of inventory.files) { + if (file.priority !== 'critical' && file.priority !== 'high') continue; + + // Read file content + const content = await Read({ file_path: file.path }); + + // Run security checks + const fileFindings = await runSecurityChecks(content, file, { + rules: securityRules, + languageRules: languageRules[file.language], + severity: args.severity || 'medium' + }); + + findings.push(...fileFindings); +} +``` + +### Step 3: Security Check Patterns + +#### A. Injection Vulnerabilities + +**SQL Injection**: +```javascript +// Pattern: String concatenation in SQL queries +const sqlInjectionPatterns = [ + /\$\{.*\}.*SELECT/, // Template literal with SELECT + /"SELECT.*\+\s*\w+/, // String concatenation + /execute\([`'"].*\$\{.*\}.*[`'"]\)/, // Parameterized query bypass + /query\(.*\+.*\)/, // Query concatenation +]; + +// Check code +for (const pattern of sqlInjectionPatterns) { + const matches = content.matchAll(new RegExp(pattern, 'g')); + for (const match of matches) { + findings.push({ + type: 'sql-injection', + severity: 'critical', + line: getLineNumber(content, match.index), + code: match[0], + file: file.path, + message: 'Potential SQL injection vulnerability', + recommendation: 'Use parameterized queries or ORM methods', + cwe: 'CWE-89', + owasp: 'A03:2021 - Injection' + }); + } +} +``` + +**Command Injection**: +```javascript +// Pattern: Unsanitized input in exec/spawn +const commandInjectionPatterns = [ + /exec\(.*\$\{.*\}/, // exec with template literal + /spawn\(.*,\s*\[.*\$\{.*\}.*\]\)/, // spawn with unsanitized args + /execSync\(.*\+.*\)/, // execSync with concatenation +]; +``` + +**XSS (Cross-Site Scripting)**: +```javascript +// Pattern: Unsanitized user input in DOM/HTML +const xssPatterns = [ + /innerHTML\s*=.*\$\{.*\}/, // innerHTML with template literal + /dangerouslySetInnerHTML/, // React dangerous prop + /document\.write\(.*\)/, // document.write + /<\w+.*\$\{.*\}.*>/, // JSX with unsanitized data +]; +``` + +#### B. Authentication & Authorization + +```javascript +// Pattern: Weak authentication +const authPatterns = [ + /password\s*===?\s*['"]/, // Hardcoded password comparison + /jwt\.sign\(.*,\s*['"][^'"]{1,16}['"]\)/, // Weak JWT secret + /bcrypt\.hash\(.*,\s*[1-9]\s*\)/, // Low bcrypt rounds + /md5\(.*password.*\)/, // MD5 for passwords + /if\s*\(\s*user\s*\)\s*\{/, // Missing auth check +]; + +// Check for missing authorization +const authzPatterns = [ + /router\.(get|post|put|delete)\(.*\)\s*=>/, // No middleware + /app\.use\([^)]*\)\s*;(?!.*auth)/, // Missing auth middleware +]; +``` + +#### C. Sensitive Data Exposure + +```javascript +// Pattern: Sensitive data in logs/responses +const sensitiveDataPatterns = [ + /(password|secret|token|key)\s*:/i, // Sensitive keys in objects + /console\.log\(.*password.*\)/i, // Password in logs + /res\.send\(.*user.*password.*\)/, // Password in response + /(api_key|apikey)\s*=\s*['"]/i, // Hardcoded API keys +]; +``` + +#### D. Security Misconfiguration + +```javascript +// Pattern: Insecure configurations +const misconfigPatterns = [ + /cors\(\{.*origin:\s*['"]?\*['"]?.*\}\)/, // CORS wildcard + /https?\s*:\s*false/, // HTTPS disabled + /helmet\(\)/, // Missing helmet config + /strictMode\s*:\s*false/, // Strict mode disabled +]; +``` + +### Step 4: Language-Specific Checks + +**TypeScript/JavaScript**: +```javascript +const jsFindings = [ + checkPrototypePollution(content), + checkEvalUsage(content), + checkUnsafeRegex(content), + checkWeakCrypto(content), +]; +``` + +**Python**: +```javascript +const pythonFindings = [ + checkPickleVulnerabilities(content), + checkYamlUnsafeLoad(content), + checkSqlAlchemy(content), + checkFlaskSecurityHeaders(content), +]; +``` + +**Java**: +```javascript +const javaFindings = [ + checkDeserialization(content), + checkXXE(content), + checkPathTraversal(content), + checkSQLInjection(content), +]; +``` + +**Go**: +```javascript +const goFindings = [ + checkRaceConditions(content), + checkSQLInjection(content), + checkPathTraversal(content), + checkCryptoWeakness(content), +]; +``` + +## Output + +### Security Findings File + +Save to `.code-review/security-findings.json`: + +```json +{ + "scan_date": "2024-01-15T11:00:00Z", + "total_findings": 24, + "by_severity": { + "critical": 3, + "high": 8, + "medium": 10, + "low": 3 + }, + "by_category": { + "injection": 5, + "authentication": 3, + "data_exposure": 4, + "misconfiguration": 6, + "xss": 3, + "other": 3 + }, + "findings": [ + { + "id": "SEC-001", + "type": "sql-injection", + "severity": "critical", + "file": "src/auth/user-service.ts", + "line": 145, + "column": 12, + "code": "const query = `SELECT * FROM users WHERE username = '${username}'`;", + "message": "SQL Injection vulnerability: User input directly concatenated in SQL query", + "cwe": "CWE-89", + "owasp": "A03:2021 - Injection", + "recommendation": { + "description": "Use parameterized queries to prevent SQL injection", + "fix_example": "const query = 'SELECT * FROM users WHERE username = ?';\ndb.execute(query, [username]);" + }, + "references": [ + "https://owasp.org/www-community/attacks/SQL_Injection", + "https://cwe.mitre.org/data/definitions/89.html" + ] + } + ] +} +``` + +### Security Report + +Generate markdown report: + +```markdown +# Security Analysis Report + +**Scan Date**: 2024-01-15 11:00:00 +**Files Analyzed**: 57 (Critical + High priority) +**Total Findings**: 24 + +## Severity Summary + +| Severity | Count | Percentage | +|----------|-------|------------| +| Critical | 3 | 12.5% | +| High | 8 | 33.3% | +| Medium | 10 | 41.7% | +| Low | 3 | 12.5% | + +## Critical Findings (Requires Immediate Action) + +### 🔴 [SEC-001] SQL Injection in User Authentication + +**File**: `src/auth/user-service.ts:145` +**CWE**: CWE-89 | **OWASP**: A03:2021 - Injection + +**Vulnerable Code**: +\`\`\`typescript +const query = \`SELECT * FROM users WHERE username = '\${username}'\`; +const user = await db.execute(query); +\`\`\` + +**Issue**: User input (`username`) is directly concatenated into SQL query, allowing attackers to inject malicious SQL commands. + +**Attack Example**: +\`\`\` +username: ' OR '1'='1' -- +Result: SELECT * FROM users WHERE username = '' OR '1'='1' --' +Effect: Bypasses authentication, returns all users +\`\`\` + +**Recommended Fix**: +\`\`\`typescript +// Use parameterized queries +const query = 'SELECT * FROM users WHERE username = ?'; +const user = await db.execute(query, [username]); + +// Or use ORM +const user = await User.findOne({ where: { username } }); +\`\`\` + +**References**: +- [OWASP SQL Injection](https://owasp.org/www-community/attacks/SQL_Injection) +- [CWE-89](https://cwe.mitre.org/data/definitions/89.html) + +--- + +### 🔴 [SEC-002] Hardcoded JWT Secret + +**File**: `src/auth/jwt.ts:23` +**CWE**: CWE-798 | **OWASP**: A07:2021 - Identification and Authentication Failures + +**Vulnerable Code**: +\`\`\`typescript +const token = jwt.sign(payload, 'mysecret123', { expiresIn: '1h' }); +\`\`\` + +**Issue**: JWT secret is hardcoded and weak (only 11 characters). + +**Recommended Fix**: +\`\`\`typescript +// Use environment variable with strong secret +const token = jwt.sign(payload, process.env.JWT_SECRET, { + expiresIn: '1h', + algorithm: 'HS256' +}); + +// Generate strong secret (32+ bytes): +// node -e "console.log(require('crypto').randomBytes(32).toString('hex'))" +\`\`\` + +--- + +## High Findings + +### 🟠 [SEC-003] Missing Input Validation + +**File**: `src/api/users.ts:67` +**CWE**: CWE-20 | **OWASP**: A03:2021 - Injection + +... + +## Medium Findings + +... + +## Remediation Priority + +1. **Critical (3)**: Fix within 24 hours +2. **High (8)**: Fix within 1 week +3. **Medium (10)**: Fix within 1 month +4. **Low (3)**: Fix in next release + +## Compliance Impact + +- **PCI DSS**: 4 findings affect compliance (SEC-001, SEC-002, SEC-008, SEC-011) +- **HIPAA**: 2 findings affect compliance (SEC-005, SEC-009) +- **GDPR**: 3 findings affect compliance (SEC-002, SEC-005, SEC-007) +``` + +## State Management + +```json +{ + "phase": "02-security-analysis", + "status": "completed", + "timestamp": "2024-01-15T11:15:00Z", + "input": { + "inventory_path": ".code-review/inventory.json", + "files_analyzed": 57 + }, + "output": { + "findings_path": ".code-review/security-findings.json", + "total_findings": 24, + "critical_count": 3, + "high_count": 8 + } +} +``` + +## Agent Instructions + +```markdown +You are in Phase 2 of the Code Review workflow. Your task is to analyze code for security vulnerabilities. + +**Instructions**: +1. Load file inventory from Phase 1 +2. Focus on Critical + High priority files +3. Run security checks for: + - OWASP Top 10 vulnerabilities + - CWE Top 25 weaknesses + - Language-specific security patterns +4. Use smart_search with mode="ripgrep" for pattern matching +5. Use mcp__ace-tool__search_context for semantic security pattern discovery +6. Classify findings by severity (Critical/High/Medium/Low) +7. Generate security-findings.json and markdown report +8. Proceed to Phase 3 (Best Practices Review) + +**Tools Available**: +- mcp__ccw_tools__smart_search (pattern search) +- mcp__ace-tool__search_context (semantic search) +- Read (read file content) +- Write (save findings and reports) +- Grep (targeted pattern matching) + +**Output Requirements**: +- security-findings.json with detailed findings +- Security report in markdown format +- Each finding must include: file, line, severity, CWE, OWASP, fix recommendation +- State file for phase tracking +``` + +## Validation + +Before proceeding to Phase 3: + +- ✅ All Critical + High priority files analyzed +- ✅ Findings categorized by severity +- ✅ Each finding has fix recommendation +- ✅ CWE and OWASP mappings included +- ✅ Security report generated +- ✅ State saved + +## Next Phase + +**Phase 3: Best Practices Review** - Analyze code quality, performance, and maintainability issues. diff --git a/.claude/skills/code-reviewer/phases/03-best-practices-review.md b/.claude/skills/code-reviewer/phases/03-best-practices-review.md new file mode 100644 index 00000000..f062300e --- /dev/null +++ b/.claude/skills/code-reviewer/phases/03-best-practices-review.md @@ -0,0 +1,36 @@ +# Phase 3: Best Practices Review + +## Objective + +Analyze code for best practices violations including code quality, performance issues, and maintainability concerns. + +## Input + +- **File Inventory**: From Phase 1 (`.code-review/inventory.json`) +- **Security Findings**: From Phase 2 (`.code-review/security-findings.json`) +- **User Arguments**: + - `--focus best-practices`: Best practices only mode + - `--check quality,performance,maintainability`: Specific areas to check + +## Process + +### Step 1: Code Quality Analysis + +Check naming conventions, function complexity, code duplication, and dead code detection. + +### Step 2: Performance Analysis + +Detect N+1 queries, inefficient algorithms, and memory leaks. + +### Step 3: Maintainability Analysis + +Check documentation coverage, test coverage, and dependency management. + +## Output + +- best-practices-findings.json +- Markdown report with recommendations + +## Next Phase + +**Phase 4: Report Generation** diff --git a/.claude/skills/code-reviewer/phases/04-report-generation.md b/.claude/skills/code-reviewer/phases/04-report-generation.md new file mode 100644 index 00000000..92bd9f3c --- /dev/null +++ b/.claude/skills/code-reviewer/phases/04-report-generation.md @@ -0,0 +1,278 @@ +# Phase 4: Report Generation + +## Objective + +Consolidate security and best practices findings into a comprehensive, actionable code review report. + +## Input + +- **Security Findings**: `.code-review/security-findings.json` +- **Best Practices Findings**: `.code-review/best-practices-findings.json` +- **File Inventory**: `.code-review/inventory.json` + +## Process + +### Step 1: Load All Findings + +```javascript +const securityFindings = JSON.parse( + await Read({ file_path: '.code-review/security-findings.json' }) +); +const bestPracticesFindings = JSON.parse( + await Read({ file_path: '.code-review/best-practices-findings.json' }) +); +const inventory = JSON.parse( + await Read({ file_path: '.code-review/inventory.json' }) +); +``` + +### Step 2: Aggregate Statistics + +```javascript +const stats = { + total_files_reviewed: inventory.total_files, + total_findings: securityFindings.total_findings + bestPracticesFindings.total_findings, + by_severity: { + critical: securityFindings.by_severity.critical, + high: securityFindings.by_severity.high + bestPracticesFindings.by_severity.high, + medium: securityFindings.by_severity.medium + bestPracticesFindings.by_severity.medium, + low: securityFindings.by_severity.low + bestPracticesFindings.by_severity.low, + }, + by_category: { + security: securityFindings.total_findings, + code_quality: bestPracticesFindings.by_category.code_quality, + performance: bestPracticesFindings.by_category.performance, + maintainability: bestPracticesFindings.by_category.maintainability, + } +}; +``` + +### Step 3: Generate Comprehensive Report + +```markdown +# Comprehensive Code Review Report + +**Generated**: {timestamp} +**Scope**: {scope} +**Files Reviewed**: {total_files} +**Total Findings**: {total_findings} + +## Executive Summary + +{Provide high-level overview of code health} + +### Risk Assessment + +{Calculate risk score based on findings} + +### Compliance Status + +{Map findings to compliance requirements} + +## Detailed Findings + +{Merge and organize security + best practices findings} + +## Action Plan + +{Prioritized list of fixes with effort estimates} + +## Appendix + +{Technical details, references, configuration} +``` + +### Step 4: Generate Fix Tracking Checklist + +Create actionable checklist for developers: + +```markdown +# Code Review Fix Checklist + +## Critical Issues (Fix Immediately) + +- [ ] [SEC-001] SQL Injection in src/auth/user-service.ts:145 +- [ ] [SEC-002] Hardcoded JWT Secret in src/auth/jwt.ts:23 +- [ ] [SEC-003] XSS Vulnerability in src/api/comments.ts:89 + +## High Priority Issues (Fix This Week) + +- [ ] [SEC-004] Missing Authorization Check in src/api/admin.ts:34 +- [ ] [BP-001] N+1 Query Pattern in src/api/orders.ts:45 +... +``` + +### Step 5: Generate Metrics Dashboard + +```markdown +## Code Health Metrics + +### Security Score: 68/100 +- Critical Issues: 3 (-30 points) +- High Issues: 8 (-2 points each) + +### Code Quality Score: 75/100 +- High Complexity Functions: 2 +- Code Duplication: 5% +- Dead Code: 3 instances + +### Performance Score: 82/100 +- N+1 Queries: 3 +- Inefficient Algorithms: 2 + +### Maintainability Score: 70/100 +- Documentation Coverage: 65% +- Test Coverage: 72% +- Missing Tests: 5 files +``` + +## Output + +### Main Report + +Save to `.code-review/REPORT.md`: + +- Executive summary +- Detailed findings (security + best practices) +- Action plan with priorities +- Metrics and scores +- References and compliance mapping + +### Fix Checklist + +Save to `.code-review/FIX-CHECKLIST.md`: + +- Organized by severity +- Checkboxes for tracking +- File:line references +- Effort estimates + +### JSON Summary + +Save to `.code-review/summary.json`: + +```json +{ + "report_date": "2024-01-15T12:00:00Z", + "scope": "src/**/*", + "statistics": { + "total_files": 247, + "total_findings": 69, + "by_severity": { "critical": 3, "high": 13, "medium": 30, "low": 23 }, + "by_category": { + "security": 24, + "code_quality": 18, + "performance": 12, + "maintainability": 15 + } + }, + "scores": { + "security": 68, + "code_quality": 75, + "performance": 82, + "maintainability": 70, + "overall": 74 + }, + "risk_level": "MEDIUM", + "action_required": true +} +``` + +## Report Template + +Full report includes: + +1. **Executive Summary** + - Overall code health + - Risk assessment + - Key recommendations + +2. **Security Findings** (from Phase 2) + - Critical/High/Medium/Low + - OWASP/CWE mappings + - Fix recommendations with code examples + +3. **Best Practices Findings** (from Phase 3) + - Code quality issues + - Performance concerns + - Maintainability gaps + +4. **Metrics Dashboard** + - Security score + - Code quality score + - Performance score + - Maintainability score + +5. **Action Plan** + - Immediate actions (critical) + - Short-term (1 week) + - Medium-term (1 month) + - Long-term (3 months) + +6. **Compliance Impact** + - PCI DSS findings + - HIPAA findings + - GDPR findings + - SOC 2 findings + +7. **Appendix** + - Full findings list + - Configuration used + - Tools and versions + - References + +## State Management + +```json +{ + "phase": "04-report-generation", + "status": "completed", + "timestamp": "2024-01-15T12:00:00Z", + "input": { + "security_findings": ".code-review/security-findings.json", + "best_practices_findings": ".code-review/best-practices-findings.json" + }, + "output": { + "report": ".code-review/REPORT.md", + "checklist": ".code-review/FIX-CHECKLIST.md", + "summary": ".code-review/summary.json" + } +} +``` + +## Agent Instructions + +```markdown +You are in Phase 4 (FINAL) of the Code Review workflow. Generate comprehensive report. + +**Instructions**: +1. Load security findings from Phase 2 +2. Load best practices findings from Phase 3 +3. Aggregate statistics and calculate scores +4. Generate comprehensive markdown report +5. Create fix tracking checklist +6. Generate JSON summary +7. Inform user of completion and output locations + +**Tools Available**: +- Read (load findings) +- Write (save reports) + +**Output Requirements**: +- REPORT.md (comprehensive markdown report) +- FIX-CHECKLIST.md (actionable checklist) +- summary.json (machine-readable summary) +- All files in .code-review/ directory +``` + +## Validation + +- ✅ All findings consolidated +- ✅ Scores calculated +- ✅ Action plan generated +- ✅ Reports saved to .code-review/ +- ✅ User notified of completion + +## Completion + +Code review complete! Outputs available in `.code-review/` directory. diff --git a/.claude/skills/code-reviewer/specs/best-practices-requirements.md b/.claude/skills/code-reviewer/specs/best-practices-requirements.md new file mode 100644 index 00000000..5f727613 --- /dev/null +++ b/.claude/skills/code-reviewer/specs/best-practices-requirements.md @@ -0,0 +1,346 @@ +# Best Practices Requirements Specification + +## Code Quality Standards + +### Naming Conventions + +**TypeScript/JavaScript**: +- Classes/Interfaces: PascalCase (`UserService`, `IUserRepository`) +- Functions/Methods: camelCase (`getUserById`, `validateEmail`) +- Constants: UPPER_SNAKE_CASE (`MAX_RETRY_COUNT`, `API_BASE_URL`) +- Private properties: prefix with `_` or `#` (`_cache`, `#secretKey`) + +**Python**: +- Classes: PascalCase (`UserService`, `DatabaseConnection`) +- Functions: snake_case (`get_user_by_id`, `validate_email`) +- Constants: UPPER_SNAKE_CASE (`MAX_RETRY_COUNT`) +- Private: prefix with `_` (`_internal_cache`) + +**Java**: +- Classes/Interfaces: PascalCase (`UserService`, `IUserRepository`) +- Methods: camelCase (`getUserById`, `validateEmail`) +- Constants: UPPER_SNAKE_CASE (`MAX_RETRY_COUNT`) +- Packages: lowercase (`com.example.service`) + +### Function Complexity + +**Cyclomatic Complexity Thresholds**: +- **Low**: 1-5 (simple functions, easy to test) +- **Medium**: 6-10 (acceptable, well-structured) +- **High**: 11-20 (needs refactoring) +- **Very High**: 21+ (critical, must refactor) + +**Calculation**: +``` +Complexity = 1 (base) + + count(if) + + count(else if) + + count(while) + + count(for) + + count(case) + + count(catch) + + count(&&) + + count(||) + + count(? :) +``` + +### Code Duplication + +**Thresholds**: +- **Acceptable**: < 3% duplication +- **Warning**: 3-5% duplication +- **Critical**: > 5% duplication + +**Detection**: +- Minimum block size: 5 lines +- Similarity threshold: 85% +- Ignore: Comments, imports, trivial getters/setters + +### Dead Code Detection + +**Targets**: +- Unused imports +- Unused variables/functions (not exported) +- Unreachable code (after return/throw) +- Commented-out code blocks (> 5 lines) + +## Performance Standards + +### N+1 Query Prevention + +**Anti-patterns**: +```javascript +// ❌ N+1 Query +for (const order of orders) { + const user = await User.findById(order.userId); +} + +// ✅ Batch Query +const userIds = orders.map(o => o.userId); +const users = await User.findByIds(userIds); +``` + +### Algorithm Efficiency + +**Common Issues**: +- Nested loops (O(n²)) when O(n) possible +- Array.indexOf in loop → use Set.has() +- Array.filter().length → use Array.some() +- Multiple array iterations → combine into one pass + +**Acceptable Complexity**: +- **O(1)**: Ideal for lookups +- **O(log n)**: Good for search +- **O(n)**: Acceptable for linear scan +- **O(n log n)**: Acceptable for sorting +- **O(n²)**: Avoid if possible, document if necessary + +### Memory Leak Prevention + +**Common Issues**: +- Event listeners without cleanup +- setInterval without clearInterval +- Global variable accumulation +- Circular references +- Large array/object allocations + +**Patterns**: +```javascript +// ❌ Memory Leak +element.addEventListener('click', handler); +// No cleanup + +// ✅ Proper Cleanup +useEffect(() => { + element.addEventListener('click', handler); + return () => element.removeEventListener('click', handler); +}, []); +``` + +### Resource Cleanup + +**Required Cleanup**: +- Database connections +- File handles +- Network sockets +- Timers (setTimeout, setInterval) +- Event listeners + +## Maintainability Standards + +### Documentation Requirements + +**Required for**: +- All exported functions/classes +- Public APIs +- Complex algorithms +- Non-obvious business logic + +**JSDoc Format**: +```javascript +/** + * Validates user credentials and generates JWT token + * + * @param {string} username - User's username or email + * @param {string} password - Plain text password + * @returns {Promise<{token: string, expiresAt: Date}>} JWT token and expiration + * @throws {AuthenticationError} If credentials are invalid + * + * @example + * const {token} = await authenticateUser('john@example.com', 'secret123'); + */ +async function authenticateUser(username, password) { + // ... +} +``` + +**Coverage Targets**: +- Critical modules: 100% +- High priority: 90% +- Medium priority: 70% +- Low priority: 50% + +### Test Coverage Requirements + +**Coverage Targets**: +- Unit tests: 80% line coverage +- Integration tests: Key workflows covered +- E2E tests: Critical user paths covered + +**Required Tests**: +- All exported functions +- All public methods +- Error handling paths +- Edge cases + +**Test File Convention**: +``` +src/auth/login.ts + → src/auth/login.test.ts (unit) + → src/auth/login.integration.test.ts (integration) +``` + +### Dependency Management + +**Best Practices**: +- Pin major versions (`"^1.2.3"` not `"*"`) +- Avoid 0.x versions in production +- Regular security audits (npm audit, snyk) +- Keep dependencies up-to-date +- Minimize dependency count + +**Version Pinning**: +```json +{ + "dependencies": { + "express": "^4.18.0", // ✅ Pinned major version + "lodash": "*", // ❌ Wildcard + "legacy-lib": "^0.5.0" // ⚠️ Unstable 0.x + } +} +``` + +### Magic Numbers + +**Definition**: Numeric literals without clear meaning + +**Anti-patterns**: +```javascript +// ❌ Magic numbers +if (user.age > 18) { } +setTimeout(() => {}, 5000); +buffer = new Array(1048576); + +// ✅ Named constants +const LEGAL_AGE = 18; +const RETRY_DELAY_MS = 5000; +const BUFFER_SIZE_1MB = 1024 * 1024; + +if (user.age > LEGAL_AGE) { } +setTimeout(() => {}, RETRY_DELAY_MS); +buffer = new Array(BUFFER_SIZE_1MB); +``` + +**Exceptions** (acceptable magic numbers): +- 0, 1, -1 (common values) +- 100, 1000 (obvious scaling factors in context) +- HTTP status codes (200, 404, 500) + +## Error Handling Standards + +### Required Error Handling + +**Categories**: +- Network errors (timeout, connection failure) +- Database errors (query failure, constraint violation) +- Validation errors (invalid input) +- Authentication/Authorization errors + +**Anti-patterns**: +```javascript +// ❌ Silent failure +try { + await saveUser(user); +} catch (err) { + // Empty catch +} + +// ❌ Generic catch +try { + await processPayment(order); +} catch (err) { + console.log('Error'); // No details +} + +// ✅ Proper handling +try { + await processPayment(order); +} catch (err) { + logger.error('Payment processing failed', { orderId: order.id, error: err }); + throw new PaymentError('Failed to process payment', { cause: err }); +} +``` + +### Logging Standards + +**Required Logs**: +- Authentication attempts (success/failure) +- Authorization failures +- Data modifications (create/update/delete) +- External API calls +- Errors and exceptions + +**Log Levels**: +- **ERROR**: System errors, exceptions +- **WARN**: Recoverable issues, deprecations +- **INFO**: Business events, state changes +- **DEBUG**: Detailed troubleshooting info + +**Sensitive Data**: +- Never log: passwords, tokens, credit cards, SSNs +- Hash/mask: emails, IPs, usernames (in production) + +## Code Structure Standards + +### File Organization + +**Max File Size**: 300 lines (excluding tests) +**Max Function Size**: 50 lines + +**Module Structure**: +``` +module/ + ├── index.ts # Public exports + ├── types.ts # Type definitions + ├── constants.ts # Constants + ├── utils.ts # Utilities + ├── service.ts # Business logic + └── service.test.ts # Tests +``` + +### Import Organization + +**Order**: +1. External dependencies +2. Internal modules (absolute imports) +3. Relative imports +4. Type imports (TypeScript) + +```typescript +// ✅ Organized imports +import express from 'express'; +import { Logger } from 'winston'; + +import { UserService } from '@/services/user'; +import { config } from '@/config'; + +import { validateEmail } from './utils'; +import { UserRepository } from './repository'; + +import type { User, UserCreateInput } from './types'; +``` + +## Scoring System + +### Overall Score Calculation + +``` +Overall Score = ( + Security Score × 0.4 + + Code Quality Score × 0.25 + + Performance Score × 0.2 + + Maintainability Score × 0.15 +) + +Security = 100 - (Critical × 30 + High × 2 + Medium × 0.5) +Code Quality = 100 - (violations / total_checks × 100) +Performance = 100 - (issues / potential_issues × 100) +Maintainability = (doc_coverage × 0.4 + test_coverage × 0.4 + dependency_health × 0.2) +``` + +### Risk Levels + +- **LOW**: Score 90-100 +- **MEDIUM**: Score 70-89 +- **HIGH**: Score 50-69 +- **CRITICAL**: Score < 50 diff --git a/.claude/skills/code-reviewer/specs/quality-standards.md b/.claude/skills/code-reviewer/specs/quality-standards.md new file mode 100644 index 00000000..92fba784 --- /dev/null +++ b/.claude/skills/code-reviewer/specs/quality-standards.md @@ -0,0 +1,252 @@ +# Quality Standards + +## Overall Quality Metrics + +### Quality Score Formula + +``` +Overall Quality = ( + Correctness × 0.30 + + Security × 0.25 + + Maintainability × 0.20 + + Performance × 0.15 + + Documentation × 0.10 +) +``` + +### Score Ranges + +| Range | Grade | Description | +|-------|-------|-------------| +| 90-100 | A | Excellent - Production ready | +| 80-89 | B | Good - Minor improvements needed | +| 70-79 | C | Acceptable - Some issues to address | +| 60-69 | D | Poor - Significant improvements required | +| 0-59 | F | Failing - Major issues, not production ready | + +## Review Completeness + +### Mandatory Checks + +**Security**: +- ✅ OWASP Top 10 coverage +- ✅ CWE Top 25 coverage +- ✅ Language-specific security patterns +- ✅ Dependency vulnerability scan + +**Code Quality**: +- ✅ Naming convention compliance +- ✅ Complexity analysis +- ✅ Code duplication detection +- ✅ Dead code identification + +**Performance**: +- ✅ N+1 query detection +- ✅ Algorithm efficiency check +- ✅ Memory leak detection +- ✅ Resource cleanup verification + +**Maintainability**: +- ✅ Documentation coverage +- ✅ Test coverage analysis +- ✅ Dependency health check +- ✅ Error handling review + +## Reporting Standards + +### Finding Requirements + +Each finding must include: +- **Unique ID**: SEC-001, BP-001, etc. +- **Type**: Specific issue type (sql-injection, high-complexity, etc.) +- **Severity**: Critical, High, Medium, Low +- **Location**: File path and line number +- **Code Snippet**: Vulnerable/problematic code +- **Message**: Clear description of the issue +- **Recommendation**: Specific fix guidance +- **Example**: Before/after code example + +### Report Structure + +**Executive Summary**: +- High-level overview +- Risk assessment +- Key statistics +- Compliance status + +**Detailed Findings**: +- Organized by severity +- Grouped by category +- Full details for each finding + +**Action Plan**: +- Prioritized fix list +- Effort estimates +- Timeline recommendations + +**Metrics Dashboard**: +- Quality scores +- Trend analysis (if historical data) +- Compliance status + +**Appendix**: +- Full findings list +- Configuration details +- Tool versions +- References + +## Output File Standards + +### File Naming + +``` +.code-review/ +├── inventory.json # File inventory +├── security-findings.json # Security findings +├── best-practices-findings.json # Best practices findings +├── summary.json # Summary statistics +├── REPORT.md # Main report +├── FIX-CHECKLIST.md # Action checklist +└── state.json # Session state +``` + +### JSON Schema + +**Finding Schema**: +```json +{ + "id": "string", + "type": "string", + "category": "security|code_quality|performance|maintainability", + "severity": "critical|high|medium|low", + "file": "string", + "line": "number", + "column": "number", + "code": "string", + "message": "string", + "recommendation": { + "description": "string", + "fix_example": "string" + }, + "references": ["string"], + "cwe": "string (optional)", + "owasp": "string (optional)" +} +``` + +## Validation Requirements + +### Phase Completion Criteria + +**Phase 1 (Code Discovery)**: +- ✅ At least 1 file discovered +- ✅ Files categorized by priority +- ✅ Metadata extracted +- ✅ Inventory JSON created + +**Phase 2 (Security Analysis)**: +- ✅ All critical/high priority files analyzed +- ✅ Findings have severity classification +- ✅ CWE/OWASP mappings included +- ✅ Fix recommendations provided + +**Phase 3 (Best Practices)**: +- ✅ Code quality checks completed +- ✅ Performance analysis done +- ✅ Maintainability assessed +- ✅ Recommendations provided + +**Phase 4 (Report Generation)**: +- ✅ All findings consolidated +- ✅ Scores calculated +- ✅ Reports generated +- ✅ Checklist created + +## Skill Execution Standards + +### Performance Targets + +- **Phase 1**: < 30 seconds per 1000 files +- **Phase 2**: < 60 seconds per 100 files (security) +- **Phase 3**: < 60 seconds per 100 files (best practices) +- **Phase 4**: < 10 seconds (report generation) + +### Resource Limits + +- **Memory**: < 2GB for projects with 1000+ files +- **CPU**: Efficient pattern matching (minimize regex complexity) +- **Disk**: Use streaming for large files (> 10MB) + +### Error Handling + +**Graceful Degradation**: +- If tool unavailable: Skip check, note in report +- If file unreadable: Log warning, continue with others +- If analysis fails: Report error, continue with next file + +**User Notification**: +- Progress updates every 10% completion +- Clear error messages with troubleshooting steps +- Final summary with metrics and file locations + +## Integration Standards + +### Git Integration + +**Pre-commit Hook**: +```bash +#!/bin/bash +ccw run code-reviewer --scope staged --severity critical,high +exit $? # Block commit if critical/high issues found +``` + +**PR Comments**: +- Automatic review comments on changed lines +- Summary comment with overall findings +- Status check (pass/fail based on threshold) + +### CI/CD Integration + +**Requirements**: +- Exit code 0 if no critical/high issues +- Exit code 1 if blocking issues found +- JSON output for parsing +- Configurable severity threshold + +### IDE Integration + +**LSP Support** (future): +- Real-time security/quality feedback +- Inline fix suggestions +- Quick actions for common fixes + +## Compliance Mapping + +### Supported Standards + +**PCI DSS**: +- Requirement 6.5: Common coding vulnerabilities +- Map findings to specific requirements + +**HIPAA**: +- Technical safeguards +- Map data exposure findings + +**GDPR**: +- Data protection by design +- Map sensitive data handling + +**SOC 2**: +- Security controls +- Map access control findings + +### Compliance Reports + +Generate compliance-specific reports: +``` +.code-review/compliance/ +├── pci-dss-report.md +├── hipaa-report.md +├── gdpr-report.md +└── soc2-report.md +``` diff --git a/.claude/skills/code-reviewer/specs/security-requirements.md b/.claude/skills/code-reviewer/specs/security-requirements.md new file mode 100644 index 00000000..59458f70 --- /dev/null +++ b/.claude/skills/code-reviewer/specs/security-requirements.md @@ -0,0 +1,243 @@ +# Security Requirements Specification + +## OWASP Top 10 Coverage + +### A01:2021 - Broken Access Control + +**Checks**: +- Missing authorization checks on protected routes +- Insecure direct object references (IDOR) +- Path traversal vulnerabilities +- Missing CSRF protection +- Elevation of privilege + +**Patterns**: +```javascript +// Missing auth middleware +router.get('/admin/*', handler); // ❌ No auth check + +// Insecure direct object reference +router.get('/user/:id', async (req, res) => { + const user = await User.findById(req.params.id); // ❌ No ownership check + res.json(user); +}); +``` + +### A02:2021 - Cryptographic Failures + +**Checks**: +- Sensitive data transmitted without encryption +- Weak cryptographic algorithms (MD5, SHA1) +- Hardcoded secrets/keys +- Insecure random number generation + +**Patterns**: +```javascript +// Weak hashing +const hash = crypto.createHash('md5').update(password); // ❌ MD5 is weak + +// Hardcoded secret +const token = jwt.sign(payload, 'secret123'); // ❌ Hardcoded secret +``` + +### A03:2021 - Injection + +**Checks**: +- SQL injection +- NoSQL injection +- Command injection +- LDAP injection +- XPath injection + +**Patterns**: +```javascript +// SQL injection +const query = `SELECT * FROM users WHERE id = ${userId}`; // ❌ + +// Command injection +exec(`git clone ${userRepo}`); // ❌ +``` + +### A04:2021 - Insecure Design + +**Checks**: +- Missing rate limiting +- Lack of input validation +- Business logic flaws +- Missing security requirements + +### A05:2021 - Security Misconfiguration + +**Checks**: +- Default credentials +- Overly permissive CORS +- Verbose error messages +- Unnecessary features enabled +- Missing security headers + +**Patterns**: +```javascript +// Overly permissive CORS +app.use(cors({ origin: '*' })); // ❌ + +// Verbose error +res.status(500).json({ error: err.stack }); // ❌ +``` + +### A06:2021 - Vulnerable and Outdated Components + +**Checks**: +- Dependencies with known vulnerabilities +- Unmaintained dependencies +- Using deprecated APIs + +### A07:2021 - Identification and Authentication Failures + +**Checks**: +- Weak password requirements +- Permits brute force attacks +- Exposed session IDs +- Weak JWT implementation + +**Patterns**: +```javascript +// Weak bcrypt rounds +bcrypt.hash(password, 4); // ❌ Too low (min: 10) + +// Session ID in URL +res.redirect(`/dashboard?sessionId=${sessionId}`); // ❌ +``` + +### A08:2021 - Software and Data Integrity Failures + +**Checks**: +- Insecure deserialization +- Unsigned/unverified updates +- CI/CD pipeline vulnerabilities + +**Patterns**: +```javascript +// Insecure deserialization +const obj = eval(userInput); // ❌ + +// Pickle vulnerability (Python) +data = pickle.loads(untrusted_data) # ❌ +``` + +### A09:2021 - Security Logging and Monitoring Failures + +**Checks**: +- Missing audit logs +- Sensitive data in logs +- Insufficient monitoring + +**Patterns**: +```javascript +// Password in logs +console.log(`Login attempt: ${username}:${password}`); // ❌ +``` + +### A10:2021 - Server-Side Request Forgery (SSRF) + +**Checks**: +- Unvalidated URLs in requests +- Internal network access +- Cloud metadata exposure + +**Patterns**: +```javascript +// SSRF vulnerability +const response = await fetch(userProvidedUrl); // ❌ +``` + +## CWE Top 25 Coverage + +### CWE-79: Cross-site Scripting (XSS) + +**Patterns**: +```javascript +element.innerHTML = userInput; // ❌ +document.write(userInput); // ❌ +``` + +### CWE-89: SQL Injection + +**Patterns**: +```javascript +query = `SELECT * FROM users WHERE name = '${name}'`; // ❌ +``` + +### CWE-20: Improper Input Validation + +**Checks**: +- Missing input sanitization +- No input length limits +- Unvalidated file uploads + +### CWE-78: OS Command Injection + +**Patterns**: +```javascript +exec(`ping ${userInput}`); // ❌ +``` + +### CWE-190: Integer Overflow + +**Checks**: +- Large number operations without bounds checking +- Array allocation with user-controlled size + +## Language-Specific Security Rules + +### TypeScript/JavaScript + +- Prototype pollution +- eval() usage +- Unsafe regex (ReDoS) +- require() with dynamic input + +### Python + +- pickle vulnerabilities +- yaml.unsafe_load() +- SQL injection in SQLAlchemy +- Command injection in subprocess + +### Java + +- Deserialization vulnerabilities +- XXE in XML parsers +- Path traversal +- SQL injection in JDBC + +### Go + +- Race conditions +- SQL injection +- Path traversal +- Weak cryptography + +## Severity Classification + +### Critical +- Remote code execution +- SQL injection with write access +- Authentication bypass +- Hardcoded credentials in production + +### High +- XSS in sensitive contexts +- Missing authorization checks +- Sensitive data exposure +- Insecure cryptography + +### Medium +- Missing rate limiting +- Weak password policy +- Security misconfiguration +- Information disclosure + +### Low +- Missing security headers +- Verbose error messages +- Outdated dependencies (no known exploits) diff --git a/.claude/skills/code-reviewer/templates/best-practice-finding.md b/.claude/skills/code-reviewer/templates/best-practice-finding.md new file mode 100644 index 00000000..bb3424c2 --- /dev/null +++ b/.claude/skills/code-reviewer/templates/best-practice-finding.md @@ -0,0 +1,234 @@ +# Best Practice Finding Template + +Use this template for documenting code quality, performance, and maintainability issues. + +## Finding Structure + +```json +{ + "id": "BP-{number}", + "type": "{issue-type}", + "category": "{code_quality|performance|maintainability}", + "severity": "{high|medium|low}", + "file": "{file-path}", + "line": {line-number}, + "function": "{function-name}", + "message": "{clear-description}", + "recommendation": { + "description": "{how-to-fix}", + "example": "{corrected-code}" + } +} +``` + +## Markdown Template + +```markdown +### 🟠 [BP-{number}] {Issue Title} + +**File**: `{file-path}:{line}` +**Category**: {Code Quality|Performance|Maintainability} + +**Issue**: {Detailed explanation of the problem} + +**Current Code**: +\`\`\`{language} +{problematic-code} +\`\`\` + +**Recommended Fix**: +\`\`\`{language} +{improved-code-with-comments} +\`\`\` + +**Impact**: {Why this matters - readability, performance, maintainability} + +--- +``` + +## Example: High Complexity + +```markdown +### 🟠 [BP-001] High Cyclomatic Complexity + +**File**: `src/utils/validator.ts:78` +**Category**: Code Quality +**Function**: `validateUserInput` +**Complexity**: 15 (threshold: 10) + +**Issue**: Function has 15 decision points, making it difficult to test and maintain. + +**Current Code**: +\`\`\`typescript +function validateUserInput(input) { + if (!input) return false; + if (!input.email) return false; + if (!input.email.includes('@')) return false; + if (input.email.length > 255) return false; + // ... 11 more conditions +} +\`\`\` + +**Recommended Fix**: +\`\`\`typescript +// Extract validation rules +const validationRules = { + email: (email) => email && email.includes('@') && email.length <= 255, + password: (pwd) => pwd && pwd.length >= 8 && /[A-Z]/.test(pwd), + username: (name) => name && /^[a-zA-Z0-9_]+$/.test(name), +}; + +// Simplified validator +function validateUserInput(input) { + return Object.entries(validationRules).every(([field, validate]) => + validate(input[field]) + ); +} +\`\`\` + +**Impact**: Reduces complexity from 15 to 3, improves testability, and makes validation rules reusable. + +--- +``` + +## Example: N+1 Query + +```markdown +### 🟠 [BP-002] N+1 Query Pattern + +**File**: `src/api/orders.ts:45` +**Category**: Performance + +**Issue**: Database query executed inside loop, causing N+1 queries problem. For 100 orders, this creates 101 database queries instead of 2. + +**Current Code**: +\`\`\`typescript +const orders = await Order.findAll(); +for (const order of orders) { + const user = await User.findById(order.userId); + order.userName = user.name; +} +\`\`\` + +**Recommended Fix**: +\`\`\`typescript +// Batch query all users at once +const orders = await Order.findAll(); +const userIds = orders.map(o => o.userId); +const users = await User.findByIds(userIds); + +// Create lookup map for O(1) access +const userMap = new Map(users.map(u => [u.id, u])); + +// Enrich orders with user data +for (const order of orders) { + order.userName = userMap.get(order.userId)?.name; +} +\`\`\` + +**Impact**: Reduces database queries from O(n) to O(1), significantly improving performance for large datasets. + +--- +``` + +## Example: Missing Documentation + +```markdown +### 🟡 [BP-003] Missing Documentation + +**File**: `src/services/PaymentService.ts:23` +**Category**: Maintainability + +**Issue**: Exported class lacks documentation, making it difficult for other developers to understand its purpose and usage. + +**Current Code**: +\`\`\`typescript +export class PaymentService { + async processPayment(orderId: string, amount: number) { + // implementation + } +} +\`\`\` + +**Recommended Fix**: +\`\`\`typescript +/** + * Service for processing payment transactions + * + * Handles payment processing, refunds, and transaction logging. + * Integrates with Stripe payment gateway. + * + * @example + * const paymentService = new PaymentService(); + * const result = await paymentService.processPayment('order-123', 99.99); + */ +export class PaymentService { + /** + * Process a payment for an order + * + * @param orderId - Unique order identifier + * @param amount - Payment amount in USD + * @returns Payment confirmation with transaction ID + * @throws {PaymentError} If payment processing fails + */ + async processPayment(orderId: string, amount: number) { + // implementation + } +} +\`\`\` + +**Impact**: Improves code discoverability and reduces onboarding time for new developers. + +--- +``` + +## Example: Memory Leak + +```markdown +### 🟠 [BP-004] Potential Memory Leak + +**File**: `src/components/Chat.tsx:56` +**Category**: Performance + +**Issue**: WebSocket event listener added without cleanup, causing memory leaks when component unmounts. + +**Current Code**: +\`\`\`tsx +useEffect(() => { + socket.on('message', handleMessage); +}, []); +\`\`\` + +**Recommended Fix**: +\`\`\`tsx +useEffect(() => { + socket.on('message', handleMessage); + + // Cleanup on unmount + return () => { + socket.off('message', handleMessage); + }; +}, []); +\`\`\` + +**Impact**: Prevents memory leaks and improves application stability in long-running sessions. + +--- +``` + +## Severity Guidelines + +### High +- Major performance impact (N+1 queries, O(n²) algorithms) +- Critical maintainability issues (complexity > 15) +- Missing error handling in critical paths + +### Medium +- Moderate performance impact +- Code quality issues (complexity 11-15, duplication) +- Missing tests for important features + +### Low +- Minor style violations +- Missing documentation +- Low-impact dead code diff --git a/.claude/skills/code-reviewer/templates/report-template.md b/.claude/skills/code-reviewer/templates/report-template.md new file mode 100644 index 00000000..4d159c49 --- /dev/null +++ b/.claude/skills/code-reviewer/templates/report-template.md @@ -0,0 +1,316 @@ +# Report Template + +## Main Report Structure (REPORT.md) + +```markdown +# Code Review Report + +**Generated**: {timestamp} +**Scope**: {scope} +**Files Reviewed**: {total_files} +**Total Findings**: {total_findings} + +--- + +## 📊 Executive Summary + +### Overall Assessment + +{Brief 2-3 paragraph assessment of code health} + +### Risk Level: {LOW|MEDIUM|HIGH|CRITICAL} + +{Risk assessment based on findings severity and count} + +### Key Statistics + +| Metric | Value | Status | +|--------|-------|--------| +| Total Files | {count} | - | +| Files with Issues | {count} | {percentage}% | +| Critical Findings | {count} | {icon} | +| High Findings | {count} | {icon} | +| Medium Findings | {count} | {icon} | +| Low Findings | {count} | {icon} | + +### Category Breakdown + +| Category | Count | Percentage | +|----------|-------|------------| +| Security | {count} | {percentage}% | +| Code Quality | {count} | {percentage}% | +| Performance | {count} | {percentage}% | +| Maintainability | {count} | {percentage}% | + +--- + +## 🎯 Quality Scores + +### Security Score: {score}/100 +{Assessment and key issues} + +### Code Quality Score: {score}/100 +{Assessment and key issues} + +### Performance Score: {score}/100 +{Assessment and key issues} + +### Maintainability Score: {score}/100 +{Assessment and key issues} + +### Overall Score: {score}/100 + +**Grade**: {A|B|C|D|F} + +--- + +## 🔴 Critical Findings (Requires Immediate Action) + +{List all critical findings using security-finding.md template} + +--- + +## 🟠 High Priority Findings + +{List all high findings} + +--- + +## 🟡 Medium Priority Findings + +{List all medium findings} + +--- + +## 🟢 Low Priority Findings + +{List all low findings} + +--- + +## 📋 Action Plan + +### Immediate (Within 24 hours) +1. {Critical issue 1} +2. {Critical issue 2} +3. {Critical issue 3} + +### Short-term (Within 1 week) +1. {High priority issue 1} +2. {High priority issue 2} +... + +### Medium-term (Within 1 month) +1. {Medium priority issue 1} +2. {Medium priority issue 2} +... + +### Long-term (Within 3 months) +1. {Low priority issue 1} +2. {Improvement initiative 1} +... + +--- + +## 📊 Metrics Dashboard + +### Code Health Trends + +{If historical data available, show trends} + +### File Hotspots + +Top files with most issues: +1. `{file-path}` - {count} issues ({severity breakdown}) +2. `{file-path}` - {count} issues +... + +### Technology Breakdown + +Issues by language/framework: +- TypeScript: {count} issues +- Python: {count} issues +... + +--- + +## ✅ Compliance Status + +### PCI DSS +- **Status**: {COMPLIANT|NON-COMPLIANT|PARTIAL} +- **Affecting Findings**: {list} + +### HIPAA +- **Status**: {COMPLIANT|NON-COMPLIANT|PARTIAL} +- **Affecting Findings**: {list} + +### GDPR +- **Status**: {COMPLIANT|NON-COMPLIANT|PARTIAL} +- **Affecting Findings**: {list} + +--- + +## 📚 Appendix + +### A. Review Configuration + +\`\`\`json +{review-config} +\`\`\` + +### B. Tools and Versions + +- Code Reviewer Skill: v1.0.0 +- Security Rules: OWASP Top 10 2021, CWE Top 25 +- Languages Analyzed: {list} + +### C. References + +- [OWASP Top 10 2021](https://owasp.org/Top10/) +- [CWE Top 25](https://cwe.mitre.org/top25/) +- {additional references} + +### D. Full Findings Index + +{Links to detailed finding JSONs} +``` + +--- + +## Fix Checklist Template (FIX-CHECKLIST.md) + +```markdown +# Code Review Fix Checklist + +**Generated**: {timestamp} +**Total Items**: {count} + +--- + +## 🔴 Critical Issues (Fix Immediately) + +- [ ] **[SEC-001]** SQL Injection in `src/auth/user-service.ts:145` + - Effort: 1 hour + - Priority: P0 + - Assignee: ___________ + +- [ ] **[SEC-002]** Hardcoded JWT Secret in `src/auth/jwt.ts:23` + - Effort: 30 minutes + - Priority: P0 + - Assignee: ___________ + +--- + +## 🟠 High Priority Issues (Fix This Week) + +- [ ] **[SEC-003]** Missing Authorization in `src/api/admin.ts:34` + - Effort: 2 hours + - Priority: P1 + - Assignee: ___________ + +- [ ] **[BP-001]** N+1 Query in `src/api/orders.ts:45` + - Effort: 1 hour + - Priority: P1 + - Assignee: ___________ + +--- + +## 🟡 Medium Priority Issues (Fix This Month) + +{List medium priority items} + +--- + +## 🟢 Low Priority Issues (Fix Next Release) + +{List low priority items} + +--- + +## Progress Tracking + +**Overall Progress**: {completed}/{total} ({percentage}%) + +- Critical: {completed}/{total} +- High: {completed}/{total} +- Medium: {completed}/{total} +- Low: {completed}/{total} + +**Estimated Total Effort**: {hours} hours +**Estimated Completion**: {date} +``` + +--- + +## Summary JSON Template (summary.json) + +```json +{ + "report_date": "2024-01-15T12:00:00Z", + "scope": "src/**/*", + "statistics": { + "total_files": 247, + "files_with_issues": 89, + "total_findings": 69, + "by_severity": { + "critical": 3, + "high": 13, + "medium": 30, + "low": 23 + }, + "by_category": { + "security": 24, + "code_quality": 18, + "performance": 12, + "maintainability": 15 + } + }, + "scores": { + "security": 68, + "code_quality": 75, + "performance": 82, + "maintainability": 70, + "overall": 74 + }, + "grade": "C", + "risk_level": "MEDIUM", + "action_required": true, + "compliance": { + "pci_dss": { + "status": "NON_COMPLIANT", + "affecting_findings": ["SEC-001", "SEC-002", "SEC-008", "SEC-011"] + }, + "hipaa": { + "status": "NON_COMPLIANT", + "affecting_findings": ["SEC-005", "SEC-009"] + }, + "gdpr": { + "status": "PARTIAL", + "affecting_findings": ["SEC-002", "SEC-005", "SEC-007"] + } + }, + "top_issues": [ + { + "id": "SEC-001", + "type": "sql-injection", + "severity": "critical", + "file": "src/auth/user-service.ts", + "line": 145 + } + ], + "hotspots": [ + { + "file": "src/auth/user-service.ts", + "issues": 5, + "severity_breakdown": { "critical": 1, "high": 2, "medium": 2 } + } + ], + "effort_estimate": { + "critical": 4.5, + "high": 18, + "medium": 35, + "low": 12, + "total_hours": 69.5 + } +} +``` diff --git a/.claude/skills/code-reviewer/templates/security-finding.md b/.claude/skills/code-reviewer/templates/security-finding.md new file mode 100644 index 00000000..083c542a --- /dev/null +++ b/.claude/skills/code-reviewer/templates/security-finding.md @@ -0,0 +1,161 @@ +# Security Finding Template + +Use this template for documenting security vulnerabilities. + +## Finding Structure + +```json +{ + "id": "SEC-{number}", + "type": "{vulnerability-type}", + "severity": "{critical|high|medium|low}", + "file": "{file-path}", + "line": {line-number}, + "column": {column-number}, + "code": "{vulnerable-code-snippet}", + "message": "{clear-description-of-issue}", + "cwe": "CWE-{number}", + "owasp": "A{number}:2021 - {category}", + "recommendation": { + "description": "{how-to-fix}", + "fix_example": "{corrected-code}" + }, + "references": [ + "https://...", + "https://..." + ] +} +``` + +## Markdown Template + +```markdown +### 🔴 [SEC-{number}] {Vulnerability Title} + +**File**: `{file-path}:{line}` +**CWE**: CWE-{number} | **OWASP**: A{number}:2021 - {category} + +**Vulnerable Code**: +\`\`\`{language} +{vulnerable-code-snippet} +\`\`\` + +**Issue**: {Detailed explanation of the vulnerability and potential impact} + +**Attack Example** (if applicable): +\`\`\` +{example-attack-payload} +Result: {what-happens} +Effect: {security-impact} +\`\`\` + +**Recommended Fix**: +\`\`\`{language} +{corrected-code-with-comments} +\`\`\` + +**References**: +- [{reference-title}]({url}) +- [{reference-title}]({url}) + +--- +``` + +## Severity Icon Mapping + +- Critical: 🔴 +- High: 🟠 +- Medium: 🟡 +- Low: 🟢 + +## Example: SQL Injection Finding + +```markdown +### 🔴 [SEC-001] SQL Injection in User Authentication + +**File**: `src/auth/user-service.ts:145` +**CWE**: CWE-89 | **OWASP**: A03:2021 - Injection + +**Vulnerable Code**: +\`\`\`typescript +const query = \`SELECT * FROM users WHERE username = '\${username}'\`; +const user = await db.execute(query); +\`\`\` + +**Issue**: User input (`username`) is directly concatenated into SQL query, allowing attackers to inject malicious SQL commands and bypass authentication. + +**Attack Example**: +\`\`\` +username: ' OR '1'='1' -- +Result: SELECT * FROM users WHERE username = '' OR '1'='1' --' +Effect: Bypasses authentication, returns all users +\`\`\` + +**Recommended Fix**: +\`\`\`typescript +// Use parameterized queries +const query = 'SELECT * FROM users WHERE username = ?'; +const user = await db.execute(query, [username]); + +// Or use ORM +const user = await User.findOne({ where: { username } }); +\`\`\` + +**References**: +- [OWASP SQL Injection](https://owasp.org/www-community/attacks/SQL_Injection) +- [CWE-89](https://cwe.mitre.org/data/definitions/89.html) + +--- +``` + +## Example: XSS Finding + +```markdown +### 🟠 [SEC-002] Cross-Site Scripting (XSS) in Comment Rendering + +**File**: `src/components/CommentList.tsx:89` +**CWE**: CWE-79 | **OWASP**: A03:2021 - Injection + +**Vulnerable Code**: +\`\`\`tsx +
+\`\`\` + +**Issue**: User-generated content rendered without sanitization, allowing script injection. + +**Attack Example**: +\`\`\` +comment.body: "" +Effect: Steals user session cookies +\`\`\` + +**Recommended Fix**: +\`\`\`tsx +import DOMPurify from 'dompurify'; + +// Sanitize HTML before rendering +
+ +// Or use text content (if HTML not needed) +
{comment.body}
+\`\`\` + +**References**: +- [OWASP XSS Prevention](https://cheatsheetseries.owasp.org/cheatsheets/Cross_Site_Scripting_Prevention_Cheat_Sheet.html) +- [CWE-79](https://cwe.mitre.org/data/definitions/79.html) + +--- +``` + +## Compliance Mapping Template + +When finding affects compliance: + +```markdown +**Compliance Impact**: +- **PCI DSS**: Requirement 6.5.1 (Injection flaws) +- **HIPAA**: Technical Safeguards - Access Control +- **GDPR**: Article 32 (Security of processing) +``` diff --git a/1.18.0 b/1.18.0 new file mode 100644 index 00000000..b0492e1a --- /dev/null +++ b/1.18.0 @@ -0,0 +1,62 @@ +Collecting onnxruntime-directml + Using cached onnxruntime_directml-1.23.0-cp310-cp310-win_amd64.whl (25.1 MB) +Collecting coloredlogs + Using cached coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB) +Collecting numpy>=1.21.6 + Using cached numpy-2.2.6-cp310-cp310-win_amd64.whl (12.9 MB) +Collecting sympy + Using cached sympy-1.14.0-py3-none-any.whl (6.3 MB) +Collecting packaging + Using cached packaging-25.0-py3-none-any.whl (66 kB) +Collecting protobuf + Using cached protobuf-6.33.2-cp310-abi3-win_amd64.whl (436 kB) +Collecting flatbuffers + Using cached flatbuffers-25.12.19-py2.py3-none-any.whl (26 kB) +Collecting humanfriendly>=9.1 + Using cached humanfriendly-10.0-py2.py3-none-any.whl (86 kB) +Collecting mpmath<1.4,>=1.1.0 + Using cached mpmath-1.3.0-py3-none-any.whl (536 kB) +Collecting pyreadline3 + Using cached pyreadline3-3.5.4-py3-none-any.whl (83 kB) +Installing collected packages: mpmath, flatbuffers, sympy, pyreadline3, protobuf, packaging, numpy, humanfriendly, coloredlogs, onnxruntime-directml + Attempting uninstall: mpmath + Found existing installation: mpmath 1.3.0 + Uninstalling mpmath-1.3.0: + Successfully uninstalled mpmath-1.3.0 + Attempting uninstall: flatbuffers + Found existing installation: flatbuffers 25.12.19 + Uninstalling flatbuffers-25.12.19: + Successfully uninstalled flatbuffers-25.12.19 + Attempting uninstall: sympy + Found existing installation: sympy 1.14.0 + Uninstalling sympy-1.14.0: + Successfully uninstalled sympy-1.14.0 + Attempting uninstall: pyreadline3 + Found existing installation: pyreadline3 3.5.4 + Uninstalling pyreadline3-3.5.4: + Successfully uninstalled pyreadline3-3.5.4 + Attempting uninstall: protobuf + Found existing installation: protobuf 6.33.2 + Uninstalling protobuf-6.33.2: + Successfully uninstalled protobuf-6.33.2 + Attempting uninstall: packaging + Found existing installation: packaging 25.0 + Uninstalling packaging-25.0: + Successfully uninstalled packaging-25.0 + Attempting uninstall: numpy + Found existing installation: numpy 2.2.6 + Uninstalling numpy-2.2.6: + Successfully uninstalled numpy-2.2.6 + Attempting uninstall: humanfriendly + Found existing installation: humanfriendly 10.0 + Uninstalling humanfriendly-10.0: + Successfully uninstalled humanfriendly-10.0 + Attempting uninstall: coloredlogs + Found existing installation: coloredlogs 15.0.1 + Uninstalling coloredlogs-15.0.1: + Successfully uninstalled coloredlogs-15.0.1 + Attempting uninstall: onnxruntime-directml + Found existing installation: onnxruntime-directml 1.23.0 + Uninstalling onnxruntime-directml-1.23.0: + Successfully uninstalled onnxruntime-directml-1.23.0 +Successfully installed coloredlogs-15.0.1 flatbuffers-25.12.19 humanfriendly-10.0 mpmath-1.3.0 numpy-2.2.6 onnxruntime-directml-1.23.0 packaging-25.0 protobuf-6.33.2 pyreadline3-3.5.4 sympy-1.14.0 diff --git a/ccw/src/commands/cli.ts b/ccw/src/commands/cli.ts index 9420d1d7..a50ed0db 100644 --- a/ccw/src/commands/cli.ts +++ b/ccw/src/commands/cli.ts @@ -791,8 +791,12 @@ async function execAction(positionalPrompt: string | undefined, options: CliExec }, onOutput); // Always pass onOutput for real-time dashboard streaming // If not streaming (default), print output now - if (!stream && result.stdout) { - console.log(result.stdout); + // Prefer parsedOutput (from stream parser) over raw stdout for better formatting + if (!stream) { + const output = result.parsedOutput || result.stdout; + if (output) { + console.log(output); + } } // Print summary with execution ID and turn info diff --git a/ccw/src/core/routes/codexlens-routes.ts b/ccw/src/core/routes/codexlens-routes.ts index 83bbeb37..b3b7ef2a 100644 --- a/ccw/src/core/routes/codexlens-routes.ts +++ b/ccw/src/core/routes/codexlens-routes.ts @@ -622,7 +622,7 @@ export async function handleCodexLensRoutes(ctx: RouteContext): Promise // API: CodexLens Init (Initialize workspace index) if (pathname === '/api/codexlens/init' && req.method === 'POST') { handlePostRequest(req, res, async (body) => { - const { path: projectPath, indexType = 'vector', embeddingModel = 'code', embeddingBackend = 'fastembed', maxWorkers = 1 } = body; + const { path: projectPath, indexType = 'vector', embeddingModel = 'code', embeddingBackend = 'fastembed', maxWorkers = 1, incremental = true } = body; const targetPath = projectPath || initialPath; // Ensure LiteLLM backend dependencies are installed before running the CLI @@ -636,6 +636,13 @@ export async function handleCodexLensRoutes(ctx: RouteContext): Promise // Build CLI arguments based on index type // Use 'index init' subcommand (new CLI structure) const args = ['index', 'init', targetPath, '--json']; + + // Force mode: when incremental=false, add --force to rebuild all files + // CLI defaults to incremental mode (skip unchanged files) + if (!incremental) { + args.push('--force'); + } + if (indexType === 'normal') { args.push('--no-embeddings'); } else { @@ -728,6 +735,98 @@ export async function handleCodexLensRoutes(ctx: RouteContext): Promise return true; } + // API: Generate embeddings only (without FTS rebuild) + if (pathname === '/api/codexlens/embeddings/generate' && req.method === 'POST') { + handlePostRequest(req, res, async (body) => { + const { path: projectPath, incremental = false, backend = 'litellm', maxWorkers = 4, model } = body; + const targetPath = projectPath || initialPath; + + // Ensure LiteLLM backend dependencies are installed + if (backend === 'litellm') { + try { + await ensureLiteLLMEmbedderReady(); + } catch (err) { + return { success: false, error: `LiteLLM embedder setup failed: ${err.message}` }; + } + } + + // Build CLI arguments for embeddings generation + // Use 'index embeddings' subcommand + const args = ['index', 'embeddings', targetPath, '--json']; + + // Add backend option + if (backend && backend !== 'fastembed') { + args.push('--backend', backend); + } + + // Add model if specified + if (model) { + args.push('--model', model); + } + + // Add max workers for API backend + if (backend === 'litellm' && maxWorkers > 1) { + args.push('--max-workers', String(maxWorkers)); + } + + // Force mode: always use --force for litellm backend to avoid model conflict + // (litellm uses different embeddings than fastembed, so regeneration is required) + // For true incremental updates with same model, use fastembed backend + if (!incremental || backend === 'litellm') { + args.push('--force'); // Force regenerate embeddings + } + + try { + // Broadcast progress start + broadcastToClients({ + type: 'CODEXLENS_INDEX_PROGRESS', + payload: { stage: 'embeddings', message: 'Generating embeddings...', percent: 10 } + }); + + const result = await executeCodexLens(args, { + cwd: targetPath, + onProgress: (progress: ProgressInfo) => { + broadcastToClients({ + type: 'CODEXLENS_INDEX_PROGRESS', + payload: { + stage: 'embeddings', + message: progress.message || 'Processing...', + percent: progress.percent || 50 + } + }); + } + }); + + if (result.success) { + broadcastToClients({ + type: 'CODEXLENS_INDEX_PROGRESS', + payload: { stage: 'complete', message: 'Embeddings generated', percent: 100 } + }); + + try { + const parsed = extractJSON(result.output || '{}'); + return { success: true, result: parsed }; + } catch { + return { success: true, result: { message: 'Embeddings generated successfully' } }; + } + } else { + broadcastToClients({ + type: 'CODEXLENS_INDEX_PROGRESS', + payload: { stage: 'error', message: result.error || 'Failed', percent: 0 } + }); + return { success: false, error: result.error }; + } + } catch (err) { + broadcastToClients({ + type: 'CODEXLENS_INDEX_PROGRESS', + payload: { stage: 'error', message: err.message, percent: 0 } + }); + return { success: false, error: err.message, status: 500 }; + } + }); + return true; + } + // API: CodexLens Semantic Search Status if (pathname === '/api/codexlens/semantic/status') { const status = await checkSemanticStatus(); diff --git a/ccw/src/core/routes/skills-routes.ts b/ccw/src/core/routes/skills-routes.ts index 7de91e7c..e6a0fd77 100644 --- a/ccw/src/core/routes/skills-routes.ts +++ b/ccw/src/core/routes/skills-routes.ts @@ -429,34 +429,45 @@ async function generateSkillViaCLI({ generationType, description, skillName, loc await fsPromises.mkdir(baseDir, { recursive: true }); } - // Build CLI prompt + // Build structured skill parameters for /skill-generator const targetLocationDisplay = location === 'project' ? '.claude/skills/' : '~/.claude/skills/'; - const prompt = `PURPOSE: Generate a complete Claude Code skill from description -TASK: • Parse skill requirements • Create SKILL.md with proper frontmatter (name, description, version, allowed-tools) • Generate supporting files if needed in skill folder -MODE: write -CONTEXT: @**/* -EXPECTED: Complete skill folder structure with SKILL.md and all necessary files -RULES: $(cat ~/.claude/workflows/cli-templates/prompts/universal/00-universal-rigorous-style.txt) | Follow Claude Code skill format | Include name, description in frontmatter | write=CREATE + // Structured fields from user input + const skillParams = { + skill_name: skillName, + description: description || 'Generate a basic skill template', + target_location: targetLocationDisplay, + target_path: targetPath, + location_type: location // 'project' | 'user' + }; -SKILL DESCRIPTION: -${description || 'Generate a basic skill template'} + // Prompt that invokes /skill-generator skill with structured parameters + const prompt = `/skill-generator -SKILL NAME: ${skillName} -TARGET LOCATION: ${targetLocationDisplay} -TARGET PATH: ${targetPath} +## Skill Parameters (Structured Input) -REQUIREMENTS: -1. Create SKILL.md with frontmatter containing: - - name: "${skillName}" - - description: Brief description of the skill - - version: "1.0.0" - - allowed-tools: List of tools this skill can use (e.g., [Read, Write, Edit, Bash]) -2. Add skill content below frontmatter explaining what the skill does and how to use it -3. If the skill requires supporting files (e.g., templates, scripts), create them in the skill folder -4. Ensure all files are properly formatted and follow best practices`; +\`\`\`json +${JSON.stringify(skillParams, null, 2)} +\`\`\` + +## User Request + +Create a new Claude Code skill with the following specifications: + +- **Skill Name**: ${skillName} +- **Description**: ${description || 'Generate a basic skill template'} +- **Target Location**: ${targetLocationDisplay}${skillName} +- **Location Type**: ${location === 'project' ? 'Project-level (.claude/skills/)' : 'User-level (~/.claude/skills/)'} + +## Instructions + +1. Use the skill-generator to create a complete skill structure +2. Generate SKILL.md with proper frontmatter (name, description, version, allowed-tools) +3. Create necessary supporting files (phases, specs, templates as needed) +4. Follow Claude Code skill design patterns and best practices +5. Output all files to: ${targetPath}`; // Execute CLI tool (Claude) with write mode const result = await executeCliTool({ diff --git a/ccw/src/templates/dashboard-css/30-core-memory.css b/ccw/src/templates/dashboard-css/30-core-memory.css index ebd38353..5b63c589 100644 --- a/ccw/src/templates/dashboard-css/30-core-memory.css +++ b/ccw/src/templates/dashboard-css/30-core-memory.css @@ -299,10 +299,30 @@ color: hsl(38 92% 50%); } +.icon-btn.favorite-active svg { + stroke: hsl(38 92% 50%); + fill: hsl(38 92% 50% / 0.2); +} + .icon-btn.favorite-active:hover { color: hsl(38 92% 40%); } +.icon-btn.favorite-active:hover svg { + stroke: hsl(38 92% 40%); + fill: hsl(38 92% 40% / 0.3); +} + +/* Favorite star icon in memory-id */ +.favorite-star { + color: hsl(38 92% 50%); +} + +.favorite-star svg { + stroke: hsl(38 92% 50%); + fill: hsl(38 92% 50% / 0.2); +} + .icon-btn i { width: 18px; height: 18px; diff --git a/ccw/src/templates/dashboard-css/33-cli-stream-viewer.css b/ccw/src/templates/dashboard-css/33-cli-stream-viewer.css index 415ed451..cf94f041 100644 --- a/ccw/src/templates/dashboard-css/33-cli-stream-viewer.css +++ b/ccw/src/templates/dashboard-css/33-cli-stream-viewer.css @@ -429,6 +429,136 @@ color: hsl(200 80% 70%); } +/* ===== Formatted Message Types ===== */ +.cli-stream-line.formatted { + display: flex; + align-items: flex-start; + gap: 8px; + padding: 6px 10px; + margin: 2px 0; + border-radius: 4px; + transition: all 0.15s ease; + color: hsl(0 0% 90%); /* Ensure text is visible */ +} + +.cli-stream-line.formatted:hover { + background: hsl(0 0% 100% / 0.05); +} + +/* Message Badge */ +.cli-msg-badge { + display: inline-flex; + align-items: center; + gap: 4px; + padding: 2px 8px; + border-radius: 4px; + font-size: 0.625rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.3px; + white-space: nowrap; + flex-shrink: 0; +} + +.cli-msg-badge i, +.cli-msg-badge svg { + width: 12px; + height: 12px; +} + +.cli-msg-content { + flex: 1; + word-break: break-word; +} + +/* System Message */ +.cli-stream-line.formatted.system { + background: hsl(210 50% 20% / 0.3); + border-left: 3px solid hsl(210 80% 55%); +} + +.cli-msg-badge.cli-msg-system { + background: hsl(210 80% 55% / 0.2); + color: hsl(210 80% 70%); +} + +/* Thinking Message */ +.cli-stream-line.formatted.thinking { + background: hsl(280 50% 20% / 0.3); + border-left: 3px solid hsl(280 70% 65%); + font-style: italic; +} + +.cli-msg-badge.cli-msg-thinking { + background: hsl(280 70% 65% / 0.2); + color: hsl(280 70% 75%); +} + +/* Response Message */ +.cli-stream-line.formatted.response { + background: hsl(145 40% 18% / 0.3); + border-left: 3px solid hsl(145 60% 50%); +} + +.cli-msg-badge.cli-msg-response { + background: hsl(145 60% 50% / 0.2); + color: hsl(145 60% 65%); +} + +/* Result Message */ +.cli-stream-line.formatted.result { + background: hsl(160 50% 18% / 0.4); + border-left: 3px solid hsl(160 80% 45%); +} + +.cli-msg-badge.cli-msg-result { + background: hsl(160 80% 45% / 0.25); + color: hsl(160 80% 60%); +} + +/* Error Message */ +.cli-stream-line.formatted.error { + background: hsl(0 50% 20% / 0.4); + border-left: 3px solid hsl(0 70% 55%); +} + +.cli-msg-badge.cli-msg-error { + background: hsl(0 70% 55% / 0.25); + color: hsl(0 70% 70%); +} + +/* Warning Message */ +.cli-stream-line.formatted.warning { + background: hsl(45 60% 18% / 0.4); + border-left: 3px solid hsl(45 80% 55%); +} + +.cli-msg-badge.cli-msg-warning { + background: hsl(45 80% 55% / 0.25); + color: hsl(45 80% 65%); +} + +/* Info Message */ +.cli-stream-line.formatted.info { + background: hsl(200 50% 18% / 0.3); + border-left: 3px solid hsl(200 70% 60%); +} + +.cli-msg-badge.cli-msg-info { + background: hsl(200 70% 60% / 0.2); + color: hsl(200 70% 70%); +} + +/* Inline Code */ +.cli-inline-code { + padding: 1px 5px; + background: hsl(0 0% 25%); + border-radius: 3px; + font-family: var(--font-mono, 'Consolas', 'Monaco', 'Courier New', monospace); + font-size: 0.85em; + color: hsl(45 80% 70%); +} + /* JSON/Code syntax coloring in output */ .cli-stream-line .json-key { color: hsl(200 80% 70%); diff --git a/ccw/src/templates/dashboard-js/components/cli-stream-viewer.js b/ccw/src/templates/dashboard-js/components/cli-stream-viewer.js index 2e869b36..9c87efc4 100644 --- a/ccw/src/templates/dashboard-js/components/cli-stream-viewer.js +++ b/ccw/src/templates/dashboard-js/components/cli-stream-viewer.js @@ -269,6 +269,106 @@ function handleCliStreamError(payload) { updateStreamBadge(); } +// ===== Message Type Parsing ===== +const MESSAGE_TYPE_PATTERNS = { + system: /^\[系统\]/, + thinking: /^\[思考\]/, + response: /^\[响应\]/, + result: /^\[结果\]/, + error: /^\[错误\]/, + warning: /^\[警告\]/, + info: /^\[信息\]/ +}; + +const MESSAGE_TYPE_ICONS = { + system: 'settings', + thinking: 'brain', + response: 'message-circle', + result: 'check-circle', + error: 'alert-circle', + warning: 'alert-triangle', + info: 'info' +}; + +const MESSAGE_TYPE_LABELS = { + system: '系统', + thinking: '思考', + response: '响应', + result: '结果', + error: '错误', + warning: '警告', + info: '信息' +}; + +/** + * Parse message content to extract type and clean content + * @param {string} content - Raw message content + * @returns {{ type: string, label: string, content: string, hasPrefix: boolean }} + */ +function parseMessageType(content) { + for (const [type, pattern] of Object.entries(MESSAGE_TYPE_PATTERNS)) { + if (pattern.test(content)) { + return { + type, + label: MESSAGE_TYPE_LABELS[type], + content: content.replace(pattern, '').trim(), + hasPrefix: true + }; + } + } + return { + type: 'default', + label: '', + content: content, + hasPrefix: false + }; +} + +/** + * Render a formatted message line with type badge + * @param {Object} line - Line object with type and content + * @param {string} searchFilter - Current search filter + * @returns {string} - HTML string + */ +function renderFormattedLine(line, searchFilter) { + const parsed = parseMessageType(line.content); + let content = escapeHtml(parsed.content); + + // Apply search highlighting + if (searchFilter && searchFilter.trim()) { + const searchRegex = new RegExp(`(${escapeRegex(searchFilter)})`, 'gi'); + content = content.replace(searchRegex, '$1'); + } + + // Format code blocks + content = formatCodeBlocks(content); + + // Format inline code + content = content.replace(/`([^`]+)`/g, '$1'); + + // Build type badge if has prefix + const typeBadge = parsed.hasPrefix ? + ` + + ${parsed.label} + ` : ''; + + // Determine line class based on original type and parsed type + const lineClass = parsed.hasPrefix ? `cli-stream-line formatted ${parsed.type}` : + `cli-stream-line ${line.type}`; + + return `
${typeBadge}${content}
`; +} + +/** + * Format code blocks in content + */ +function formatCodeBlocks(content) { + // Handle multi-line code blocks (already escaped) + // Just apply styling class for now + return content; +} + // ===== UI Rendering ===== function renderStreamTabs() { const tabsContainer = document.getElementById('cliStreamTabs'); @@ -351,16 +451,15 @@ function renderStreamContent(executionId) { ); } - // Render output lines with search highlighting - contentContainer.innerHTML = filteredOutput.map(line => { - let content = escapeHtml(line.content); - // Highlight search matches - if (searchFilter.trim()) { - const searchRegex = new RegExp(`(${escapeRegex(searchFilter)})`, 'gi'); - content = content.replace(searchRegex, '$1'); - } - return `
${content}
`; - }).join(''); + // Render output lines with formatted styling + contentContainer.innerHTML = filteredOutput.map(line => + renderFormattedLine(line, searchFilter) + ).join(''); + + // Initialize Lucide icons for message badges + if (typeof lucide !== 'undefined') { + lucide.createIcons({ attrs: { class: 'cli-msg-icon' } }); + } // Show filter result count if filtering if (searchFilter.trim() && filteredOutput.length !== exec.output.length) { diff --git a/ccw/src/templates/dashboard-js/i18n.js b/ccw/src/templates/dashboard-js/i18n.js index b4ecb499..7c9da789 100644 --- a/ccw/src/templates/dashboard-js/i18n.js +++ b/ccw/src/templates/dashboard-js/i18n.js @@ -298,6 +298,8 @@ const i18n = { 'codexlens.configuredInApiSettings': 'Configured in API Settings', 'codexlens.commonModels': 'Common Models', 'codexlens.selectApiModel': 'Select API model...', + 'codexlens.selectLocalModel': 'Select local model...', + 'codexlens.noConfiguredModels': 'No models configured in API Settings', 'codexlens.autoDownloadHint': 'Models are auto-downloaded on first use', 'codexlens.embeddingBackend': 'Embedding Backend', 'codexlens.localFastembed': 'Local (FastEmbed)', @@ -2305,6 +2307,8 @@ const i18n = { 'codexlens.configuredInApiSettings': '已在 API 设置中配置', 'codexlens.commonModels': '常用模型', 'codexlens.selectApiModel': '选择 API 模型...', + 'codexlens.selectLocalModel': '选择本地模型...', + 'codexlens.noConfiguredModels': '未在 API 设置中配置模型', 'codexlens.autoDownloadHint': '模型会在首次使用时自动下载', 'codexlens.embeddingBackend': '嵌入后端', 'codexlens.localFastembed': '本地 (FastEmbed)', diff --git a/ccw/src/templates/dashboard-js/views/codexlens-manager.js b/ccw/src/templates/dashboard-js/views/codexlens-manager.js index 56c65d2d..4788a872 100644 --- a/ccw/src/templates/dashboard-js/views/codexlens-manager.js +++ b/ccw/src/templates/dashboard-js/views/codexlens-manager.js @@ -147,14 +147,40 @@ function buildCodexLensConfigContent(config) { '
' + '
' + - // Quick Actions + // Index Operations - 4 buttons grid '
' + - '

Quick Actions

' + - '
' + - (isInstalled - ? '' + + // FTS Incremental + '' + + // Vector Full Index + '' + + // Vector Incremental + '' + + '
' + : '
' + + '' + + '
') + + '
' + + + // Quick Actions + '
' + + '

' + (t('codexlens.quickActions') || 'Quick Actions') + '

' + + (isInstalled + ? '
' + '' + @@ -163,11 +189,9 @@ function buildCodexLensConfigContent(config) { '' + '' - : '') + - '
' + + '' + + '
' + : '') + '' + '' + @@ -684,9 +708,10 @@ var ENV_VAR_GROUPS = { { group: 'Jina', items: ['jina-embeddings-v3', 'jina-embeddings-v2-base-en', 'jina-embeddings-v2-base-zh'] } ] }, - 'CODEXLENS_USE_GPU': { label: 'Use GPU', type: 'select', options: ['true', 'false'], default: 'true', settingsPath: 'embedding.use_gpu', showWhen: function(env) { return env['CODEXLENS_EMBEDDING_BACKEND'] !== 'litellm'; } }, - 'CODEXLENS_EMBEDDING_STRATEGY': { label: 'Load Balance', type: 'select', options: ['round_robin', 'latency_aware', 'weighted_random'], default: 'latency_aware', settingsPath: 'embedding.strategy', showWhen: function(env) { return env['CODEXLENS_EMBEDDING_BACKEND'] === 'litellm'; } }, - 'CODEXLENS_EMBEDDING_COOLDOWN': { label: 'Rate Limit Cooldown (s)', type: 'number', placeholder: '60', default: '60', settingsPath: 'embedding.cooldown', min: 0, max: 300, showWhen: function(env) { return env['CODEXLENS_EMBEDDING_BACKEND'] === 'litellm'; } } + 'CODEXLENS_USE_GPU': { label: 'Use GPU', type: 'select', options: ['true', 'false'], default: 'true', settingsPath: 'embedding.use_gpu', showWhen: function(env) { return env['CODEXLENS_EMBEDDING_BACKEND'] === 'local'; } }, + 'CODEXLENS_EMBEDDING_POOL_ENABLED': { label: 'High Availability', type: 'select', options: ['true', 'false'], default: 'false', settingsPath: 'embedding.pool_enabled', showWhen: function(env) { return env['CODEXLENS_EMBEDDING_BACKEND'] === 'api'; } }, + 'CODEXLENS_EMBEDDING_STRATEGY': { label: 'Load Balance Strategy', type: 'select', options: ['round_robin', 'latency_aware', 'weighted_random'], default: 'latency_aware', settingsPath: 'embedding.strategy', showWhen: function(env) { return env['CODEXLENS_EMBEDDING_BACKEND'] === 'api' && env['CODEXLENS_EMBEDDING_POOL_ENABLED'] === 'true'; } }, + 'CODEXLENS_EMBEDDING_COOLDOWN': { label: 'Rate Limit Cooldown (s)', type: 'number', placeholder: '60', default: '60', settingsPath: 'embedding.cooldown', min: 0, max: 300, showWhen: function(env) { return env['CODEXLENS_EMBEDDING_BACKEND'] === 'api' && env['CODEXLENS_EMBEDDING_POOL_ENABLED'] === 'true'; } } } }, reranker: { @@ -711,7 +736,10 @@ var ENV_VAR_GROUPS = { { group: 'Jina', items: ['jina-reranker-v2-base-multilingual', 'jina-reranker-v1-base-en'] } ] }, - 'CODEXLENS_RERANKER_TOP_K': { label: 'Top K Results', type: 'number', placeholder: '50', default: '50', settingsPath: 'reranker.top_k', min: 5, max: 200 } + 'CODEXLENS_RERANKER_TOP_K': { label: 'Top K Results', type: 'number', placeholder: '50', default: '50', settingsPath: 'reranker.top_k', min: 5, max: 200 }, + 'CODEXLENS_RERANKER_POOL_ENABLED': { label: 'High Availability', type: 'select', options: ['true', 'false'], default: 'false', settingsPath: 'reranker.pool_enabled', showWhen: function(env) { return env['CODEXLENS_RERANKER_BACKEND'] === 'api'; } }, + 'CODEXLENS_RERANKER_STRATEGY': { label: 'Load Balance Strategy', type: 'select', options: ['round_robin', 'latency_aware', 'weighted_random'], default: 'latency_aware', settingsPath: 'reranker.strategy', showWhen: function(env) { return env['CODEXLENS_RERANKER_BACKEND'] === 'api' && env['CODEXLENS_RERANKER_POOL_ENABLED'] === 'true'; } }, + 'CODEXLENS_RERANKER_COOLDOWN': { label: 'Rate Limit Cooldown (s)', type: 'number', placeholder: '60', default: '60', settingsPath: 'reranker.cooldown', min: 0, max: 300, showWhen: function(env) { return env['CODEXLENS_RERANKER_BACKEND'] === 'api' && env['CODEXLENS_RERANKER_POOL_ENABLED'] === 'true'; } } } }, concurrency: { @@ -730,15 +758,6 @@ var ENV_VAR_GROUPS = { 'CODEXLENS_CASCADE_COARSE_K': { label: 'Coarse K (1st stage)', type: 'number', placeholder: '100', default: '100', settingsPath: 'cascade.coarse_k', min: 10, max: 500 }, 'CODEXLENS_CASCADE_FINE_K': { label: 'Fine K (final)', type: 'number', placeholder: '10', default: '10', settingsPath: 'cascade.fine_k', min: 1, max: 100 } } - }, - llm: { - labelKey: 'codexlens.envGroup.llm', - icon: 'sparkles', - collapsed: true, - vars: { - 'CODEXLENS_LLM_ENABLED': { label: 'Enable LLM', type: 'select', options: ['true', 'false'], default: 'false', settingsPath: 'llm.enabled' }, - 'CODEXLENS_LLM_BATCH_SIZE': { label: 'Batch Size', type: 'number', placeholder: '5', default: '5', settingsPath: 'llm.batch_size', min: 1, max: 20 } - } } }; @@ -859,12 +878,11 @@ async function loadEnvVariables() { for (var key in group.vars) { var config = group.vars[key]; - - // Check variable-level showWhen condition - if (config.showWhen && !config.showWhen(env)) { - continue; - } - + + // Check variable-level showWhen condition - render but hide if condition is false + var shouldShow = !config.showWhen || config.showWhen(env); + var hiddenStyle = shouldShow ? '' : ' style="display:none"'; + // Priority: env file > settings.json > hardcoded default var value = env[key] || settings[key] || config.default || ''; @@ -874,7 +892,7 @@ async function loadEnvVariables() { if (key === 'CODEXLENS_EMBEDDING_BACKEND' || key === 'CODEXLENS_RERANKER_BACKEND') { onchangeHandler = ' onchange="updateModelOptionsOnBackendChange(\'' + key + '\', this.value)"'; } - html += '
' + + html += '
' + '' + ''; if (isApiBackend) { - // For API backend: show configured models from API settings first + // For API backend: show ONLY configured models from API settings + // (don't show unconfigured preset models - they won't work without configuration) if (configuredModels.length > 0) { html += ''; configuredModels.forEach(function(model) { @@ -918,19 +937,8 @@ async function loadEnvVariables() { (providers ? ' (' + escapeHtml(providers) + ')' : '') + ''; }); - } - // Then show common API models as suggestions - if (apiModelList.length > 0) { - html += ''; - apiModelList.forEach(function(group) { - group.items.forEach(function(model) { - // Skip if already in configured list - var exists = configuredModels.some(function(m) { return m.modelId === model; }); - if (!exists) { - html += ''; - } - }); - }); + } else { + html += ''; } } else { // For local backend (fastembed): show actually downloaded models @@ -959,7 +967,7 @@ async function loadEnvVariables() { if (config.max !== undefined) extraAttrs += ' max="' + config.max + '"'; extraAttrs += ' step="1"'; } - html += '
' + + html += '
' + '' + '' + @@ -1021,7 +1029,8 @@ async function loadEnvVariables() { var optionsHtml = ''; if (isApiBackend) { - // For API backend: show configured models from API settings first + // For API backend: show ONLY configured models from API settings + // (don't show unconfigured preset models - they won't work without configuration) if (apiConfiguredModels.length > 0) { optionsHtml += ''; apiConfiguredModels.forEach(function(model) { @@ -1031,18 +1040,8 @@ async function loadEnvVariables() { (providers ? ' (' + escapeHtml(providers) + ')' : '') + ''; }); - } - // Then show common API models as suggestions - if (apiModelList.length > 0) { - optionsHtml += ''; - apiModelList.forEach(function(group) { - group.items.forEach(function(model) { - var exists = apiConfiguredModels.some(function(m) { return m.modelId === model; }); - if (!exists) { - optionsHtml += ''; - } - }); - }); + } else { + optionsHtml += ''; } } else { // For local backend: show actually downloaded models @@ -1070,9 +1069,65 @@ async function loadEnvVariables() { } } } + + // Update visibility of dependent fields based on new backend value + var prefix = isEmbedding ? 'CODEXLENS_EMBEDDING_' : 'CODEXLENS_RERANKER_'; + var gpuField = document.querySelector('[data-env-key="' + prefix + 'USE_GPU"]'); + var poolField = document.querySelector('[data-env-key="' + prefix + 'POOL_ENABLED"]'); + var strategyField = document.querySelector('[data-env-key="' + prefix + 'STRATEGY"]'); + var cooldownField = document.querySelector('[data-env-key="' + prefix + 'COOLDOWN"]'); + + // GPU only for local backend + if (gpuField) { + var gpuRow = gpuField.closest('.flex.items-center'); + if (gpuRow) gpuRow.style.display = isApiBackend ? 'none' : ''; + } + + // Pool, Strategy, Cooldown only for API backend + if (poolField) { + var poolRow = poolField.closest('.flex.items-center'); + if (poolRow) poolRow.style.display = isApiBackend ? '' : 'none'; + // Reset pool value when switching to local + if (!isApiBackend) poolField.value = 'false'; + } + + // Strategy and Cooldown depend on pool being enabled + var poolEnabled = poolField && poolField.value === 'true'; + if (strategyField) { + var strategyRow = strategyField.closest('.flex.items-center'); + if (strategyRow) strategyRow.style.display = (isApiBackend && poolEnabled) ? '' : 'none'; + } + if (cooldownField) { + var cooldownRow = cooldownField.closest('.flex.items-center'); + if (cooldownRow) cooldownRow.style.display = (isApiBackend && poolEnabled) ? '' : 'none'; + } + // Note: No auto-save here - user must click Save button }); }); + + // Add change handler for pool_enabled selects to show/hide strategy and cooldown + var poolSelects = container.querySelectorAll('select[data-env-key*="POOL_ENABLED"]'); + poolSelects.forEach(function(select) { + select.addEventListener('change', function() { + var poolKey = select.getAttribute('data-env-key'); + var poolEnabled = select.value === 'true'; + var isEmbedding = poolKey.indexOf('EMBEDDING') !== -1; + var prefix = isEmbedding ? 'CODEXLENS_EMBEDDING_' : 'CODEXLENS_RERANKER_'; + + var strategyField = document.querySelector('[data-env-key="' + prefix + 'STRATEGY"]'); + var cooldownField = document.querySelector('[data-env-key="' + prefix + 'COOLDOWN"]'); + + if (strategyField) { + var strategyRow = strategyField.closest('.flex.items-center'); + if (strategyRow) strategyRow.style.display = poolEnabled ? '' : 'none'; + } + if (cooldownField) { + var cooldownRow = cooldownField.closest('.flex.items-center'); + if (cooldownRow) cooldownRow.style.display = poolEnabled ? '' : 'none'; + } + }); + }); } catch (err) { container.innerHTML = '
' + escapeHtml(err.message) + '
'; } @@ -2213,6 +2268,9 @@ async function loadModelList() { '
' + statusIcon + '' + model.profile + '' + + '' + '' + model.dimensions + 'd' + '
' + '
' + @@ -2491,6 +2549,9 @@ async function loadRerankerModelList() { '
' + statusIcon + '' + model.id + recBadge + '' + + '' + '' + model.desc + '' + '
' + '
' + @@ -2901,12 +2962,14 @@ async function updateSemanticStatusBadge() { * @param {string} embeddingModel - Model profile: 'code', 'fast' * @param {string} embeddingBackend - Backend: 'fastembed' (local) or 'litellm' (API) * @param {number} maxWorkers - Max concurrent API calls for embedding generation (default: 1) + * @param {boolean} incremental - Incremental mode: true=skip unchanged, false=full rebuild (default: false) */ -async function initCodexLensIndex(indexType, embeddingModel, embeddingBackend, maxWorkers) { +async function initCodexLensIndex(indexType, embeddingModel, embeddingBackend, maxWorkers, incremental) { indexType = indexType || 'vector'; embeddingModel = embeddingModel || 'code'; embeddingBackend = embeddingBackend || 'fastembed'; maxWorkers = maxWorkers || 1; + incremental = incremental !== undefined ? incremental : false; // Default: full rebuild // For vector/full index with local backend, check if semantic dependencies are available // LiteLLM backend uses remote embeddings and does not require fastembed/ONNX deps. @@ -3011,7 +3074,7 @@ async function initCodexLensIndex(indexType, embeddingModel, embeddingBackend, m var apiIndexType = (indexType === 'full') ? 'vector' : indexType; // Start indexing with specified type and model - startCodexLensIndexing(apiIndexType, embeddingModel, embeddingBackend, maxWorkers); + startCodexLensIndexing(apiIndexType, embeddingModel, embeddingBackend, maxWorkers, incremental); } /** @@ -3020,12 +3083,14 @@ async function initCodexLensIndex(indexType, embeddingModel, embeddingBackend, m * @param {string} embeddingModel - Model profile: 'code', 'fast' * @param {string} embeddingBackend - Backend: 'fastembed' (local) or 'litellm' (API) * @param {number} maxWorkers - Max concurrent API calls for embedding generation (default: 1) + * @param {boolean} incremental - Incremental mode (default: false for full rebuild) */ -async function startCodexLensIndexing(indexType, embeddingModel, embeddingBackend, maxWorkers) { +async function startCodexLensIndexing(indexType, embeddingModel, embeddingBackend, maxWorkers, incremental) { indexType = indexType || 'vector'; embeddingModel = embeddingModel || 'code'; embeddingBackend = embeddingBackend || 'fastembed'; maxWorkers = maxWorkers || 1; + incremental = incremental !== undefined ? incremental : false; // Default: full rebuild var statusText = document.getElementById('codexlensIndexStatus'); var progressBar = document.getElementById('codexlensIndexProgressBar'); var percentText = document.getElementById('codexlensIndexPercent'); @@ -3057,11 +3122,11 @@ async function startCodexLensIndexing(indexType, embeddingModel, embeddingBacken } try { - console.log('[CodexLens] Starting index for:', projectPath, 'type:', indexType, 'model:', embeddingModel, 'backend:', embeddingBackend, 'maxWorkers:', maxWorkers); + console.log('[CodexLens] Starting index for:', projectPath, 'type:', indexType, 'model:', embeddingModel, 'backend:', embeddingBackend, 'maxWorkers:', maxWorkers, 'incremental:', incremental); var response = await fetch('/api/codexlens/init', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ path: projectPath, indexType: indexType, embeddingModel: embeddingModel, embeddingBackend: embeddingBackend, maxWorkers: maxWorkers }) + body: JSON.stringify({ path: projectPath, indexType: indexType, embeddingModel: embeddingModel, embeddingBackend: embeddingBackend, maxWorkers: maxWorkers, incremental: incremental }) }); var result = await response.json(); @@ -4165,6 +4230,121 @@ function initCodexLensIndexFromPage(indexType) { } } +// ============================================================ +// INDEX OPERATIONS - 4 Button Functions +// ============================================================ + +/** + * Run FTS full index (rebuild full-text search index) + * Creates FTS index without embeddings + */ +window.runFtsFullIndex = async function runFtsFullIndex() { + showRefreshToast(t('codexlens.startingFtsFullIndex') || 'Starting FTS full index...', 'info'); + // FTS only, no embeddings, full rebuild (incremental=false) + initCodexLensIndex('normal', null, 'fastembed', 1, false); +} + +/** + * Run FTS incremental update + * Updates FTS index for changed files only + */ +window.runFtsIncrementalUpdate = async function runFtsIncrementalUpdate() { + var projectPath = window.CCW_PROJECT_ROOT || '.'; + showRefreshToast(t('codexlens.startingFtsIncremental') || 'Starting FTS incremental update...', 'info'); + + try { + // Use index update endpoint for FTS incremental + var response = await fetch('/api/codexlens/init', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + path: projectPath, + indexType: 'normal', // FTS only + incremental: true + }) + }); + var result = await response.json(); + + if (result.success) { + showRefreshToast(t('codexlens.ftsIncrementalComplete') || 'FTS incremental update completed', 'success'); + renderCodexLensManager(); + } else { + showRefreshToast((t('codexlens.ftsIncrementalFailed') || 'FTS incremental failed') + ': ' + (result.error || 'Unknown error'), 'error'); + } + } catch (err) { + showRefreshToast((t('common.error') || 'Error') + ': ' + err.message, 'error'); + } +} + +/** + * Run Vector full index (generate all embeddings) + * Generates embeddings for all files + */ +window.runVectorFullIndex = async function runVectorFullIndex() { + showRefreshToast(t('codexlens.startingVectorFullIndex') || 'Starting Vector full index...', 'info'); + + try { + // Fetch env settings to get the configured embedding model + var envResponse = await fetch('/api/codexlens/env'); + var envData = await envResponse.json(); + var embeddingModel = envData.CODEXLENS_EMBEDDING_MODEL || envData.LITELLM_EMBEDDING_MODEL || 'code'; + + // Use litellm backend with env-configured model, full rebuild (incremental=false) + initCodexLensIndex('vector', embeddingModel, 'litellm', 4, false); + } catch (err) { + // Fallback to default model if env fetch fails + initCodexLensIndex('vector', 'code', 'litellm', 4, false); + } +} + +/** + * Run Vector incremental update + * Generates embeddings for new/changed files only + */ +window.runVectorIncrementalUpdate = async function runVectorIncrementalUpdate() { + var projectPath = window.CCW_PROJECT_ROOT || '.'; + showRefreshToast(t('codexlens.startingVectorIncremental') || 'Starting Vector incremental update...', 'info'); + + try { + // Fetch env settings to get the configured embedding model + var envResponse = await fetch('/api/codexlens/env'); + var envData = await envResponse.json(); + var embeddingModel = envData.CODEXLENS_EMBEDDING_MODEL || envData.LITELLM_EMBEDDING_MODEL || null; + + // Use embeddings endpoint for vector incremental + var requestBody = { + path: projectPath, + incremental: true, // Only new/changed files + backend: 'litellm', + maxWorkers: 4 + }; + + // Add model if configured in env + if (embeddingModel) { + requestBody.model = embeddingModel; + } + + var response = await fetch('/api/codexlens/embeddings/generate', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(requestBody) + }); + var result = await response.json(); + + if (result.success) { + var stats = result.result || {}; + var msg = (t('codexlens.vectorIncrementalComplete') || 'Vector incremental completed') + + (stats.chunks_created ? ': ' + stats.chunks_created + ' chunks' : ''); + showRefreshToast(msg, 'success'); + renderCodexLensManager(); + } else { + showRefreshToast((t('codexlens.vectorIncrementalFailed') || 'Vector incremental failed') + ': ' + (result.error || 'Unknown error'), 'error'); + } + } catch (err) { + showRefreshToast((t('common.error') || 'Error') + ': ' + err.message, 'error'); + } +} + /** * Run incremental update on the current workspace index */ diff --git a/ccw/src/templates/dashboard-js/views/core-memory.js b/ccw/src/templates/dashboard-js/views/core-memory.js index 99fced53..af1db942 100644 --- a/ccw/src/templates/dashboard-js/views/core-memory.js +++ b/ccw/src/templates/dashboard-js/views/core-memory.js @@ -228,15 +228,31 @@ function renderMemoryCard(memory) { const updatedDate = memory.updated_at ? new Date(memory.updated_at).toLocaleString() : createdDate; const isArchived = memory.archived || false; - const metadata = memory.metadata || {}; + // Parse metadata - it may be double-encoded JSON string from the backend + let metadata = {}; + if (memory.metadata) { + try { + let parsed = typeof memory.metadata === 'string' ? JSON.parse(memory.metadata) : memory.metadata; + // Handle double-encoded JSON (string within string) + if (typeof parsed === 'string') { + parsed = JSON.parse(parsed); + } + metadata = parsed; + console.log('[DEBUG] Memory', memory.id, 'metadata parsed:', metadata, 'favorite:', metadata.favorite); + } catch (e) { + console.warn('Failed to parse memory metadata:', e); + } + } const tags = metadata.tags || []; const priority = metadata.priority || 'medium'; + const isFavorite = metadata.favorite === true; + console.log('[DEBUG] Memory', memory.id, 'isFavorite:', isFavorite); return `
- ${metadata.favorite ? '' : ''} + ${isFavorite ? '' : ''} ${memory.id} ${isArchived ? `${t('common.archived')}` : ''} ${priority !== 'medium' ? `${priority}` : ''} @@ -245,7 +261,7 @@ function renderMemoryCard(memory) { - ${!isArchived @@ -312,7 +328,8 @@ function renderMemoryCard(memory) { // API Functions async function fetchCoreMemories(archived = false) { try { - const response = await fetch(`/api/core-memory/memories?path=${encodeURIComponent(projectPath)}&archived=${archived}`); + // Add timestamp to prevent browser caching + const response = await fetch(`/api/core-memory/memories?path=${encodeURIComponent(projectPath)}&archived=${archived}&_t=${Date.now()}`); if (!response.ok) throw new Error(`HTTP ${response.status}`); const data = await response.json(); return data.memories || []; @@ -325,7 +342,8 @@ async function fetchCoreMemories(archived = false) { async function fetchMemoryById(memoryId) { try { - const response = await fetch(`/api/core-memory/memories/${memoryId}?path=${encodeURIComponent(projectPath)}`); + // Add timestamp to prevent browser caching + const response = await fetch(`/api/core-memory/memories/${memoryId}?path=${encodeURIComponent(projectPath)}&_t=${Date.now()}`); if (!response.ok) throw new Error(`HTTP ${response.status}`); const data = await response.json(); return data.memory || null; @@ -356,7 +374,9 @@ async function editMemory(memoryId) { document.getElementById('memoryModalTitle').textContent = t('coreMemory.edit'); document.getElementById('memoryContent').value = memory.content || ''; document.getElementById('memorySummary').value = memory.summary || ''; - document.getElementById('memoryMetadata').value = memory.metadata ? JSON.stringify(memory.metadata, null, 2) : ''; + document.getElementById('memoryMetadata').value = memory.metadata + ? (typeof memory.metadata === 'string' ? memory.metadata : JSON.stringify(memory.metadata, null, 2)) + : ''; modal.dataset.editId = memoryId; modal.style.display = 'flex'; lucide.createIcons(); @@ -523,13 +543,23 @@ async function viewMemoryDetail(memoryId) { 
${escapeHtml(memory.content)}
- ${memory.metadata && Object.keys(memory.metadata).length > 0 - ? `
+ ${(() => { + if (!memory.metadata) return ''; + try { + let metadataObj = typeof memory.metadata === 'string' ? JSON.parse(memory.metadata) : memory.metadata; + // Handle double-encoded JSON + if (typeof metadataObj === 'string') { + metadataObj = JSON.parse(metadataObj); + } + if (Object.keys(metadataObj).length === 0) return ''; + return `

${t('coreMemory.metadata')}

- 
${escapeHtml(JSON.stringify(memory.metadata, null, 2))}
-
` - : '' - } + 
${escapeHtml(JSON.stringify(metadataObj, null, 2))}
+
`; + } catch (e) { + return ''; + } + })()} ${memory.raw_output ? `
@@ -644,7 +674,19 @@ function showClustersView() { // Favorites Functions async function refreshFavorites() { const allMemories = await fetchCoreMemories(false); - const favorites = allMemories.filter(m => m.metadata && m.metadata.favorite); + const favorites = allMemories.filter(m => { + if (!m.metadata) return false; + try { + let parsed = typeof m.metadata === 'string' ? JSON.parse(m.metadata) : m.metadata; + // Handle double-encoded JSON + if (typeof parsed === 'string') { + parsed = JSON.parse(parsed); + } + return parsed.favorite === true; + } catch (e) { + return false; + } + }); const countEl = document.getElementById('totalFavoritesCount'); const gridEl = document.getElementById('favoritesGridContent'); @@ -670,7 +712,7 @@ async function refreshFavorites() { async function showMemoryRelations(memoryId) { try { // Fetch all clusters - const response = await fetch(`/api/core-memory/clusters?path=${encodeURIComponent(projectPath)}`); + const response = await fetch(`/api/core-memory/clusters?path=${encodeURIComponent(projectPath)}&_t=${Date.now()}`); if (!response.ok) throw new Error(`HTTP ${response.status}`); const result = await response.json(); @@ -679,7 +721,7 @@ async function showMemoryRelations(memoryId) { // Find clusters containing this memory const relatedClusters = []; for (const cluster of clusters) { - const detailRes = await fetch(`/api/core-memory/clusters/${cluster.id}?path=${encodeURIComponent(projectPath)}`); + const detailRes = await fetch(`/api/core-memory/clusters/${cluster.id}?path=${encodeURIComponent(projectPath)}&_t=${Date.now()}`); if (detailRes.ok) { const detail = await detailRes.json(); const members = detail.members || []; @@ -749,7 +791,20 @@ async function toggleFavorite(memoryId) { const memory = await fetchMemoryById(memoryId); if (!memory) return; - const metadata = memory.metadata || {}; + // Parse metadata - it may be double-encoded JSON string from the backend + let metadata = {}; + if (memory.metadata) { + try { + let parsed = typeof memory.metadata === 'string' ? JSON.parse(memory.metadata) : memory.metadata; + // Handle double-encoded JSON + if (typeof parsed === 'string') { + parsed = JSON.parse(parsed); + } + metadata = parsed; + } catch (e) { + console.warn('Failed to parse memory metadata:', e); + } + } metadata.favorite = !metadata.favorite; const response = await fetch('/api/core-memory/memories', { diff --git a/ccw/src/tools/cli-config-manager.ts b/ccw/src/tools/cli-config-manager.ts index ab7a66cd..62fc429e 100644 --- a/ccw/src/tools/cli-config-manager.ts +++ b/ccw/src/tools/cli-config-manager.ts @@ -20,14 +20,15 @@ export interface CliConfig { tools: Record; } -export type CliToolName = 'gemini' | 'qwen' | 'codex'; +export type CliToolName = 'gemini' | 'qwen' | 'codex' | 'claude'; // ========== Constants ========== export const PREDEFINED_MODELS: Record = { gemini: ['gemini-2.5-pro', 'gemini-2.5-flash', 'gemini-2.0-flash', 'gemini-1.5-pro', 'gemini-1.5-flash'], qwen: ['coder-model', 'vision-model', 'qwen2.5-coder-32b'], - codex: ['gpt-5.2', 'gpt-4.1', 'o4-mini', 'o3'] + codex: ['gpt-5.2', 'gpt-4.1', 'o4-mini', 'o3'], + claude: ['sonnet', 'opus', 'haiku', 'claude-sonnet-4-5-20250929', 'claude-opus-4-5-20251101'] }; export const DEFAULT_CONFIG: CliConfig = { @@ -47,6 +48,11 @@ export const DEFAULT_CONFIG: CliConfig = { enabled: true, primaryModel: 'gpt-5.2', secondaryModel: 'gpt-5.2' + }, + claude: { + enabled: true, + primaryModel: 'sonnet', + secondaryModel: 'haiku' } } }; @@ -63,7 +69,7 @@ function ensureConfigDirForProject(baseDir: string): void { } function isValidToolName(tool: string): tool is CliToolName { - return ['gemini', 'qwen', 'codex'].includes(tool); + return ['gemini', 'qwen', 'codex', 'claude'].includes(tool); } function validateConfig(config: unknown): config is CliConfig { @@ -74,7 +80,7 @@ function validateConfig(config: unknown): config is CliConfig { if (!c.tools || typeof c.tools !== 'object') return false; const tools = c.tools as Record; - for (const toolName of ['gemini', 'qwen', 'codex']) { + for (const toolName of ['gemini', 'qwen', 'codex', 'claude']) { const tool = tools[toolName]; if (!tool || typeof tool !== 'object') return false; diff --git a/ccw/src/tools/cli-executor.ts b/ccw/src/tools/cli-executor.ts index c1819746..4688e3b2 100644 --- a/ccw/src/tools/cli-executor.ts +++ b/ccw/src/tools/cli-executor.ts @@ -66,6 +66,309 @@ function errorLog(category: string, message: string, error?: Error | unknown, co } } +// ========== Unified Stream-JSON Parser ========== + +/** + * Claude CLI stream-json message types + */ +interface ClaudeStreamMessage { + type: 'system' | 'assistant' | 'result' | 'error'; + subtype?: 'init' | 'success' | 'error'; + session_id?: string; + model?: string; + message?: { + content: Array<{ type: 'text'; text: string }>; + }; + result?: string; + total_cost_usd?: number; + usage?: { + input_tokens?: number; + output_tokens?: number; + }; + error?: string; +} + +/** + * Gemini/Qwen CLI stream-json message types + */ +interface GeminiStreamMessage { + type: 'init' | 'message' | 'result'; + timestamp?: string; + session_id?: string; + model?: string; + role?: 'user' | 'assistant'; + content?: string; + delta?: boolean; + status?: 'success' | 'error'; + stats?: { + total_tokens?: number; + input_tokens?: number; + output_tokens?: number; + }; +} + +/** + * Codex CLI JSON message types + */ +interface CodexStreamMessage { + type: 'thread.started' | 'turn.started' | 'item.completed' | 'turn.completed'; + thread_id?: string; + item?: { + type: 'reasoning' | 'agent_message'; + text: string; + }; + usage?: { + input_tokens?: number; + output_tokens?: number; + }; +} + +/** + * Unified Stream-JSON Parser for Claude, Gemini, Qwen, and Codex + * Supports different JSON formats and extracts text, session info, and usage data + */ +class UnifiedStreamParser { + private tool: 'claude' | 'gemini' | 'qwen' | 'codex'; + private lineBuffer = ''; + private extractedText = ''; + private sessionInfo: { session_id?: string; model?: string; thread_id?: string } = {}; + private usageInfo: { cost?: number; tokens?: { input: number; output: number } } = {}; + + constructor(tool: 'claude' | 'gemini' | 'qwen' | 'codex') { + this.tool = tool; + } + + /** + * Process incoming data chunk + * @returns Extracted text to output with message type prefixes + */ + processChunk(data: string): string { + this.lineBuffer += data; + const lines = this.lineBuffer.split('\n'); + + // Keep last incomplete line in buffer + this.lineBuffer = lines.pop() || ''; + + let output = ''; + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) continue; + + try { + output += this.parseJsonLine(trimmed); + } catch (err) { + // Not valid JSON or not a stream-json line - pass through as-is + debugLog('STREAM_PARSER', `Non-JSON line (passing through): ${trimmed.substring(0, 100)}`); + output += line + '\n'; + } + } + + return output; + } + + /** + * Parse a single JSON line based on tool type + */ + private parseJsonLine(line: string): string { + switch (this.tool) { + case 'claude': + return this.parseClaudeLine(line); + case 'gemini': + case 'qwen': + return this.parseGeminiQwenLine(line); + case 'codex': + return this.parseCodexLine(line); + default: + return ''; + } + } + + /** + * Parse Claude stream-json format + */ + private parseClaudeLine(line: string): string { + const msg: ClaudeStreamMessage = JSON.parse(line); + let output = ''; + + // Extract session metadata + if (msg.type === 'system' && msg.subtype === 'init') { + this.sessionInfo.session_id = msg.session_id; + this.sessionInfo.model = msg.model; + debugLog('STREAM_PARSER', 'Claude session initialized', this.sessionInfo); + output += `[系统] 会话初始化: ${msg.model || 'unknown'}\n`; + } + + // Extract assistant response text + if (msg.type === 'assistant' && msg.message?.content) { + for (const item of msg.message.content) { + if (item.type === 'text' && item.text && item.text.trim()) { // Filter empty/whitespace-only text + this.extractedText += item.text; + output += `[响应] ${item.text}\n`; // Add newline for proper line separation + } + } + } + + // Extract result metadata + if (msg.type === 'result') { + if (msg.total_cost_usd !== undefined) { + this.usageInfo.cost = msg.total_cost_usd; + } + if (msg.usage) { + this.usageInfo.tokens = { + input: msg.usage.input_tokens || 0, + output: msg.usage.output_tokens || 0 + }; + } + debugLog('STREAM_PARSER', 'Claude execution result received', { + subtype: msg.subtype, + cost: this.usageInfo.cost, + tokens: this.usageInfo.tokens + }); + output += `[结果] 状态: ${msg.subtype || 'completed'}\n`; + } + + // Handle errors + if (msg.type === 'error') { + errorLog('STREAM_PARSER', `Claude error in stream: ${msg.error || 'Unknown error'}`); + output += `[错误] ${msg.error || 'Unknown error'}\n`; + } + + return output; + } + + private lastMessageType: string = ''; // Track last message type for delta mode + + /** + * Parse Gemini/Qwen stream-json format + */ + private parseGeminiQwenLine(line: string): string { + const msg: GeminiStreamMessage = JSON.parse(line); + let output = ''; + + // Extract session metadata + if (msg.type === 'init') { + this.sessionInfo.session_id = msg.session_id; + this.sessionInfo.model = msg.model; + debugLog('STREAM_PARSER', `${this.tool} session initialized`, this.sessionInfo); + output += `[系统] 会话初始化: ${msg.model || 'unknown'}\n`; + this.lastMessageType = 'init'; + } + + // Extract assistant message + if (msg.type === 'message' && msg.role === 'assistant' && msg.content) { + const contentText = msg.content.trim(); // Filter empty/whitespace-only content + if (contentText) { + this.extractedText += msg.content; + if (msg.delta) { + // Delta mode: add prefix only for first chunk + if (this.lastMessageType !== 'assistant') { + output += `[响应] ${msg.content}`; + } else { + output += msg.content; + } + } else { + // Full message mode + output += `[响应] ${msg.content}\n`; + } + this.lastMessageType = 'assistant'; + } + } + + // Extract result statistics + if (msg.type === 'result') { + // Add newline before result if last was delta streaming + if (this.lastMessageType === 'assistant') { + output += '\n'; + } + if (msg.stats) { + this.usageInfo.tokens = { + input: msg.stats.input_tokens || 0, + output: msg.stats.output_tokens || 0 + }; + } + debugLog('STREAM_PARSER', `${this.tool} execution result received`, { + status: msg.status, + tokens: this.usageInfo.tokens + }); + output += `[结果] 状态: ${msg.status || 'success'}\n`; + this.lastMessageType = 'result'; + } + + return output; + } + + /** + * Parse Codex JSON format + */ + private parseCodexLine(line: string): string { + const msg: CodexStreamMessage = JSON.parse(line); + let output = ''; + + // Extract thread metadata + if (msg.type === 'thread.started' && msg.thread_id) { + this.sessionInfo.thread_id = msg.thread_id; + debugLog('STREAM_PARSER', 'Codex thread started', { thread_id: msg.thread_id }); + output += `[系统] 线程启动: ${msg.thread_id}\n`; + } + + // Extract reasoning text + if (msg.type === 'item.completed' && msg.item?.type === 'reasoning') { + output += `[思考] ${msg.item.text}\n`; + } + + // Extract agent message + if (msg.type === 'item.completed' && msg.item?.type === 'agent_message') { + this.extractedText += msg.item.text; + output += `[响应] ${msg.item.text}\n`; + } + + // Extract usage statistics + if (msg.type === 'turn.completed' && msg.usage) { + this.usageInfo.tokens = { + input: msg.usage.input_tokens || 0, + output: msg.usage.output_tokens || 0 + }; + debugLog('STREAM_PARSER', 'Codex turn completed', { + tokens: this.usageInfo.tokens + }); + output += `[结果] 回合完成\n`; + } + + return output; + } + + /** + * Flush remaining buffer on stream end + */ + flush(): string { + if (this.lineBuffer.trim()) { + return this.processChunk('\n'); // Force process remaining line + } + return ''; + } + + /** + * Get full extracted text + */ + getExtractedText(): string { + return this.extractedText; + } + + /** + * Get session metadata + */ + getSessionInfo() { + return this.sessionInfo; + } + + /** + * Get usage metadata + */ + getUsageInfo() { + return this.usageInfo; + } +} + // LiteLLM integration import { executeLiteLLMEndpoint } from './litellm-executor.js'; import { findEndpointById } from '../config/litellm-api-config-manager.js'; @@ -116,7 +419,7 @@ function getSqliteStoreSync(baseDir: string) { // Define Zod schema for validation const ParamsSchema = z.object({ - tool: z.enum(['gemini', 'qwen', 'codex']), + tool: z.enum(['gemini', 'qwen', 'codex', 'claude']), prompt: z.string().min(1, 'Prompt is required'), mode: z.enum(['analysis', 'write', 'auto']).default('analysis'), format: z.enum(['plain', 'yaml', 'json']).default('plain'), // Multi-turn prompt concatenation format @@ -255,6 +558,7 @@ interface ExecutionOutput { conversation: ConversationRecord; // Full conversation record stdout: string; stderr: string; + parsedOutput?: string; // Parsed output from stream parser (for stream-json tools) } /** @@ -380,6 +684,8 @@ function buildCommand(params: { if (include) { args.push('--include-directories', include); } + // Enable stream-json output for unified parsing + args.push('--output-format', 'stream-json'); break; case 'qwen': @@ -400,6 +706,8 @@ function buildCommand(params: { if (include) { args.push('--include-directories', include); } + // Enable stream-json output for unified parsing + args.push('--output-format', 'stream-json'); break; case 'codex': @@ -434,6 +742,8 @@ function buildCommand(params: { args.push('--add-dir', addDir); } } + // Enable JSON output for unified parsing + args.push('--json'); // Use `-` to indicate reading prompt from stdin args.push('-'); } else { @@ -458,6 +768,8 @@ function buildCommand(params: { args.push('--add-dir', addDir); } } + // Enable JSON output for unified parsing + args.push('--json'); // Use `-` to indicate reading prompt from stdin (avoids Windows escaping issues) args.push('-'); } @@ -483,8 +795,9 @@ function buildCommand(params: { } else { args.push('--permission-mode', 'default'); } - // Output format for better parsing - args.push('--output-format', 'text'); + // Output format: stream-json for real-time parsing, text for backward compatibility + args.push('--output-format', 'stream-json'); + args.push('--verbose'); // Required for stream-json format // Add directories if (include) { const dirs = include.split(',').map(d => d.trim()).filter(d => d); @@ -962,11 +1275,23 @@ async function executeCliTool( let stderr = ''; let timedOut = false; + // Initialize unified stream parser for all tools + const streamParser = ['claude', 'gemini', 'qwen', 'codex'].includes(tool) + ? new UnifiedStreamParser(tool as 'claude' | 'gemini' | 'qwen' | 'codex') + : null; + // Handle stdout child.stdout!.on('data', (data) => { const text = data.toString(); stdout += text; - if (onOutput) { + + // Parse stream-json for all supported tools + if (streamParser && onOutput) { + const parsedText = streamParser.processChunk(text); + if (parsedText) { + onOutput({ type: 'stdout', data: parsedText }); + } + } else if (onOutput) { onOutput({ type: 'stdout', data: text }); } }); @@ -985,6 +1310,23 @@ async function executeCliTool( // Clear current child process reference currentChildProcess = null; + // Flush unified parser buffer if present + if (streamParser && onOutput) { + const remaining = streamParser.flush(); + if (remaining) { + onOutput({ type: 'stdout', data: remaining }); + } + + // Log usage information if available + const usageInfo = streamParser.getUsageInfo(); + if (usageInfo.cost !== undefined || usageInfo.tokens) { + debugLog('STREAM_USAGE', `${tool} execution usage`, { + cost_usd: usageInfo.cost, + tokens: usageInfo.tokens + }); + } + } + const endTime = Date.now(); const duration = endTime - startTime; @@ -1212,7 +1554,8 @@ async function executeCliTool( execution, conversation, stdout, - stderr + stderr, + parsedOutput: streamParser?.getExtractedText() || undefined }); }); diff --git a/ccw/src/tools/smart-search.ts b/ccw/src/tools/smart-search.ts index 92eea258..53c908a0 100644 --- a/ccw/src/tools/smart-search.ts +++ b/ccw/src/tools/smart-search.ts @@ -2,7 +2,8 @@ * Smart Search Tool - Unified intelligent search with CodexLens integration * * Features: - * - Intent classification with automatic mode selection + * - Fuzzy mode: FTS + ripgrep fusion with RRF ranking (default) + * - Semantic mode: Dense coarse retrieval + cross-encoder reranking * - CodexLens integration (init, dense_rerank, fts) * - Ripgrep fallback for exact mode * - Index status checking and warnings @@ -10,7 +11,7 @@ * * Actions: * - init: Initialize CodexLens index - * - search: Intelligent search with auto mode selection + * - search: Intelligent search with fuzzy (default) or semantic mode * - status: Check index status * - update: Incremental index update for changed files * - watch: Start file watcher for automatic updates @@ -66,7 +67,7 @@ const ParamsSchema = z.object({ action: z.enum(['init', 'search', 'search_files', 'find_files', 'status', 'update', 'watch']).default('search'), query: z.string().optional().describe('Content search query (for action="search")'), pattern: z.string().optional().describe('Glob pattern for path matching (for action="find_files")'), - mode: z.enum(['auto', 'hybrid', 'exact', 'ripgrep', 'priority']).default('auto'), + mode: z.enum(['fuzzy', 'semantic']).default('fuzzy'), output_mode: z.enum(['full', 'files_only', 'count']).default('full'), path: z.string().optional(), paths: z.array(z.string()).default([]), @@ -94,7 +95,7 @@ const ParamsSchema = z.object({ type Params = z.infer; // Search mode constants -const SEARCH_MODES = ['auto', 'hybrid', 'exact', 'ripgrep', 'priority'] as const; +const SEARCH_MODES = ['fuzzy', 'semantic'] as const; // Classification confidence threshold const CONFIDENCE_THRESHOLD = 0.7; @@ -850,6 +851,93 @@ async function executeWatchAction(params: Params): Promise { }; } +/** + * Mode: fuzzy - FTS + ripgrep fusion with RRF ranking + * Runs both exact (FTS) and ripgrep searches in parallel, merges and ranks results + */ +async function executeFuzzyMode(params: Params): Promise { + const { query, path = '.', maxResults = 5, extraFilesCount = 10 } = params; + + if (!query) { + return { + success: false, + error: 'Query is required for search', + }; + } + + const timer = createTimer(); + + // Run both searches in parallel + const [ftsResult, ripgrepResult] = await Promise.allSettled([ + executeCodexLensExactMode(params), + executeRipgrepMode(params), + ]); + timer.mark('parallel_search'); + + // Collect results from both sources + const resultsMap = new Map(); + + // Add FTS results if successful + if (ftsResult.status === 'fulfilled' && ftsResult.value.success && ftsResult.value.results) { + resultsMap.set('exact', ftsResult.value.results as any[]); + } + + // Add ripgrep results if successful + if (ripgrepResult.status === 'fulfilled' && ripgrepResult.value.success && ripgrepResult.value.results) { + resultsMap.set('ripgrep', ripgrepResult.value.results as any[]); + } + + // If both failed, return error + if (resultsMap.size === 0) { + const errors: string[] = []; + if (ftsResult.status === 'rejected') errors.push(`FTS: ${ftsResult.reason}`); + if (ripgrepResult.status === 'rejected') errors.push(`Ripgrep: ${ripgrepResult.reason}`); + return { + success: false, + error: `Both search backends failed: ${errors.join('; ')}`, + }; + } + + // Apply RRF fusion with fuzzy-optimized weights + // Fuzzy mode: balanced between exact and ripgrep + const fusionWeights = { exact: 0.5, ripgrep: 0.5 }; + const totalToFetch = maxResults + extraFilesCount; + const fusedResults = applyRRFFusion(resultsMap, fusionWeights, totalToFetch); + timer.mark('rrf_fusion'); + + // Normalize results format + const normalizedResults = fusedResults.map((item: any) => ({ + file: item.file || item.path, + line: item.line || 0, + column: item.column || 0, + content: item.content || '', + score: item.fusion_score || 0, + matchCount: item.matchCount, + matchScore: item.matchScore, + })); + + // Split results: first N with full content, rest as file paths only + const { results, extra_files } = splitResultsWithExtraFiles(normalizedResults, maxResults, extraFilesCount); + + // Log timing + timer.log(); + const timings = timer.getTimings(); + + return { + success: true, + results, + extra_files: extra_files.length > 0 ? extra_files : undefined, + metadata: { + mode: 'fuzzy', + backend: 'fts+ripgrep', + count: results.length, + query, + note: `Fuzzy search using RRF fusion of FTS and ripgrep (weights: exact=${fusionWeights.exact}, ripgrep=${fusionWeights.ripgrep})`, + timing: TIMING_ENABLED ? timings : undefined, + }, + }; +} + /** * Mode: auto - Intent classification and mode selection * Routes to: hybrid (NL + index) | exact (index) | ripgrep (no index) @@ -1832,10 +1920,9 @@ export const schema: ToolSchema = { - watch: Start file watcher for automatic updates **Content Search (action="search"):** - smart_search(query="authentication logic") # auto mode - routes to best backend - smart_search(query="MyClass", mode="exact") # exact mode - precise FTS matching - smart_search(query="auth", mode="ripgrep") # ripgrep mode - fast literal search - smart_search(query="how to auth", mode="hybrid") # hybrid mode - semantic + fuzzy search + smart_search(query="authentication logic") # fuzzy mode (default) - FTS + ripgrep fusion + smart_search(query="MyClass", mode="fuzzy") # fuzzy mode - fast hybrid search + smart_search(query="how to auth", mode="semantic") # semantic mode - dense + reranker **File Discovery (action="find_files"):** smart_search(action="find_files", pattern="*.ts") # find all TypeScript files @@ -1852,17 +1939,7 @@ export const schema: ToolSchema = { smart_search(query="auth", limit=10, offset=0) # first page smart_search(query="auth", limit=10, offset=10) # second page -**Multi-Word Search (ripgrep mode with tokenization):** - smart_search(query="CCW_PROJECT_ROOT CCW_ALLOWED_DIRS", mode="ripgrep") # tokenized OR matching - smart_search(query="auth login user", mode="ripgrep") # matches any token, ranks by match count - smart_search(query="exact phrase", mode="ripgrep", tokenize=false) # disable tokenization - -**Regex Search (ripgrep mode):** - smart_search(query="class.*Builder") # auto-detects regex pattern - smart_search(query="def.*\\(.*\\):") # find function definitions - smart_search(query="import.*from", caseSensitive=false) # case-insensitive - -**Modes:** auto (intelligent routing), hybrid (semantic+fuzzy), exact (FTS), ripgrep (fast with tokenization), priority (fallback chain)`, +**Modes:** fuzzy (FTS + ripgrep fusion, default), semantic (dense + reranker)`, inputSchema: { type: 'object', properties: { @@ -1883,8 +1960,8 @@ export const schema: ToolSchema = { mode: { type: 'string', enum: SEARCH_MODES, - description: 'Search mode: auto, hybrid (best quality), exact (CodexLens FTS), ripgrep (fast, no index), priority (fallback chain)', - default: 'auto', + description: 'Search mode: fuzzy (FTS + ripgrep fusion, default), semantic (dense + reranker for natural language queries)', + default: 'fuzzy', }, output_mode: { type: 'string', @@ -2323,25 +2400,16 @@ export async function handler(params: Record): Promise "Config": """Load config with settings from file.""" diff --git a/codex-lens/src/codexlens/env_config.py b/codex-lens/src/codexlens/env_config.py index 992a04dd..987294d3 100644 --- a/codex-lens/src/codexlens/env_config.py +++ b/codex-lens/src/codexlens/env_config.py @@ -27,11 +27,17 @@ ENV_VARS = { "RERANKER_API_KEY": "API key for reranker service (SiliconFlow/Cohere/Jina)", "RERANKER_API_BASE": "Base URL for reranker API (overrides provider default)", "RERANKER_PROVIDER": "Reranker provider: siliconflow, cohere, jina", + "RERANKER_POOL_ENABLED": "Enable reranker high availability pool: true/false", + "RERANKER_STRATEGY": "Reranker load balance strategy: round_robin, latency_aware, weighted_random", + "RERANKER_COOLDOWN": "Reranker rate limit cooldown in seconds", # Embedding configuration (overrides settings.json) "EMBEDDING_MODEL": "Embedding model/profile name (overrides settings.json)", "EMBEDDING_BACKEND": "Embedding backend: fastembed, litellm", "EMBEDDING_API_KEY": "API key for embedding service", "EMBEDDING_API_BASE": "Base URL for embedding API", + "EMBEDDING_POOL_ENABLED": "Enable embedding high availability pool: true/false", + "EMBEDDING_STRATEGY": "Embedding load balance strategy: round_robin, latency_aware, weighted_random", + "EMBEDDING_COOLDOWN": "Embedding rate limit cooldown in seconds", # LiteLLM configuration "LITELLM_API_KEY": "API key for LiteLLM", "LITELLM_API_BASE": "Base URL for LiteLLM", diff --git a/codex-lens/src/codexlens/search/chain_search.py b/codex-lens/src/codexlens/search/chain_search.py index e04455ee..fa8a19b4 100644 --- a/codex-lens/src/codexlens/search/chain_search.py +++ b/codex-lens/src/codexlens/search/chain_search.py @@ -1338,8 +1338,9 @@ class ChainSearchEngine: (d for cid, d, _ in coarse_candidates if cid == chunk_id), 1.0 ) - # Convert cosine distance to score - score = 1.0 - distance + # Convert cosine distance to score (clamp to [0, 1] for Pydantic validation) + # Cosine distance can be > 1 for anti-correlated vectors, causing negative scores + score = max(0.0, 1.0 - distance) content = chunk.get("content", "") result = SearchResult( diff --git a/test_index_dir/README.md b/test_index_dir/README.md new file mode 100644 index 00000000..89e85e15 --- /dev/null +++ b/test_index_dir/README.md @@ -0,0 +1 @@ +# Test README\n\nThis is a test file for vector indexing. diff --git a/test_index_dir/test.js b/test_index_dir/test.js new file mode 100644 index 00000000..c8a3b2f6 --- /dev/null +++ b/test_index_dir/test.js @@ -0,0 +1 @@ +// Test file for embedding generation\nfunction testEmbedding() {\n console.log('Hello world');\n}\n\nexport default testEmbedding;