feat(discovery): enhance discovery index reading and issue exporting

- Improved the reading of the discovery index by adding a fallback mechanism to scan directories for discovery folders if the index.json is invalid or missing.
- Added sorting of discoveries by creation time in descending order.
- Enhanced the `appendToIssuesJsonl` function to include deduplication logic for issues based on ID and source finding ID.
- Updated the discovery route handler to reflect the number of issues added and skipped during export.
- Introduced UI elements for selecting and deselecting findings in the dashboard.
- Added CSS styles for exported findings and action buttons.
- Implemented search functionality for filtering findings based on title, file, and description.
- Added internationalization support for new UI elements.
- Created scripts for automated API extraction from various project types, including FastAPI and TypeScript.
- Documented the API extraction process and library bundling instructions.
This commit is contained in:
catlog22
2025-12-28 19:27:34 +08:00
parent 3ef1e54412
commit 169f218f7a
18 changed files with 1602 additions and 612 deletions

View File

@@ -0,0 +1,245 @@
# API 文档提取脚本
根据项目类型自动提取 API 文档,支持 FastAPI、Next.js、Python 模块。
## 支持的技术栈
| 类型 | 技术栈 | 工具 | 输出格式 |
|------|--------|------|----------|
| Backend | FastAPI | openapi-to-md | Markdown |
| Frontend | Next.js/TypeScript | TypeDoc | Markdown |
| Python Module | Python | pdoc | Markdown/HTML |
## 使用方法
### 1. FastAPI Backend (OpenAPI)
```bash
# 提取 OpenAPI JSON
cd D:/dongdiankaifa9/backend
python -c "
from app.main import app
import json
print(json.dumps(app.openapi(), indent=2))
" > api-docs/openapi.json
# 转换为 Markdown (使用 widdershins)
npx widdershins api-docs/openapi.json -o api-docs/API_REFERENCE.md --language_tabs 'python:Python' 'javascript:JavaScript' 'bash:cURL'
```
**备选方案 (无需启动服务)**:
```python
# scripts/extract_fastapi_openapi.py
import sys
sys.path.insert(0, 'D:/dongdiankaifa9/backend')
from app.main import app
import json
openapi_schema = app.openapi()
with open('api-docs/openapi.json', 'w', encoding='utf-8') as f:
json.dump(openapi_schema, f, indent=2, ensure_ascii=False)
print(f"Extracted {len(openapi_schema.get('paths', {}))} endpoints")
```
### 2. Next.js Frontend (TypeDoc)
```bash
cd D:/dongdiankaifa9/frontend
# 安装 TypeDoc
npm install --save-dev typedoc typedoc-plugin-markdown
# 生成文档
npx typedoc --plugin typedoc-plugin-markdown \
--out api-docs \
--entryPoints "./lib" "./hooks" "./components" \
--entryPointStrategy expand \
--exclude "**/node_modules/**" \
--exclude "**/*.test.*" \
--readme none
```
**typedoc.json 配置**:
```json
{
"$schema": "https://typedoc.org/schema.json",
"entryPoints": ["./lib", "./hooks", "./components"],
"entryPointStrategy": "expand",
"out": "api-docs",
"plugin": ["typedoc-plugin-markdown"],
"exclude": ["**/node_modules/**", "**/*.test.*", "**/*.spec.*"],
"excludePrivate": true,
"excludeInternal": true,
"readme": "none",
"name": "Frontend API Reference"
}
```
### 3. Python Module (pdoc)
```bash
# 安装 pdoc
pip install pdoc
# hydro_generator_module
cd D:/dongdiankaifa9
pdoc hydro_generator_module \
--output-dir api-docs/hydro_generator \
--format markdown \
--no-show-source
# multiphysics_network
pdoc multiphysics_network \
--output-dir api-docs/multiphysics \
--format markdown \
--no-show-source
```
**备选: Sphinx (更强大)**:
```bash
# 安装 Sphinx
pip install sphinx sphinx-markdown-builder
# 生成 API 文档
sphinx-apidoc -o docs/source hydro_generator_module
cd docs && make markdown
```
## 集成脚本
```python
#!/usr/bin/env python3
# scripts/extract_all_apis.py
import subprocess
import sys
from pathlib import Path
PROJECTS = {
'backend': {
'path': 'D:/dongdiankaifa9/backend',
'type': 'fastapi',
'output': 'api-docs/backend'
},
'frontend': {
'path': 'D:/dongdiankaifa9/frontend',
'type': 'typescript',
'output': 'api-docs/frontend'
},
'hydro_generator_module': {
'path': 'D:/dongdiankaifa9/hydro_generator_module',
'type': 'python',
'output': 'api-docs/hydro_generator'
},
'multiphysics_network': {
'path': 'D:/dongdiankaifa9/multiphysics_network',
'type': 'python',
'output': 'api-docs/multiphysics'
}
}
def extract_fastapi(config):
"""提取 FastAPI OpenAPI 文档"""
path = Path(config['path'])
sys.path.insert(0, str(path))
try:
from app.main import app
import json
output_dir = Path(config['output'])
output_dir.mkdir(parents=True, exist_ok=True)
# 导出 OpenAPI JSON
with open(output_dir / 'openapi.json', 'w', encoding='utf-8') as f:
json.dump(app.openapi(), f, indent=2, ensure_ascii=False)
print(f"✓ FastAPI: {len(app.openapi().get('paths', {}))} endpoints")
return True
except Exception as e:
print(f"✗ FastAPI error: {e}")
return False
def extract_typescript(config):
"""提取 TypeScript 文档"""
try:
subprocess.run([
'npx', 'typedoc',
'--plugin', 'typedoc-plugin-markdown',
'--out', config['output'],
'--entryPoints', './lib', './hooks',
'--entryPointStrategy', 'expand'
], cwd=config['path'], check=True)
print(f"✓ TypeDoc: {config['path']}")
return True
except Exception as e:
print(f"✗ TypeDoc error: {e}")
return False
def extract_python(config):
"""提取 Python 模块文档"""
try:
module_name = Path(config['path']).name
subprocess.run([
'pdoc', module_name,
'--output-dir', config['output'],
'--format', 'markdown'
], cwd=Path(config['path']).parent, check=True)
print(f"✓ pdoc: {module_name}")
return True
except Exception as e:
print(f"✗ pdoc error: {e}")
return False
EXTRACTORS = {
'fastapi': extract_fastapi,
'typescript': extract_typescript,
'python': extract_python
}
if __name__ == '__main__':
for name, config in PROJECTS.items():
print(f"\n[{name}]")
extractor = EXTRACTORS.get(config['type'])
if extractor:
extractor(config)
```
## Phase 3 集成
`api-reference` Agent 提示词中添加:
```
[PRE-EXTRACTION]
运行 API 提取脚本获取结构化文档:
- python scripts/extract_all_apis.py
[INPUT FILES]
- api-docs/backend/openapi.json (FastAPI endpoints)
- api-docs/frontend/*.md (TypeDoc output)
- api-docs/hydro_generator/*.md (pdoc output)
- api-docs/multiphysics/*.md (pdoc output)
```
## 输出结构
```
api-docs/
├── backend/
│ ├── openapi.json # Raw OpenAPI spec
│ └── API_REFERENCE.md # Converted Markdown
├── frontend/
│ ├── modules.md
│ ├── functions.md
│ └── classes/
├── hydro_generator/
│ ├── assembler.md
│ ├── blueprint.md
│ └── builders/
└── multiphysics/
├── analysis_domain.md
├── builders.md
└── compilers.md
```

View File

@@ -0,0 +1,85 @@
# 库文件打包说明
## 依赖库
HTML 组装阶段需要内嵌以下成熟库(无 CDN 依赖):
### 1. marked.js - Markdown 解析
```bash
# 获取最新版本
curl -o templates/libs/marked.min.js https://unpkg.com/marked/marked.min.js
```
### 2. highlight.js - 代码语法高亮
```bash
# 获取核心 + 常用语言包
curl -o templates/libs/highlight.min.js https://unpkg.com/@highlightjs/cdn-assets/highlight.min.js
# 获取 github-dark 主题
curl -o templates/libs/github-dark.min.css https://unpkg.com/@highlightjs/cdn-assets/styles/github-dark.min.css
```
## 内嵌方式
Phase 5 Agent 应:
1. 读取 `templates/libs/*.js``*.css`
2. 将内容嵌入 HTML 的 `<script>``<style>` 标签
3.`DOMContentLoaded` 后初始化:
```javascript
// 初始化 marked
marked.setOptions({
highlight: function(code, lang) {
if (lang && hljs.getLanguage(lang)) {
return hljs.highlight(code, { language: lang }).value;
}
return hljs.highlightAuto(code).value;
},
breaks: true,
gfm: true
});
// 应用高亮
document.querySelectorAll('pre code').forEach(block => {
hljs.highlightElement(block);
});
```
## 备选方案
如果无法获取外部库,使用内置的简化 Markdown 转换:
```javascript
function simpleMarkdown(md) {
return md
.replace(/^### (.+)$/gm, '<h3>$1</h3>')
.replace(/^## (.+)$/gm, '<h2>$1</h2>')
.replace(/^# (.+)$/gm, '<h1>$1</h1>')
.replace(/```(\w+)?\n([\s\S]*?)```/g, (m, lang, code) =>
`<pre data-language="${lang || ''}"><code class="language-${lang || ''}">${escapeHtml(code)}</code></pre>`)
.replace(/`([^`]+)`/g, '<code>$1</code>')
.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>')
.replace(/\*(.+?)\*/g, '<em>$1</em>')
.replace(/\[([^\]]+)\]\(([^)]+)\)/g, '<a href="$2">$1</a>')
.replace(/^\|(.+)\|$/gm, processTableRow)
.replace(/^- (.+)$/gm, '<li>$1</li>')
.replace(/^\d+\. (.+)$/gm, '<li>$1</li>');
}
```
## 文件结构
```
templates/
├── libs/
│ ├── marked.min.js # Markdown parser
│ ├── highlight.min.js # Syntax highlighting
│ └── github-dark.min.css # Code theme
├── tiddlywiki-shell.html
└── css/
├── wiki-base.css
└── wiki-dark.css
```

View File

@@ -0,0 +1,270 @@
#!/usr/bin/env python3
"""
API 文档提取脚本
支持 FastAPI、TypeScript、Python 模块
"""
import subprocess
import sys
import json
from pathlib import Path
from typing import Dict, Any, Optional
# 项目配置
PROJECTS = {
'backend': {
'path': Path('D:/dongdiankaifa9/backend'),
'type': 'fastapi',
'entry': 'app.main:app',
'output': 'api-docs/backend'
},
'frontend': {
'path': Path('D:/dongdiankaifa9/frontend'),
'type': 'typescript',
'entries': ['lib', 'hooks', 'components'],
'output': 'api-docs/frontend'
},
'hydro_generator_module': {
'path': Path('D:/dongdiankaifa9/hydro_generator_module'),
'type': 'python',
'output': 'api-docs/hydro_generator'
},
'multiphysics_network': {
'path': Path('D:/dongdiankaifa9/multiphysics_network'),
'type': 'python',
'output': 'api-docs/multiphysics'
}
}
def extract_fastapi(name: str, config: Dict[str, Any], output_base: Path) -> bool:
"""提取 FastAPI OpenAPI 文档"""
path = config['path']
output_dir = output_base / config['output']
output_dir.mkdir(parents=True, exist_ok=True)
# 添加路径到 sys.path
if str(path) not in sys.path:
sys.path.insert(0, str(path))
try:
# 动态导入 app
from app.main import app
# 获取 OpenAPI schema
openapi_schema = app.openapi()
# 保存 JSON
json_path = output_dir / 'openapi.json'
with open(json_path, 'w', encoding='utf-8') as f:
json.dump(openapi_schema, f, indent=2, ensure_ascii=False)
# 生成 Markdown 摘要
md_path = output_dir / 'API_SUMMARY.md'
generate_api_markdown(openapi_schema, md_path)
endpoints = len(openapi_schema.get('paths', {}))
print(f" ✓ Extracted {endpoints} endpoints → {output_dir}")
return True
except ImportError as e:
print(f" ✗ Import error: {e}")
return False
except Exception as e:
print(f" ✗ Error: {e}")
return False
def generate_api_markdown(schema: Dict, output_path: Path):
"""从 OpenAPI schema 生成 Markdown"""
lines = [
f"# {schema.get('info', {}).get('title', 'API Reference')}",
"",
f"Version: {schema.get('info', {}).get('version', '1.0.0')}",
"",
"## Endpoints",
"",
"| Method | Path | Summary |",
"|--------|------|---------|"
]
for path, methods in schema.get('paths', {}).items():
for method, details in methods.items():
if method in ('get', 'post', 'put', 'delete', 'patch'):
summary = details.get('summary', details.get('operationId', '-'))
lines.append(f"| `{method.upper()}` | `{path}` | {summary} |")
lines.extend([
"",
"## Schemas",
""
])
for name, schema_def in schema.get('components', {}).get('schemas', {}).items():
lines.append(f"### {name}")
lines.append("")
if 'properties' in schema_def:
lines.append("| Property | Type | Required |")
lines.append("|----------|------|----------|")
required = schema_def.get('required', [])
for prop, prop_def in schema_def['properties'].items():
prop_type = prop_def.get('type', prop_def.get('$ref', 'any'))
is_required = '' if prop in required else ''
lines.append(f"| `{prop}` | {prop_type} | {is_required} |")
lines.append("")
with open(output_path, 'w', encoding='utf-8') as f:
f.write('\n'.join(lines))
def extract_typescript(name: str, config: Dict[str, Any], output_base: Path) -> bool:
"""提取 TypeScript 文档 (TypeDoc)"""
path = config['path']
output_dir = output_base / config['output']
# 检查 TypeDoc 是否已安装
try:
result = subprocess.run(
['npx', 'typedoc', '--version'],
cwd=path,
capture_output=True,
text=True
)
if result.returncode != 0:
print(f" ⚠ TypeDoc not installed, installing...")
subprocess.run(
['npm', 'install', '--save-dev', 'typedoc', 'typedoc-plugin-markdown'],
cwd=path,
check=True
)
except FileNotFoundError:
print(f" ✗ npm/npx not found")
return False
# 运行 TypeDoc
try:
entries = config.get('entries', ['lib'])
cmd = [
'npx', 'typedoc',
'--plugin', 'typedoc-plugin-markdown',
'--out', str(output_dir),
'--entryPointStrategy', 'expand',
'--exclude', '**/node_modules/**',
'--exclude', '**/*.test.*',
'--readme', 'none'
]
for entry in entries:
entry_path = path / entry
if entry_path.exists():
cmd.extend(['--entryPoints', str(entry_path)])
result = subprocess.run(cmd, cwd=path, capture_output=True, text=True)
if result.returncode == 0:
print(f" ✓ TypeDoc generated → {output_dir}")
return True
else:
print(f" ✗ TypeDoc error: {result.stderr[:200]}")
return False
except Exception as e:
print(f" ✗ Error: {e}")
return False
def extract_python_module(name: str, config: Dict[str, Any], output_base: Path) -> bool:
"""提取 Python 模块文档 (pdoc)"""
path = config['path']
output_dir = output_base / config['output']
module_name = path.name
# 检查 pdoc
try:
subprocess.run(['pdoc', '--version'], capture_output=True, check=True)
except (FileNotFoundError, subprocess.CalledProcessError):
print(f" ⚠ pdoc not installed, installing...")
subprocess.run([sys.executable, '-m', 'pip', 'install', 'pdoc'], check=True)
# 运行 pdoc
try:
result = subprocess.run(
[
'pdoc', module_name,
'--output-dir', str(output_dir),
'--format', 'markdown'
],
cwd=path.parent,
capture_output=True,
text=True
)
if result.returncode == 0:
# 统计生成的文件
md_files = list(output_dir.glob('**/*.md'))
print(f" ✓ pdoc generated {len(md_files)} files → {output_dir}")
return True
else:
print(f" ✗ pdoc error: {result.stderr[:200]}")
return False
except Exception as e:
print(f" ✗ Error: {e}")
return False
EXTRACTORS = {
'fastapi': extract_fastapi,
'typescript': extract_typescript,
'python': extract_python_module
}
def main(output_base: Optional[str] = None, projects: Optional[list] = None):
"""主入口"""
base = Path(output_base) if output_base else Path.cwd()
print("=" * 50)
print("API Documentation Extraction")
print("=" * 50)
results = {}
for name, config in PROJECTS.items():
if projects and name not in projects:
continue
print(f"\n[{name}] ({config['type']})")
if not config['path'].exists():
print(f" ✗ Path not found: {config['path']}")
results[name] = False
continue
extractor = EXTRACTORS.get(config['type'])
if extractor:
results[name] = extractor(name, config, base)
else:
print(f" ✗ Unknown type: {config['type']}")
results[name] = False
# 汇总
print("\n" + "=" * 50)
print("Summary")
print("=" * 50)
success = sum(1 for v in results.values() if v)
print(f"Success: {success}/{len(results)}")
return all(results.values())
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description='Extract API documentation')
parser.add_argument('--output', '-o', default='.', help='Output base directory')
parser.add_argument('--projects', '-p', nargs='+', help='Specific projects to extract')
args = parser.parse_args()
success = main(args.output, args.projects)
sys.exit(0 if success else 1)