mirror of
https://github.com/catlog22/Claude-Code-Workflow.git
synced 2026-03-19 18:58:47 +08:00
Implement search and reranking functionality with FTS and embedding support
- Added BaseReranker abstract class for defining reranking interfaces. - Implemented FastEmbedReranker using fastembed's TextCrossEncoder for scoring document-query pairs. - Introduced FTSEngine for full-text search capabilities using SQLite FTS5. - Developed SearchPipeline to integrate embedding, binary search, ANN indexing, FTS, and reranking. - Added fusion methods for combining results from different search strategies using Reciprocal Rank Fusion. - Created unit and integration tests for the new search and reranking components. - Established configuration management for search parameters and models.
This commit is contained in:
25
codex-lens-v2/src/codexlens/rerank/local.py
Normal file
25
codex-lens-v2/src/codexlens/rerank/local.py
Normal file
@@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from codexlens.config import Config
|
||||
from .base import BaseReranker
|
||||
|
||||
|
||||
class FastEmbedReranker(BaseReranker):
|
||||
"""Local reranker backed by fastembed TextCrossEncoder."""
|
||||
|
||||
def __init__(self, config: Config) -> None:
|
||||
self._config = config
|
||||
self._model = None
|
||||
|
||||
def _load(self) -> None:
|
||||
if self._model is None:
|
||||
from fastembed.rerank.cross_encoder import TextCrossEncoder
|
||||
self._model = TextCrossEncoder(model_name=self._config.reranker_model)
|
||||
|
||||
def score_pairs(self, query: str, documents: list[str]) -> list[float]:
|
||||
self._load()
|
||||
results = list(self._model.rerank(query, documents))
|
||||
scores = [0.0] * len(documents)
|
||||
for r in results:
|
||||
scores[r.index] = float(r.score)
|
||||
return scores
|
||||
Reference in New Issue
Block a user