Add comprehensive tests for query parsing and Reciprocal Rank Fusion

- Implemented tests for the QueryParser class, covering various identifier splitting methods (CamelCase, snake_case, kebab-case), OR expansion, and FTS5 operator preservation.
- Added parameterized tests to validate expected token outputs for different query formats.
- Created edge case tests to ensure robustness against unusual input scenarios.
- Developed tests for the Reciprocal Rank Fusion (RRF) algorithm, including score computation, weight handling, and result ranking across multiple sources.
- Included tests for normalization of BM25 scores and tagging search results with source metadata.

codex-lens/pyproject.toml

@@ -30,6 +30,11 @@ semantic = [
     "fastembed>=0.2",
 ]
 
+# Encoding detection for non-UTF8 files
+encoding = [
+    "chardet>=5.0",
+]
+
 # Full features including tiktoken for accurate token counting
 full = [
     "tiktoken>=0.5.0",