Add comprehensive tests for tokenizer, performance benchmarks, and TreeSitter parser functionality

- Implemented unit tests for the Tokenizer class, covering various text inputs, edge cases, and fallback mechanisms.
- Created performance benchmarks comparing tiktoken and pure Python implementations for token counting.
- Developed extensive tests for TreeSitterSymbolParser across Python, JavaScript, and TypeScript, ensuring accurate symbol extraction and parsing.
- Added configuration documentation for MCP integration and custom prompts, enhancing usability and flexibility.
- Introduced a refactor script for GraphAnalyzer to streamline future improvements.

codex-lens/pyproject.toml

@@ -30,6 +30,11 @@ semantic = [
     "fastembed>=0.2",
 ]
 
+# Full features including tiktoken for accurate token counting
+full = [
+    "tiktoken>=0.5.0",
+]
+
 [project.urls]
 Homepage = "https://github.com/openai/codex-lens"