Initial setup: Knowledge base RAG system with LlamaIndex and ChromaDB
- Add Python project with uv package manager - Implement LlamaIndex + ChromaDB RAG pipeline - Add sentence-transformers for local embeddings (all-MiniLM-L6-v2) - Create MCP server with semantic search, indexing, and stats tools - Add Markdown chunker with heading/wikilink/frontmatter support - Add Dockerfile and docker-compose.yaml for self-hosted deployment - Include sample Obsidian vault files for testing - Add .gitignore and .env.example
This commit is contained in:
15
.env.example
Normal file
15
.env.example
Normal file
@ -0,0 +1,15 @@
|
||||
# Knowledge RAG Configuration
|
||||
|
||||
# Path to your Obsidian vault (must contain markdown files)
|
||||
# This should be an absolute path or relative to where you run docker-compose
|
||||
VAULT_PATH=./knowledge
|
||||
|
||||
# Embedding model to use
|
||||
# Default: all-MiniLM-L6-v2 (fast, good quality, ~90MB)
|
||||
# Other options:
|
||||
# - all-mpnet-base-v2 (higher quality, slower, ~420MB)
|
||||
# - BAAI/bge-small-en-v1.5 (good quality, ~130MB)
|
||||
EMBEDDING_MODEL=all-MiniLM-L6-v2
|
||||
|
||||
# Optional: Log level (DEBUG, INFO, WARNING, ERROR)
|
||||
LOG_LEVEL=INFO
|
||||
47
.gitignore
vendored
Normal file
47
.gitignore
vendored
Normal file
@ -0,0 +1,47 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
|
||||
# Virtual environments
|
||||
venv/
|
||||
.venv/
|
||||
env/
|
||||
.env/
|
||||
|
||||
# IDEs
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# uv
|
||||
.ruff_cache/
|
||||
.mypy_cache/
|
||||
.pytest_cache/
|
||||
|
||||
# Data directories (should be mounted externally)
|
||||
data/
|
||||
knowledge/
|
||||
|
||||
# Environment
|
||||
.env
|
||||
.env.local
|
||||
33
Dockerfile
Normal file
33
Dockerfile
Normal file
@ -0,0 +1,33 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
# Install system dependencies for sentence-transformers
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
g++ \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Install uv
|
||||
RUN pip install uv
|
||||
|
||||
# Copy pyproject.toml
|
||||
COPY pyproject.toml .
|
||||
|
||||
# Install dependencies
|
||||
RUN uv sync --frozen --no-dev
|
||||
|
||||
# Copy source code
|
||||
COPY src/ ./src/
|
||||
|
||||
# Create data directories
|
||||
RUN mkdir -p /data/vault /data/chroma_db /data/embeddings_cache
|
||||
|
||||
# Set environment variables
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
VAULT_PATH=/data/vault \
|
||||
EMBEDDINGS_CACHE_DIR=/data/embeddings_cache
|
||||
|
||||
# Default command runs the MCP server
|
||||
CMD ["python", "-m", "knowledge_rag.server"]
|
||||
32
docker-compose.yaml
Normal file
32
docker-compose.yaml
Normal file
@ -0,0 +1,32 @@
|
||||
version: "3.8"
|
||||
|
||||
services:
|
||||
knowledge-rag:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: knowledge-rag
|
||||
volumes:
|
||||
# Mount your obsidian vault here
|
||||
- ${VAULT_PATH:-./knowledge}:/data/vault
|
||||
# Persist ChromaDB vector store
|
||||
- ./data/chroma_db:/data/chroma_db
|
||||
# Persist embeddings cache
|
||||
- ./data/embeddings_cache:/data/embeddings_cache
|
||||
environment:
|
||||
- VAULT_PATH=/data/vault
|
||||
- EMBEDDING_MODEL=${EMBEDDING_MODEL:-all-MiniLM-L6-v2}
|
||||
- EMBEDDINGS_CACHE_DIR=/data/embeddings_cache
|
||||
restart: unless-stopped
|
||||
|
||||
# Optional: Watchtower for auto-updates
|
||||
# watchtower:
|
||||
# image: containrr/watchtower
|
||||
# container_name: watchtower
|
||||
# volumes:
|
||||
# - /var/run/docker.sock:/var/run/docker.sock
|
||||
# environment:
|
||||
# - WATCHTOWER_CLEANUP=true
|
||||
# - WATCHTOWER_INCLUDE_STOPPED=true
|
||||
# command: --interval 3600 knowledge-rag
|
||||
# restart: unless-stopped
|
||||
36
pyproject.toml
Normal file
36
pyproject.toml
Normal file
@ -0,0 +1,36 @@
|
||||
[project]
|
||||
name = "knowledge-rag"
|
||||
version = "0.1.0"
|
||||
description = "RAG system for Obsidian vault knowledge base with MCP server"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"llama-index>=0.10.0",
|
||||
"llama-index-vector-stores-chroma>=0.1.0",
|
||||
"chromadb>=0.4.0",
|
||||
"sentence-transformers>=2.2.0",
|
||||
"mcp>=1.0.0",
|
||||
"python-dotenv>=1.0.0",
|
||||
"pydantic>=2.0.0",
|
||||
"watchdog>=3.0.0",
|
||||
"httpx>=0.25.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=7.0.0",
|
||||
"pytest-asyncio>=0.21.0",
|
||||
"ruff>=0.1.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py311"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "N", "W"]
|
||||
ignore = ["E501"]
|
||||
3
src/knowledge_rag/__init__.py
Normal file
3
src/knowledge_rag/__init__.py
Normal file
@ -0,0 +1,3 @@
|
||||
"""Knowledge RAG - RAG system for Obsidian vault knowledge base."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
181
src/knowledge_rag/chunker.py
Normal file
181
src/knowledge_rag/chunker.py
Normal file
@ -0,0 +1,181 @@
|
||||
"""Markdown-aware document chunking for Obsidian vault."""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from llama_index.core.schema import TextNode
|
||||
|
||||
|
||||
class MarkdownChunker:
|
||||
"""Intelligent markdown chunker for Obsidian vaults.
|
||||
|
||||
Chunks markdown files while preserving:
|
||||
- Document/folder structure context
|
||||
- Code blocks as atomic units
|
||||
- Heading hierarchy
|
||||
- Wiki links as metadata
|
||||
"""
|
||||
|
||||
# Default chunk settings
|
||||
DEFAULT_CHUNK_SIZE = 512
|
||||
DEFAULT_CHUNK_OVERLAP = 50
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
chunk_size: int = DEFAULT_CHUNK_SIZE,
|
||||
chunk_overlap: int = DEFAULT_CHUNK_OVERLAP,
|
||||
):
|
||||
self.chunk_size = chunk_size
|
||||
self.chunk_overlap = chunk_overlap
|
||||
|
||||
def chunk_file(self, file_path: str, content: str) -> List[TextNode]:
|
||||
"""Chunk a single markdown file.
|
||||
|
||||
Args:
|
||||
file_path: Path to the markdown file
|
||||
content: Raw markdown content
|
||||
|
||||
Returns:
|
||||
List of TextNode chunks with metadata
|
||||
"""
|
||||
# Extract frontmatter if present
|
||||
frontmatter, body = self._extract_frontmatter(content)
|
||||
|
||||
# Extract wiki links for metadata
|
||||
wiki_links = self._extract_wiki_links(body)
|
||||
|
||||
# Get relative path for context
|
||||
rel_path = os.path.relpath(file_path)
|
||||
|
||||
# Split into sections based on headings
|
||||
sections = self._split_by_headings(body)
|
||||
|
||||
chunks = []
|
||||
for i, section in enumerate(sections):
|
||||
if not section["content"].strip():
|
||||
continue
|
||||
|
||||
# Create chunk with metadata
|
||||
# Note: wiki_links must be a string for ChromaDB compatibility
|
||||
node = TextNode(
|
||||
text=section["content"],
|
||||
metadata={
|
||||
"source": rel_path,
|
||||
"file_name": os.path.basename(file_path),
|
||||
"heading": section.get("heading", ""),
|
||||
"section_index": i,
|
||||
"wiki_links": ",".join(wiki_links) if wiki_links else "",
|
||||
"has_frontmatter": frontmatter is not None,
|
||||
},
|
||||
excluded_embed_metadata_keys=["wiki_links"],
|
||||
excluded_search_metadata_keys=["wiki_links"],
|
||||
)
|
||||
chunks.append(node)
|
||||
|
||||
return chunks
|
||||
|
||||
def chunk_directory(self, dir_path: str) -> List[TextNode]:
|
||||
"""Chunk all markdown files in a directory recursively.
|
||||
|
||||
Args:
|
||||
dir_path: Root directory containing markdown files
|
||||
|
||||
Returns:
|
||||
List of all TextNode chunks
|
||||
"""
|
||||
all_chunks = []
|
||||
dir_path = Path(dir_path)
|
||||
|
||||
if not dir_path.exists():
|
||||
raise FileNotFoundError(f"Directory not found: {dir_path}")
|
||||
|
||||
# Find all .md files
|
||||
md_files = list(dir_path.rglob("*.md"))
|
||||
|
||||
for md_file in md_files:
|
||||
try:
|
||||
content = md_file.read_text(encoding="utf-8")
|
||||
chunks = self.chunk_file(str(md_file), content)
|
||||
all_chunks.extend(chunks)
|
||||
except Exception as e:
|
||||
print(f"Error chunking {md_file}: {e}")
|
||||
continue
|
||||
|
||||
return all_chunks
|
||||
|
||||
def _extract_frontmatter(
|
||||
self, content: str
|
||||
) -> tuple[Optional[dict], str]:
|
||||
"""Extract YAML frontmatter from markdown."""
|
||||
if not content.startswith("---"):
|
||||
return None, content
|
||||
|
||||
# Find closing ---
|
||||
lines = content.split("\n")
|
||||
if len(lines) < 3:
|
||||
return None, content
|
||||
|
||||
frontmatter_lines = []
|
||||
body_start = 2
|
||||
|
||||
for i in range(1, len(lines)):
|
||||
if lines[i].strip() == "---":
|
||||
body_start = i + 1
|
||||
break
|
||||
frontmatter_lines.append(lines[i])
|
||||
|
||||
# Parse simple key-value frontmatter
|
||||
frontmatter = {}
|
||||
for line in frontmatter_lines:
|
||||
if ":" in line:
|
||||
key, value = line.split(":", 1)
|
||||
frontmatter[key.strip()] = value.strip()
|
||||
|
||||
body = "\n".join(lines[body_start:])
|
||||
return frontmatter, body
|
||||
|
||||
def _extract_wiki_links(self, content: str) -> List[str]:
|
||||
"""Extract [[wiki links]] from markdown content."""
|
||||
wiki_link_pattern = r"\[\[([^\]|]+)(?:\|[^\]]+)?\]]"
|
||||
return re.findall(wiki_link_pattern, content)
|
||||
|
||||
def _split_by_headings(self, content: str) -> List[dict]:
|
||||
"""Split content by markdown headings while preserving context."""
|
||||
# Split by heading lines (# ## ### etc)
|
||||
heading_pattern = r"^(#{1,6})\s+(.+)$"
|
||||
|
||||
sections = []
|
||||
current_section = {
|
||||
"heading": "",
|
||||
"content": "",
|
||||
}
|
||||
|
||||
lines = content.split("\n")
|
||||
for line in lines:
|
||||
match = re.match(heading_pattern, line)
|
||||
if match:
|
||||
# Save current section if non-empty
|
||||
if current_section["content"].strip():
|
||||
sections.append(current_section)
|
||||
|
||||
# Start new section
|
||||
level = len(match.group(1))
|
||||
heading_text = match.group(2).strip()
|
||||
current_section = {
|
||||
"heading": heading_text,
|
||||
"content": line + "\n",
|
||||
}
|
||||
else:
|
||||
current_section["content"] += line + "\n"
|
||||
|
||||
# Don't forget the last section
|
||||
if current_section["content"].strip():
|
||||
sections.append(current_section)
|
||||
|
||||
# If no headings found, treat entire content as one section
|
||||
if not sections:
|
||||
sections = [{"heading": "", "content": content}]
|
||||
|
||||
return sections
|
||||
75
src/knowledge_rag/embeddings.py
Normal file
75
src/knowledge_rag/embeddings.py
Normal file
@ -0,0 +1,75 @@
|
||||
"""Embedding model wrapper using sentence-transformers."""
|
||||
|
||||
import os
|
||||
from typing import List, Any
|
||||
|
||||
from llama_index.core.embeddings import BaseEmbedding
|
||||
from sentence_transformers import SentenceTransformer
|
||||
|
||||
|
||||
class LocalEmbeddingModel(BaseEmbedding):
|
||||
"""Local embedding model using sentence-transformers.
|
||||
|
||||
Uses a lightweight, high-quality model for semantic similarity.
|
||||
Default model: 'all-MiniLM-L6-v2' - fast and good quality.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str = "all-MiniLM-L6-v2",
|
||||
cache_folder: str | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
# Store model name before super init
|
||||
self._model_name = model_name
|
||||
|
||||
# Use persistent cache directory for Docker, or local cache for development
|
||||
if cache_folder is None:
|
||||
if os.path.exists("/data"):
|
||||
cache_folder = "/data/embeddings_cache"
|
||||
else:
|
||||
cache_folder = None
|
||||
|
||||
# Load model first
|
||||
model = SentenceTransformer(model_name, cache_folder=cache_folder)
|
||||
embed_dim = model.get_sentence_embedding_dimension()
|
||||
|
||||
# Initialize pydantic model with required fields
|
||||
super().__init__(
|
||||
embed_dim=embed_dim,
|
||||
model_name=model_name,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Now set the model after pydantic init
|
||||
object.__setattr__(self, '_model', model)
|
||||
|
||||
def _get_text_embedding(self, text: str) -> List[float]:
|
||||
"""Get embedding for a single text."""
|
||||
return self._model.encode(text, convert_to_numpy=True).tolist()
|
||||
|
||||
async def _aget_text_embedding(self, text: str) -> List[float]:
|
||||
"""Async get embedding - synchronous for local model."""
|
||||
return self._get_text_embedding(text)
|
||||
|
||||
def _get_query_embedding(self, query: str) -> List[float]:
|
||||
"""Get embedding for a query."""
|
||||
return self._model.encode(query, convert_to_numpy=True).tolist()
|
||||
|
||||
async def _aget_query_embedding(self, query: str) -> List[float]:
|
||||
"""Async get query embedding - synchronous for local model."""
|
||||
return self._get_query_embedding(query)
|
||||
|
||||
def _get_text_embeddings(self, texts: List[str]) -> List[List[float]]:
|
||||
"""Get embeddings for multiple texts."""
|
||||
return self._model.encode(texts, convert_to_numpy=True).tolist()
|
||||
|
||||
async def _aget_text_embeddings(self, texts: List[str]) -> List[List[float]]:
|
||||
"""Async get embeddings - synchronous for local model."""
|
||||
return self._get_text_embeddings(texts)
|
||||
|
||||
|
||||
def get_embedding_model() -> LocalEmbeddingModel:
|
||||
"""Factory function to create the embedding model."""
|
||||
model_name = os.environ.get("EMBEDDING_MODEL", "all-MiniLM-L6-v2")
|
||||
return LocalEmbeddingModel(model_name=model_name)
|
||||
282
src/knowledge_rag/server.py
Normal file
282
src/knowledge_rag/server.py
Normal file
@ -0,0 +1,282 @@
|
||||
"""MCP server for knowledge base RAG system."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from mcp.server import Server
|
||||
from mcp.server.stdio import stdio_server
|
||||
from mcp.types import Tool, TextContent
|
||||
from pydantic import AnyUrl
|
||||
|
||||
from .chunker import MarkdownChunker
|
||||
from .embeddings import get_embedding_model
|
||||
from .vector_store import KnowledgeVectorStore
|
||||
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KnowledgeMCPServer:
|
||||
"""MCP server for semantic search in Obsidian vault.
|
||||
|
||||
Provides tools to:
|
||||
- Search the knowledge base semantically
|
||||
- Index/update the knowledge base
|
||||
- Get statistics about indexed content
|
||||
"""
|
||||
|
||||
def __init__(self, vault_path: str | None = None):
|
||||
# Get vault path from environment or use default
|
||||
self.vault_path = vault_path or os.environ.get(
|
||||
"VAULT_PATH", "/data/vault"
|
||||
)
|
||||
|
||||
# Ensure vault path exists
|
||||
if not Path(self.vault_path).exists():
|
||||
logger.warning(f"Vault path does not exist: {self.vault_path}")
|
||||
|
||||
# Initialize components
|
||||
self.embedding_model = get_embedding_model()
|
||||
self.vector_store = KnowledgeVectorStore(
|
||||
embedding_model=self.embedding_model
|
||||
)
|
||||
self.chunker = MarkdownChunker()
|
||||
|
||||
# Track indexing status
|
||||
self._indexed = False
|
||||
|
||||
# Create MCP server
|
||||
self.server = Server("knowledge-rag")
|
||||
|
||||
# Register handlers
|
||||
self._register_handlers()
|
||||
|
||||
def _register_handlers(self):
|
||||
"""Register MCP request handlers."""
|
||||
|
||||
@self.server.list_tools()
|
||||
async def list_tools() -> list[Tool]:
|
||||
"""List available MCP tools."""
|
||||
return [
|
||||
Tool(
|
||||
name="search_knowledge",
|
||||
description="Semantic search through the knowledge base. "
|
||||
"Uses embeddings to find relevant content based on meaning, "
|
||||
"not just keywords. Best for answering questions or finding "
|
||||
"related concepts.",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "The search query in natural language",
|
||||
},
|
||||
"top_k": {
|
||||
"type": "integer",
|
||||
"description": "Number of results to return",
|
||||
"default": 5,
|
||||
},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="index_knowledge",
|
||||
description="Index or re-index the knowledge base. "
|
||||
"Run this after adding new files to the vault. "
|
||||
"Scans all markdown files and builds the search index.",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"force": {
|
||||
"type": "boolean",
|
||||
"description": "Force re-index (clear existing index first)",
|
||||
"default": False,
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="get_knowledge_stats",
|
||||
description="Get statistics about the indexed knowledge base.",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
@self.server.call_tool()
|
||||
async def call_tool(
|
||||
name: str, arguments: dict | None
|
||||
) -> list[TextContent]:
|
||||
"""Handle tool calls."""
|
||||
if name == "search_knowledge":
|
||||
return await self._search_knowledge(arguments or {})
|
||||
elif name == "index_knowledge":
|
||||
return await self._index_knowledge(arguments or {})
|
||||
elif name == "get_knowledge_stats":
|
||||
return await self._get_stats()
|
||||
else:
|
||||
raise ValueError(f"Unknown tool: {name}")
|
||||
|
||||
async def _search_knowledge(
|
||||
self, arguments: dict[str, Any]
|
||||
) -> list[TextContent]:
|
||||
"""Search the knowledge base semantically."""
|
||||
query = arguments.get("query", "")
|
||||
top_k = arguments.get("top_k", 5)
|
||||
|
||||
if not query:
|
||||
return [TextContent(type="text", text="Query cannot be empty.")]
|
||||
|
||||
# Ensure we've indexed
|
||||
if not self._indexed:
|
||||
await self._index_knowledge({})
|
||||
|
||||
try:
|
||||
# Search with embeddings
|
||||
results = self.vector_store.search(
|
||||
query=query,
|
||||
top_k=top_k,
|
||||
)
|
||||
|
||||
if not results:
|
||||
return [
|
||||
TextContent(
|
||||
type="text",
|
||||
text="No results found. Try indexing your knowledge base first."
|
||||
)
|
||||
]
|
||||
|
||||
# Format results
|
||||
output = []
|
||||
for i, result in enumerate(results, 1):
|
||||
source = result["metadata"].get("file_name", "unknown")
|
||||
heading = result["metadata"].get("heading", "")
|
||||
score = result.get("score", 0)
|
||||
|
||||
text = result["text"][:500] # Truncate long text
|
||||
if len(result["text"]) > 500:
|
||||
text += "..."
|
||||
|
||||
output.append(
|
||||
f"--- Result {i} ---\n"
|
||||
f"Source: {source}"
|
||||
+ (f" > {heading}" if heading else "")
|
||||
+ f"\nRelevance: {score:.2f}\n\n{text}\n"
|
||||
)
|
||||
|
||||
return [TextContent(type="text", text="\n".join(output))]
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Search error")
|
||||
return [TextContent(type="text", text=f"Search error: {str(e)}")]
|
||||
|
||||
async def _index_knowledge(
|
||||
self, arguments: dict[str, Any]
|
||||
) -> list[TextContent]:
|
||||
"""Index the knowledge base."""
|
||||
force = arguments.get("force", False)
|
||||
|
||||
vault_path = Path(self.vault_path)
|
||||
|
||||
if not vault_path.exists():
|
||||
return [
|
||||
TextContent(
|
||||
type="text",
|
||||
text=f"Vault path does not exist: {self.vault_path}"
|
||||
)
|
||||
]
|
||||
|
||||
try:
|
||||
# Clear existing index if forced
|
||||
if force:
|
||||
logger.info("Force re-indexing...")
|
||||
self.vector_store.clear()
|
||||
else:
|
||||
logger.info("Indexing knowledge base...")
|
||||
|
||||
# Chunk all markdown files
|
||||
chunks = self.chunker.chunk_directory(str(vault_path))
|
||||
|
||||
if not chunks:
|
||||
return [
|
||||
TextContent(
|
||||
type="text",
|
||||
text="No markdown files found in vault."
|
||||
)
|
||||
]
|
||||
|
||||
logger.info(f"Created {len(chunks)} chunks, adding to vector store...")
|
||||
|
||||
# Add to vector store (this embeds them)
|
||||
self.vector_store.add_nodes(chunks, embedding_model=self.embedding_model)
|
||||
|
||||
self._indexed = True
|
||||
|
||||
stats = self.vector_store.get_stats()
|
||||
return [
|
||||
TextContent(
|
||||
type="text",
|
||||
text=f"Successfully indexed {len(chunks)} chunks from the knowledge base.\n"
|
||||
f"Total chunks in index: {stats['total_chunks']}"
|
||||
)
|
||||
]
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Indexing error")
|
||||
return [TextContent(type="text", text=f"Indexing error: {str(e)}")]
|
||||
|
||||
async def _get_stats(self) -> list[TextContent]:
|
||||
"""Get knowledge base statistics."""
|
||||
stats = self.vector_store.get_stats()
|
||||
|
||||
vault_path = Path(self.vault_path)
|
||||
md_files = list(vault_path.rglob("*.md")) if vault_path.exists() else []
|
||||
|
||||
return [
|
||||
TextContent(
|
||||
type="text",
|
||||
text=f"Knowledge Base Statistics:\n"
|
||||
f"- Vault path: {self.vault_path}\n"
|
||||
f"- Markdown files: {len(md_files)}\n"
|
||||
f"- Indexed chunks: {stats['total_chunks']}\n"
|
||||
f"- Index status: {'Ready' if self._indexed else 'Not indexed'}"
|
||||
)
|
||||
]
|
||||
|
||||
async def run(self):
|
||||
"""Run the MCP server."""
|
||||
logger.info(f"Starting Knowledge RAG MCP Server")
|
||||
logger.info(f"Vault path: {self.vault_path}")
|
||||
|
||||
# Auto-index on startup
|
||||
await self._index_knowledge({})
|
||||
|
||||
# Run stdio server
|
||||
async with stdio_server() as (read_stream, write_stream):
|
||||
await self.server.run(
|
||||
read_stream,
|
||||
write_stream,
|
||||
self.server.create_initialization_options(),
|
||||
)
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main entry point."""
|
||||
server = KnowledgeMCPServer()
|
||||
await server.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import asyncio
|
||||
asyncio.run(main())
|
||||
137
src/knowledge_rag/vector_store.py
Normal file
137
src/knowledge_rag/vector_store.py
Normal file
@ -0,0 +1,137 @@
|
||||
"""ChromaDB vector store wrapper for knowledge base."""
|
||||
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Any, List, Optional
|
||||
|
||||
from llama_index.core.schema import TextNode
|
||||
from llama_index.vector_stores.chroma import ChromaVectorStore
|
||||
import chromadb
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from llama_index.core.embeddings import BaseEmbedding
|
||||
|
||||
|
||||
class KnowledgeVectorStore:
|
||||
"""ChromaDB vector store for the knowledge base.
|
||||
|
||||
Handles persistence of embeddings and semantic search.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
persist_dir: str | None = None,
|
||||
collection_name: str = "knowledge_base",
|
||||
embedding_model: "BaseEmbedding | None" = None,
|
||||
):
|
||||
self._collection_name = collection_name
|
||||
self._embedding_model = embedding_model
|
||||
|
||||
# Use Docker path if available, otherwise use local data dir
|
||||
if persist_dir is None:
|
||||
if os.path.exists("/data"):
|
||||
persist_dir = "/data/chroma_db"
|
||||
else:
|
||||
persist_dir = "./data/chroma_db"
|
||||
|
||||
self._persist_dir = persist_dir
|
||||
|
||||
# Ensure persist directory exists
|
||||
os.makedirs(persist_dir, exist_ok=True)
|
||||
|
||||
# Initialize ChromaDB client
|
||||
self._client = chromadb.PersistentClient(path=persist_dir)
|
||||
|
||||
# Get or create collection
|
||||
self._collection = self._client.get_or_create_collection(
|
||||
name=collection_name,
|
||||
metadata={"description": "Knowledge base embeddings"}
|
||||
)
|
||||
|
||||
# Wrap in LlamaIndex vector store
|
||||
# Pass the chroma_collection directly for PersistentClient
|
||||
self._vector_store = ChromaVectorStore(
|
||||
chroma_collection=self._collection,
|
||||
)
|
||||
|
||||
def set_embedding_model(self, embedding_model: "BaseEmbedding") -> None:
|
||||
"""Set the embedding model for query embedding."""
|
||||
self._embedding_model = embedding_model
|
||||
|
||||
@property
|
||||
def vector_store(self) -> ChromaVectorStore:
|
||||
"""Get the LlamaIndex ChromaVectorStore."""
|
||||
return self._vector_store
|
||||
|
||||
def add_nodes(self, nodes: List[TextNode], embedding_model: "BaseEmbedding | None" = None) -> None:
|
||||
"""Add nodes to the vector store."""
|
||||
from llama_index.core import VectorStoreIndex, StorageContext
|
||||
|
||||
# Use provided embedding model or the stored one
|
||||
model = embedding_model or self._embedding_model
|
||||
|
||||
if model is None:
|
||||
raise ValueError("No embedding model provided")
|
||||
|
||||
# First embed the nodes
|
||||
for node in nodes:
|
||||
node.embedding = model.get_text_embedding(node.text)
|
||||
|
||||
# Then add to vector store
|
||||
self._vector_store.add(nodes)
|
||||
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
top_k: int = 5,
|
||||
filter: Optional[dict[str, Any]] = None,
|
||||
) -> List[dict[str, Any]]:
|
||||
"""Semantic search for similar chunks.
|
||||
|
||||
Args:
|
||||
query: The search query
|
||||
top_k: Number of results to return
|
||||
filter: Optional metadata filters
|
||||
|
||||
Returns:
|
||||
List of search results with text and metadata
|
||||
"""
|
||||
from llama_index.core import VectorStoreIndex
|
||||
|
||||
# Use embedding model if provided, otherwise use the one from storage
|
||||
embed_model = self._embedding_model
|
||||
|
||||
index = VectorStoreIndex.from_vector_store(
|
||||
self._vector_store,
|
||||
embed_model=embed_model,
|
||||
)
|
||||
|
||||
query_engine = index.as_retriever(
|
||||
similarity_top_k=top_k,
|
||||
filters=filter,
|
||||
)
|
||||
|
||||
results = query_engine.retrieve(query)
|
||||
|
||||
return [
|
||||
{
|
||||
"text": node.text,
|
||||
"score": node.score,
|
||||
"metadata": node.metadata,
|
||||
}
|
||||
for node in results
|
||||
]
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear all embeddings from the store."""
|
||||
self._client.delete_collection(self._collection_name)
|
||||
self._collection = self._client.get_or_create_collection(
|
||||
name=self._collection_name,
|
||||
metadata={"description": "Knowledge base embeddings"}
|
||||
)
|
||||
|
||||
def get_stats(self) -> dict[str, Any]:
|
||||
"""Get vector store statistics."""
|
||||
return {
|
||||
"total_chunks": self._collection.count(),
|
||||
"collection_name": self._collection_name,
|
||||
}
|
||||
Reference in New Issue
Block a user