from langchain.text_splitter import RecursiveCharacterTextSplitter

splitter = RecursiveCharacterTextSplitter(
    chunk_size=512,          # PROTECTED
    chunk_overlap=50,        # PROTECTED
    separators=["\n\n", "\n", " ", ""],
    length_function=len
)

chunks = splitter.split_text(document)

from langchain.text_splitter import TokenTextSplitter

splitter = TokenTextSplitter(
    chunk_size=512,
    chunk_overlap=50
)

from langchain.text_splitter import MarkdownTextSplitter

splitter = MarkdownTextSplitter(
    chunk_size=512,
    chunk_overlap=50
)

Without overlap (BAD):
Chunk 1: "...end of sentence A."
Chunk 2: "Start of sentence B..."
→ Context lost between chunks

With overlap (GOOD):
Chunk 1: "...end of sentence A. Start of sentence B..."
Chunk 2: "...end of sentence A. Start of sentence B. More context..."
→ Continuity preserved

from ragas.metrics import ContextRelevance

# Test different chunk sizes
chunk_sizes = [256, 512, 1024]
results = {}

for size in chunk_sizes:
    splitter = RecursiveCharacterTextSplitter(chunk_size=size)
    # Re-index with new chunk size
    # Run evaluation
    score = evaluate_retrieval(splitter)
    results[size] = score

# Choose best
best_size = max(results, key=results.get)
print(f"Best chunk size: {best_size}")

import re

def clean_text(text: str) -> str:
    # Remove extra whitespace
    text = re.sub(r'\s+', ' ', text)

    # Remove special characters
    text = re.sub(r'[^\w\s.,!?-]', '', text)

    # Normalize line breaks
    text = text.replace('\r\n', '\n')

    return text.strip()

def remove_boilerplate(text: str) -> str:
    # Remove headers/footers
    text = re.sub(r'Page \d+ of \d+', '', text)

    # Remove navigation
    text = re.sub(r'Home \| About \| Contact', '', text)

    return text