from ragas import faithfulness
from ragas.metrics import Faithfulness

metric = Faithfulness()

score = metric.score({
    "question": "What is RAG?",
    "answer": "RAG is Retrieval Augmented Generation, a technique...",
    "contexts": ["RAG combines retrieval with generation..."]
})

print(f"Faithfulness: {score}")  # 0.0-1.0

# Good faithfulness (0.95)
question = "What is the capital of France?"
answer = "The capital of France is Paris."
contexts = ["Paris is the capital and largest city of France."]

# Bad faithfulness (0.3)
question = "What is the capital of France?"
answer = "The capital of France is London and it has 10 million people."
contexts = ["Paris is the capital and largest city of France."]
# Hallucination: London (wrong), 10 million (not in context)

from ragas.metrics import ContextRelevance

metric = ContextRelevance()

score = metric.score({
    "question": "How does vector search work?",
    "contexts": [
        "Vector search uses embeddings to find similar documents.",
        "Embeddings are numerical representations of text."
    ]
})

print(f"Context Relevance: {score}")  # 0.0-1.0

# Good relevance (0.92)
question = "How do I make a cake?"
contexts = [
    "Mix flour, eggs, and sugar. Bake at 350°F for 30 minutes.",
    "Cake baking requires preheating the oven first."
]

# Bad relevance (0.2)
question = "How do I make a cake?"
contexts = [
    "The history of bread dates back to ancient Egypt.",
    "Different types of pasta include spaghetti and penne."
]

from ragas.metrics import AnswerRelevance

metric = AnswerRelevance()

score = metric.score({
    "question": "What is machine learning?",
    "answer": "Machine learning is a subset of AI that enables systems to learn from data."
})

print(f"Answer Relevance: {score}")  # 0.0-1.0

pip install ragas

from ragas import evaluate
from ragas.metrics import (
    faithfulness,
    answer_relevance,
    context_relevance,
    context_recall,
    context_precision
)

# Prepare dataset
dataset = {
    "question": [
        "What is RAG?",
        "How does retrieval work?"
    ],
    "answer": [
        "RAG is Retrieval Augmented Generation...",
        "Retrieval uses vector embeddings..."
    ],
    "contexts": [
        ["RAG combines retrieval with generation..."],
        ["Vector embeddings enable semantic search..."]
    ],
    "ground_truths": [
        ["RAG is a technique that..."],
        ["Retrieval finds relevant documents..."]
    ]
}

# Evaluate
results = evaluate(
    dataset,
    metrics=[
        faithfulness,
        answer_relevance,
        context_relevance,
        context_recall,
        context_precision
    ]
)

print(results)

{
    'faithfulness': 0.85,
    'answer_relevance': 0.92,
    'context_relevance': 0.88,
    'context_recall': 0.90,
    'context_precision': 0.87
}

from langchain_openai import ChatOpenAI

def calculate_faithfulness(answer: str, contexts: list[str]) -> float:
    """
    Check if answer claims are supported by context.
    """
    llm = ChatOpenAI(model="gpt-4", temperature=0)

    prompt = f"""
    Given the following context and answer, rate how faithful the answer is to the context.
    Return a score from 0.0 to 1.0.

    Context:
    {' '.join(contexts)}

    Answer:
    {answer}

    Faithfulness score (0.0-1.0):
    """

    response = llm.invoke(prompt)
    score = float(response.content.strip())

    return score

from langchain_openai import ChatOpenAI

def calculate_relevance(question: str, contexts: list[str]) -> float:
    """
    Check if retrieved contexts are relevant to question.
    """
    llm = ChatOpenAI(model="gpt-4", temperature=0)

    prompt = f"""
    Given the following question and retrieved documents, rate how relevant the documents are.
    Return a score from 0.0 to 1.0.

    Question:
    {question}

    Retrieved Documents:
    {' | '.join(contexts)}

    Relevance score (0.0-1.0):
    """

    response = llm.invoke(prompt)
    score = float(response.content.strip())

    return score

from langsmith import Client
from ragas.metrics import Faithfulness, ContextRelevance

client = Client()
faithfulness_metric = Faithfulness()
relevance_metric = ContextRelevance()

# After RAG query
result = rag_chain.invoke(query)

# Calculate metrics
faithfulness_score = faithfulness_metric.score({
    "question": query,
    "answer": result["answer"],
    "contexts": [doc.page_content for doc in result["source_documents"]]
})

relevance_score = relevance_metric.score({
    "question": query,
    "contexts": [doc.page_content for doc in result["source_documents"]]
})

# Send to LangSmith
client.create_feedback(
    run_id=run_id,
    key="faithfulness",
    score=faithfulness_score
)

client.create_feedback(
    run_id=run_id,
    key="context_relevance",
    score=relevance_score
)

from ragas import evaluate

# Strategy A: Semantic only
results_a = evaluate(
    dataset_a,
    metrics=[faithfulness, context_relevance]
)

# Strategy B: Hybrid (BM25 + Semantic)
results_b = evaluate(
    dataset_b,
    metrics=[faithfulness, context_relevance]
)

# Compare
print(f"Semantic only: {results_a}")
print(f"Hybrid search: {results_b}")

# Decision: Choose strategy with higher context_relevance

# Template A: Simple
prompt_a = "Answer: {question}\nContext: {context}"

# Template B: Few-shot
prompt_b = """
Examples:
Q: What is X?
A: X is...

Now answer:
Q: {question}
Context: {context}
"""

# Evaluate both
results_a = evaluate(dataset_a, metrics=[faithfulness, answer_relevance])
results_b = evaluate(dataset_b, metrics=[faithfulness, answer_relevance])

# Decision: Choose template with higher faithfulness

from langsmith import Client
from datetime import datetime, timedelta

client = Client()

# Get today's runs
runs = client.list_runs(
    project_name="rag-production",
    start_time=datetime.now() - timedelta(days=1)
)

# Calculate avg metrics
faithfulness_scores = []
relevance_scores = []

for run in runs:
    feedbacks = client.list_feedback(run_id=run.id)
    for fb in feedbacks:
        if fb.key == "faithfulness":
            faithfulness_scores.append(fb.score)
        elif fb.key == "context_relevance":
            relevance_scores.append(fb.score)

avg_faithfulness = sum(faithfulness_scores) / len(faithfulness_scores)
avg_relevance = sum(relevance_scores) / len(relevance_scores)

print(f"Avg Faithfulness: {avg_faithfulness:.2f}")
print(f"Avg Relevance: {avg_relevance:.2f}")

# Alert if below threshold
if avg_faithfulness < 0.7:
    print("WARNING: Faithfulness below threshold!")
if avg_relevance < 0.7:
    print("WARNING: Relevance below threshold!")

# Bad: Only simple questions
dataset = [
    "What is X?",
    "What is Y?"
]

# Good: Mix of simple, complex, edge cases
dataset = [
    "What is X?",                          # Simple
    "Compare X and Y",                     # Complex
    "What is the capital of Atlantis?",    # No answer in knowledge base
    ""                                     # Empty query
]

import pandas as pd
from datetime import datetime

# Log daily metrics
metrics_log = []

def log_metrics(date, faithfulness, relevance):
    metrics_log.append({
        "date": date,
        "faithfulness": faithfulness,
        "relevance": relevance
    })

# Visualize trends
df = pd.DataFrame(metrics_log)
df.plot(x="date", y=["faithfulness", "relevance"])

# Include expected answers in dataset
dataset = {
    "question": ["What is RAG?"],
    "ground_truths": [["RAG is Retrieval Augmented Generation..."]],
    # ... rest of dataset
}

# Evaluate with context_recall (how much of ground truth is retrieved)
from ragas.metrics import context_recall

results = evaluate(dataset, metrics=[context_recall])

# Reduce temperature
llm = ChatOpenAI(model="gpt-4", temperature=0)  # 0 = deterministic

# Add "stick to context" instruction
prompt = """
Use ONLY the provided context to answer. Do not use external knowledge.
If the answer is not in the context, say "I don't know based on the provided context."

Context: {context}
Question: {question}
"""

# Use hybrid search (BM25 + Semantic)
from langchain.retrievers import EnsembleRetriever

retriever = EnsembleRetriever(
    retrievers=[bm25_retriever, semantic_retriever],
    weights=[0.5, 0.5]
)

# Or add reranking
from langchain.retrievers import ContextualCompressionRetriever
from langchain.retrievers.document_compressors import CrossEncoderReranker

compressor = CrossEncoderReranker()
retriever = ContextualCompressionRetriever(
    base_compressor=compressor,
    base_retriever=base_retriever
)

RAG Query
   ↓
LangSmith Trace (langsmith-testing-SKILL.md)
   ↓
Get Result
   ↓
Calculate Metrics (THIS SKILL)
   ↓
Send Feedback to LangSmith
   ↓
Monitor Quality Thresholds