{
    "question": "用户问题",
    "answer": "RAG系统的回答",
    "contexts": ["检索到的文档1", "检索到的文档2", ...],
    "ground_truth": "标准答案（Context Recall需要）"
}

from ragas.testset import TestsetGenerator
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from langchain_community.document_loaders import DirectoryLoader

# 加载你的知识库文档
loader = DirectoryLoader("./docs", glob="**/*.txt")
documents = loader.load()

# 自动生成测试集（RAGAS v0.2+ API）
generator = TestsetGenerator.from_langchain(
    generator_llm=ChatOpenAI(model="gpt-4o-mini"),
    critic_llm=ChatOpenAI(model="gpt-4o-mini"),
    embeddings=OpenAIEmbeddings()
)
testset = generator.generate_with_langchain_docs(
    documents,
    test_size=50
)

# evaluate_rag.py
# 以下代码适用于 RAGAS v0.2+
# v0.1 的 evaluate() 接口有所不同，请参考对应版本文档
import os
from datasets import Dataset
from ragas import evaluate
from ragas.metrics import (
    faithfulness,
    answer_relevancy,
    context_precision,
    context_recall
)
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from langchain_community.vectorstores import Chroma
from langchain.chains import RetrievalQA

os.environ["OPENAI_API_KEY"] = "sk-your-key"

# ——— 初始化RAG系统 ———
embeddings = OpenAIEmbeddings()
vectorstore = Chroma(
    persist_directory="./chroma_db",
    embedding_function=embeddings
)
llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
rag_chain = RetrievalQA.from_chain_type(
    llm=llm,
    retriever=vectorstore.as_retriever(search_kwargs={"k": 3}),
    return_source_documents=True
)

# ——— 准备评估数据集 ———
# 实际使用时，这里应该是你标注好的测试集
test_questions = [
    "公司的年假政策是什么？",
    "如何申请请假？",
    "试用期是多少天？",
    "加班费怎么计算？",
    "员工福利有哪些？"
]

ground_truths = [
    "入职满一年后每年10天年假，满三年15天，满五年20天。",
    "需要提前三天在OA系统提交申请，经直属上级审批。",
    "试用期为三个月，双方协商一致可以缩短。",
    "工作日加班按1.5倍，休息日加班按2倍，法定节假日按3倍。",
    "包括五险一金、补充医疗保险、年度体检、餐补、交通补贴。"
]

# ——— 运行RAG并收集结果 ———
eval_data = {
    "question": [],
    "answer": [],
    "contexts": [],
    "ground_truth": []
}

for question, ground_truth in zip(test_questions, ground_truths):
    result = rag_chain.invoke({"query": question})

    eval_data["question"].append(question)
    eval_data["answer"].append(result["result"])
    eval_data["contexts"].append([
        doc.page_content for doc in result["source_documents"]
    ])
    eval_data["ground_truth"].append(ground_truth)
    print(f"已评估：{question}")

# ——— 转成Dataset格式 ———
dataset = Dataset.from_dict(eval_data)

# ——— 运行RAGAS评估 ———
result = evaluate(
    dataset,
    metrics=[
        faithfulness,
        answer_relevancy,
        context_precision,
        context_recall
    ],
    llm=ChatOpenAI(model="gpt-4o-mini"),
    embeddings=OpenAIEmbeddings()
)

# ——— 输出结果 ———
print("\n===== RAGAS评估结果 =====")
print(f"Faithfulness（忠实度）:      {result['faithfulness']:.3f}")
print(f"Answer Relevancy（答案相关性）: {result['answer_relevancy']:.3f}")
print(f"Context Precision（上下文精确率）: {result['context_precision']:.3f}")
print(f"Context Recall（上下文召回率）: {result['context_recall']:.3f}")

# 保存详细结果
df = result.to_pandas()
df.to_csv("rag_evaluation_results.csv", index=False, encoding="utf-8")
print("\n详细结果已保存到 rag_evaluation_results.csv")

pip install ragas langchain langchain-openai langchain-community chromadb datasets
python evaluate_rag.py

===== RAGAS评估结果 =====
Faithfulness（忠实度）:       0.823
Answer Relevancy（答案相关性）: 0.756
Context Precision（上下文精确率）: 0.667
Context Recall（上下文召回率）:  0.714