feat: Update Ragas integration

integrations/ragas/example/evaluation_from_pipeline_example.py

@@ -0,0 +1,140 @@
+# A valid OpenAI API key must be provided as an environment variable "OPENAI_API_KEY" to run this example.
+
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+from haystack import Document
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.components.embedders import OpenAITextEmbedder, OpenAIDocumentEmbedder
+from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever
+from haystack.components.builders import ChatPromptBuilder
+from haystack.dataclasses import ChatMessage
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.builders import AnswerBuilder
+from haystack import Pipeline
+from haystack_integrations.components.evaluators.ragas import RagasEvaluator
+
+from langchain_openai import ChatOpenAI
+from ragas.llms import LangchainLLMWrapper
+from ragas.metrics import AnswerRelevancy, ContextPrecision, Faithfulness
+from ragas import evaluate
+from ragas.dataset_schema import EvaluationDataset
+
+
+dataset = [
+    "OpenAI is one of the most recognized names in the large language model space, known for its GPT series of models. These models excel at generating human-like text and performing tasks like creative writing, answering questions, and summarizing content. GPT-4, their latest release, has set benchmarks in understanding context and delivering detailed responses.",
+    "Anthropic is well-known for its Claude series of language models, designed with a strong focus on safety and ethical AI behavior. Claude is particularly praised for its ability to follow complex instructions and generate text that aligns closely with user intent.",
+    "DeepMind, a division of Google, is recognized for its cutting-edge Gemini models, which are integrated into various Google products like Bard and Workspace tools. These models are renowned for their conversational abilities and their capacity to handle complex, multi-turn dialogues.",
+    "Meta AI is best known for its LLaMA (Large Language Model Meta AI) series, which has been made open-source for researchers and developers. LLaMA models are praised for their ability to support innovation and experimentation due to their accessibility and strong performance.",
+    "Meta AI with it's LLaMA models aims to democratize AI development by making high-quality models available for free, fostering collaboration across industries. Their open-source approach has been a game-changer for researchers without access to expensive resources.",
+    "Microsoft’s Azure AI platform is famous for integrating OpenAI’s GPT models, enabling businesses to use these advanced models in a scalable and secure cloud environment. Azure AI powers applications like Copilot in Office 365, helping users draft emails, generate summaries, and more.",
+    "Amazon’s Bedrock platform is recognized for providing access to various language models, including its own models and third-party ones like Anthropic’s Claude and AI21’s Jurassic. Bedrock is especially valued for its flexibility, allowing users to choose models based on their specific needs.",
+    "Cohere is well-known for its language models tailored for business use, excelling in tasks like search, summarization, and customer support. Their models are recognized for being efficient, cost-effective, and easy to integrate into workflows.",
+    "AI21 Labs is famous for its Jurassic series of language models, which are highly versatile and capable of handling tasks like content creation and code generation. The Jurassic models stand out for their natural language understanding and ability to generate detailed and coherent responses.",
+    "In the rapidly advancing field of artificial intelligence, several companies have made significant contributions with their large language models. Notable players include OpenAI, known for its GPT Series (including GPT-4); Anthropic, which offers the Claude Series; Google DeepMind with its Gemini Models; Meta AI, recognized for its LLaMA Series; Microsoft Azure AI, which integrates OpenAI’s GPT Models; Amazon AWS (Bedrock), providing access to various models including Claude (Anthropic) and Jurassic (AI21 Labs); Cohere, which offers its own models tailored for business use; and AI21 Labs, known for its Jurassic Series. These companies are shaping the landscape of AI by providing powerful models with diverse capabilities.",
+]
+
+# Initialize components for RAG pipeline
+document_store = InMemoryDocumentStore()
+docs = [Document(content=doc) for doc in dataset]
+
+document_embedder = OpenAIDocumentEmbedder(model="text-embedding-3-small")
+text_embedder = OpenAITextEmbedder(model="text-embedding-3-small")
+
+docs_with_embeddings = document_embedder.run(docs)
+document_store.write_documents(docs_with_embeddings["documents"])
+
+retriever = InMemoryEmbeddingRetriever(document_store, top_k=2)
+
+template = [
+    ChatMessage.from_user(
+        """
+Given the following information, answer the question.
+
+Context:
+{% for document in documents %}
+    {{ document.content }}
+{% endfor %}
+
+Question: {{question}}
+Answer:
+"""
+    )
+]
+
+prompt_builder = ChatPromptBuilder(template=template)
+chat_generator = OpenAIChatGenerator(model="gpt-4o-mini")
+
+# Creating the Pipeline
+rag_pipeline = Pipeline()
+
+# Adding the components
+rag_pipeline.add_component("text_embedder", text_embedder)
+rag_pipeline.add_component("retriever", retriever)
+rag_pipeline.add_component("prompt_builder", prompt_builder)
+rag_pipeline.add_component("llm", chat_generator)
+rag_pipeline.add_component("answer_builder", AnswerBuilder())
+
+# Connecting the components
+rag_pipeline.connect("text_embedder.embedding", "retriever.query_embedding")
+rag_pipeline.connect("retriever", "prompt_builder")
+rag_pipeline.connect("prompt_builder.prompt", "llm.messages")
+rag_pipeline.connect("llm.replies", "answer_builder.replies")
+rag_pipeline.connect("retriever", "answer_builder.documents")
+rag_pipeline.connect("llm.replies", "answer_builder.replies")
+rag_pipeline.connect("retriever", "answer_builder.documents")
+
+
+questions = [
+    "Who are the major players in the large language model space?",
+    "What is Microsoft’s Azure AI platform known for?",
+    "What kind of models does Cohere provide?",
+]
+
+references = [
+    "The major players include OpenAI (GPT Series), Anthropic (Claude Series), Google DeepMind (Gemini Models), Meta AI (LLaMA Series), Microsoft Azure AI (integrating GPT Models), Amazon AWS (Bedrock with Claude and Jurassic), Cohere (business-focused models), and AI21 Labs (Jurassic Series).",
+    "Microsoft’s Azure AI platform is known for integrating OpenAI’s GPT models, enabling businesses to use these models in a scalable and secure cloud environment.",
+    "Cohere provides language models tailored for business use, excelling in tasks like search, summarization, and customer support.",
+]
+
+
+evals_list = []
+
+for que_idx in range(len(questions)):
+
+    single_turn = {}
+    single_turn['user_input'] = questions[que_idx]
+    single_turn['reference'] = references[que_idx]
+
+    # Running the pipeline
+    response = rag_pipeline.run(
+        {
+            "text_embedder": {"text": questions[que_idx]},
+            "prompt_builder": {"question": questions[que_idx]},
+            "answer_builder": {"query": questions[que_idx]},
+        }
+    )
+
+    single_turn['response'] = response["answer_builder"]["answers"][0].data
+
+    haystack_documents = response["answer_builder"]["answers"][0].documents
+    # extracting context from haystack documents
+    single_turn['retrieved_contexts'] = [doc.content for doc in haystack_documents]
+
+    evals_list.append(single_turn)
+
+evaluation_dataset = EvaluationDataset.from_list(evals_list)
+
+llm = ChatOpenAI(model="gpt-4o-mini")
+evaluator_llm = LangchainLLMWrapper(llm)
+
+result = evaluate(
+    dataset=evaluation_dataset,
+    metrics=[AnswerRelevancy(), ContextPrecision(), Faithfulness()],
+    llm=evaluator_llm,
+)
+
+print(result)
+
+result.to_pandas()

integrations/ragas/example/evaluation_with_components_example.py

@@ -0,0 +1,115 @@
+# A valid OpenAI API key must be provided as an environment variable "OPENAI_API_KEY" to run this example.
+
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+from haystack import Document
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.components.embedders import OpenAITextEmbedder, OpenAIDocumentEmbedder
+from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever
+from haystack.components.builders import ChatPromptBuilder
+from haystack.dataclasses import ChatMessage
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.builders import AnswerBuilder
+from haystack import Pipeline
+from haystack_integrations.components.evaluators.ragas import RagasEvaluator
+
+from langchain_openai import ChatOpenAI
+from ragas.llms import LangchainLLMWrapper
+from ragas.metrics import AnswerRelevancy, ContextPrecision, Faithfulness
+
+
+dataset = [
+    "OpenAI is one of the most recognized names in the large language model space, known for its GPT series of models. These models excel at generating human-like text and performing tasks like creative writing, answering questions, and summarizing content. GPT-4, their latest release, has set benchmarks in understanding context and delivering detailed responses.",
+    "Anthropic is well-known for its Claude series of language models, designed with a strong focus on safety and ethical AI behavior. Claude is particularly praised for its ability to follow complex instructions and generate text that aligns closely with user intent.",
+    "DeepMind, a division of Google, is recognized for its cutting-edge Gemini models, which are integrated into various Google products like Bard and Workspace tools. These models are renowned for their conversational abilities and their capacity to handle complex, multi-turn dialogues.",
+    "Meta AI is best known for its LLaMA (Large Language Model Meta AI) series, which has been made open-source for researchers and developers. LLaMA models are praised for their ability to support innovation and experimentation due to their accessibility and strong performance.",
+    "Meta AI with it's LLaMA models aims to democratize AI development by making high-quality models available for free, fostering collaboration across industries. Their open-source approach has been a game-changer for researchers without access to expensive resources.",
+    "Microsoft’s Azure AI platform is famous for integrating OpenAI’s GPT models, enabling businesses to use these advanced models in a scalable and secure cloud environment. Azure AI powers applications like Copilot in Office 365, helping users draft emails, generate summaries, and more.",
+    "Amazon’s Bedrock platform is recognized for providing access to various language models, including its own models and third-party ones like Anthropic’s Claude and AI21’s Jurassic. Bedrock is especially valued for its flexibility, allowing users to choose models based on their specific needs.",
+    "Cohere is well-known for its language models tailored for business use, excelling in tasks like search, summarization, and customer support. Their models are recognized for being efficient, cost-effective, and easy to integrate into workflows.",
+    "AI21 Labs is famous for its Jurassic series of language models, which are highly versatile and capable of handling tasks like content creation and code generation. The Jurassic models stand out for their natural language understanding and ability to generate detailed and coherent responses.",
+    "In the rapidly advancing field of artificial intelligence, several companies have made significant contributions with their large language models. Notable players include OpenAI, known for its GPT Series (including GPT-4); Anthropic, which offers the Claude Series; Google DeepMind with its Gemini Models; Meta AI, recognized for its LLaMA Series; Microsoft Azure AI, which integrates OpenAI’s GPT Models; Amazon AWS (Bedrock), providing access to various models including Claude (Anthropic) and Jurassic (AI21 Labs); Cohere, which offers its own models tailored for business use; and AI21 Labs, known for its Jurassic Series. These companies are shaping the landscape of AI by providing powerful models with diverse capabilities.",
+]
+
+# Initialize components for RAG pipeline
+document_store = InMemoryDocumentStore()
+docs = [Document(content=doc) for doc in dataset]
+
+document_embedder = OpenAIDocumentEmbedder(model="text-embedding-3-small")
+text_embedder = OpenAITextEmbedder(model="text-embedding-3-small")
+
+docs_with_embeddings = document_embedder.run(docs)
+document_store.write_documents(docs_with_embeddings["documents"])
+
+retriever = InMemoryEmbeddingRetriever(document_store, top_k=2)
+
+template = [
+    ChatMessage.from_user(
+        """
+Given the following information, answer the question.
+
+Context:
+{% for document in documents %}
+    {{ document.content }}
+{% endfor %}
+
+Question: {{question}}
+Answer:
+"""
+    )
+]
+
+prompt_builder = ChatPromptBuilder(template=template)
+chat_generator = OpenAIChatGenerator(model="gpt-4o-mini")
+
+# Setting the RagasEvaluator Component
+llm = ChatOpenAI(model="gpt-4o-mini")
+evaluator_llm = LangchainLLMWrapper(llm)
+
+ragas_evaluator = RagasEvaluator(
+    ragas_metrics=[AnswerRelevancy(), ContextPrecision(), Faithfulness()], evaluator_llm=evaluator_llm
+)
+
+# Creating the Pipeline
+rag_pipeline = Pipeline()
+
+# Adding the components
+rag_pipeline.add_component("text_embedder", text_embedder)
+rag_pipeline.add_component("retriever", retriever)
+rag_pipeline.add_component("prompt_builder", prompt_builder)
+rag_pipeline.add_component("llm", chat_generator)
+rag_pipeline.add_component("answer_builder", AnswerBuilder())
+rag_pipeline.add_component("ragas_evaluator", ragas_evaluator)
+
+# Connecting the components
+rag_pipeline.connect("text_embedder.embedding", "retriever.query_embedding")
+rag_pipeline.connect("retriever", "prompt_builder")
+rag_pipeline.connect("prompt_builder.prompt", "llm.messages")
+rag_pipeline.connect("llm.replies", "answer_builder.replies")
+rag_pipeline.connect("retriever", "answer_builder.documents")
+rag_pipeline.connect("llm.replies", "answer_builder.replies")
+rag_pipeline.connect("retriever", "answer_builder.documents")
+rag_pipeline.connect("retriever", "ragas_evaluator.documents")
+rag_pipeline.connect("llm.replies", "ragas_evaluator.response")
+
+# Run the pipeline
+question = "What makes Meta AI’s LLaMA models stand out?"
+
+reference = "Meta AI’s LLaMA models stand out for being open-source, supporting innovation and experimentation due to their accessibility and strong performance."
+
+
+result = rag_pipeline.run(
+    {
+        "text_embedder": {"text": question},
+        "prompt_builder": {"question": question},
+        "answer_builder": {"query": question},
+        "ragas_evaluator": {"query": question, "reference": reference},
+        # Each metric expects a specific set of parameters as input. Refer to the
+        # Ragas class' documentation for more details.
+    }
+)
+
+print(result['answer_builder']['answers'][0].data, '\n')
+print(result['ragas_evaluator']['result'])

integrations/ragas/pydoc/config.yml

@@ -3,8 +3,7 @@ loaders:
     search_path: [../src]
     modules:
       [
-        "haystack_integrations.components.evaluators.ragas.evaluator",
-        "haystack_integrations.components.evaluators.ragas.metrics",
+        "haystack_integrations.components.evaluators.ragas.evaluator"
       ]
     ignore_when_discovered: ["__init__"]
 processors:
@@ -14,7 +13,6 @@ processors:
     do_not_filter_modules: false
     skip_empty_modules: true
   - type: filter
-    expression: "name not in ['InputConverters', 'MetricDescriptor', 'MetricParamsValidators', 'OutputConverters', 'METRIC_DESCRIPTORS']"
   - type: smart
   - type: crossref
 renderer:

integrations/ragas/pyproject.toml

@@ -22,7 +22,7 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
-dependencies = ["haystack-ai", "ragas>=0.1.11,<=0.1.16"]
+dependencies = ["haystack-ai", "ragas>=0.2.0,<0.3.0", "langchain_openai"]
 
 [project.urls]
 Source = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/ragas"
@@ -48,6 +48,7 @@ dependencies = [
   "pytest-rerunfailures",
   "haystack-pydoc-tools",
   "pytest-asyncio",
+  "pydantic"
 ]
 [tool.hatch.envs.default.scripts]
 test = "pytest {args:tests}"

integrations/ragas/src/haystack_integrations/components/evaluators/ragas/__init__.py

@@ -1,4 +1,3 @@
 from .evaluator import RagasEvaluator
-from .metrics import RagasMetric
 
-__all__ = ("RagasEvaluator", "RagasMetric")
+__all__ = ["RagasEvaluator"]