Advent of Haystack: Day 10

!pip install -q haystack-ai datasets

import os
from getpass import getpass

os.environ["HF_API_TOKEN"] = getpass("Your Hugging Face token")
os.environ["OPENAI_API_KEY"] = getpass("Enter OpenAI API key:")

from datasets import load_dataset

data = load_dataset(
    "hotpotqa/hotpot_qa", "distractor", split="train[:50]", trust_remote_code=True
)

data[4]

from haystack.dataclasses.document import Document


def convert_hotpot_to_docs(data):
    question = []
    true_answer = []
    doc_chunks = []

    for item in data:

        question.append(item["question"])
        true_answer.append(item["answer"])

        # Documents
        assert len(item["context"]["title"]) == len(item["context"]["sentences"])
        context_dict = {
            item["context"]["title"][i]: item["context"]["sentences"][i]
            for i in range(len(item["context"]["title"]))
        }

        # Add default rank (irrelevant docs will have the rank of 1000)
        for k, v in context_dict.items():
            context_dict[k] = [
                (vv, 1000, item["id"], item["question"], item["answer"])
                for vv in v
                if vv.strip()
            ]  # remove empty contents

        # Add True rank (irrelevant docs will have the the true rank)
        assert len(item["supporting_facts"]["title"]) == len(
            item["supporting_facts"]["sent_id"]
        )

        for i in range(len(item["supporting_facts"]["title"])):

            k = item["supporting_facts"]["title"][i]  # Key
            v_pos = item["supporting_facts"]["sent_id"][1]  # Value
            rank = i

            if v_pos < len(context_dict[k]):
                context_dict[k][v_pos] = (
                    context_dict[k][v_pos][0],
                    rank,
                    context_dict[k][v_pos][2],
                    context_dict[k][v_pos][3],
                    context_dict[k][v_pos][4],
                )

        nested_list = [v for k, v in context_dict.items()]
        chunks = [x for xs in nested_list for x in xs]

        # Sort by rank (for MRR)
        sorted_chunks = sorted(chunks, key=lambda tup: tup[1])

        # Create Haystack Documents
        doc_chunks.append(
            [
                Document(
                    content=tup[0],
                    meta={
                        "rank": tup[1],
                        "data_id": tup[2],
                        "question": tup[3],
                        "ground_truth_answer": tup[4],
                    },
                )
                for tup in sorted_chunks
            ]
        )

    return question, true_answer, doc_chunks

questions, ground_truth_answers, doc_chunks = convert_hotpot_to_docs(data)

documents = [doc for chunk in doc_chunks for doc in chunk]

from haystack import Pipeline
from haystack.components.embedders import HuggingFaceAPIDocumentEmbedder
from haystack.components.writers import DocumentWriter
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.document_stores.types import DuplicatePolicy

document_store = InMemoryDocumentStore()
document_embedder = HuggingFaceAPIDocumentEmbedder(
    api_type="serverless_inference_api", api_params={"model": "BAAI/bge-m3"}
)
document_writer = DocumentWriter(
    document_store=document_store, policy=DuplicatePolicy.SKIP
)

indexing = Pipeline()
indexing.add_component(instance=document_embedder, name="document_embedder")
indexing.add_component(instance=document_writer, name="document_writer")

indexing.connect("document_embedder.documents", "document_writer.documents")

indexing.run({"document_embedder": {"documents": documents}})

from haystack import Pipeline
from haystack.components.builders import PromptBuilder
from haystack.components.embedders import HuggingFaceAPITextEmbedder
from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever
from haystack.components.generators import HuggingFaceAPIGenerator

generator = HuggingFaceAPIGenerator(
    api_type="serverless_inference_api",
    api_params={"model": "meta-llama/Llama-3.2-1B-Instruct"},
)

template = """
<|start_header_id|>assistant<|end_header_id|>
Answer the following question based on the given context information only.

Context:
{% for document in documents %}
    {{ document.content }}
{% endfor %}

Question: {{question}}
Answer:
"""

rag_pipeline = Pipeline()
rag_pipeline.add_component(
    "query_embedder",
    HuggingFaceAPITextEmbedder(
        api_type="serverless_inference_api", api_params={"model": "BAAI/bge-m3"}
    ),
)
rag_pipeline.add_component(
    "retriever", InMemoryEmbeddingRetriever(document_store, top_k=3)
)
rag_pipeline.add_component("prompt_builder", PromptBuilder(template=template))
rag_pipeline.add_component("generator", generator)

rag_pipeline.connect("query_embedder", "retriever.query_embedding")
rag_pipeline.connect("retriever", "prompt_builder.documents")
rag_pipeline.connect("prompt_builder", "generator")

question = (
    "Cadmium Chloride is slightly soluble in this chemical, it is also called what?"
)

rag_pipeline.run(
    {"query_embedder": {"text": question}, "prompt_builder": {"question": question}}
)

# https://docs.haystack.deepset.ai/reference/experimental-evaluation-harness-api

from haystack_experimental.evaluation.harness.rag import (
    DefaultRAGArchitecture,
    RAGEvaluationHarness,
    RAGEvaluationMetric,
    RAGEvaluationInput,
)

# Use Semantic Answer Similarity, Faithfulness, Context Relevance, Document Recall (Single hit) and Document MRR
# Feel free to skip the Faithfulness and Context Relevance if you don't have an OPENAI_API_KEY
# HINT: You need questions, ground_truth_answers and ground_truth_document
pipeline_eval_harness = RAGEvaluationHarness(
    rag_pipeline=rag_pipeline,
    rag_components=DefaultRAGArchitecture.GENERATION_WITH_EMBEDDING_RETRIEVAL,
    metrics={
        RAGEvaluationMetric.SEMANTIC_ANSWER_SIMILARITY,
        RAGEvaluationMetric.FAITHFULNESS,
        RAGEvaluationMetric.CONTEXT_RELEVANCE,
        RAGEvaluationMetric.DOCUMENT_RECALL_SINGLE_HIT,
        RAGEvaluationMetric.DOCUMENT_MRR,
    },
)

eval_harness_input = RAGEvaluationInput(
    queries=questions,
    ground_truth_answers=ground_truth_answers,
    ground_truth_documents=ground_truth_documents,
    rag_pipeline_inputs={"prompt_builder": {"question": questions}},
)

harness_eval_run = pipeline_eval_harness.run(
    inputs=eval_harness_input, run_name="rag_eval"
)

dir(RAGEvaluationMetric)

dir(RAGEvaluationInput)

from haystack import Pipeline
from haystack.components.builders import PromptBuilder
from haystack.components.embedders import HuggingFaceAPITextEmbedder
from haystack.components.retrievers.in_memory import (
    InMemoryEmbeddingRetriever,
    InMemoryBM25Retriever,
)
from haystack.components.generators import HuggingFaceAPIGenerator
from haystack.components.joiners import DocumentJoiner

generator = HuggingFaceAPIGenerator(
    api_type="serverless_inference_api",
    api_params={"model": "meta-llama/Llama-3.2-1B-Instruct"},
)

template = """
<|start_header_id|>assistant<|end_header_id|>
Answer the following question based on the given context information only.

Context:
{% for document in documents %}
    {{ document.content }}
{% endfor %}

Question: {{question}}
Answer:
"""

hybrid_pipeline = Pipeline()
hybrid_pipeline.add_component(
    "query_embedder",
    HuggingFaceAPITextEmbedder(
        api_type="serverless_inference_api", api_params={"model": "BAAI/bge-m3"}
    ),
)
hybrid_pipeline.add_component(
    "embedding_retriever", InMemoryEmbeddingRetriever(document_store, top_k=3)
)
hybrid_pipeline.add_component(
    "bm25_retriever", InMemoryBM25Retriever(document_store, top_k=3)
)
hybrid_pipeline.add_component("joiner", DocumentJoiner())
hybrid_pipeline.add_component("prompt_builder", PromptBuilder(template=template))
hybrid_pipeline.add_component("generator", generator)

hybrid_pipeline.connect("query_embedder", "embedding_retriever.query_embedding")
hybrid_pipeline.connect("embedding_retriever", "joiner")
hybrid_pipeline.connect("bm25_retriever", "joiner")
hybrid_pipeline.connect("joiner", "prompt_builder.documents")
hybrid_pipeline.connect("prompt_builder", "generator")

hybrid_pipeline.show()

question = (
    "Cadmium Chloride is slightly soluble in this chemical, it is also called what?"
)

hybrid_pipeline.run(
    {
        "query_embedder": {"text": question},
        "bm25_retriever": {"query": question},
        "prompt_builder": {"question": question},
    }
)