Advent of Haystack - Day 6

import pandas as pd

elf_profiles_df = pd.read_csv("challenge_files/task1_elf_profiles.csv")
elf_profiles_df.head(3)

task_descriptions_df = pd.read_csv("challenge_files/task1_descriptions.csv")
task_descriptions_df.head(3)

elf_profiles = elf_profiles_df.to_dict()

elf_names = list(elf_profiles["Elf Name"].values())
elf_descriptions = list(elf_profiles["Profile Description"].values())

task_descriptions = task_descriptions_df.to_dict()
task_descriptions = list(task_descriptions["Task Description"].values())

from haystack_integrations.components.rankers.nvidia import NvidiaRanker
from haystack import Document
from haystack.utils import Secret

tasks = []

for idx, t in enumerate(task_descriptions):
    tasks.append(Document(content=t, id=idx + 1))

ranker = NvidiaRanker(
    model="nvidia/nv-rerankqa-mistral-4b-v3",
    api_key=Secret.from_env_var("NVIDIA_API_KEY"),
)
ranker.warm_up()

all_elf_compatibility = []
for i, elf in enumerate(elf_descriptions):
    # For each elf, run the ranker model on all tasks to find the best match
    elf_compatibility = ranker.run(elf, tasks, top_k=1)
    all_elf_compatibility.append(elf_compatibility)

tasks[0]

elf_descriptions[0]

from IPython.display import display, Markdown

# Printing the solution! - Best task for each elf
for i in range(len(all_elf_compatibility)):
    display(
        Markdown(
            f"The best task for elf **{elf_names[i]}** is: *{all_elf_compatibility[i]['documents'][0].content}*"
        )
    )

# Repeat the exercise with an alternative ranker model in build.nvidia.com
# TIP💡: If you used nv-rerankqa-mistral-4b-v3, use llama-3.2-nv-rerankqa-1b-v1 now or vice-versa)
alternative_ranker = NvidiaRanker(
    model="nvidia/llama-3.2-nv-rerankqa-1b-v1",
    api_key=Secret.from_env_var("NVIDIA_API_KEY"),
)
alternative_ranker.warm_up()

alternative_all_elf_compatibility = []
for i, elf in enumerate(elf_descriptions):
    # For each elf, run the alternative ranker model on all tasks and append them to the list alternative_all_elf_compatibility
    elf_compatibility = alternative_ranker.run(elf, tasks, top_k=1)
    alternative_all_elf_compatibility.append(elf_compatibility)

# Printing the solution! - Best task for each elf
for i in range(len(alternative_all_elf_compatibility)):
    display(
        Markdown(
            f"The best task for elf **{elf_names[i]}** is: *{alternative_all_elf_compatibility[i]['documents'][0].content}*"
        )
    )

from haystack.components.evaluators import DocumentMRREvaluator

# Getting the ground truth: optimal task-elf placement
ground_truth = pd.read_csv("challenge_files/task1_solution_key.csv").to_dict()
ground_truth = list(ground_truth["Task ID"].values())

# Getting the ranking and alternative_ranking actual results
ranking = [
    all_elf_compatibility[index]["documents"][0].id
    for index in range(len(all_elf_compatibility))
]
alternative_ranking = [
    alternative_all_elf_compatibility[index]["documents"][0].id
    for index in range(len(alternative_all_elf_compatibility))
]

# Compare the results of the two ranker models using the Mean Reciprocal Rank (MRR) score. This is a ranking quality metric that only considers the position of the first relevant item in the ranked list.
mrr_evaluator = DocumentMRREvaluator()

ground_truth_documents = []
ranking_documents = []
alternative_ranking_documents = []

for idx, gt in enumerate(ground_truth):
    ground_truth_documents.append([Document(content=gt)])
    ranking_documents.append([Document(content=ranking[idx])])
    alternative_ranking_documents.append([Document(content=alternative_ranking[idx])])

### COMPLETE ###
# Run the mrr_evaluator over the original and the alternative rankers' results
result_ranker = mrr_evaluator.run(
    ground_truth_documents=ground_truth_documents,
    retrieved_documents=ranking_documents,
)
result_alternative_ranker = mrr_evaluator.run(
    ground_truth_documents=ground_truth_documents,
    retrieved_documents=alternative_ranking_documents,
)

print(f"MRR for the first ranker is {result_ranker['score']}")
print(f"MRR for the second ranker is {result_alternative_ranker['score']}")

result_ranker

result_alternative_ranker

parent_notes_df = pd.read_excel("challenge_files/task2_parent_notes.xlsx")
parent_notes_df.head(3)

parent_notes = parent_notes_df.to_dict()
parent_notes = list(parent_notes["Notes"].values())

# Based on the task description above, input the possible locations for the presents
# TIP💡: They are 4
location_options = [
    "Under the Christmas Tree",
    "Under the Children's Beds",
    "Near the Fireplace",
    "In the Stockings",
]

from haystack_integrations.components.embedders.nvidia import NvidiaTextEmbedder

### COMPLETE ###
# Look into the NvidiaTextEmbedder component and choose an embedding model from https://build.nvidia.com/explore/retrieval
embedder = NvidiaTextEmbedder(
    model="nvidia/llama-3.2-nv-embedqa-1b-v2",
    api_url="https://integrate.api.nvidia.com/v1",
    api_key=Secret.from_env_var("NVIDIA_API_KEY"),
)
embedder.warm_up()

embedded_notes = []
for note in parent_notes:
    embedded_notes.append(embedder.run(note))

embedded_locations = []
for l in location_options:
    embedded_locations.append(embedder.run(l))

from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

max_cos_similarity_per_note = []
for i in range(len(embedded_notes)):
    cos_similarity_per_note = []
    # Extract the embedding vector for the current note
    note_embedding = np.array(embedded_notes[i]["embedding"]).reshape(1, -1)

    for j in range(len(embedded_locations)):
        # Extract the embedding vector for the current location
        location_embedding = np.array(embedded_locations[j]["embedding"]).reshape(1, -1)

        # Compute cosine similarity between the note and location embeddings
        similarity = cosine_similarity(note_embedding, location_embedding)[0][0]
        cos_similarity_per_note.append(similarity)

    # Get the index of the best matching location
    max_cos_similarity_per_note.append(np.argmax(cos_similarity_per_note))

# Read the ground truth data
ground_truth_data = pd.read_excel("challenge_files/task2_solution_key.xlsx").to_dict()
ground_truth_data = list(ground_truth_data["Key"].values())

print(ground_truth_data)
print(max_cos_similarity_per_note)

### COMPLETE ###
# Calculate the accuracy of the solution by comparing max_cos_similarity_per_note to ground_truth_data
# TIP💡: The accuracy is the number of True Positives divided by all solutions!

# Count the number of correct predictions
correct_predictions = sum(
    1
    for pred, true in zip(max_cos_similarity_per_note, ground_truth_data)
    if pred == true
)

# Calculate accuracy by dividing correct predictions by total number of predictions
accuracy = correct_predictions / len(ground_truth_data)

print(f"The accuracy of this embedder is {accuracy}")