Judge Images and Audio with Your LLM

pip install ai-evaluation
export GOOGLE_API_KEY=your-gemini-api-key

from fi.evals import evaluate

MODEL = "gemini/gemini-2.5-flash"

FLOWER_IMAGE = "https://storage.googleapis.com/cloud-samples-data/ai-platform/flowers/daisy/100080576_f52e8ee070_n.jpg"

result = evaluate(
    prompt="""Rate how accurately the text description matches the image.
    Score 1.0 if every detail in the description is visible in the image.
    Score 0.5 if the description is partially correct but has some inaccuracies.
    Score 0.0 if the description is completely wrong or describes something else.""",
    output="A white daisy flower with a yellow center, growing in a garden.",
    image_url=FLOWER_IMAGE,
    engine="llm",
    model=MODEL,
)
print(f"Score: {result.score}")
print(f"Reason: {result.reason[:150]}")

result = evaluate(
    prompt="""Rate how accurately the text description matches the image.
    Score 1.0 if every detail in the description is visible in the image.
    Score 0.5 if the description is partially correct but has some inaccuracies.
    Score 0.0 if the description is completely wrong or describes something else.""",
    output="A golden retriever puppy playing fetch on a sandy beach.",
    image_url=FLOWER_IMAGE,
    engine="llm",
    model=MODEL,
)
print(f"Score: {result.score}")
print(f"Reason: {result.reason[:150]}")

TULIP_IMAGE = "https://storage.googleapis.com/cloud-samples-data/ai-platform/flowers/tulips/10791227_7168491604.jpg"

result = evaluate(
    prompt="flower species identification accuracy from photos",
    output="This image shows a bright red tulip in full bloom with green stems.",
    image_url=TULIP_IMAGE,
    engine="llm",
    model=MODEL,
    generate_prompt=True,
)
print(f"Score: {result.score}")
print(f"Reason: {result.reason[:150]}")

from fi.evals.core.prompt_generator import generate_grading_criteria

criteria = generate_grading_criteria(
    "product photo quality assessment for e-commerce listings",
    MODEL,
    {"image_url": "...", "output": "..."},
)
print(f"Generated criteria:\n{criteria[:300]}...")

ROSE_IMAGE = "https://storage.googleapis.com/cloud-samples-data/ai-platform/flowers/roses/12240303_80d87f77a3_n.jpg"

result = evaluate(
    prompt="""You are given two images and a text description.
    input_image_url is the reference product photo.
    output_image_url is what the AI selected as matching.
    Rate whether the text description matches the input_image_url (1.0) or
    actually describes the output_image_url instead (0.0).""",
    output="A pink rose flower in a garden.",
    input_image_url=FLOWER_IMAGE,
    output_image_url=ROSE_IMAGE,
    engine="llm",
    model=MODEL,
)
print(f"Score: {result.score}")
print(f"Reason: {result.reason[:150]}")

AUDIO_URL = "https://storage.googleapis.com/cloud-samples-data/speech/brooklyn_bridge.flac"

result = evaluate(
    prompt="""Rate how accurately the transcription captures the audio content.
    Score 1.0 if the transcription is accurate and complete.
    Score 0.5 if partially correct. Score 0.0 if completely wrong.""",
    output="How old is the Brooklyn Bridge?",
    audio_url=AUDIO_URL,
    engine="llm",
    model=MODEL,
)
print(f"Score: {result.score}")
print(f"Reason: {result.reason[:150]}")

result = evaluate(
    prompt="Rate the factual accuracy of the response given the context.",
    output="The Eiffel Tower is 330 meters tall and was built in 1889.",
    context="The Eiffel Tower, completed in 1889, stands at 330 metres.",
    engine="llm",
    model=MODEL,
)
print(f"Score: {result.score}")
print(f"Reason: {result.reason[:150]}")