Use agent-opt Python SDK for Prompt Optimization

pip install agent-opt

export FI_API_KEY="your_api_key"
export FI_SECRET_KEY="your_secret_key"

dataset = [
    {
        "article": "The James Webb Space Telescope has captured stunning new images of the Pillars of Creation...",
        "target_summary": "The JWST has taken new pictures of the Pillars of Creation."
    },
    {
        "article": "Researchers have discovered a new enzyme that can break down plastics at record speed...",
        "target_summary": "A new enzyme that rapidly breaks down plastics has been found."
    },
    # ... more data points
]

from fi.opt.base.evaluator import Evaluator

evaluator = Evaluator(
    eval_template="summary_quality",  # A built-in template for summarization
    eval_model_name="turing_flash",   # The model to perform the evaluation
    fi_api_key="your_key",
    fi_secret_key="your_secret"
)

from fi.opt.datamappers import BasicDataMapper

data_mapper = BasicDataMapper(
    key_map={
        "input": "article",          # Maps the 'input' to the 'article' field in the dataset
        "output": "generated_output" # The key for the model's generated text
    }
)

from fi.opt.optimizers import MetaPromptOptimizer
from fi.opt.generators import LiteLLMGenerator

# The teacher model is a powerful LLM that guides the optimization
teacher = LiteLLMGenerator(model="gpt-4o", prompt_template="{prompt}")

optimizer = MetaPromptOptimizer(
    teacher_generator=teacher,
    num_rounds=5  # Number of refinement iterations
)

initial_prompt = "Summarize the following article: {article}"

result = optimizer.optimize(
    evaluator=evaluator,
    data_mapper=data_mapper,
    dataset=dataset,
    initial_prompts=[initial_prompt],
    task_description="Generate a concise, one-sentence summary of the article.",
    eval_subset_size=10  # Use a subset of the data for faster evaluation per round
)

# Print the final score and the best prompt found
print(f"Final Score: {result.final_score:.4f}")
print(f"Best Prompt:\n{result.best_generator.get_prompt_template()}")

# Review the history of the optimization
for i, iteration in enumerate(result.history):
    print(f"\n--- Round {i+1} ---")
    print(f"Score: {iteration.average_score:.4f}")
    print(f"Prompt: {iteration.prompt}")

from fi.opt.optimizers import BayesianSearchOptimizer

# Dataset with examples for a classification task
dataset = [
    {"text": "This movie was fantastic!", "label": "Positive"},
    {"text": "I would not recommend this product.", "label": "Negative"},
    # ... more examples
]

# Initialize the optimizer to search for 2 to 5 few-shot examples
bayesian_optimizer = BayesianSearchOptimizer(
    inference_model_name="gpt-4o-mini",
    n_trials=20,          # Number of configurations to test
    min_examples=2,
    max_examples=5,
    example_template="Text: {text}\nSentiment: {label}" # How to format each example
)

# Run the optimization
result = bayesian_optimizer.optimize(
    evaluator=evaluator,
    data_mapper=BasicDataMapper(key_map={"input": "text", "output": "generated_output"}),
    dataset=dataset,
    initial_prompts=["Classify the sentiment of the following text:"]
)

print(f"Best few-shot prompt:\n{result.best_generator.get_prompt_template()}")

from fi.opt.optimizers import ProTeGi

protegi_optimizer = ProTeGi(
    teacher_generator=LiteLLMGenerator(model="gpt-4o", prompt_template="{prompt}"),
    num_gradients=4,       # Number of critiques to generate per failure
    beam_size=4,           # Number of candidate prompts to maintain
    num_rounds=3
)

# Run the optimization
result = protegi_optimizer.optimize(
    evaluator=evaluator,
    data_mapper=data_mapper,
    dataset=dataset,
    initial_prompts=["Your initial prompt with known issues."],
    eval_subset_size=20
)

print(f"Refined prompt after error correction:\n{result.best_generator.get_prompt_template()}")