Meta-Prompt Optimizer

from fi.opt.optimizers import MetaPromptOptimizer
from fi.opt.generators import LiteLLMGenerator
from fi.opt.datamappers import BasicDataMapper
from fi.opt.base.evaluator import Evaluator

# Setup teacher model (use a powerful model for analysis)
teacher = LiteLLMGenerator(
    model="gpt-4o",
    prompt_template="{prompt}"
)

# Setup evaluator
evaluator = Evaluator(
    eval_template="summary_quality",
    eval_model_name="turing_flash",
    fi_api_key="your_key",
    fi_secret_key="your_secret"
)

# Setup data mapper
data_mapper = BasicDataMapper(
    key_map={"input": "text", "output": "generated_output"}
)

# Create optimizer
optimizer = MetaPromptOptimizer(
    teacher_generator=teacher
)

# Run optimization
result = optimizer.optimize(
    evaluator=evaluator,
    data_mapper=data_mapper,
    dataset=dataset,
    initial_prompts=["Summarize this text: {text}"],
    task_description="Create concise, informative summaries",
    num_rounds=5,
    eval_subset_size=40
)

print(f"Improvement: {result.final_score:.2%}")
print(f"Best prompt:\n{result.best_generator.get_prompt_template()}")

{
  "hypothesis": "The prompt fails on complex multi-sentence texts because it doesn't specify a structure. Adding explicit instruction to identify main points first should improve clarity.",
  "improved_prompt": "First identify the 2-3 main points in the following text. Then write a single concise sentence that captures these points:\n\n{text}"
}

optimizer = MetaPromptOptimizer(
    teacher_generator=LiteLLMGenerator("gpt-4o", "{prompt}")
)

result = optimizer.optimize(
    evaluator=evaluator,
    data_mapper=data_mapper,
    dataset=dataset,
    initial_prompts=[initial_prompt],
    
    # Provide detailed context
    task_description="""
    I want to extract structured information from customer support tickets.
    The prompt should:
    - Identify the main issue
    - Extract customer sentiment (positive/negative/neutral)
    - Determine urgency level (low/medium/high)
    - Suggest appropriate department routing
    
    The output must be in JSON format and handle incomplete information gracefully.
    """,
    
    num_rounds=7,
    eval_subset_size=30
)

# For very complex tasks, use more rounds
optimizer = MetaPromptOptimizer(
    teacher_generator=LiteLLMGenerator("gpt-4o", "{prompt}")
)

result = optimizer.optimize(
    evaluator=evaluator,
    data_mapper=data_mapper,
    dataset=complex_dataset,
    initial_prompts=[initial_prompt],
    task_description=detailed_description,
    num_rounds=10,  # More iterations for complex refinement
    eval_subset_size=50  # More examples for reliable signal
)

# Analyze the evolution
for i, iteration in enumerate(result.history):
    print(f"\nRound {i+1} Score: {iteration.average_score:.4f}")
    print(f"Prompt: {iteration.prompt[:150]}...")

# Stage 1: Quick exploration
random_result = random_search_optimizer.optimize(...)

# Stage 2: Deep refinement on best candidate
meta_optimizer = MetaPromptOptimizer(
    teacher_generator=LiteLLMGenerator("gpt-4o", "{prompt}")
)

final_result = meta_optimizer.optimize(
    evaluator=evaluator,
    data_mapper=data_mapper,
    dataset=dataset,
    initial_prompts=[random_result.best_generator.get_prompt_template()],
    task_description="Refine for clarity and consistency",
    num_rounds=5
)

result = optimizer.optimize(...)

# View the optimization journey
for i, iteration in enumerate(result.history):
    print(f"\n{'='*60}")
    print(f"Round {i+1}")
    print(f"Score: {iteration.average_score:.4f}")
    print(f"\nPrompt:\n{iteration.prompt}")
    
    # Note: Hypothesis is internal to teacher model, 
    # but you can infer it from prompt evolution

scores = [iteration.average_score for iteration in result.history]

import matplotlib.pyplot as plt
plt.plot(scores, marker='o')
plt.xlabel('Round')
plt.ylabel('Score')
plt.title('Meta-Prompt Optimization Progress')
plt.show()

# Calculate improvement
initial_score = scores[0]
final_score = scores[-1]
improvement = ((final_score - initial_score) / initial_score) * 100
print(f"Total improvement: {improvement:.1f}%")

dataset = [
    {
        "problem": "Multi-step math word problem...",
        "solution": "Step-by-step solution..."
    }
]

optimizer = MetaPromptOptimizer(
    teacher_generator=LiteLLMGenerator("gpt-4o", "{prompt}")
)

result = optimizer.optimize(
    evaluator=evaluator,
    data_mapper=BasicDataMapper({
        "input": "problem",
        "output": "generated_output"
    }),
    dataset=dataset,
    initial_prompts=["Solve this problem: {problem}"],
    task_description="""
    Generate step-by-step solutions that:
    - Show clear reasoning at each step
    - Explain why each step is necessary
    - Arrive at the correct final answer
    """,
    num_rounds=8
)

optimizer = MetaPromptOptimizer(
    teacher_generator=LiteLLMGenerator("gpt-4o", "{prompt}")
)

result = optimizer.optimize(
    evaluator=evaluator,
    data_mapper=data_mapper,
    dataset=creative_dataset,
    initial_prompts=["Write a story based on: {prompt}"],
    task_description="""
    Generate engaging short stories (200-300 words) that:
    - Have a clear beginning, middle, and end
    - Include vivid sensory details
    - Match the tone specified in the prompt
    - Are appropriate for a general audience
    """,
    num_rounds=6,
    eval_subset_size=25
)

optimizer = MetaPromptOptimizer(
    teacher_generator=LiteLLMGenerator("gpt-4o", "{prompt}")
)

result = optimizer.optimize(
    evaluator=evaluator,
    data_mapper=data_mapper,
    dataset=transformation_dataset,
    initial_prompts=["Convert this data: {input_data}"],
    task_description="""
    Transform unstructured text into JSON format with these fields:
    - name (string)
    - date (YYYY-MM-DD format)
    - amount (number)
    - category (one of: personal, business, travel)
    
    Handle missing fields by using null. Infer dates from context when possible.
    """,
    num_rounds=5
)