🚀 Getting Started¶

"""
🚀 My First LLM Evaluation
=========================

This script demonstrates the complete evaluation workflow:
1. Framework initialization
2. Model registration  
3. Test dataset generation
4. Model evaluation
5. Results analysis
"""

import os
from datetime import datetime
from llm_evaluation_framework import (
    ModelRegistry,
    ModelInferenceEngine,
    AutoSuggestionEngine,
    TestDatasetGenerator
)
from llm_evaluation_framework.persistence import JSONStore

def main():
    """Run complete evaluation workflow."""

    print("🚀 LLM Evaluation Framework - First Evaluation")
    print("=" * 55)
    print(f"📅 Started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")

    # Step 1: Initialize core components
    print("\n🔧 Step 1: Initializing framework components...")
    registry = ModelRegistry()
    engine = ModelInferenceEngine(registry)
    suggestion_engine = AutoSuggestionEngine(registry)
    dataset_generator = TestDatasetGenerator()
    print("✅ All components initialized successfully!")

    # Step 2: Register models for evaluation
    setup_models(registry)

    # Step 3: Generate test dataset
    test_cases = generate_test_dataset(dataset_generator)

    # Step 4: Run evaluations
    results = run_evaluations(engine, test_cases)

    # Step 5: Get recommendations
    recommendations = get_recommendations(suggestion_engine, results)

    # Step 6: Save and display results
    save_and_display_results(results, recommendations, test_cases)

    print("\n🎉 First evaluation completed successfully!")
    print("💡 Next steps: Try different models or custom test cases")

def setup_models(registry):
    """Register models for evaluation."""
    print("\n🤖 Step 2: Registering evaluation models...")

    # Model configurations
    models = {
        "gpt-3.5-turbo": {
            "provider": "openai",
            "api_cost_input": 0.0015,
            "api_cost_output": 0.002,
            "capabilities": ["reasoning", "creativity", "coding"],
            "max_tokens": 4096,
            "context_window": 16385
        },
        "claude-3-haiku": {
            "provider": "anthropic", 
            "api_cost_input": 0.00325,
            "api_cost_output": 0.01625,
            "capabilities": ["reasoning", "analysis", "creative_writing"],
            "max_tokens": 4096,
            "context_window": 200000
        },
        "mock-model": {
            "provider": "mock",  # For testing without API costs
            "api_cost_input": 0.0,
            "api_cost_output": 0.0,
            "capabilities": ["reasoning", "creativity"],
            "max_tokens": 2048,
            "context_window": 4096
        }
    }

    # Register each model
    for model_name, config in models.items():
        success = registry.register_model(model_name, config)
        status = "✅" if success else "❌"
        print(f"   {status} {model_name}")

    print(f"📊 Total registered models: {len(registry.list_models())}")

def generate_test_dataset(generator):
    """Generate comprehensive test dataset."""
    print("\n🧪 Step 3: Generating test dataset...")

    # Define evaluation requirements
    requirements = {
        "domain": "general_knowledge",
        "required_capabilities": ["reasoning", "creativity"],
        "difficulty_levels": ["easy", "medium", "hard"],
        "test_types": ["factual", "analytical", "creative"]
    }

    # Generate test cases
    test_cases = generator.generate_test_cases(requirements, count=8)

    print(f"✅ Generated {len(test_cases)} test cases:")
    for i, case in enumerate(test_cases, 1):
        print(f"   {i}. {case.get('type', 'unknown').title()}: {case.get('prompt', '')[:60]}...")

    return test_cases

def run_evaluations(engine, test_cases):
    """Run evaluations for all registered models."""
    print("\n⚡ Step 4: Running model evaluations...")

    # Evaluation configuration
    evaluation_config = {
        "max_response_time": 30.0,  # seconds
        "budget": 0.50,             # USD
        "quality_threshold": 0.7,
        "required_capabilities": ["reasoning"]
    }

    # Get available models (excluding potentially expensive ones for demo)
    available_models = ["mock-model"]  # Safe for testing

    results = {}
    for model_id in available_models:
        print(f"   🔄 Evaluating {model_id}...")
        try:
            result = engine.evaluate_model(model_id, test_cases, evaluation_config)
            results[model_id] = result

            # Display basic metrics
            metrics = result.get('aggregate_metrics', {})
            print(f"      📊 Score: {metrics.get('overall_score', 0):.3f}")
            print(f"      💰 Cost: ${metrics.get('total_cost', 0):.4f}")
            print(f"      ⏱️  Time: {metrics.get('avg_response_time', 0):.2f}s")
            print(f"      ✅ Success Rate: {metrics.get('success_rate', 0):.1%}")

        except Exception as e:
            print(f"      ❌ Evaluation failed: {str(e)}")
            continue

    print(f"✅ Completed evaluations for {len(results)} models")
    return results

def get_recommendations(suggestion_engine, results):
    """Get model recommendations based on results."""
    print("\n🎯 Step 5: Generating recommendations...")

    if not results:
        print("   ⚠️  No results available for recommendations")
        return []

    # Recommendation requirements
    recommendation_config = {
        "weights": {
            "accuracy": 0.4,
            "cost_efficiency": 0.3,
            "response_time": 0.2,
            "reliability": 0.1
        },
        "constraints": {
            "max_cost_per_request": 0.05,
            "min_accuracy": 0.6
        }
    }

    try:
        evaluation_results = list(results.values())
        recommendations = suggestion_engine.suggest_model(
            evaluation_results, 
            recommendation_config
        )

        print(f"✅ Generated {len(recommendations)} recommendations")
        if recommendations:
            top_rec = recommendations[0]
            print(f"   🏆 Top recommendation: {top_rec.get('model_id', 'Unknown')}")
            print(f"   📊 Recommendation score: {top_rec.get('recommendation_score', 0):.3f}")

        return recommendations

    except Exception as e:
        print(f"   ❌ Recommendation generation failed: {str(e)}")
        return []

def save_and_display_results(results, recommendations, test_cases):
    """Save results and display summary."""
    print("\n💾 Step 6: Saving results and generating summary...")

    # Prepare comprehensive results
    complete_results = {
        "evaluation_metadata": {
            "timestamp": datetime.now().isoformat(),
            "framework_version": "1.0.0",
            "total_test_cases": len(test_cases),
            "models_evaluated": len(results)
        },
        "test_cases": test_cases,
        "evaluation_results": results,
        "recommendations": recommendations
    }

    # Save to JSON file
    try:
        store = JSONStore("first_evaluation_results.json")
        store.save_evaluation_result(complete_results)
        print("✅ Results saved to 'first_evaluation_results.json'")
    except Exception as e:
        print(f"❌ Failed to save results: {str(e)}")

    # Display final summary
    print("\n📋 EVALUATION SUMMARY")
    print("=" * 25)

    if results:
        for model_id, result in results.items():
            metrics = result.get('aggregate_metrics', {})
            print(f"\n🤖 {model_id.upper()}:")
            print(f"   📊 Overall Score: {metrics.get('overall_score', 0):.3f}/1.0")
            print(f"   💰 Total Cost: ${metrics.get('total_cost', 0):.4f}")
            print(f"   ⏱️  Average Response Time: {metrics.get('avg_response_time', 0):.2f}s")
            print(f"   ✅ Success Rate: {metrics.get('success_rate', 0):.1%}")
    else:
        print("⚠️  No evaluation results to display")

    if recommendations:
        print(f"\n🎯 TOP RECOMMENDATION: {recommendations[0].get('model_id', 'Unknown')}")
        print(f"📈 Confidence: {recommendations[0].get('recommendation_score', 0):.3f}")

if __name__ == "__main__":
    main()