📚 API Reference¶

class ModelInferenceEngine:
    """
    Core engine for evaluating Language Learning Models (LLMs).

    Handles prompt execution, response collection, metrics calculation,
    and cost analysis across different model providers.
    """

def __init__(self, model_registry: ModelRegistry) -> None

from llm_evaluation_framework import ModelRegistry, ModelInferenceEngine

registry = ModelRegistry()
registry.register("gpt-3.5-turbo", {
    "provider": "openai",
    "api_cost_input": 0.001,
    "api_cost_output": 0.002,
    "capabilities": ["reasoning", "creativity"]
})

engine = ModelInferenceEngine(registry)

def evaluate_model(
    self,
    model_id: str,
    test_cases: List[Dict[str, Any]],
    use_case_requirements: Dict[str, Any]
) -> Dict[str, Any]

{
    "model_id": str,
    "test_results": List[Dict],
    "aggregate_metrics": {
        "total_cost": float,
        "avg_response_time": float,
        "success_rate": float,
        "overall_score": float
    },
    "timestamp": str,
    "metadata": Dict
}

# Define test cases
test_cases = [
    {
        "id": "tc001",
        "type": "reasoning",
        "prompt": "If it takes 5 machines 5 minutes to make 5 widgets, how long would it take 100 machines to make 100 widgets?",
        "expected_output": "5 minutes",
        "evaluation_criteria": {
            "accuracy": 0.8,
            "reasoning_quality": 0.7
        }
    },
    {
        "id": "tc002", 
        "type": "creativity",
        "prompt": "Write a haiku about machine learning",
        "evaluation_criteria": {
            "creativity": 0.9,
            "format_compliance": 1.0
        }
    }
]

# Define requirements
requirements = {
    "max_response_time": 30.0,  # seconds
    "budget": 0.10,             # USD
    "required_capabilities": ["reasoning", "creativity"],
    "quality_threshold": 0.7
}

# Run evaluation
results = engine.evaluate_model("gpt-3.5-turbo", test_cases, requirements)

print(f"Overall Score: {results['aggregate_metrics']['overall_score']}")
print(f"Total Cost: ${results['aggregate_metrics']['total_cost']:.4f}")

def evaluate_multiple_models(
    self,
    model_ids: List[str],
    test_cases: List[Dict[str, Any]],
    use_case_requirements: Dict[str, Any]
) -> Dict[str, Dict[str, Any]]

models = ["gpt-3.5-turbo", "gpt-4", "claude-3"]
comparative_results = engine.evaluate_multiple_models(models, test_cases, requirements)

for model_id, results in comparative_results.items():
    score = results['aggregate_metrics']['overall_score']
    cost = results['aggregate_metrics']['total_cost']
    print(f"{model_id}: Score={score:.3f}, Cost=${cost:.4f}")

class AutoSuggestionEngine:
    """
    Intelligent recommendation engine for model selection.

    Analyzes evaluation results and suggests optimal models based on
    performance metrics, cost constraints, and use case requirements.
    """

def __init__(self, model_registry: ModelRegistry) -> None

def suggest_model(
    self,
    evaluation_results: List[Dict[str, Any]], 
    use_case_requirements: Dict[str, Any]
) -> List[Dict[str, Any]]

[
    {
        "model_id": str,
        "recommendation_score": float,
        "strengths": List[str],
        "weaknesses": List[str],
        "cost_analysis": Dict,
        "performance_summary": Dict,
        "confidence": float
    }
]

from llm_evaluation_framework import AutoSuggestionEngine

# Initialize suggestion engine
suggestion_engine = AutoSuggestionEngine(registry)

# Define prioritization requirements
requirements = {
    "weights": {
        "accuracy": 0.4,      # 40% weight on accuracy
        "cost": 0.3,          # 30% weight on cost efficiency  
        "response_time": 0.2, # 20% weight on speed
        "creativity": 0.1     # 10% weight on creativity
    },
    "constraints": {
        "max_cost_per_request": 0.01,
        "max_response_time": 15.0,
        "min_accuracy": 0.7
    },
    "use_case": "content_generation"
}

# Get recommendations (using previous evaluation results)
recommendations = suggestion_engine.suggest_model(comparative_results, requirements)

# Display top recommendation
top_choice = recommendations[0]
print(f"🏆 Recommended Model: {top_choice['model_id']}")
print(f"📊 Recommendation Score: {top_choice['recommendation_score']:.3f}")
print(f"💪 Strengths: {', '.join(top_choice['strengths'])}")
print(f"⚠️  Considerations: {', '.join(top_choice['weaknesses'])}")

def compare_models(
    self,
    model_results: Dict[str, Dict],
    comparison_criteria: List[str]
) -> Dict[str, Any]

comparison = suggestion_engine.compare_models(
    comparative_results,
    criteria=["accuracy", "cost_efficiency", "response_time", "creativity"]
)

print("📋 Model Comparison Summary:")
for criterion, analysis in comparison.items():
    print(f"  {criterion}: Best = {analysis['best_model']}")

class TestDatasetGenerator:
    """
    Generates synthetic test datasets for model evaluation.

    Creates domain-specific prompts, edge cases, and evaluation criteria
    to ensure comprehensive model testing across various scenarios.
    """

def generate_test_cases(
    self,
    use_case_requirements: Dict[str, Any],
    num_cases: int = 10
) -> List[Dict[str, Any]]

[
    {
        "id": str,
        "type": str,
        "prompt": str,
        "expected_output": str,
        "evaluation_criteria": Dict[str, float],
        "difficulty": str,
        "domain": str,
        "metadata": Dict
    }
]

from llm_evaluation_framework import TestDatasetGenerator

generator = TestDatasetGenerator()

# Define generation requirements
requirements = {
    "domain": "customer_service",
    "required_capabilities": ["reasoning", "empathy", "problem_solving"],
    "difficulty_levels": ["easy", "medium", "hard"],
    "topics": ["product_issues", "billing_questions", "technical_support"],
    "response_types": ["informative", "empathetic", "solution_oriented"]
}

# Generate test cases
test_cases = generator.generate_test_cases(requirements, num_cases=15)

print(f"Generated {len(test_cases)} test cases:")
for case in test_cases[:3]:  # Show first 3
    print(f"  📝 {case['id']}: {case['type']} ({case['difficulty']})")
    print(f"     Prompt: {case['prompt'][:80]}...")

def generate_edge_cases(
    self,
    base_requirements: Dict[str, Any],
    edge_case_types: List[str]
) -> List[Dict[str, Any]]

edge_cases = generator.generate_edge_cases(
    requirements,
    edge_case_types=["empty_input", "very_long_input", "ambiguous_prompt"]
)

print(f"Generated {len(edge_cases)} edge cases for robustness testing")

def load_real_world_dataset(
    self,
    dataset_name: str,
    sample_size: Optional[int] = None
) -> List[Dict[str, Any]]

# Load subset of real-world data
real_data = generator.load_real_world_dataset("customer_support_tickets", sample_size=50)

class ModelRegistry:
    """
    Centralized registry for model configurations and metadata.

    Manages model registration, configuration validation, capability tracking,
    and provides consistent access to model information across the framework.
    """

def __init__(self, config_file: Optional[str] = None) -> None

def register(self, name: str, config: Dict[str, Any]) -> bool

{
    "provider": str,                    # e.g., "openai", "anthropic", "azure"
    "api_endpoint": Optional[str],      # Custom endpoint URL
    "api_key_env": Optional[str],       # Environment variable for API key
    "api_cost_input": float,            # Cost per 1K input tokens
    "api_cost_output": float,           # Cost per 1K output tokens
    "max_tokens": int,                  # Maximum tokens per request
    "capabilities": List[str],          # Model capabilities
    "rate_limits": Dict[str, int],      # Rate limiting configuration
    "context_window": int,              # Maximum context window size
    "metadata": Dict[str, Any]          # Additional metadata
}

from llm_evaluation_framework.registry import ModelRegistry

registry = ModelRegistry()

# Register OpenAI model
openai_config = {
    "provider": "openai",
    "api_key_env": "OPENAI_API_KEY",
    "api_cost_input": 0.001,     # $0.001 per 1K input tokens
    "api_cost_output": 0.002,    # $0.002 per 1K output tokens
    "max_tokens": 4096,
    "capabilities": ["reasoning", "creativity", "code_generation"],
    "rate_limits": {
        "requests_per_minute": 3500,
        "tokens_per_minute": 90000
    },
    "context_window": 16385,
    "metadata": {
        "version": "gpt-3.5-turbo-0125",
        "training_cutoff": "2023-09",
        "multimodal": False
    }
}

success = registry.register("gpt-3.5-turbo", openai_config)
print(f"Registration successful: {success}")

# Register Anthropic model
anthropic_config = {
    "provider": "anthropic",
    "api_key_env": "ANTHROPIC_API_KEY", 
    "api_cost_input": 0.00325,
    "api_cost_output": 0.01625,
    "max_tokens": 4096,
    "capabilities": ["reasoning", "analysis", "creative_writing"],
    "context_window": 200000,
    "metadata": {
        "version": "claude-3-haiku-20240307",
        "safety_filtered": True
    }
}

registry.register("claude-3-haiku", anthropic_config)

def list_models(self, filter_by: Optional[Dict[str, Any]] = None) -> List[str]

# List all models
all_models = registry.list_models()
print(f"Registered models: {all_models}")

# Filter by capability
reasoning_models = registry.list_models(filter_by={"capabilities": "reasoning"})
print(f"Reasoning capable models: {reasoning_models}")

# Filter by cost range
budget_models = registry.list_models(filter_by={"max_input_cost": 0.002})

def get_model(self, name: str) -> Dict[str, Any]

model_config = registry.get_model("gpt-3.5-turbo")
print(f"Provider: {model_config['provider']}")
print(f"Capabilities: {model_config['capabilities']}")
print(f"Input cost: ${model_config['api_cost_input']}/1K tokens")

def update_model(self, name: str, updates: Dict[str, Any]) -> bool

# Update pricing
registry.update_model("gpt-3.5-turbo", {
    "api_cost_input": 0.0015,   # Price increase
    "api_cost_output": 0.002
})

def validate_model(self, name: str) -> Dict[str, Any]

validation_result = registry.validate_model("gpt-3.5-turbo")
if validation_result["valid"]:
    print("✅ Model configuration is valid and accessible")
else:
    print(f"❌ Validation failed: {validation_result['errors']}")

class AccuracyScorer:
    """Exact match accuracy scoring for factual responses."""

    def score(self, predicted: str, reference: str) -> float

from llm_evaluation_framework.evaluation.scoring_strategies import AccuracyScorer

scorer = AccuracyScorer()
score = scorer.score("Paris", "Paris")  # Returns 1.0
score = scorer.score("London", "Paris")  # Returns 0.0

class SemanticSimilarityScorer:
    """Semantic similarity scoring using embeddings."""

    def score(self, predicted: str, reference: str) -> float

from llm_evaluation_framework.evaluation.scoring_strategies import SemanticSimilarityScorer

scorer = SemanticSimilarityScorer(model="sentence-transformers/all-MiniLM-L6-v2")
score = scorer.score("The sky is blue", "Blue is the color of the sky")  # Returns ~0.85

class BLEUScorer:
    """BLEU score for translation and text generation quality."""

    def score(self, predicted: str, reference: str, n_gram: int = 4) -> float

class ROUGEScorer:
    """ROUGE score for summarization quality."""

    def score(self, predicted: str, reference: str, rouge_type: str = "rouge-l") -> float

class CustomScorer:
    """Base class for implementing custom scoring strategies."""

    def score(self, predicted: str, reference: str, **kwargs) -> float:
        """Implement custom scoring logic."""
        raise NotImplementedError

class DomainSpecificScorer(CustomScorer):
    """Custom scorer for domain-specific evaluation."""

    def __init__(self, domain_keywords: List[str]):
        self.domain_keywords = domain_keywords

    def score(self, predicted: str, reference: str, **kwargs) -> float:
        # Custom scoring logic
        keyword_match_score = self._calculate_keyword_match(predicted)
        semantic_score = self._calculate_semantic_similarity(predicted, reference)

        return 0.6 * semantic_score + 0.4 * keyword_match_score

    def _calculate_keyword_match(self, text: str) -> float:
        # Implementation details
        pass

    def _calculate_semantic_similarity(self, pred: str, ref: str) -> float:
        # Implementation details  
        pass

# Usage
custom_scorer = DomainSpecificScorer(["finance", "investment", "portfolio"])
score = custom_scorer.score(predicted_response, reference_response)