🤝 Contributing to LLM Evaluation Framework¶

"""
Example test following best practices
"""
import pytest
from unittest.mock import Mock, patch
from llm_evaluation_framework import ModelRegistry

class TestModelRegistry:
    """Comprehensive tests for ModelRegistry"""

    def setup_method(self):
        """Setup before each test"""
        self.registry = ModelRegistry()
        self.valid_config = {
            "provider": "openai",
            "api_cost_input": 0.001,
            "api_cost_output": 0.002,
            "capabilities": ["reasoning", "creativity"]
        }

    def test_register_model_success(self):
        """Test successful model registration with valid config"""
        # Arrange
        model_name = "test-gpt-3.5"

        # Act
        result = self.registry.register_model(model_name, self.valid_config)

        # Assert
        assert result is True
        assert model_name in self.registry._models
        assert self.registry.get_model(model_name) == self.valid_config

    @pytest.mark.parametrize("invalid_config,expected_error", [
        ({"provider": "unknown"}, "Invalid provider"),
        ({"provider": "openai"}, "Missing required fields"),
        ({"provider": "openai", "api_cost_input": -1}, "Invalid cost"),
    ])
    def test_register_model_validation_errors(self, invalid_config, expected_error):
        """Test model registration validation with various invalid configs"""
        with pytest.raises(ValueError, match=expected_error):
            self.registry.register_model("test-model", invalid_config)

    @patch('llm_evaluation_framework.model_registry.validate_api_key')
    def test_register_model_with_api_validation(self, mock_validate):
        """Test model registration with API key validation"""
        # Arrange
        mock_validate.return_value = True
        config = self.valid_config.copy()
        config["api_key"] = "test-key"

        # Act
        result = self.registry.register_model("test-model", config)

        # Assert
        assert result is True
        mock_validate.assert_called_once_with("test-key", "openai")

# Integration test example
class TestRegistryEngineIntegration:
    """Integration tests between registry and engine"""

    @pytest.fixture
    def configured_system(self):
        """Fixture providing configured registry and engine"""
        registry = ModelRegistry()
        engine = ModelInferenceEngine(registry)

        registry.register_model("test-model", {
            "provider": "openai",
            "capabilities": ["reasoning"]
        })

        return registry, engine

    def test_end_to_end_evaluation(self, configured_system):
        """Test complete evaluation workflow"""
        registry, engine = configured_system

        test_cases = [
            {"prompt": "2+2=?", "expected": "4"},
            {"prompt": "Capital of France?", "expected": "Paris"}
        ]

        # This would use mocked API calls in real tests
        with patch('llm_evaluation_framework.engine.call_model_api') as mock_api:
            mock_api.return_value = "4"

            results = engine.evaluate_model("test-model", test_cases)

            assert results is not None
            assert "aggregate_metrics" in results
            assert results["aggregate_metrics"]["test_count"] == 2