🖥️ CLI Usage Guide¶

# Simple model evaluation
llm-eval evaluate --model gpt-3.5-turbo

# Output:
# 🚀 Starting evaluation for gpt-3.5-turbo...
# ✅ Generated 5 test cases for reasoning
# ⚡ Running evaluation...
# 📊 Results: 80% accuracy, $0.0032 cost, 2.5s time

# Evaluate with more test cases
llm-eval evaluate \
  --model gpt-4 \
  --test-cases 20 \
  --capability creativity

# Output:
# 🚀 Starting evaluation for gpt-4...
# ✅ Generated 20 test cases for creativity
# ⚡ Running evaluation...
# 📊 Results: 92% accuracy, $0.0156 cost, 8.2s time

# Save detailed results
llm-eval evaluate \
  --model claude-3 \
  --test-cases 15 \
  --capability reasoning \
  --output claude_evaluation.json \
  --verbose

# Creates: claude_evaluation.json with full results

# Test different capabilities
for capability in reasoning creativity coding; do
  llm-eval evaluate \
    --model gpt-3.5-turbo \
    --capability $capability \
    --output "results_${capability}.json"
done

# Generate basic reasoning tests
llm-eval generate --capability reasoning

# Output:
# 🧪 Generating test dataset...
# ✅ Generated 5 test cases for reasoning
# 
# 1. Solve this logic puzzle: If all roses are flowers...
#    Criteria: Logical reasoning, step-by-step thinking
# 
# 2. What comes next in the sequence: 2, 4, 8, 16...
#    Criteria: Pattern recognition, mathematical reasoning

# Generate large coding dataset
llm-eval generate \
  --capability coding \
  --count 100 \
  --domain "web development" \
  --output coding_tests.json

# Output:
# 🧪 Generating 100 test cases for coding...
# 📊 Domain: web development
# ✅ Generated 100 test cases successfully
# 💾 Saved to: coding_tests.json

# Generate tests for all capabilities
for cap in reasoning creativity coding factual instruction; do
  llm-eval generate \
    --capability $cap \
    --count 25 \
    --output "tests_${cap}.json"
done

# Simple accuracy calculation
llm-eval score \
  --predictions "The sky is blue" "Paris is in France" \
  --references "The sky is blue" "Paris is the capital of France"

# Output:
# 📊 Scoring Results
# Average Score
# Accuracy score: 0.5000 (50%)

# Use F1 scoring metric
llm-eval score \
  --predictions "Machine learning is AI" "Python is great" \
  --references "ML is artificial intelligence" "Python is awesome" \
  --metric f1

# Output:
# 📊 Scoring Results
# Average Score
# F1 score: 0.7500 (75%)

# Score predictions from files (using shell)
predictions=$(cat predictions.txt | tr '\n' ' ')
references=$(cat references.txt | tr '\n' ' ')

llm-eval score \
  --predictions $predictions \
  --references $references \
  --metric accuracy \
  --output batch_scores.json

# Show all available resources
llm-eval list

# Output:
# Available Capabilities
#   - reasoning
#   - creativity
#   - factual
#   - instruction
#   - coding

# List models only
llm-eval list --type models

# List scoring strategies
llm-eval list --type strategies

# List capabilities
llm-eval list --type capabilities

# Get machine-readable output
llm-eval list --format json

# Output:
# {
#   "capabilities": ["reasoning", "creativity", "coding"],
#   "models": [],
#   "strategies": ["accuracy", "f1"]
# }