Round-trip Eval: Pre/Post Quant Table — TR-MMLU + MT-Bench + Niche Benchmark

# === Round-trip eval — cookbook tarafından sağlanan script ===
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from lm_eval import simple_evaluate
 
models_to_test = {
    "bf16": "meta-llama/Meta-Llama-3.1-8B-Instruct",
    "awq_int4": "llama-3.1-8b-int4-awq",
    "gptq_int4": "llama-3.1-8b-int4-gptq",
    "fp8": "llama-3.1-8b-fp8",
}
 
tasks = ["mmlu_tr", "mt_bench_tr", "wikitext_perplexity"]
 
results = {}
for name, path in models_to_test.items():
    print(f"Evaluating {name}...")
    result = simple_evaluate(
        model="hf",
        model_args=f"pretrained={path},dtype=auto",
        tasks=tasks,
        device="cuda",
        batch_size="auto",
    )
    results[name] = result
 
# Print comparison table
print(f"{'Model':<15} {'TR-MMLU':<10} {'MT-Bench-TR':<15} {'WikiText PPL':<15}")
for name, result in results.items():
    print(f"{name:<15} {result['mmlu_tr']:<10.2f} {result['mt_bench_tr']:<15.2f} {result['wikitext_ppl']:<15.2f}")