DPO Math: Bradley-Terry → Loss Function Derivation — Why No Reward Model?

# === DPO Lab — TRL DPOTrainer RTX 4090 ===
import torch
from unsloth import FastLanguageModel
from datasets import load_dataset
from trl import DPOConfig, DPOTrainer
 
# 1. Load SFT'ed model (DPO için reference base — SFT'den geldiyse iyi)
model, tok = FastLanguageModel.from_pretrained(
    "llama-3.1-8b-tr-instruct/final",      # önceki SFT'den
    max_seq_length=4096,
    dtype="bfloat16", load_in_4bit=True,
)
model = FastLanguageModel.get_peft_model(
    model, r=32, lora_alpha=64,
    target_modules=["q_proj","k_proj","v_proj","o_proj"],
    use_gradient_checkpointing="unsloth",
)
 
# 2. Preference dataset
dataset = load_dataset("HuggingFaceH4/ultrafeedback_binarized", split="train_prefs")
# Format: {"prompt": "...", "chosen": "...", "rejected": "..."}
 
# 3. DPO Config
cfg = DPOConfig(
    output_dir="llama-3.1-8b-tr-dpo",
    num_train_epochs=1,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,
    learning_rate=5e-6,                       # DPO için DÜŞÜK lr (SFT'nin 1/50'si)
    warmup_ratio=0.05,
    lr_scheduler_type="cosine",
    bf16=True, optim="paged_adamw_8bit",
    max_length=4096,
    max_prompt_length=2048,
    beta=0.1,                                  # KL strength
    loss_type="sigmoid",                       # standart DPO loss
    logging_steps=5,
    save_steps=100, report_to="wandb",
)
 
# 4. Train
trainer = DPOTrainer(
    model=model,
    ref_model=None,                            # PEFT için — base model otomatik ref
    args=cfg,
    train_dataset=dataset,
    tokenizer=tok,
)
trainer.train()
 
# Bench:
# - 50K pref pairs, 1 epoch, ~80 dakika RTX 4090
# - TR-MMLU: SFT 39.8 → DPO 40.2 (+0.4)
# - MT-Bench-TR: SFT 7.18 → DPO 7.51 (+0.33)