ORPO: Odds Ratio Preference Optimization — Single-Stage SFT+Alignment

# === ORPO Lab — TRL ORPOTrainer ===
from trl import ORPOConfig, ORPOTrainer
from unsloth import FastLanguageModel
 
model, tok = FastLanguageModel.from_pretrained(
    "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit",
    max_seq_length=4096, dtype="bfloat16", load_in_4bit=True,
)
model = FastLanguageModel.get_peft_model(
    model, r=32, lora_alpha=64,
    target_modules=["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj","down_proj"],
)
 
dataset = load_dataset("HuggingFaceH4/ultrafeedback_binarized", split="train_prefs")
 
cfg = ORPOConfig(
    output_dir="llama-3.1-8b-orpo",
    num_train_epochs=2,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,
    learning_rate=8e-6,                      # ORPO için 5e-6 ila 1e-5
    bf16=True, optim="paged_adamw_8bit",
    max_length=4096,
    max_prompt_length=2048,
    beta=0.1,                                # OR loss weight (≡ λ)
    logging_steps=5,
    save_steps=100, report_to="wandb",
)
 
trainer = ORPOTrainer(model=model, args=cfg, train_dataset=dataset, tokenizer=tok)
trainer.train()
# 60-70 dakika, peak 14 GB