Capstone Module 18: Turkish Mixtral DPO — Bend Open MoE to Turkish

# Modül 18 Capstone — Türkçe Mixtral DPO
# QLoRA + DPO + 2× H100 FSDP + vLLM deployment
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
from trl import DPOTrainer, DPOConfig
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
from datasets import load_dataset
import wandb
 
MODEL_BASE = 'mistralai/Mixtral-8x7B-Instruct-v0.1'
DATASET_TR = 'sukruyusufkaya/turkish-preferences-5k'  # Modül 15.6'dan
OUTPUT_DIR = './mixtral-8x7b-tr-dpo'
 
wandb.init(project='mixtral-tr-capstone', name='mixtral-tr-dpo-v1')
 
# 1. Quantization (Mixtral 47B → 4-bit ≈ 25 GB, single H100 sınırda)
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_quant_type='nf4',
    bnb_4bit_use_double_quant=True,
)
 
# 2. Model
tokenizer = AutoTokenizer.from_pretrained(MODEL_BASE)
tokenizer.pad_token = tokenizer.eos_token
 
model = AutoModelForCausalLM.from_pretrained(
    MODEL_BASE,
    quantization_config=bnb_config,
    device_map='auto',  # FSDP otomatik
    torch_dtype=torch.bfloat16,
)
model = prepare_model_for_kbit_training(model)
 
# 3. LoRA — MoE için tüm expert FFN'leri target
lora_config = LoraConfig(
    r=16,
    lora_alpha=32,
    target_modules=[
        'q_proj', 'k_proj', 'v_proj', 'o_proj',
        # MoE expert FFN'leri
        'gate_proj', 'up_proj', 'down_proj',
        # Gating network (kritik!)
        'gate',  # Mixtral gating layer adı
    ],
    lora_dropout=0.05,
    bias='none',
    task_type='CAUSAL_LM',
)
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()  # ~50M trainable (47B'nin %0.1'i)
 
# 4. Dataset
dataset = load_dataset(DATASET_TR, split='train')
split = dataset.train_test_split(test_size=0.05, seed=42)
train_dataset = split['train']  # 4,750
val_dataset = split['test']  # 250
 
# 5. DPO config
dpo_config = DPOConfig(
    output_dir=OUTPUT_DIR,
    num_train_epochs=1,
    per_device_train_batch_size=1,
    gradient_accumulation_steps=32,    # effective batch 32
    learning_rate=3e-6,                # MoE daha hassas, biraz düşük
    warmup_steps=100,
    lr_scheduler_type='cosine',
    optim='paged_adamw_8bit',
    bf16=True,
    
    beta=0.1,
    max_length=4096,
    max_prompt_length=2048,
    
    logging_steps=10,
    eval_strategy='steps',
    eval_steps=100,
    save_steps=200,
    save_total_limit=3,
    
    report_to='wandb',
    run_name='mixtral-tr-dpo-v1',
    load_best_model_at_end=True,
)
 
# 6. DPO trainer
trainer = DPOTrainer(
    model=model,
    ref_model=None,
    args=dpo_config,
    train_dataset=train_dataset,
    eval_dataset=val_dataset,
    tokenizer=tokenizer,
    peft_config=lora_config,
)
 
# 7. Train — 2× H100 FSDP, ~36 saat
# Komut: torchrun --nproc_per_node=2 capstone.py
trainer.train()
trainer.save_model(OUTPUT_DIR + '/final')
 
# 8. LoRA merge
from peft import PeftModel
base_model = AutoModelForCausalLM.from_pretrained(
    MODEL_BASE,
    torch_dtype=torch.bfloat16,
    device_map='auto',
)
model_with_lora = PeftModel.from_pretrained(base_model, OUTPUT_DIR + '/final')
merged_model = model_with_lora.merge_and_unload()
merged_model.save_pretrained(OUTPUT_DIR + '/merged')
tokenizer.save_pretrained(OUTPUT_DIR + '/merged')
 
# 9. HuggingFace Hub push
merged_model.push_to_hub('sukruyusufkaya/mixtral-8x7b-tr-dpo')
tokenizer.push_to_hub('sukruyusufkaya/mixtral-8x7b-tr-dpo')
 
# 10. vLLM deployment (production)
# vllm serve sukruyusufkaya/mixtral-8x7b-tr-dpo \
#   --tensor-parallel-size 2 \
#   --gpu-memory-utilization 0.9 \
#   --max-model-len 8192 \
#   --port 8000
 
print('🎉 Capstone tamamlandı!')
print('Model: https://huggingface.co/sukruyusufkaya/mixtral-8x7b-tr-dpo')
print('Maliyet: ~$200-500 (2× H100 spot, 36-48 saat)')