Llama 3.1 / 3.2 / 3.3 8B — RTX 4090'ın İş Atı: GQA + 128K Context + Türkçe Reçete

# === Ders 3.1 Lab — Llama 3.1 8B QLoRA + Türkçe SFT ===
# Stage: Reference
# Hardware: RTX 4090 (24GB)
# Tahmini süre: 1 epoch, 50K Türkçe Alpaca, ~50 dakika
 
import os, torch
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
 
from unsloth import FastLanguageModel
from unsloth.chat_templates import get_chat_template, train_on_responses_only
from datasets import load_dataset
from trl import SFTTrainer, SFTConfig
 
# 1. Model + tokenizer (Unsloth: 2x faster, 70% less mem)
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit",
    max_seq_length=4096,
    dtype=torch.bfloat16,
    load_in_4bit=True,                      # NF4
)
 
# 2. LoRA adapters — Unsloth fused (q/k/v/o + gate/up/down)
model = FastLanguageModel.get_peft_model(
    model,
    r=32,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
                    "gate_proj", "up_proj", "down_proj"],
    lora_alpha=64,
    lora_dropout=0.05,
    bias="none",
    use_gradient_checkpointing="unsloth",   # selective ckpt — Unsloth Triton
    random_state=42,
)
 
# 3. Tokenizer + chat template
tokenizer = get_chat_template(tokenizer, chat_template="llama-3.1")
 
# 4. Dataset — TR Alpaca
def to_chat(example):
    messages = [
        {"role": "user", "content": example["instruction"]},
        {"role": "assistant", "content": example["output"]},
    ]
    return {"text": tokenizer.apply_chat_template(messages, tokenize=False)}
 
dataset = load_dataset("malhajar/alpaca-gpt4-tr", split="train")
dataset = dataset.map(to_chat, num_proc=8)
 
# 5. SFTConfig
cfg = SFTConfig(
    output_dir="llama-3.1-8b-tr-instruct",
    num_train_epochs=1,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,          # effective=8
    learning_rate=2e-4,                     # QLoRA için yüksek (LoRA gerçek lr)
    warmup_ratio=0.03,
    lr_scheduler_type="cosine",
    weight_decay=0.0,
    bf16=True,
    optim="paged_adamw_8bit",
    max_seq_length=4096,
    packing=True,                            # variable-length, +%40 throughput
    dataset_text_field="text",
    logging_steps=5,
    save_steps=100,
    save_total_limit=2,
    report_to="wandb",
    run_name="ftc-3.1-llama-8b-tr",
    seed=42,
)
 
# 6. Trainer + loss masking
trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=dataset,
    args=cfg,
)
 
# Sadece response üzerinde loss
trainer = train_on_responses_only(
    trainer,
    instruction_part="<|start_header_id|>user<|end_header_id|>\n\n",
    response_part="<|start_header_id|>assistant<|end_header_id|>\n\n",
)
 
# 7. Train
trainer.train()
trainer.save_model("llama-3.1-8b-tr-instruct/final")
 
# 8. Inference test
FastLanguageModel.for_inference(model)
prompt = "İstanbul'un yedi tepesi nelerdir, kısaca anlat."
inputs = tokenizer.apply_chat_template(
    [{"role": "user", "content": prompt}],
    tokenize=True, add_generation_prompt=True, return_tensors="pt",
).to("cuda")
out = model.generate(inputs, max_new_tokens=300, temperature=0.7,
                     do_sample=True, top_p=0.9)
print(tokenizer.decode(out[0], skip_special_tokens=True))