DeepSeek-R1-Distill (Llama-8B / Qwen-7B): Reasoning Trace Distillation — 'Think Token'ları Öğrenmek

# === Kendi R1-style FT'ni yap — DeepSeek-R1 ile distill ===
# Önce: R1'i çalıştır + traces topla (cloud veya HF API)
# Sonra: bu traces ile küçük base'i FT et
 
# Adım 1 — 1000 problem için R1 traces toplama (örnek, gerçekte cloud/API)
import openai  # placeholder; gerçekte DeepSeek API
from datasets import load_dataset
 
math_problems = load_dataset("openai/gsm8k", "main", split="train[:1000]")
r1_traces = []
for problem in math_problems:
    response = call_r1_api(problem["question"])   # returns "<think>...</think>...answer..."
    r1_traces.append({
        "question": problem["question"],
        "r1_response": response,                   # includes <think>
        "gold_answer": problem["answer"],
    })
 
# Adım 2 — Trace dataset'ini SFT format'ına çevir
def to_sft(ex):
    messages = [
        {"role": "user", "content": ex["question"]},
        {"role": "assistant", "content": ex["r1_response"]},   # think dahil
    ]
    return {"text": tok.apply_chat_template(messages, tokenize=False)}
 
trace_ds = trace_ds.map(to_sft, num_proc=8)
 
# Adım 3 — Llama 3.1 8B'yi traces ile FT et (think'i öğrenecek)
from unsloth import FastLanguageModel
model, tok = FastLanguageModel.from_pretrained(
    "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit",
    max_seq_length=8192,                          # think traces uzun
    load_in_4bit=True, dtype="bfloat16",
)
model = FastLanguageModel.get_peft_model(
    model, r=64,                                  # rank yüksek — reasoning capacity
    lora_alpha=128, lora_dropout=0.05,
    target_modules=["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj","down_proj"],
)
 
# Tokenizer'a <think>, </think> special token'larını ekle
new_tokens = ["<think>", "</think>"]
n_added = tok.add_tokens(new_tokens, special_tokens=True)
model.resize_token_embeddings(len(tok))
# Yeni embedding'ler için byte-decomp init (Ders 2.2)
 
from trl import SFTTrainer, SFTConfig
cfg = SFTConfig(
    output_dir="llama-8b-reasoning",
    num_train_epochs=3,
    per_device_train_batch_size=1,                # 8K seq → batch=1
    gradient_accumulation_steps=8,
    learning_rate=1e-4,
    bf16=True, optim="paged_adamw_8bit",
    max_seq_length=8192, packing=False,           # think trace bütünlüğü
    dataset_text_field="text",
    logging_steps=5, save_steps=100, report_to="wandb",
)
SFTTrainer(model=model, tokenizer=tok, train_dataset=trace_ds, args=cfg).train()