Tokenizer Extension Lab: Llama-3 → +8K TR Token + Embedding Init

# === Tam TR Tokenizer Extension + Continual Pre-train ===
# Aşama 1: TR-only BPE eğit (Part II Ders 2.1'den)
from tokenizers import Tokenizer, models, normalizers, pre_tokenizers, trainers
 
tr_tok = Tokenizer(models.BPE(unk_token="<|unk|>"))
tr_tok.normalizer = normalizers.NFC()
tr_tok.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
 
trainer = trainers.BpeTrainer(
    vocab_size=50_000, min_frequency=10,
    initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
    special_tokens=["<|pad|>"],
)
tr_tok.train(["/data/tr-cleaned-50gb.txt"], trainer)
tr_tok.save("tr_50k.json")
 
# Aşama 2: Llama-3'de OLMAYAN TR-spesifik 8K token belirle
from transformers import AutoTokenizer
from collections import Counter
 
base = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3.1-8B")
base_vocab = set(base.get_vocab().keys())
 
# 50M cümle sample'da TR-BPE ile tokenize
sample_text = open("/data/tr-cleaned-50gb.txt").read(50_000_000)
tr_tokens = tr_tok.encode(sample_text).tokens
tr_freq = Counter(tr_tokens)
 
# Llama-3'de olmayan, sık olanları al
candidates = []
for tok, freq in tr_freq.most_common():
    if tok not in base_vocab and freq >= 100:
        candidates.append(tok)
    if len(candidates) >= 8000:
        break
 
# Aşama 3: Base tokenizer'a ekle
n = base.add_tokens(candidates)
base.save_pretrained("llama3-tr-tok-136k")
print(f"Eklenen TR token: {n} → toplam vocab: {len(base)}")
 
# Aşama 4: Embedding init (byte-decomposition)
from transformers import AutoModelForCausalLM
import torch
 
model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3.1-8B",
    torch_dtype=torch.bfloat16, device_map="cuda",
)
old_vocab = 128256
new_vocab = len(base)
model.resize_token_embeddings(new_vocab)
 
old_emb = model.get_input_embeddings().weight.data
new_emb = model.get_input_embeddings().weight.data
for i, tok_str in enumerate(base.convert_ids_to_tokens(range(old_vocab, new_vocab))):
    decomp_ids = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3.1-8B").encode(
        tok_str.replace("Ġ", " "), add_special_tokens=False)
    if decomp_ids:
        new_emb[old_vocab + i] = old_emb[decomp_ids].mean(dim=0).to(torch.bfloat16)
 
# Aşama 5: Continual pre-train — 500M token TR
# Bu cookbook'un Part III/IX'taki most-resource-intensive Lab
# RTX 4090 + Llama 8B + 500M token + LoRA-style efficient continual:
 
from peft import LoraConfig, get_peft_model
lora = LoraConfig(
    r=64, lora_alpha=128, lora_dropout=0.0,
    target_modules=["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj","down_proj","embed_tokens","lm_head"],
    task_type="CAUSAL_LM",
)
model = get_peft_model(model, lora)
# embed_tokens + lm_head LoRA — yeni token'ları öğrenmek için kritik
 
# Train — TR cleaned 50GB üzerinde packed sequences
# 500M token / (8K batch effective) = 62K step
# step/s ≈ 1.5 → 12 saat 4090'da
# ya da 1 saat H100'de