Tokenizer Distillation: Cross-Model Token Mapping and TR Token Efficiency Measurement

# === Cross-tokenizer mapping table — distill için ===
from transformers import AutoTokenizer
import numpy as np
 
teacher_tok = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-72B-Instruct")
student_tok = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3.1-8B")
 
# Tek bir prompt için iki tokenizer'ın output'larını align et
prompt = "Yapay zekâ ile insan kararının kavşağında..."
 
t_ids = teacher_tok.encode(prompt, add_special_tokens=False)
s_ids = student_tok.encode(prompt, add_special_tokens=False)
 
t_tokens = teacher_tok.convert_ids_to_tokens(t_ids)
s_tokens = student_tok.convert_ids_to_tokens(s_ids)
 
print(f"Teacher tokens: {len(t_tokens)}  | Student tokens: {len(s_tokens)}")
# Teacher 14 token, Student 28 token (TR ile Llama-3 daha parçalı)
 
# Yaklaşım: 'span alignment' — substring matching
# Teacher token i. → student token range [j_start, j_end]
def align_spans(t_tokens, s_tokens, t_str, s_str):
    """O(n+m) span alignment by reconstructing original text."""
    t_offsets = []
    pos = 0
    for tok in t_tokens:
        clean = tok.replace("Ġ", " ").lstrip()
        t_offsets.append((pos, pos + len(clean)))
        pos += len(clean)
 
    s_offsets = []
    pos = 0
    for tok in s_tokens:
        clean = tok.replace("Ġ", " ").lstrip()
        s_offsets.append((pos, pos + len(clean)))
        pos += len(clean)
 
    # Map each teacher span to student span(s) by overlap
    mapping = []
    j = 0
    for ti, (ts, te) in enumerate(t_offsets):
        student_spans = []
        while j < len(s_offsets) and s_offsets[j][1] <= te:
            student_spans.append(j)
            j += 1
        mapping.append((ti, student_spans))
    return mapping
 
# Bu mapping ile teacher'ın logit distribution'unu student'ın token uzayına project edebilirsin
# Detay: Part XI — Distillation Recipes