GPTQ Algoritması: Optimal Brain Quantization + Hessian Update — RTX 4090'da 12 Dakikada Llama 8B

# === GPTQ ile Llama 3.1 8B int4 quantization — RTX 4090, 12 dakika ===
from transformers import AutoTokenizer
from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig
from datasets import load_dataset
 
# 1. Calibration dataset (önemli! dağılım modelin gerçek inference'ıyla benzer olmalı)
calibration_data = load_dataset("wikitext", "wikitext-2-raw-v1", split="train[:512]")
 
tok = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3.1-8B-Instruct")
 
def to_examples(data):
    examples = []
    for row in data:
        text = row["text"]
        if len(text.strip()) > 10:
            inputs = tok(text, return_tensors="pt", max_length=2048, truncation=True)
            examples.append({"input_ids": inputs["input_ids"][0],
                            "attention_mask": inputs["attention_mask"][0]})
    return examples[:128]
 
calibration = to_examples(calibration_data)
 
# 2. Quantization config
quant_cfg = BaseQuantizeConfig(
    bits=4,
    group_size=128,
    desc_act=False,                    # activation reorder (true daha iyi ama yavaş)
    sym=True,
    true_sequential=True,              # sequential layer-by-layer
)
 
# 3. Load + quantize
model = AutoGPTQForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3.1-8B-Instruct",
    quantize_config=quant_cfg,
    torch_dtype="bfloat16",
    device_map="cuda",
)
 
# 4. Quantize — RTX 4090, ~12 dakika
model.quantize(calibration)
 
# 5. Save (~4.5 GB instead of 16 GB)
model.save_quantized("llama-3.1-8b-int4-gptq")
tok.save_pretrained("llama-3.1-8b-int4-gptq")
 
# Bench:
# - bf16 size: 16 GB
# - GPTQ int4 size: 4.5 GB (-72%)
# - WikiText-2 perplexity: bf16 5.93 → int4 6.04 (+1.9%)
# - Inference speed (vLLM): bf16 95 tok/s → int4 165 tok/s (+74%)