Int4 QLoRA NF4 Internals: Double Quantization + Paged Optimizer + Bitsandbytes Source Tour

# === bitsandbytes NF4 internal — adım adım ===
import torch
import bitsandbytes as bnb
 
# 1. NF4 levels
NF4_LEVELS = torch.tensor([
    -1.0, -0.6962, -0.5251, -0.3949, -0.2844, -0.1848, -0.0911, 0.0,
    0.0796, 0.1609, 0.2461, 0.3379, 0.4407, 0.5626, 0.7230, 1.0
], dtype=torch.float32)
 
# 2. Forward quantize
def nf4_quantize(W, block_size=64):
    """W: bf16 tensor → 4-bit NF4 + scale per block."""
    W_flat = W.flatten()
    n = W_flat.numel()
    n_blocks = n // block_size
 
    # Block-wise absolute max
    W_blocks = W_flat.reshape(n_blocks, block_size)
    scales = W_blocks.abs().max(dim=-1).values         # [n_blocks]
 
    # Normalize → [-1, 1]
    W_normalized = W_blocks / scales.unsqueeze(-1)
 
    # Find closest NF4 level for each weight
    distances = torch.abs(W_normalized.unsqueeze(-1) - NF4_LEVELS.unsqueeze(0).unsqueeze(0))
    indices = distances.argmin(dim=-1)                 # [n_blocks, block_size]
 
    # Pack 2 indices per byte (4-bit)
    indices_4bit = (indices[..., ::2] << 4) | indices[..., 1::2]    # [n_blocks, block_size//2]
 
    return indices_4bit, scales
 
# 3. Dequantize (inference)
def nf4_dequantize(indices_4bit, scales):
    high = (indices_4bit >> 4) & 0xF
    low = indices_4bit & 0xF
    indices = torch.stack([high, low], dim=-1).flatten(-2)
    values = NF4_LEVELS[indices]                       # [n_blocks, block_size]
    W = values * scales.unsqueeze(-1)
    return W.flatten()
 
# 4. bitsandbytes equivalent
linear_4bit = bnb.nn.Linear4bit(
    input_features=4096,
    output_features=4096,
    bias=False,
    quant_type="nf4",                                  # vs "fp4"
    compute_dtype=torch.bfloat16,                       # forward bf16
    quant_storage=torch.uint8,                          # storage
)
# Internally bunu yapar — sadece daha optimized + CUDA kernel ile