Hybrid SSM Modelleri: Falcon-Mamba + Zamba2 — KV-Cache Olmadan Long Context

# === Falcon-Mamba 7B FT ===
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import LoraConfig, get_peft_model
 
model = AutoModelForCausalLM.from_pretrained(
    "tiiuae/falcon-mamba-7b",
    torch_dtype="bfloat16",
    device_map="cuda",
    # SSM'lerin Mamba-spesifik kernel'ları var — flash-attn yerine selective_scan
)
tok = AutoTokenizer.from_pretrained("tiiuae/falcon-mamba-7b")
 
# LoRA — SSM layer target
lora = LoraConfig(
    r=32, lora_alpha=64, lora_dropout=0.05,
    target_modules=["in_proj", "x_proj", "dt_proj", "out_proj"],   # Mamba layer adları
    task_type="CAUSAL_LM",
)
model = get_peft_model(model, lora)
 
# Training — lr daha düşük (Mamba hassas)
cfg = SFTConfig(
    output_dir="falcon-mamba-7b-tr",
    learning_rate=5e-5,                       # Transformer'daki 2e-4'ün 1/4'ü
    warmup_ratio=0.1,                          # Mamba'da uzun warmup
    bf16=True, optim="paged_adamw_8bit",
    max_seq_length=8192,                       # long-context advantage
    ...
)