Mistral 7B v0.3 + Mistral Small 3 (24B): Sliding Window Deprecation + Tool-Calling

# === Mistral 7B v0.3 Function-Calling FT Lab ===
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import LoraConfig, get_peft_model
from trl import SFTTrainer, SFTConfig
from datasets import load_dataset
from transformers import BitsAndBytesConfig
import torch
 
bnb = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4",
                         bnb_4bit_use_double_quant=True,
                         bnb_4bit_compute_dtype=torch.bfloat16)
 
model = AutoModelForCausalLM.from_pretrained(
    "mistralai/Mistral-7B-Instruct-v0.3",
    quantization_config=bnb,
    attn_implementation="flash_attention_2",
    torch_dtype=torch.bfloat16,
    device_map="cuda",
)
tok = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.3")
tok.pad_token = tok.eos_token
 
lora = LoraConfig(r=32, lora_alpha=64, lora_dropout=0.05,
                  target_modules=["q_proj","k_proj","v_proj","o_proj",
                                  "gate_proj","up_proj","down_proj"],
                  task_type="CAUSAL_LM")
model = get_peft_model(model, lora)
 
# Function-calling dataset (e.g. Glaive-Function-Calling-v2 TR-translated)
dataset = load_dataset("user/tr-function-calling", split="train")
# Format: {"messages": [...]}
 
def format_fc(example):
    return {"text": tok.apply_chat_template(
        example["messages"],
        tools=example.get("tools"),
        tokenize=False,
    )}
 
dataset = dataset.map(format_fc, num_proc=8)
 
cfg = SFTConfig(
    output_dir="mistral-7b-v03-fc-tr",
    num_train_epochs=2,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,
    learning_rate=2e-4,
    bf16=True, optim="paged_adamw_8bit",
    max_seq_length=4096, packing=True,
    dataset_text_field="text",
    logging_steps=5, report_to="wandb",
)
trainer = SFTTrainer(model=model, tokenizer=tok, train_dataset=dataset, args=cfg)
trainer.train()