Experiment Tracking Architecture: W&B + Hydra + DVC — The Engineering of Sweeps

# === configs/ — Cookbook Hydra layout (her Lab aynı şema) ===
# configs/config.yaml (default)
defaults:
  - model: llama3_8b
  - data: tr_alpaca
  - trainer: qlora
  - optim: adamw_8bit
  - logging: wandb
  - _self_
 
run:
  name: ${model.name}_${data.name}_${trainer.name}
  seed: 42
  output_dir: outputs/${run.name}/${now:%Y-%m-%d_%H-%M-%S}
  stage: reference                  # spike | reference | production | research
 
# configs/model/llama3_8b.yaml
name: llama-3-8b
hf_id: meta-llama/Meta-Llama-3-8B
torch_dtype: bfloat16
quantization:
  enabled: true
  bnb_4bit_quant_type: nf4
  bnb_4bit_use_double_quant: true
  bnb_4bit_compute_dtype: bfloat16
 
# configs/data/tr_alpaca.yaml
name: tr-alpaca
hf_id: malhajar/alpaca-gpt4-tr
split: train
max_seq_len: 2048
packing: true
text_field: text
chat_template: llama3
 
# configs/trainer/qlora.yaml
name: qlora-r32
lora:
  r: 32
  alpha: 64
  dropout: 0.05
  target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]
training:
  num_epochs: 3
  per_device_train_batch_size: 2
  gradient_accumulation_steps: 8
  learning_rate: 2.0e-4
  warmup_ratio: 0.03
  lr_scheduler_type: cosine
  gradient_checkpointing: true
  bf16: true
  logging_steps: 5
  save_steps: 100
  eval_steps: 100
 
# configs/logging/wandb.yaml
project: ftc-llama-tr-cookbook
entity: ${oc.env:WANDB_ENTITY,null}
mode: online                        # offline mode: cluster'da internet yoksa
tags: [${run.stage}, ${model.name}, ${trainer.name}]

# === train.py — W&B + Hydra entegrasyonu (cookbook iskeleti) ===
import os, json, hashlib
from pathlib import Path
 
import hydra
from omegaconf import DictConfig, OmegaConf
import wandb
import torch
 
@hydra.main(version_base=None, config_path="configs", config_name="config")
def main(cfg: DictConfig):
    # 1) Config'i resolve et + freeze
    cfg = OmegaConf.to_container(cfg, resolve=True)
    OmegaConf.save(cfg, Path(cfg["run"]["output_dir"]) / "config.frozen.yaml")
 
    # 2) W&B init — config'in tamamı + git fingerprint
    git_sha = os.popen("git rev-parse HEAD").read().strip()
    data_hash = compute_dataset_hash(cfg["data"])  # aşağıda
 
    wandb.init(
        project=cfg["logging"]["project"],
        entity=cfg["logging"]["entity"],
        name=cfg["run"]["name"],
        tags=cfg["logging"]["tags"],
        mode=cfg["logging"]["mode"],
        config={
            **cfg,
            "_git_sha": git_sha,
            "_data_sha256": data_hash,
            "_gpu": torch.cuda.get_device_name(0),
            "_torch": torch.__version__,
            "_cuda": torch.version.cuda,
        },
        # cookbook standardı: her run unique ID, lineage'a bağlı
        id=hashlib.sha256(
            f"{git_sha}-{cfg['run']['seed']}-{data_hash}".encode()
        ).hexdigest()[:16],
        resume="allow",
    )
 
    # 3) Sistem metriklerini agresif kaydet
    wandb.define_metric("step")
    wandb.define_metric("train/*", step_metric="step")
    wandb.define_metric("eval/*", step_metric="step", summary="best")
    wandb.define_metric("gpu_mem/peak_gb", step_metric="step", summary="max")
    wandb.define_metric("throughput/tokens_per_sec", step_metric="step", summary="mean")
 
    # 4) Modeli yükle, training loop
    model, tokenizer = build_model_and_tokenizer(cfg["model"])
    dataset = build_dataset(cfg["data"], tokenizer)
    trainer = build_trainer(cfg["trainer"], model, dataset)
 
    # 5) Trainer callback ile metrik log'la
    trainer.add_callback(WandbDetailedCallback(cfg))
    trainer.train()
 
    # 6) Final artifact — model + config + frozen state
    art = wandb.Artifact(name=f"model-{wandb.run.id}", type="model",
                        metadata={"git_sha": git_sha, "data_sha256": data_hash})
    art.add_dir(cfg["run"]["output_dir"])
    wandb.log_artifact(art, aliases=["latest", cfg["run"]["stage"]])
    wandb.finish()
 
def compute_dataset_hash(data_cfg: dict) -> str:
    """Dataset'in 'state'ini hash'le: HF dataset id + split + version."""
    key = f"{data_cfg['hf_id']}::{data_cfg['split']}::{data_cfg.get('revision', 'main')}"
    return hashlib.sha256(key.encode()).hexdigest()[:32]
 
if __name__ == "__main__":
    main()

# === sweep.yaml — W&B Bayesian sweep (8B QLoRA tuning) ===
program: train.py
method: bayes
metric:
  name: eval/loss
  goal: minimize
 
parameters:
  trainer.training.learning_rate:
    distribution: log_uniform_values
    min: 1.0e-5
    max: 5.0e-4
  trainer.lora.r:
    values: [8, 16, 32, 64, 128]
  trainer.lora.alpha:
    distribution: int_uniform
    min: 8
    max: 256
  trainer.lora.dropout:
    distribution: uniform
    min: 0.0
    max: 0.2
  trainer.training.warmup_ratio:
    distribution: uniform
    min: 0.0
    max: 0.1
 
early_terminate:
  type: hyperband
  min_iter: 100              # 100 step görmeden agent'ı erken durdurma
  s: 2
 
# Run komutu
command:
  - ${env}
  - uv
  - run
  - python
  - ${program}
  - ${args_no_hyphens}      # Hydra için "--" olmadan key=value