Dataset Accounts & Cloud: Kaggle, HuggingFace, Numenta, MVTec, NASA — The Anomaly Detection Data Arsenal

# Credit Card Fraud Detection — Modül 3, 5, 7 için
kaggle datasets download -d mlg-ulb/creditcardfraud
unzip creditcardfraud.zip -d data/raw/credit-card-fraud/
rm creditcardfraud.zip
 
# IEEE-CIS Fraud — Capstone 1 için
kaggle competitions download -c ieee-fraud-detection
unzip ieee-fraud-detection.zip -d data/raw/ieee-cis-fraud/
rm ieee-fraud-detection.zip
 
# NYC Taxi (anomaly streaming)
kaggle datasets download -d boltzmannbrain/nyc-taxi
unzip nyc-taxi.zip -d data/raw/nyc-taxi/
rm nyc-taxi.zip

# HuggingFace datasets — Python kullanımı
from datasets import load_dataset
 
# Örnek 1: HDFS log anomaly (LogBERT için)
hdfs = load_dataset("nielsr/hdfs-logs", split="train")
print(f"HDFS satır sayısı: {len(hdfs)}")
 
# Örnek 2: PTB-XL ECG
ptbxl = load_dataset("PTB-XL/ptbxl", split="train")
print(f"PTB-XL kayıt sayısı: {len(ptbxl)}")
 
# Örnek 3: Custom dataset cache yolunu kontrol et
import os
print(f"HF cache: {os.environ.get('HF_HOME', '~/.cache/huggingface')}")
 
# Disk doluyorsa cache'i temizle
# huggingface-cli delete-cache

mkdir -p data/raw/numenta
cd data/raw/numenta
git clone --depth=1 https://github.com/numenta/NAB.git
cd NAB
 
# Veri burası:
# data/realAWSCloudwatch/  — AWS cloud metrics
# data/realKnownCause/     — bilinen sebepli gerçek anomaliler
# data/realTraffic/        — trafik/araç sayma
# data/realTweets/         — tweet volume
# data/artificialNoAnomaly/ — kontrol grubu (anomalisiz)
# data/artificialWithAnomaly/ — sentetik anomalili
 
# Etiketler:
# labels/combined_labels.json   — birleşik etiket dosyası
# labels/combined_windows.json  — anomaly window'lar

# Resmi indirme — biraz manuel
# 1. https://www.mvtec.com/company/research/datasets/mvtec-ad adresine git
# 2. Form doldur (email ile araştırma amaçlı talep)
# 3. Onay sonrası link gelir
 
# Alternatif: Hugging Face üzerinden (resmi mirror)
uv pip install --upgrade datasets
 
python -c "
from datasets import load_dataset
ds = load_dataset('Voxel51/mvtec-ad', split='train', cache_dir='./data/raw/mvtec')
print(f'MVTec train: {len(ds)} sample')
"
 
# Veya anomalib ile (Modül 24'te göstereceğiz)
python -c "
from anomalib.data import MVTec
data = MVTec(
    root='./data/raw/mvtec',
    category='bottle',
    image_size=(256, 256),
)
data.setup()
print(data.train_data)
"

# Komut satırından PhysioNet
uv pip install wfdb
 
# Tam dataset (~75 MB)
python -c "
import wfdb
wfdb.dl_database('mitdb', dl_dir='./data/raw/mit-bih', records='all')
print('İndirme tamam')
"
 
# Veya HuggingFace
python -c "
from datasets import load_dataset
ds = load_dataset('lukestanley/mit-bih-arrhythmia-database')
print(ds)
"

# Kaggle mirror (önerilir)
kaggle datasets download -d brjapon/cwru-bearing-datasets
unzip cwru-bearing-datasets.zip -d data/raw/cwru/
 
# .mat dosyalarını okumak için
uv pip install scipy
 
# Python ile okuma örneği
python -c "
from scipy.io import loadmat
import numpy as np
 
mat = loadmat('data/raw/cwru/Normal_0.mat')
print('Keys:', [k for k in mat.keys() if not k.startswith('__')])
# Genelde 'X097_DE_time' gibi anahtarlar — Drive End sensor
"

# NASA resmi — ZIP ile
mkdir -p data/raw/nasa-turbofan
cd data/raw/nasa-turbofan
wget -O cmapss.zip 'https://data.nasa.gov/download/ff5v-kuh6/application%2Fzip'
unzip cmapss.zip
rm cmapss.zip
 
# 4 alt-dataset gelir: FD001, FD002, FD003, FD004
ls
# train_FD001.txt  test_FD001.txt  RUL_FD001.txt  ...
 
# Kaggle mirror alternatifi
kaggle datasets download -d behrad3d/nasa-cmaps
unzip nasa-cmaps.zip -d data/raw/nasa-turbofan-kaggle/

# Tennessee Eastman Process — alternatif olarak
kaggle datasets download -d averkij/tennessee-eastman-process-simulation-dataset
unzip tennessee-eastman-process-simulation-dataset.zip -d data/raw/tep/

mkdir -p data/raw/loghub
cd data/raw/loghub
git clone --depth=1 https://github.com/logpai/loghub.git
cd loghub
 
# Zip'ler genellikle SourceForge / Zenodo'da; LogHub README'de linkler var
# Önerilen direkt download (Zenodo mirror)
 
# HDFS
wget -O HDFS.zip "https://zenodo.org/records/8196385/files/HDFS_v1.zip"
unzip HDFS.zip -d HDFS/
rm HDFS.zip
 
# BGL
wget -O BGL.zip "https://zenodo.org/records/8196385/files/BGL.zip"
unzip BGL.zip -d BGL/
rm BGL.zip
 
# Bir göz at
head -5 HDFS/HDFS.log
# 081109 203518 143 INFO dfs.DataNode$DataXceiver: Receiving block ...