Hands-on Lab: 4 Sampling Strategy Benchmark on IEEE-CIS Fraud Data

import numpy as np
import pandas as pd
import warnings
warnings.filterwarnings('ignore')
 
DATA_DIR = '../data/raw/ieee-cis-fraud/'
 
# Veri yükleme — büyük dosyalar (~600 MB toplam)
print("Veri yükleniyor...")
trans = pd.read_csv(DATA_DIR + 'train_transaction.csv')
identity = pd.read_csv(DATA_DIR + 'train_identity.csv')
 
# Birleştir
df = trans.merge(identity, on='TransactionID', how='left')
 
print(f"Toplam satır: {len(df):,}")
print(f"Toplam feature: {df.shape[1]}")
print(f"Fraud oranı: {df['isFraud'].mean()*100:.2f}%")
print(f"Fraud sayısı: {df['isFraud'].sum():,}")
print(f"Normal sayısı: {(df['isFraud']==0).sum():,}")

# Time-based features
df['hour'] = (df['TransactionDT'] / 3600) % 24
df['day'] = (df['TransactionDT'] / 86400) % 7
 
# Log-transform tutar (right-skewed)
df['log_amount'] = np.log1p(df['TransactionAmt'])
 
# Email domain feature'ları
for col in ['P_emaildomain', 'R_emaildomain']:
    if col in df.columns:
        df[col + '_is_null'] = df[col].isnull().astype(int)
        df[col] = df[col].fillna('unknown')
 
# Kategorik kodlama (basit label encode)
from sklearn.preprocessing import LabelEncoder
cat_cols = df.select_dtypes(include=['object']).columns
for col in cat_cols:
    if df[col].nunique() < 5000:
        df[col] = LabelEncoder().fit_transform(df[col].astype(str))
    else:
        df = df.drop(col, axis=1)
 
# NaN doldur
df = df.fillna(-999)
 
print(f"Sonuç şekil: {df.shape}")
 
# Train/Val split (zaman-bazlı)
df = df.sort_values('TransactionDT').reset_index(drop=True)
split_idx = int(0.8 * len(df))
train_df = df.iloc[:split_idx]
val_df = df.iloc[split_idx:]
 
X_train = train_df.drop(['isFraud', 'TransactionID', 'TransactionDT'], axis=1, errors='ignore')
y_train = train_df['isFraud']
X_val = val_df.drop(['isFraud', 'TransactionID', 'TransactionDT'], axis=1, errors='ignore')
y_val = val_df['isFraud']
 
print(f"Train: {len(X_train):,}, Val: {len(X_val):,}")
print(f"Train fraud oranı: {y_train.mean()*100:.2f}%")
print(f"Val fraud oranı:   {y_val.mean()*100:.2f}%")

import xgboost as xgb
from sklearn.metrics import (average_precision_score, roc_auc_score,
                              precision_recall_curve, recall_score, precision_score)
import time
 
def evaluate(model, X_val, y_val, name):
    """Standart değerlendirme suite."""
    probs = model.predict_proba(X_val)[:, 1]
    preds = (probs > 0.5).astype(int)
 
    return {
        'Strategy': name,
        'PR-AUC':   average_precision_score(y_val, probs),
        'ROC-AUC':  roc_auc_score(y_val, probs),
        'Recall@0.5':    recall_score(y_val, preds),
        'Precision@0.5': precision_score(y_val, preds),
        'Recall@top1%':  recall_at_k(y_val, probs, 0.01),
        'Recall@top5%':  recall_at_k(y_val, probs, 0.05),
        'probs': probs,
    }
 
def recall_at_k(y_true, probs, k):
    """Top-k%'ye karşılık gelen recall."""
    threshold = np.percentile(probs, 100 - k * 100)
    preds = (probs >= threshold).astype(int)
    return recall_score(y_true, preds)
 
# Strateji 1: Baseline XGBoost
print("\n=== Strateji 1: Baseline ===")
t0 = time.time()
clf1 = xgb.XGBClassifier(
    n_estimators=300, max_depth=6, learning_rate=0.1,
    random_state=42, n_jobs=-1, eval_metric='aucpr'
)
clf1.fit(X_train, y_train)
print(f"Eğitim süresi: {time.time()-t0:.1f}s")
 
results = [evaluate(clf1, X_val, y_val, 'Baseline')]
print(f"PR-AUC: {results[0]['PR-AUC']:.4f}")

from imblearn.over_sampling import SMOTE
 
print("\n=== Strateji 2: SMOTE ===")
t0 = time.time()
 
# SMOTE (1:5 hedef oran)
smote = SMOTE(sampling_strategy=0.2, k_neighbors=5, random_state=42)
X_train_smote, y_train_smote = smote.fit_resample(X_train, y_train)
print(f"SMOTE sonrası: {len(X_train_smote):,} satır, fraud oranı: {y_train_smote.mean()*100:.2f}%")
 
clf2 = xgb.XGBClassifier(
    n_estimators=300, max_depth=6, learning_rate=0.1,
    random_state=42, n_jobs=-1, eval_metric='aucpr'
)
clf2.fit(X_train_smote, y_train_smote)
print(f"Eğitim süresi: {time.time()-t0:.1f}s")
 
results.append(evaluate(clf2, X_val, y_val, 'SMOTE (1:5)'))
print(f"PR-AUC: {results[-1]['PR-AUC']:.4f}")

print("\n=== Strateji 3: class_weight (scale_pos_weight) ===")
t0 = time.time()
 
# scale_pos_weight = N_negative / N_positive
n_neg, n_pos = (y_train == 0).sum(), (y_train == 1).sum()
scale_pos_weight = n_neg / n_pos
print(f"scale_pos_weight: {scale_pos_weight:.1f}")
 
clf3 = xgb.XGBClassifier(
    n_estimators=300, max_depth=6, learning_rate=0.1,
    scale_pos_weight=scale_pos_weight,
    random_state=42, n_jobs=-1, eval_metric='aucpr'
)
clf3.fit(X_train, y_train)
print(f"Eğitim süresi: {time.time()-t0:.1f}s")
 
results.append(evaluate(clf3, X_val, y_val, 'class_weight'))
print(f"PR-AUC: {results[-1]['PR-AUC']:.4f}")

def focal_loss_obj(alpha=0.25, gamma=2.0):
    """
    XGBoost custom objective: focal loss.
    Lin et al., 2017 — sigmoid focal.
    """
    def fobj(preds, dtrain):
        labels = dtrain.get_label()
        # XGBoost preds = log-odds; sigmoid'le
        p = 1.0 / (1.0 + np.exp(-preds))
        # Focal weight
        p_t = p * labels + (1 - p) * (1 - labels)
        alpha_t = alpha * labels + (1 - alpha) * (1 - labels)
        focal = alpha_t * (1 - p_t) ** gamma
 
        # Gradient ve hessian (focal CE)
        grad = focal * (p - labels)
        hess = focal * p * (1 - p)
        return grad, hess
    return fobj
 
print("\n=== Strateji 4: Focal Loss ===")
t0 = time.time()
 
dtrain = xgb.DMatrix(X_train, label=y_train)
dval = xgb.DMatrix(X_val, label=y_val)
 
params = {
    'max_depth': 6,
    'eta': 0.1,
    'verbosity': 1,
    'tree_method': 'hist',
    'eval_metric': 'aucpr',
}
 
booster = xgb.train(
    params=params,
    dtrain=dtrain,
    num_boost_round=300,
    obj=focal_loss_obj(alpha=0.25, gamma=2.0),
    evals=[(dval, 'val')],
    verbose_eval=50,
)
print(f"Eğitim süresi: {time.time()-t0:.1f}s")
 
# Custom objective için predict_proba yok — kendi yapıyoruz
probs4 = 1.0 / (1.0 + np.exp(-booster.predict(dval)))
preds4 = (probs4 > 0.5).astype(int)
 
results.append({
    'Strategy':     'Focal Loss',
    'PR-AUC':       average_precision_score(y_val, probs4),
    'ROC-AUC':      roc_auc_score(y_val, probs4),
    'Recall@0.5':   recall_score(y_val, preds4),
    'Precision@0.5':precision_score(y_val, preds4),
    'Recall@top1%': recall_at_k(y_val, probs4, 0.01),
    'Recall@top5%': recall_at_k(y_val, probs4, 0.05),
    'probs':        probs4,
})
print(f"PR-AUC: {results[-1]['PR-AUC']:.4f}")

# Sonuç tablosu
summary = pd.DataFrame([{k: v for k, v in r.items() if k != 'probs'} for r in results])
print("\n" + "=" * 70)
print(summary.round(4).to_string(index=False))
print("=" * 70)

# Sektör gerçeği: ortalama fraud kaybı + analist saati maliyeti
COST_FN = 2500   # ortalama fraud kaybı TL
COST_FP = 100    # analist 3 dakikası TL
 
def cost_analysis(y_true, probs, name, k_percent=1.0):
    """Top-k% incelendiğinde yıllık beklenen maliyet."""
    threshold = np.percentile(probs, 100 - k_percent)
    preds = (probs >= threshold).astype(int)
 
    tp = ((preds == 1) & (y_true == 1)).sum()
    fp = ((preds == 1) & (y_true == 0)).sum()
    fn = ((preds == 0) & (y_true == 1)).sum()
 
    cost = fn * COST_FN + fp * COST_FP
    # Validation 20% verinin
    annual_cost = cost / 0.2 * 12  # 1 yıllık extrapolation
 
    return {
        'Strategy': name,
        'TP': tp, 'FP': fp, 'FN': fn,
        'Cost (val)': f"{cost:,.0f} TL",
        'Annual Est.': f"{annual_cost/1e6:.1f}M TL",
    }
 
print("\n=== Top-1% incelendiğinde yıllık beklenen maliyet ===")
for r in results:
    print(cost_analysis(y_val.values, r['probs'], r['Strategy'], k_percent=1))

import matplotlib.pyplot as plt
 
fig, ax = plt.subplots(figsize=(10, 7))
colors = ['steelblue', 'orange', 'green', 'crimson']
 
for r, color in zip(results, colors):
    p, rec, _ = precision_recall_curve(y_val, r['probs'])
    ap = r['PR-AUC']
    ax.plot(rec, p, color=color, lw=2, label=f"{r['Strategy']} (AP={ap:.3f})")
 
# Baseline (random)
baseline_ap = y_val.mean()
ax.axhline(y=baseline_ap, color='gray', linestyle='--', alpha=0.5,
           label=f'Random ({baseline_ap:.3f})')
 
ax.set_xlabel('Recall')
ax.set_ylabel('Precision')
ax.set_title('IEEE-CIS Fraud — 4 Strateji PR Curve Karşılaştırması')
ax.legend(loc='upper right')
ax.grid(alpha=0.3)
plt.tight_layout()
plt.savefig('reports/ieee_cis_pr_curves.png', dpi=120)
plt.show()