IQR, Tukey's Fences and Adjusted Boxplot: Outlier Detection in Skewed Data

import numpy as np
 
def iqr_outliers(x, k=1.5):
    """
    Tukey's fences ile outlier tespiti.
 
    Returns: outlier indices array, ve sınırlar (low, high)
    """
    q1, q3 = np.percentile(x, [25, 75])
    iqr_value = q3 - q1
    lower = q1 - k * iqr_value
    upper = q3 + k * iqr_value
 
    is_outlier = (x < lower) | (x > upper)
    return np.where(is_outlier)[0], (lower, upper)
 
# Örnek
np.random.seed(42)
data = np.concatenate([
    np.random.normal(50, 5, 100),    # normal
    [80, 85, 90, 15, 10],            # outlier'lar
])
 
idx, bounds = iqr_outliers(data, k=1.5)
print(f"Outlier sayısı: {len(idx)}")
print(f"Sınırlar: ({bounds[0]:.1f}, {bounds[1]:.1f})")
print(f"Outlier değerler: {data[idx]}")

import matplotlib.pyplot as plt
import numpy as np
 
# Üç farklı dağılım simüle et
np.random.seed(42)
normal_data = np.random.normal(50, 10, 1000)
skewed_data = np.random.exponential(scale=20, size=1000) + 30  # right-skewed
heavy_tail = np.concatenate([np.random.normal(50, 5, 950),
                              np.random.normal(50, 30, 50)])  # heavy tail
 
fig, axes = plt.subplots(1, 3, figsize=(15, 5))
axes[0].boxplot(normal_data, vert=True)
axes[0].set_title('Normal Dağılım')
 
axes[1].boxplot(skewed_data, vert=True)
axes[1].set_title('Sağa Skewed (exponential)')
 
axes[2].boxplot(heavy_tail, vert=True)
axes[2].set_title('Heavy-tail')
 
plt.tight_layout()
plt.savefig('reports/boxplots_comparison.png', dpi=120)
plt.show()

from statsmodels.stats.stattools import medcouple
 
def adjusted_boxplot_outliers(x, k=1.5):
    """
    Hubert-Vandervieren adjusted boxplot.
    Skewed veride klasik IQR'a göre çok daha az false positive verir.
    """
    q1, q3 = np.percentile(x, [25, 75])
    iqr_v = q3 - q1
    mc = medcouple(x)
 
    if mc >= 0:
        lower = q1 - k * np.exp(-4 * mc) * iqr_v
        upper = q3 + k * np.exp(3 * mc) * iqr_v
    else:
        lower = q1 - k * np.exp(-3 * mc) * iqr_v
        upper = q3 + k * np.exp(4 * mc) * iqr_v
 
    is_outlier = (x < lower) | (x > upper)
    return np.where(is_outlier)[0], (lower, upper, mc)
 
# Maaş örneği
np.random.seed(42)
salaries = np.concatenate([
    np.random.lognormal(mean=10, sigma=0.5, size=900),  # ~22-180K TL
    [400000, 500000, 800000],  # gerçek outlier'lar (CEO seviyesi)
])
 
# Klasik IQR
idx_iqr, b_iqr = iqr_outliers(salaries, k=1.5)
print(f"Klasik IQR: {len(idx_iqr)} outlier")
 
# Adjusted boxplot
idx_adj, (lo, hi, mc) = adjusted_boxplot_outliers(salaries, k=1.5)
print(f"Adjusted boxplot: {len(idx_adj)} outlier (MC={mc:.3f})")