Online Evaluation: A/B Test, Interleaving, CUPED and Statistical Power

# evaluation/ab_test.py — Sample size calculator
import numpy as np
from scipy import stats
 
def required_sample_size(
    baseline_mean: float,
    baseline_std: float,
    min_detectable_effect: float,  # absolute or relative?
    relative: bool = True,
    alpha: float = 0.05,
    power: float = 0.80,
    two_sided: bool = True,
) -> int:
    """
    Her grup için gereken sample size'ı hesaplar.
 
    Returns: int (sample size per group)
    """
    z_alpha = stats.norm.ppf(1 - alpha / (2 if two_sided else 1))
    z_beta = stats.norm.ppf(power)
 
    if relative:
        delta = baseline_mean * min_detectable_effect
    else:
        delta = min_detectable_effect
 
    n = 2 * ((z_alpha + z_beta) ** 2) * (baseline_std ** 2) / (delta ** 2)
    return int(np.ceil(n))
 
 
# Pratik örnek: CTR 5%'ten 5.05%'e
n = required_sample_size(
    baseline_mean=0.05,
    baseline_std=np.sqrt(0.05 * 0.95),  # Bernoulli std
    min_detectable_effect=0.01,         # 1% relative iyileşme
)
print(f"CTR test için her grup: {n:,} user")
# CTR test için her grup: 6,232,448 user — 12M total, bayağı çok
 
# Yüksek effect ile:
n2 = required_sample_size(
    baseline_mean=0.05,
    baseline_std=np.sqrt(0.05 * 0.95),
    min_detectable_effect=0.10,         # 10% relative iyileşme
)
print(f"CTR (10% effect) test için her grup: {n2:,} user")
# CTR (10% effect) için her grup: 62,325 user — toplam 125K, gerçekçi
 

# evaluation/interleaving.py — Team Draft Interleaving
import random
from typing import Literal
 
def team_draft_interleave(
    list_a: list[int],
    list_b: list[int],
    seed: int = None,
) -> tuple[list[int], list[Literal["A", "B"]]]:
    """
    Team Draft interleaving — Joachims 2003.
 
    Returns:
        interleaved_list: final ranking shown to user
        ownership: each position'ın hangi takıma ait olduğunu söyler
    """
    if seed is not None:
        random.seed(seed)
 
    interleaved = []
    ownership = []
    used = set()
    a_idx, b_idx = 0, 0
 
    # Coin flip — who picks first
    a_first = random.random() < 0.5
 
    while a_idx < len(list_a) and b_idx < len(list_b):
        # A's turn
        if a_first:
            # A picks
            while a_idx < len(list_a) and list_a[a_idx] in used:
                a_idx += 1
            if a_idx < len(list_a):
                interleaved.append(list_a[a_idx])
                ownership.append("A")
                used.add(list_a[a_idx])
                a_idx += 1
            # B picks
            while b_idx < len(list_b) and list_b[b_idx] in used:
                b_idx += 1
            if b_idx < len(list_b):
                interleaved.append(list_b[b_idx])
                ownership.append("B")
                used.add(list_b[b_idx])
                b_idx += 1
        else:
            # B picks first
            while b_idx < len(list_b) and list_b[b_idx] in used:
                b_idx += 1
            if b_idx < len(list_b):
                interleaved.append(list_b[b_idx])
                ownership.append("B")
                used.add(list_b[b_idx])
                b_idx += 1
            while a_idx < len(list_a) and list_a[a_idx] in used:
                a_idx += 1
            if a_idx < len(list_a):
                interleaved.append(list_a[a_idx])
                ownership.append("A")
                used.add(list_a[a_idx])
                a_idx += 1
 
    return interleaved, ownership
 
 
def score_interleaving(
    clicks: list[int],
    ownership_map: dict[int, Literal["A", "B"]],
) -> Literal["A", "B", "tie"]:
    """Click'leri team A ve B arasında say."""
    a_score = sum(1 for c in clicks if ownership_map.get(c) == "A")
    b_score = sum(1 for c in clicks if ownership_map.get(c) == "B")
    if a_score > b_score:
        return "A"
    elif b_score > a_score:
        return "B"
    return "tie"
 
 
# Test
list_a = [101, 102, 103, 104, 105]
list_b = [201, 102, 202, 203, 105]   # 102 ve 105 ortak
 
interleaved, ownership = team_draft_interleave(list_a, list_b, seed=42)
print(f"Interleaved: {interleaved}")
print(f"Ownership:   {ownership}")
 
ownership_map = dict(zip(interleaved, ownership))
clicks = [interleaved[0], interleaved[2]]  # user 1. ve 3. sıraya tıkladı
print(f"Winner: {score_interleaving(clicks, ownership_map)}")