MovieLens from Zero: Schema, EDA, and Efficient Loading with Polars

# data/movielens_loader.py — Reusable MovieLens loader
import polars as pl
from pathlib import Path
from typing import Literal
 
DATA_DIR = Path("data")
 
def load_ml_100k() -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame]:
    """ML-100K yükler. Returns: (ratings, items, users)"""
    base = DATA_DIR / "ml-100k"
 
    ratings = pl.read_csv(
        base / "u.data",
        separator="\t",
        has_header=False,
        new_columns=["user_id", "item_id", "rating", "timestamp"],
        schema_overrides={
            "user_id": pl.Int32,
            "item_id": pl.Int32,
            "rating": pl.Int8,           # 1-5 -> int8 yeter
            "timestamp": pl.Int64,        # Unix epoch
        },
    )
 
    # u.item: 24 sütun, pipe separator
    item_cols = ["item_id", "title", "release_date", "video_release_date", "imdb_url"]
    genre_cols = [
        "unknown", "Action", "Adventure", "Animation", "Children",
        "Comedy", "Crime", "Documentary", "Drama", "Fantasy",
        "Film-Noir", "Horror", "Musical", "Mystery", "Romance",
        "Sci-Fi", "Thriller", "War", "Western"
    ]
 
    items = pl.read_csv(
        base / "u.item",
        separator="|",
        has_header=False,
        new_columns=item_cols + genre_cols,
        encoding="latin-1",   # ascii değil, ML-100K Latin-1
        schema_overrides={"item_id": pl.Int32},
    )
 
    users = pl.read_csv(
        base / "u.user",
        separator="|",
        has_header=False,
        new_columns=["user_id", "age", "gender", "occupation", "zip_code"],
        schema_overrides={"user_id": pl.Int32, "age": pl.Int8},
    )
 
    return ratings, items, users
 
 
def load_ml_1m() -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame]:
    """ML-1M yükler. ::-separated, no header."""
    base = DATA_DIR / "ml-1m"
 
    ratings = pl.read_csv(
        base / "ratings.dat",
        separator="::",
        has_header=False,
        new_columns=["user_id", "item_id", "rating", "timestamp"],
        encoding="latin-1",
    ).with_columns([
        pl.col("user_id").cast(pl.Int32),
        pl.col("item_id").cast(pl.Int32),
        pl.col("rating").cast(pl.Int8),
        pl.col("timestamp").cast(pl.Int64),
    ])
 
    items = pl.read_csv(
        base / "movies.dat",
        separator="::",
        has_header=False,
        new_columns=["item_id", "title", "genres"],
        encoding="latin-1",
    )
 
    users = pl.read_csv(
        base / "users.dat",
        separator="::",
        has_header=False,
        new_columns=["user_id", "gender", "age", "occupation", "zip_code"],
        encoding="latin-1",
    )
 
    return ratings, items, users
 
 
def load_ml_25m_lazy() -> pl.LazyFrame:
    """ML-25M lazy load — 250 MB CSV, RAM'e tam alma."""
    return pl.scan_csv(
        DATA_DIR / "ml-25m" / "ratings.csv",
        schema_overrides={
            "userId": pl.Int32,
            "movieId": pl.Int32,
            "rating": pl.Float32,    # 0.5 step var ML-25M'de
            "timestamp": pl.Int64,
        },
    )
 
 
if __name__ == "__main__":
    # Smoke test
    r100k, i100k, u100k = load_ml_100k()
    print(f"100K ratings shape: {r100k.shape}")
    print(f"100K items:  {i100k.shape}")
    print(f"100K users:  {u100k.shape}")
 
    r1m, i1m, u1m = load_ml_1m()
    print(f"1M ratings shape:  {r1m.shape}")
 
    r25m_lazy = load_ml_25m_lazy()
    print(f"25M row count: {r25m_lazy.select(pl.count()).collect().item():,}")
 

# eda_movielens.py — İlk bakış
import polars as pl
import matplotlib.pyplot as plt
from data.movielens_loader import load_ml_1m
 
ratings, items, users = load_ml_1m()
 
# === 1. Temel istatistikler ===
print(f"Total ratings:     {ratings.height:,}")
print(f"Unique users:      {ratings['user_id'].n_unique():,}")
print(f"Unique items:      {ratings['item_id'].n_unique():,}")
print(f"Rating range:      {ratings['rating'].min()} - {ratings['rating'].max()}")
print(f"Mean rating:       {ratings['rating'].mean():.3f}")
print(f"Sparsity:          {1 - ratings.height / (6040 * 3706):.4%}")
 
# Çıktı:
# Total ratings:     1,000,209
# Unique users:      6,040
# Unique items:      3,706
# Rating range:      1 - 5
# Mean rating:       3.582
# Sparsity:          95.5318%
 
# === 2. Rating dağılımı (J-curve kontrolü) ===
rating_dist = (
    ratings.group_by("rating")
    .agg(pl.count().alias("count"))
    .sort("rating")
)
print(rating_dist)
# shape: (5, 2)
# ┌────────┬────────┐
# │ rating │ count  │
# ├────────┼────────┤
# │ 1      │ 56174  │
# │ 2      │ 107557 │
# │ 3      │ 261197 │
# │ 4      │ 348971 │  ← peak
# │ 5      │ 226310 │
# └────────┴────────┘
 
# === 3. Per-user activity dağılımı (long-tail kontrolü) ===
user_activity = (
    ratings.group_by("user_id")
    .agg(pl.count().alias("n_ratings"))
    .sort("n_ratings", descending=True)
)
print("Top 10 active users:")
print(user_activity.head(10))
 
# Percentile dağılımı
percentiles = [10, 25, 50, 75, 90, 95, 99]
for p in percentiles:
    val = user_activity["n_ratings"].quantile(p / 100)
    print(f"  P{p}: {int(val)} ratings")
# P10: 24, P25: 44, P50: 96, P75: 208, P90: 437, P95: 668, P99: 1466
 
# === 4. Per-item popularity dağılımı (long-tail) ===
item_pop = (
    ratings.group_by("item_id")
    .agg(pl.count().alias("n_ratings"))
    .sort("n_ratings", descending=True)
)
 
# 80/20 testi: top-20% item kaç rating alıyor?
top_20pct_count = item_pop["n_ratings"].head(int(item_pop.height * 0.2)).sum()
share = top_20pct_count / ratings.height
print(f"Top 20% items get {share:.2%} of all ratings")
# Top 20% items get 75% of all ratings — popularity bias gerçek
 
# === 5. Time range ===
from datetime import datetime
min_ts = ratings["timestamp"].min()
max_ts = ratings["timestamp"].max()
print(f"Time range: {datetime.fromtimestamp(min_ts)} → {datetime.fromtimestamp(max_ts)}")
# Time range: 2000-04-25 → 2003-02-28
 
# === 6. Cold user/item sayısı (5'ten az interaction) ===
cold_users = user_activity.filter(pl.col("n_ratings") < 5).height
cold_items = item_pop.filter(pl.col("n_ratings") < 5).height
print(f"Cold users (<5 ratings): {cold_users}")
print(f"Cold items (<5 ratings): {cold_items}")
# Cold users: 0   (ML-1M'de minimum 20 rating filter var)
# Cold items: 285
 

# data/sparse_builder.py — Sparse matrix builder
import numpy as np
import polars as pl
from scipy.sparse import coo_matrix, csr_matrix
 
def ratings_to_sparse(
    ratings: pl.DataFrame,
    n_users: int = None,
    n_items: int = None,
    binarize: bool = False,
) -> tuple[csr_matrix, dict, dict]:
    """
    Polars rating DataFrame'i CSR sparse matrix'e çevirir.
 
    Args:
        ratings: pl.DataFrame with columns [user_id, item_id, rating]
        n_users, n_items: matrix boyutu (None ise max+1)
        binarize: True ise rating'ler 1'e çevrilir (implicit feedback)
 
    Returns:
        R: CSR matrix (n_users × n_items)
        user_to_idx: original user_id → matrix row mapping
        item_to_idx: original item_id → matrix col mapping
    """
    # ID'leri 0-indexli yeniden numaralandır (modeller için kritik)
    unique_users = sorted(ratings["user_id"].unique().to_list())
    unique_items = sorted(ratings["item_id"].unique().to_list())
    user_to_idx = {u: i for i, u in enumerate(unique_users)}
    item_to_idx = {it: i for i, it in enumerate(unique_items)}
 
    if n_users is None:
        n_users = len(unique_users)
    if n_items is None:
        n_items = len(unique_items)
 
    # Vectorized mapping (Polars sayesinde hızlı)
    rows = ratings["user_id"].map_elements(lambda u: user_to_idx[u], return_dtype=pl.Int32).to_numpy()
    cols = ratings["item_id"].map_elements(lambda i: item_to_idx[i], return_dtype=pl.Int32).to_numpy()
 
    if binarize:
        data = np.ones(len(rows), dtype=np.float32)
    else:
        data = ratings["rating"].cast(pl.Float32).to_numpy()
 
    R_coo = coo_matrix((data, (rows, cols)), shape=(n_users, n_items))
    R_csr = R_coo.tocsr()  # CSR'a dönüştür
    R_csr.eliminate_zeros()
 
    return R_csr, user_to_idx, item_to_idx
 
 
# Smoke test
from data.movielens_loader import load_ml_1m
ratings, _, _ = load_ml_1m()
R, u_map, i_map = ratings_to_sparse(ratings)
 
print(f"Sparse matrix: {R.shape}")
print(f"NNZ:           {R.nnz:,}")
print(f"Density:       {R.nnz / (R.shape[0] * R.shape[1]):.4%}")
print(f"Memory (CSR):  {(R.data.nbytes + R.indices.nbytes + R.indptr.nbytes) / 1e6:.1f} MB")
 
# Sparse matrix: (6040, 3706)
# NNZ:           1,000,209
# Density:       4.4682%
# Memory (CSR):  9.1 MB  ← dense olsaydı: 89.6 MB