From-Scratch NumPy Content-Based Recommender: 150 Lines on MovieLens-100K

# step_1_load.py — Veri yükleme + train/test split
import polars as pl
import numpy as np
from pathlib import Path
from datetime import datetime
 
DATA_DIR = Path("data/ml-100k")
 
def load_ratings() -> pl.DataFrame:
    return pl.read_csv(
        DATA_DIR / "u.data",
        separator="\t",
        has_header=False,
        new_columns=["user_id", "item_id", "rating", "timestamp"],
        schema_overrides={
            "user_id": pl.Int32, "item_id": pl.Int32,
            "rating": pl.Int8, "timestamp": pl.Int64
        },
    )
 
def load_items() -> pl.DataFrame:
    genre_cols = [
        "unknown", "Action", "Adventure", "Animation", "Children",
        "Comedy", "Crime", "Documentary", "Drama", "Fantasy",
        "Film-Noir", "Horror", "Musical", "Mystery", "Romance",
        "Sci-Fi", "Thriller", "War", "Western"
    ]
    item_cols = ["item_id", "title", "release_date", "video_release_date", "imdb_url"]
    return pl.read_csv(
        DATA_DIR / "u.item",
        separator="|", has_header=False,
        new_columns=item_cols + genre_cols,
        encoding="latin-1",
        schema_overrides={"item_id": pl.Int32},
    )
 
# Yükle
ratings = load_ratings()
items = load_items()
 
print(f"Ratings: {ratings.height:,}")
print(f"Items:   {items.height:,}")
 
# Time split — son %20 test
threshold = ratings["timestamp"].quantile(0.8)
train = ratings.filter(pl.col("timestamp") < threshold)
test = ratings.filter(pl.col("timestamp") >= threshold)
print(f"Train: {train.height:,} | Test: {test.height:,}")
 
# Cold-item filter (test'te train'de olmayan item'ları çıkar)
train_items_set = set(train["item_id"].unique().to_list())
test_filtered = test.filter(pl.col("item_id").is_in(train_items_set))
print(f"Test after cold filter: {test_filtered.height:,}")
 

# step_2_item_profile.py — Item profil vektörü
import numpy as np
import polars as pl
from sklearn.feature_extraction.text import TfidfVectorizer
from scipy.sparse import hstack, csr_matrix
 
def build_item_profiles(items_df: pl.DataFrame):
    """
    Item profile = genre multi-hot + title TF-IDF.
 
    Returns:
        item_profiles: (n_items, n_features) sparse matrix
        item_id_to_idx: {item_id: row_idx} mapping
    """
    GENRE_COLS = [
        "Action", "Adventure", "Animation", "Children", "Comedy",
        "Crime", "Documentary", "Drama", "Fantasy", "Film-Noir",
        "Horror", "Musical", "Mystery", "Romance", "Sci-Fi",
        "Thriller", "War", "Western", "unknown",
    ]
 
    items_sorted = items_df.sort("item_id")
    item_ids = items_sorted["item_id"].to_list()
    item_id_to_idx = {iid: i for i, iid in enumerate(item_ids)}
 
    # 1) Genre multi-hot
    genre_matrix = items_sorted.select(GENRE_COLS).to_numpy().astype(np.float32)
    print(f"Genre matrix: {genre_matrix.shape}")
 
    # 2) Title TF-IDF (1-2 grams)
    titles = items_sorted["title"].to_list()
    # Yıl etiketini başlıktan çıkar — "Toy Story (1995)" → "Toy Story"
    titles_clean = [t.split(" (")[0] if " (" in t else t for t in titles]
 
    tfidf = TfidfVectorizer(
        lowercase=True,
        ngram_range=(1, 2),
        min_df=2,
        max_features=500,
        stop_words="english",
    )
    title_matrix = tfidf.fit_transform(titles_clean)
    print(f"Title TF-IDF matrix: {title_matrix.shape}, NNZ: {title_matrix.nnz}")
 
    # 3) Concatenate (genre dense + tfidf sparse)
    # Genre matrix'i sparse'a çevir, sonra horizontal stack
    genre_sparse = csr_matrix(genre_matrix)
    item_profiles = hstack([genre_sparse, title_matrix]).tocsr()
    print(f"Combined item profiles: {item_profiles.shape}")
 
    # 4) L2 normalize (cosine similarity için kritik)
    norms = np.array(np.sqrt(item_profiles.multiply(item_profiles).sum(axis=1))).flatten()
    norms[norms == 0] = 1.0  # 0 division önle
    # Sparse'da row-wise division — diag matrix trick
    from scipy.sparse import diags
    inv_norms = diags(1.0 / norms)
    item_profiles_normalized = inv_norms @ item_profiles
 
    return item_profiles_normalized, item_id_to_idx, tfidf, GENRE_COLS
 
 
# Test
item_profiles, item_id_to_idx, _, _ = build_item_profiles(items)
print(f"Item profiles shape: {item_profiles.shape}")
print(f"First item's norm:   {np.sqrt(item_profiles[0].multiply(item_profiles[0]).sum()):.4f}")
# First item's norm: 1.0000  ← L2 normalize çalışıyor
 

# step_3_user_profile.py — User profile vektörü
import numpy as np
from scipy.sparse import csr_matrix, vstack
import polars as pl
 
def build_user_profiles(
    train_df: pl.DataFrame,
    item_profiles: csr_matrix,
    item_id_to_idx: dict,
    rating_threshold: int = 4,
):
    """
    User profile = rating'i >=threshold olan item'ların ortalaması.
 
    Yüksek rating'leri pozitif sinyal, düşükleri exclude (basit yaklaşım).
 
    Returns:
        user_profiles: dict {user_id: profile_vector}
        user_history: dict {user_id: set of item_ids}
    """
    # Pozitif rating'leri filtrele
    positives = train_df.filter(pl.col("rating") >= rating_threshold)
    print(f"Positive interactions (rating >= {rating_threshold}): {positives.height:,}")
 
    user_profiles = {}
    user_history = {}
 
    # User bazında grupla
    grouped = positives.group_by("user_id").agg([
        pl.col("item_id").alias("item_ids"),
        pl.col("rating").alias("ratings"),
    ])
 
    for row in grouped.iter_rows(named=True):
        user_id = row["user_id"]
        item_ids = row["item_ids"]
        ratings = np.array(row["ratings"], dtype=np.float32)
 
        # Item profil vektörlerini al
        item_indices = [item_id_to_idx.get(iid) for iid in item_ids if iid in item_id_to_idx]
        if not item_indices:
            continue
 
        # Rating-weighted average
        item_vecs = item_profiles[item_indices]  # (n_pos, n_features)
        weights = ratings[:len(item_indices)] / ratings[:len(item_indices)].sum()
 
        # Sparse * dense weights → dense user profile
        weighted_sum = (item_vecs.T @ weights.reshape(-1, 1)).flatten()
        # L2 normalize
        norm = np.linalg.norm(weighted_sum)
        if norm > 0:
            weighted_sum /= norm
 
        user_profiles[user_id] = weighted_sum
        user_history[user_id] = set(item_ids)
 
    print(f"Profiles built for {len(user_profiles):,} users")
    return user_profiles, user_history
 
 
# Test
user_profiles, user_history = build_user_profiles(
    train, item_profiles, item_id_to_idx
)
print(f"Example user 1 profile shape: {user_profiles[1].shape}")
print(f"Example user 1 history size:  {len(user_history[1])}")
 

# step_4_scoring.py — Top-N öneri scoring
import numpy as np
from scipy.sparse import csr_matrix
 
def recommend_top_n(
    user_id: int,
    user_profile: np.ndarray,
    item_profiles: csr_matrix,
    item_id_to_idx: dict,
    user_history: set,
    n: int = 10,
) -> list[int]:
    """
    Top-N öneri.
 
    user_profile: (n_features,) dense vector
    item_profiles: (n_items, n_features) sparse matrix
    user_history: izlenmiş item_id'lar (filter out edilecek)
 
    Returns: top N item_id list
    """
    # Cosine similarity: user_profile (dense) @ item_profiles.T (sparse)
    scores = (item_profiles @ user_profile.reshape(-1, 1)).flatten()
 
    # Geçmişi filter et
    idx_to_item = {v: k for k, v in item_id_to_idx.items()}
    for hist_iid in user_history:
        if hist_iid in item_id_to_idx:
            scores[item_id_to_idx[hist_iid]] = -np.inf
 
    # Top-N
    top_indices = np.argpartition(-scores, n)[:n]
    top_indices = top_indices[np.argsort(-scores[top_indices])]
    return [idx_to_item[i] for i in top_indices]
 
 
# Test
example_user = 1
recommendations = recommend_top_n(
    user_id=example_user,
    user_profile=user_profiles[example_user],
    item_profiles=item_profiles,
    item_id_to_idx=item_id_to_idx,
    user_history=user_history[example_user],
    n=10,
)
print(f"Top-10 for user {example_user}: {recommendations}")
 
# Önerinin başlıklarını gör
for rid in recommendations:
    title = items.filter(pl.col("item_id") == rid)["title"][0]
    print(f"  {rid}: {title}")
 

# step_5_evaluation.py — NDCG, Recall, Coverage
import numpy as np
import polars as pl
from collections import defaultdict
 
def dcg(rels: np.ndarray) -> float:
    if len(rels) == 0:
        return 0.0
    discounts = 1.0 / np.log2(np.arange(2, len(rels) + 2))
    return float(np.sum(rels * discounts))
 
def ndcg_at_k(recommended: list[int], gt_set: set[int], k: int = 10) -> float:
    if not gt_set:
        return 0.0
    rels = np.array([1.0 if iid in gt_set else 0.0 for iid in recommended[:k]])
    ideal = np.array([1.0] * min(len(gt_set), k))
    idcg_val = dcg(ideal)
    return dcg(rels) / idcg_val if idcg_val > 0 else 0.0
 
def recall_at_k(recommended: list[int], gt_set: set[int], k: int = 20) -> float:
    if not gt_set:
        return 0.0
    hits = sum(1 for iid in recommended[:k] if iid in gt_set)
    return hits / len(gt_set)
 
def evaluate(
    test_df: pl.DataFrame,
    user_profiles: dict,
    user_history: dict,
    item_profiles,
    item_id_to_idx: dict,
    k: int = 10,
    rating_threshold: int = 4,
):
    """Tüm kullanıcılar için NDCG@k, Recall@k, Coverage hesapla."""
    # Test set'inde her user'ın pozitif item'ları
    test_pos = test_df.filter(pl.col("rating") >= rating_threshold)
    test_gt = defaultdict(set)
    for row in test_pos.iter_rows(named=True):
        test_gt[row["user_id"]].add(row["item_id"])
 
    ndcgs = []
    recalls = []
    all_recommended = set()
 
    for user_id, gt_set in test_gt.items():
        if user_id not in user_profiles:
            continue  # cold user
        recs = recommend_top_n(
            user_id,
            user_profiles[user_id],
            item_profiles,
            item_id_to_idx,
            user_history[user_id],
            n=max(k, 20),
        )
        ndcgs.append(ndcg_at_k(recs, gt_set, k))
        recalls.append(recall_at_k(recs, gt_set, 20))
        all_recommended.update(recs[:k])
 
    return {
        f"NDCG@{k}": float(np.mean(ndcgs)),
        f"Recall@20": float(np.mean(recalls)),
        f"Coverage@{k}": len(all_recommended) / len(item_id_to_idx),
        "Evaluated users": len(ndcgs),
    }
 
 
# Çalıştır
results = evaluate(
    test_filtered,
    user_profiles, user_history,
    item_profiles, item_id_to_idx,
    k=10
)
print("\n📊 Content-Based Recommender Results (MovieLens-100K):")
for metric, value in results.items():
    if isinstance(value, float):
        print(f"  {metric}: {value:.4f}")
    else:
        print(f"  {metric}: {value}")
 
# Beklenen çıktı:
# NDCG@10:        0.0892
# Recall@20:      0.1245
# Coverage@10:    0.3104
# Evaluated users: ~700