Item Profiling: TF-IDF, BM25, n-grams, and Categorical Feature Encoding — Math + NumPy

# vectorizers/tfidf.py — Sıfırdan NumPy TF-IDF
import numpy as np
from collections import Counter
from typing import Iterable
 
class NumpyTfIdfVectorizer:
    """
    Sıfırdan NumPy TF-IDF. Eğitsel amaçlı — production'da sklearn kullanın.
 
    Args:
        min_df: Bir term en az kaç dokümanda geçmeli (vocabulary filter)
        max_df: Bir term en fazla yüzde kaç dokümanda geçmeli
        smooth_idf: IDF'te +1 yapısı
        sublinear_tf: log(1 + tf) kullan (smooth TF)
    """
 
    def __init__(
        self,
        min_df: int = 1,
        max_df: float = 1.0,
        smooth_idf: bool = True,
        sublinear_tf: bool = False,
    ):
        self.min_df = min_df
        self.max_df = max_df
        self.smooth_idf = smooth_idf
        self.sublinear_tf = sublinear_tf
        self.vocabulary_ = {}  # term -> index
        self.idf_ = None        # numpy array
 
    def _tokenize(self, doc: str) -> list[str]:
        """Simple whitespace + lowercase tokenizer."""
        return doc.lower().split()
 
    def fit(self, documents: list[str]) -> "NumpyTfIdfVectorizer":
        # 1) Vocabulary + document frequency
        df_counter = Counter()
        for doc in documents:
            unique_terms = set(self._tokenize(doc))
            df_counter.update(unique_terms)
 
        N = len(documents)
 
        # 2) Filter by min_df and max_df
        filtered = {}
        for term, df in df_counter.items():
            if df < self.min_df:
                continue
            if df / N > self.max_df:
                continue
            filtered[term] = df
 
        # 3) Build vocabulary (sorted for determinism)
        terms_sorted = sorted(filtered.keys())
        self.vocabulary_ = {t: i for i, t in enumerate(terms_sorted)}
 
        # 4) Compute IDF
        idf = np.zeros(len(terms_sorted), dtype=np.float64)
        for term, idx in self.vocabulary_.items():
            df = filtered[term]
            if self.smooth_idf:
                idf[idx] = np.log((N + 1) / (df + 1)) + 1
            else:
                idf[idx] = np.log(N / df) + 1
        self.idf_ = idf
 
        return self
 
    def transform(self, documents: list[str]) -> np.ndarray:
        """
        Returns dense (n_docs, vocab_size) TF-IDF matrix.
        Production'da sparse matrix kullan!
        """
        n_docs = len(documents)
        vocab_size = len(self.vocabulary_)
        X = np.zeros((n_docs, vocab_size), dtype=np.float64)
 
        for doc_idx, doc in enumerate(documents):
            tokens = self._tokenize(doc)
            counts = Counter(tokens)
 
            for term, count in counts.items():
                if term not in self.vocabulary_:
                    continue
                term_idx = self.vocabulary_[term]
 
                # TF
                if self.sublinear_tf:
                    tf = np.log(1 + count)
                else:
                    tf = float(count)
 
                # TF * IDF
                X[doc_idx, term_idx] = tf * self.idf_[term_idx]
 
            # L2 normalize
            norm = np.linalg.norm(X[doc_idx])
            if norm > 0:
                X[doc_idx] /= norm
 
        return X
 
    def fit_transform(self, documents: list[str]) -> np.ndarray:
        return self.fit(documents).transform(documents)
 
 
# Test — yukarıdaki örneği reproduce et
docs = [
    "machine learning",
    "deep learning",
    "machine deep model",
]
vec = NumpyTfIdfVectorizer(smooth_idf=False)
X = vec.fit_transform(docs)
print(f"Vocabulary: {vec.vocabulary_}")
print(f"IDF:        {vec.idf_}")
print(f"TF-IDF:\n{X}")
 
# Cosine similarity
from numpy.linalg import norm
def cosine(a, b):
    return float(np.dot(a, b) / (norm(a) * norm(b)))
print(f"D1 vs D3: {cosine(X[0], X[2]):.3f}")
# D1 vs D3: 0.341  ← yukarıdaki manuel hesapla aynı
 
# sklearn karşılaştırması
from sklearn.feature_extraction.text import TfidfVectorizer
skl = TfidfVectorizer(token_pattern=r"\S+", smooth_idf=False)
X_skl = skl.fit_transform(docs).toarray()
print(f"sklearn TF-IDF:\n{X_skl}")
# Yaklaşık aynı (sklearn'ün default tokenizer'ı farklı olduğu için tam aynı değil)
 

# vectorizers/bm25.py — Sıfırdan BM25
import numpy as np
from collections import Counter
 
class NumpyBM25Vectorizer:
    """Okapi BM25 — Robertson 1994."""
 
    def __init__(self, k1: float = 1.5, b: float = 0.75):
        self.k1 = k1
        self.b = b
        self.vocabulary_ = {}
        self.idf_ = None
        self.avg_doc_len_ = None
        self.doc_lens_ = None  # her doc'un uzunluğu (fit sonrası set)
 
    def _tokenize(self, doc: str) -> list[str]:
        return doc.lower().split()
 
    def fit(self, documents: list[str]):
        df_counter = Counter()
        doc_lens = []
        for doc in documents:
            tokens = self._tokenize(doc)
            doc_lens.append(len(tokens))
            df_counter.update(set(tokens))
 
        N = len(documents)
        terms_sorted = sorted(df_counter.keys())
        self.vocabulary_ = {t: i for i, t in enumerate(terms_sorted)}
 
        # BM25 IDF
        idf = np.zeros(len(terms_sorted), dtype=np.float64)
        for term, idx in self.vocabulary_.items():
            df = df_counter[term]
            idf[idx] = np.log((N - df + 0.5) / (df + 0.5))
        self.idf_ = idf
 
        self.doc_lens_ = np.array(doc_lens)
        self.avg_doc_len_ = self.doc_lens_.mean()
 
        return self
 
    def transform(self, documents: list[str]) -> np.ndarray:
        n_docs = len(documents)
        vocab_size = len(self.vocabulary_)
        X = np.zeros((n_docs, vocab_size), dtype=np.float64)
 
        for doc_idx, doc in enumerate(documents):
            tokens = self._tokenize(doc)
            counts = Counter(tokens)
            doc_len = len(tokens)
 
            for term, count in counts.items():
                if term not in self.vocabulary_:
                    continue
                term_idx = self.vocabulary_[term]
                idf = self.idf_[term_idx]
 
                # BM25 score
                numerator = count * (self.k1 + 1)
                denominator = count + self.k1 * (
                    1 - self.b + self.b * doc_len / self.avg_doc_len_
                )
                X[doc_idx, term_idx] = idf * numerator / denominator
 
        return X
 
    def fit_transform(self, documents: list[str]):
        return self.fit(documents).transform(documents)
 
 
# Karşılaştırma
docs = [
    "machine learning algorithm",
    "deep learning neural network",
    "machine learning machine learning machine learning",  # TF spam
]
 
tfidf = NumpyTfIdfVectorizer()
X_tfidf = tfidf.fit_transform(docs)
 
bm25 = NumpyBM25Vectorizer()
X_bm25 = bm25.fit_transform(docs)
 
# D2'nin (deep neural) D3 (machine spam)'ye benzerliği TF-IDF'te yüksek görünebilir
# çünkü D3 çok "machine learning" var
# BM25'te satürasyon var, dengeli sonuç verir