TR Reranker FT: bge-reranker + jina-reranker — Pair Generation Recipe

# === bge-reranker-v2-m3 TR FT ===
from sentence_transformers import CrossEncoder, InputExample
from torch.utils.data import DataLoader
 
model = CrossEncoder("BAAI/bge-reranker-v2-m3", device="cuda", max_length=512)
 
# Dataset — (query, doc, label) çiftleri (label: 0 = irrelevant, 1 = relevant)
train_examples = []
for query, pos_doc, neg_docs in tr_dataset:
    train_examples.append(InputExample(texts=[query, pos_doc], label=1.0))
    for neg in neg_docs[:5]:
        train_examples.append(InputExample(texts=[query, neg], label=0.0))
 
train_dataloader = DataLoader(train_examples, batch_size=16, shuffle=True)
 
model.fit(
    train_dataloader=train_dataloader,
    epochs=2,
    warmup_steps=500,
    optimizer_params={"lr": 1e-5},
    output_path="bge-reranker-tr",
)
 
# RAG pipeline'da kullanım:
# 1. Embedding ile top-50 candidate al
# 2. Reranker ile top-50'yi rerank → top-5
# 3. LLM'e top-5'i ver
candidates = retriever.retrieve(query, top_k=50)
scores = reranker.predict([(query, c) for c in candidates])
top5 = [c for c, s in sorted(zip(candidates, scores), key=lambda x: -x[1])[:5]]