TR Corpus Building: mC4-TR + OSCAR-TR + KAPAR + Wikipedia + Common Crawl + Library Scraping

# === TR Corpus Download Pipeline ===
from datasets import load_dataset
 
# 1. mC4-TR (35 GB raw → ~25 GB cleaned)
mc4_tr = load_dataset("allenai/c4", "tr", split="train",
                      streaming=True)   # streaming için 5TB disk gerek olmaz
 
# 2. OSCAR-TR
oscar_tr = load_dataset("oscar-corpus/OSCAR-2301", "tr",
                        split="train", streaming=True)
 
# 3. CulturaX-TR (en kaliteli)
culturax_tr = load_dataset("uonlp/CulturaX", "tr",
                           split="train", streaming=True)
 
# 4. Wikipedia TR
wiki_tr = load_dataset("wikipedia", "20231101.tr", split="train")
 
# 5. KAPAR (TBMM)
# Manuel scrape gerek — https://www.tbmm.gov.tr/develop/owa/tutanak_g.tutanaklar
# Cookbook tarafından sağlanan script: scripts/scrape_kapar.py
 
# Combine + shard
import webdataset as wds
import json, gzip
 
def to_shards(streams, shard_path, max_per_shard=10000):
    shard_idx = 0
    count = 0
    writer = wds.TarWriter(f"{shard_path}/tr-{shard_idx:06d}.tar")
    for s in streams:
        for example in s:
            text = example["text"]
            if 50 < len(text) < 100000:
                writer.write({"__key__": f"{shard_idx:06d}-{count}",
                            "text.txt": text})
                count += 1
                if count >= max_per_shard:
                    writer.close()
                    shard_idx += 1
                    count = 0
                    writer = wds.TarWriter(f"{shard_path}/tr-{shard_idx:06d}.tar")
    writer.close()
 
to_shards([mc4_tr, oscar_tr, culturax_tr, wiki_tr], "/data/tr-shards/")