Custom Stack FT Lab: Repo-Tuned Model on Mid-Size Repo (~50K LoC)

# === Custom repo dataset üretimi ===
import os, json
 
def walk_repo(repo_path, include_exts=(".py", ".ts", ".tsx", ".js")):
    samples = []
    for root, dirs, files in os.walk(repo_path):
        dirs[:] = [d for d in dirs if not d.startswith(".") and d != "node_modules"]
        for f in files:
            if f.endswith(include_exts):
                path = os.path.join(root, f)
                with open(path) as fp:
                    code = fp.read()
                rel = os.path.relpath(path, repo_path)
                # Each file: path + content header
                samples.append({
                    "text": f"# File: {rel}\n\n{code}",
                })
    return samples
 
samples = walk_repo("/path/to/my-repo")
# 50K LoC repo → ~3000-5000 file samples
 
# Save as HF dataset
from datasets import Dataset
ds = Dataset.from_list(samples)
ds.save_to_disk("custom-repo-ds")
 
# Then: FIM transform + Qwen2.5-Coder 7B FT (8.2'deki gibi)