Matrix Decompositions: Eigendecomposition, SVD, PCA, and the Secret of LoRA

import torch
 
# Basit 2x2 örnek — simetrik
A = torch.tensor([[4., 2.],
                  [2., 3.]])
 
eigenvalues, eigenvectors = torch.linalg.eig(A)
print("Eigenvalues:", eigenvalues.real)         # tensor([5.5616, 1.4384])
print("Eigenvectors:")
print(eigenvectors.real)
# tensor([[ 0.7882, -0.6154],
#         [ 0.6154,  0.7882]])
 
# Doğrulama: A @ v ≈ λ * v
v1 = eigenvectors[:, 0].real
lam1 = eigenvalues[0].real
print("A v1 =", A @ v1)               # tensor([4.3835, 3.4221])
print("λ v1 =", lam1 * v1)            # tensor([4.3835, 3.4221]) ✓
 
# Eigenvalue'lar bize geometrik bilgi verir:
# - Pozitif → eksende uzatma
# - Negatif → ters çevirme
# - Tüm |λ| < 1 → her uygulamada vektör küçülür (contraction)
# - Tüm |λ| = 1 → izometri (uzunluk korunur, örn. rotation)
# - Bazı |λ| > 1 → bazı yönlerde uzatma

import torch
 
# (4, 3) bir matris
torch.manual_seed(0)
A = torch.randn(4, 3)
print("A =")
print(A)
 
U, S, Vh = torch.linalg.svd(A, full_matrices=False)
# Vh = V^T (PyTorch konvansiyonu)
 
print("\nU shape:", U.shape)       # (4, 3)
print("S (singular values):", S)    # (3,)
print("V^T shape:", Vh.shape)        # (3, 3)
 
# A'yı geri inşa et
A_reconstructed = U @ torch.diag(S) @ Vh
print("\nReconstruction error:", (A - A_reconstructed).norm().item())  # ~0
 
# rank-1 dekompozisyona ayır
# A = sigma_1 * u_1 * v_1^T + sigma_2 * u_2 * v_2^T + ...
A_rank1 = S[0] * torch.outer(U[:, 0], Vh[0, :])
A_rank2 = A_rank1 + S[1] * torch.outer(U[:, 1], Vh[1, :])
A_rank3 = A_rank2 + S[2] * torch.outer(U[:, 2], Vh[2, :])
 
print("\nrank-1 yaklaşım hatası:", (A - A_rank1).norm().item())
print("rank-2 yaklaşım hatası:", (A - A_rank2).norm().item())
print("rank-3 yaklaşım hatası:", (A - A_rank3).norm().item())  # ~0

import torch
import matplotlib.pyplot as plt
 
torch.manual_seed(42)
 
# Sentetik veri: 2D Gaussian, x ekseninde daha geniş
N = 200
data = torch.randn(N, 2) * torch.tensor([3.0, 0.5])
# Hafif rotation
theta = 0.5
R = torch.tensor([[torch.cos(torch.tensor(theta)), -torch.sin(torch.tensor(theta))],
                  [torch.sin(torch.tensor(theta)),  torch.cos(torch.tensor(theta))]])
data = data @ R.T
 
# 1. Merkezle
mean = data.mean(dim=0)
X_c = data - mean
 
# 2. SVD
U, S, Vh = torch.linalg.svd(X_c, full_matrices=False)
 
# 3. Principal components
print("Variance per component:", (S ** 2 / (N - 1)))
print("First PC direction:", Vh[0])
print("Second PC direction:", Vh[1])
 
# 4. 1D'ye indir
X_reduced = X_c @ Vh.T[:, 0:1]   # (N, 1)
print("Reduced shape:", X_reduced.shape)
 
# 5. Geri yansıt (lossy)
X_restored = X_reduced @ Vh[0:1, :] + mean
print("Reconstruction error:", (data - X_restored).norm().item())
 
# Görselleştirme (Colab/Jupyter'da)
# plt.scatter(data[:, 0], data[:, 1], alpha=0.5, label="Original")
# plt.scatter(X_restored[:, 0], X_restored[:, 1], alpha=0.5, label="1D Projection")
# plt.legend()
# plt.axis('equal')
# plt.show()

import torch
 
# Bir "doğal" düşük-rank matris simüle et:
# Gerçek rank 5, ama 64x64 boyutunda
torch.manual_seed(0)
A_low = torch.randn(64, 5) @ torch.randn(5, 64)  # rank ≤ 5
A_noise = A_low + 0.1 * torch.randn(64, 64)       # gürültü ekle
 
# SVD
U, S, Vh = torch.linalg.svd(A_noise, full_matrices=False)
 
# Singular value spectrum'unu yazdır
print("İlk 10 singular value:")
print(S[:10].tolist())
# İlk 5 büyük, sonra hızla düşer — "elbow" rank=5 civarında
 
# Truncated reconstruction
def truncate_svd(U, S, Vh, k):
    return U[:, :k] @ torch.diag(S[:k]) @ Vh[:k, :]
 
for k in [1, 3, 5, 10, 20]:
    A_k = truncate_svd(U, S, Vh, k)
    err = (A_noise - A_k).norm() / A_noise.norm()
    print(f"k={k:2d}: relative error = {err*100:.2f}%")
 
# Çıktı (yaklaşık):
# k= 1: relative error = 80.50%
# k= 3: relative error = 40.10%
# k= 5: relative error = 12.40%  <- elbow
# k=10: relative error = 11.95%
# k=20: relative error = 11.85%

import torch
import torch.nn as nn
 
class LoRALinear(nn.Module):
    """Pre-trained Linear üzerine LoRA adapter."""
    def __init__(self, base_layer: nn.Linear, rank: int = 16, alpha: float = 32):
        super().__init__()
        self.base = base_layer
        # Base'i dondur
        for p in self.base.parameters():
            p.requires_grad = False
 
        d_out, d_in = base_layer.weight.shape
        # A: küçük → büyük; init: gaussian
        self.A = nn.Parameter(torch.randn(rank, d_in) * 0.01)
        # B: büyük → küçük; init: sıfır → başta ΔW = 0
        self.B = nn.Parameter(torch.zeros(d_out, rank))
 
        self.rank = rank
        self.alpha = alpha
        self.scaling = alpha / rank
 
    def forward(self, x):
        # Base output
        out = self.base(x)
        # LoRA update: x @ A^T @ B^T (ya da B @ A operasyonunu transpose'lı yaz)
        lora_out = (x @ self.A.T) @ self.B.T
        return out + lora_out * self.scaling
 
# Test
base = nn.Linear(4096, 4096, bias=False)
lora = LoRALinear(base, rank=16)
print("Base params:", sum(p.numel() for p in base.parameters()))           # 16.7M
print("LoRA trainable:", sum(p.numel() for p in lora.parameters() if p.requires_grad))  # 131K
 
x = torch.randn(2, 8, 4096)
y = lora(x)
print(y.shape)   # (2, 8, 4096)
 
# Başlangıçta y = base(x) çünkü B = 0 → ΔW = 0
y_base = base(x)
print(torch.allclose(y, y_base))  # True (eğitim başlangıcında)

import torch
 
# İyi kondisyonlu matris
A_good = torch.randn(100, 100)
print("Good κ:", torch.linalg.cond(A_good).item())   # ~100-1000 (rastgele Gaussian)
 
# Kötü kondisyonlu — son satır birinciye yakın
A_bad = torch.randn(100, 100)
A_bad[-1] = A_bad[0] + 1e-8 * torch.randn(100)  # neredeyse aynı
print("Bad κ:", torch.linalg.cond(A_bad).item())     # 10^7+ — tehlikeli
 
# Hilbert matrisi — klasik kötü kondisyonlu örnek
H = torch.tensor([[1 / (i + j + 1) for j in range(5)] for i in range(5)])
print("Hilbert(5) κ:", torch.linalg.cond(H).item())   # 4.8e5 — zaten 5x5'te kötü