LoRA Hot-Swap Lab: Tek Base + N Adapter — Tek 4090'da 50 Müşteri Servisi

# === vLLM LoRA serving ===
from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest
 
llm = LLM(
    model="meta-llama/Meta-Llama-3.1-8B-Instruct",
    enable_lora=True,
    max_loras=8,                              # GPU'da aynı anda max LoRA sayısı
    max_lora_rank=64,                         # destekli en yüksek rank
    max_cpu_loras=32,                         # CPU RAM'de tutulan LoRA
    gpu_memory_utilization=0.92,
)
 
# Adapter pool — disk'te 30 LoRA
lora_paths = {
    "customer-01": "/loras/cust01",
    "customer-02": "/loras/cust02",
    # ... 30 müşteri
}
 
# Request handling
def handle_request(customer_id, prompt):
    lora = LoRARequest(
        lora_name=customer_id,
        lora_int_id=hash(customer_id) % 10000,
        lora_path=lora_paths[customer_id],
    )
    sampling = SamplingParams(temperature=0.7, max_tokens=500)
    output = llm.generate([prompt], sampling, lora_request=lora)
    return output[0].outputs[0].text
 
# Usage
response = handle_request("customer-01", "Türkçe sohbet öneri ver")