# Python venv
python -m venv vllm-env
source vllm-env/bin/activate
 
# vLLM kur
pip install vllm
 
# Hugging Face login (Llama-3.1 gated)
huggingface-cli login
 
# Modeli indir + serve
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --enable-prefix-caching \
  --block-size 16 \
  --max-model-len 32768 \
  --port 8000

import time
from openai import OpenAI
 
client = OpenAI(base_url="http://localhost:8000/v1", api_key="EMPTY")
 
LONG_KB = open("knowledge_base.txt").read()  # ~10K token
 
def run_query(system, user):
    start = time.perf_counter()
    resp = client.chat.completions.create(
        model="meta-llama/Llama-3.1-8B-Instruct",
        max_tokens=200,
        messages=[
            {"role": "system", "content": system},
            {"role": "user", "content": user}
        ],
    )
    return time.perf_counter() - start
 
# 100 farklı user query, hep aynı system
USER_QUERIES = [f"Soru {i}: detayları açıkla" for i in range(100)]
 
latencies = []
for q in USER_QUERIES:
    latencies.append(run_query(LONG_KB, q))
 
print(f"İlk istek (cache miss):    {latencies[0]:.2f}s")
print(f"Avg sonraki 99 (cache hit): {sum(latencies[1:])/99:.2f}s")
print(f"Throughput improvement: {latencies[0]/sum(latencies[1:])*99:.1f}×")

import asyncio
from openai import AsyncOpenAI
 
async_client = AsyncOpenAI(base_url="http://localhost:8000/v1", api_key="EMPTY")
 
async def query(system, user):
    resp = await async_client.chat.completions.create(
        model="meta-llama/Llama-3.1-8B-Instruct",
        max_tokens=200,
        messages=[
            {"role": "system", "content": system},
            {"role": "user", "content": user}
        ],
    )
    return resp
 
async def benchmark(concurrent_users):
    # Hepsi aynı system, farklı user query
    tasks = [
        query(LONG_KB, f"Soru {i}")
        for i in range(concurrent_users)
    ]
    start = time.perf_counter()
    await asyncio.gather(*tasks)
    return time.perf_counter() - start
 
# 1, 10, 100 concurrent test
for n in [1, 10, 50, 100]:
    duration = asyncio.run(benchmark(n))
    print(f"{n:>3} concurrent: {duration:.2f}s, {n/duration:.1f} req/s")