Sıfırdan Self-Hosted LLM Observability: ClickHouse + Grafana ile $/Request Dashboard

# docker-compose.yml
version: "3.9"
 
services:
  clickhouse:
    image: clickhouse/clickhouse-server:24.10
    container_name: ch
    ports:
      - "8123:8123"   # HTTP
      - "9000:9000"   # native
    volumes:
      - clickhouse_data:/var/lib/clickhouse
      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
    environment:
      CLICKHOUSE_USER: telemetry
      CLICKHOUSE_PASSWORD: ${CH_PASSWORD}
      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
    ulimits:
      nofile:
        soft: 262144
        hard: 262144
 
  grafana:
    image: grafana/grafana-oss:11.3.0
    container_name: grafana
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana-datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml
      - ./grafana-dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml
      - ./dashboards:/var/lib/grafana/dashboards
    environment:
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD}
      GF_INSTALL_PLUGINS: vertamedia-clickhouse-datasource
 
volumes:
  clickhouse_data:
  grafana_data:

CREATE DATABASE IF NOT EXISTS llm_telemetry;
 
CREATE TABLE IF NOT EXISTS llm_telemetry.requests (
    ts                       DateTime64(3) DEFAULT now64(3),
    request_id               String,
    trace_id                 String,
    user_id                  String,
    tenant_id                String,
    feature                  String,
    team                     String,
    model                    String,
    provider                 String,
 
    input_tokens             UInt32,
    cached_input_tokens      UInt32 DEFAULT 0,
    cache_creation_tokens    UInt32 DEFAULT 0,
    output_tokens            UInt32,
    reasoning_tokens         UInt32 DEFAULT 0,
    tool_tokens              UInt32 DEFAULT 0,
    image_tokens             UInt32 DEFAULT 0,
 
    cost_input_usd           Decimal(10, 6),
    cost_output_usd          Decimal(10, 6),
    cost_total_usd           Decimal(10, 6),
 
    latency_ms               UInt32,
    ttft_ms                  UInt32 DEFAULT 0,
    status_code              UInt16 DEFAULT 200,
    error_type               String DEFAULT '',
 
    cache_hit                Bool DEFAULT false,
    streamed                 Bool DEFAULT false,
    cancelled                Bool DEFAULT false,
 
    raw_metadata             String -- JSON for extra fields
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(ts)
ORDER BY (ts, model, user_id)
TTL ts + INTERVAL 90 DAY;  -- 90 gün veri saklama
 
CREATE INDEX idx_user ON llm_telemetry.requests (user_id) TYPE bloom_filter GRANULARITY 4;
CREATE INDEX idx_feature ON llm_telemetry.requests (feature) TYPE bloom_filter GRANULARITY 4;

# llm_telemetry.py
import os
import json
import uuid
import time
import litellm
from clickhouse_connect import get_client
from datetime import datetime
 
ch_client = get_client(
    host="localhost",
    port=8123,
    username="telemetry",
    password=os.environ["CH_PASSWORD"],
    database="llm_telemetry",
)
 
def telemetry_callback(
    kwargs,
    response,
    start_time,
    end_time,
):
    """LiteLLM her isteğin sonrası çağırır."""
    try:
        usage = response.usage if hasattr(response, "usage") else None
        if not usage:
            return  # error / cancelled
 
        metadata = kwargs.get("metadata", {}) or {}
        litellm_params = kwargs.get("litellm_params", {}) or {}
        model = kwargs.get("model", "")
        provider = model.split("/")[0] if "/" in model else infer_provider(model)
 
        # Maliyet — LiteLLM otomatik hesaplar
        cost = response._hidden_params.get("response_cost", 0)
 
        # Cache hit
        cached = (
            getattr(usage, "prompt_tokens_details", None) and
            getattr(usage.prompt_tokens_details, "cached_tokens", 0)
        ) or (
            getattr(usage, "cache_read_input_tokens", 0)
        ) or (
            getattr(usage, "cached_content_token_count", 0)
        ) or 0
 
        # Reasoning
        reasoning = (
            getattr(usage, "completion_tokens_details", None) and
            getattr(usage.completion_tokens_details, "reasoning_tokens", 0)
        ) or (
            getattr(usage, "thoughts_token_count", 0)
        ) or 0
 
        latency_ms = int((end_time - start_time) * 1000)
 
        ch_client.insert(
            "llm_telemetry.requests",
            [[
                datetime.utcnow(),
                metadata.get("trace_id", str(uuid.uuid4())),
                metadata.get("trace_id", ""),
                metadata.get("user_id", ""),
                metadata.get("tenant_id", ""),
                metadata.get("feature", ""),
                metadata.get("team", ""),
                model,
                provider,
                getattr(usage, "prompt_tokens", 0) or getattr(usage, "input_tokens", 0),
                cached,
                getattr(usage, "cache_creation_input_tokens", 0) or 0,
                getattr(usage, "completion_tokens", 0) or getattr(usage, "output_tokens", 0),
                reasoning,
                0, 0,  # tool, image (extract from metadata gerekiyorsa)
                0, 0, cost,  # input, output cost ayrımı (varsa)
                latency_ms,
                0,  # ttft (stream-only)
                200, "",
                cached > 0,
                kwargs.get("stream", False),
                False,
                json.dumps(metadata),
            ]],
            column_names=[
                "ts","request_id","trace_id","user_id","tenant_id","feature","team","model","provider",
                "input_tokens","cached_input_tokens","cache_creation_tokens","output_tokens","reasoning_tokens",
                "tool_tokens","image_tokens","cost_input_usd","cost_output_usd","cost_total_usd",
                "latency_ms","ttft_ms","status_code","error_type","cache_hit","streamed","cancelled","raw_metadata"
            ],
        )
    except Exception as e:
        print(f"telemetry callback failed: {e}")
 
def infer_provider(model: str) -> str:
    if model.startswith("gpt") or model.startswith("o3"):
        return "openai"
    if model.startswith("claude"):
        return "anthropic"
    if model.startswith("gemini"):
        return "google"
    return "unknown"
 
# LiteLLM'e callback olarak register et
litellm.success_callback = [telemetry_callback]