Streaming Token Sayım Tuzakları: Üretimde Sıkça Karşılaşılan 7 Hata

Hata #1 — Streaming'de Usage Yok Sanmak#

Doğru kullanım#

stream = openai.chat.completions.create(
    model="gpt-5",
    messages=[...],
    stream=True,
    stream_options={"include_usage": True},  # ← MUTLAKA
)

usage = None
for chunk in stream:
    if chunk.choices:
        # content token'ı
        delta = chunk.choices[0].delta
        ...
    if chunk.usage:  # ← son chunk
        usage = chunk.usage

if usage:
    log_cost(usage)

Hata #2 — Cancelled Stream'in Token Maliyeti#

Doğru pattern#

import asyncio
from contextlib import suppress

partial_output_tokens = 0

async def stream_with_cleanup():
    try:
        async for chunk in stream:
            if chunk.choices and chunk.choices[0].delta.content:
                partial_output_tokens += 1  # her chunk yaklaşık 1 token
                yield chunk.choices[0].delta.content
            if chunk.usage:
                log_cost(chunk.usage)
                return
    except asyncio.CancelledError:
        # User cancelled — log partial cost
        estimated_output = partial_output_tokens
        log_partial_cost(
            input_tokens=request.estimated_input,
            output_tokens=estimated_output,
            cancelled=True,
        )
        raise

Hata #3 — Anthropic Stream Event'leri Kaçırmak#

message_start          ← model_id, usage.input_tokens burada
content_block_start    ← her content block için
content_block_delta    ← token-by-token
content_block_stop
message_delta          ← usage.output_tokens burada (cumulative)
message_stop           ← final

Doğru parse#

from anthropic import Anthropic

with client.messages.stream(model="claude-sonnet-4-6", ...) as stream:
    input_tokens = None
    output_tokens = None

    for event in stream:
        if event.type == "message_start":
            input_tokens = event.message.usage.input_tokens
            cache_read = event.message.usage.cache_read_input_tokens or 0
            cache_create = event.message.usage.cache_creation_input_tokens or 0
        elif event.type == "message_delta":
            output_tokens = event.usage.output_tokens  # cumulative
        elif event.type == "content_block_delta":
            print(event.delta.text, end="", flush=True)

    log_cost(input_tokens, output_tokens, cache_read, cache_create)

Hata #4 — Gemini Stream'de Usage Sadece Son Chunk#

stream = client.models.generate_content_stream(
    model="gemini-2.5-pro",
    contents="...",
)

usage = None
full_text = ""
for chunk in stream:
    if chunk.text:
        full_text += chunk.text
    if chunk.usage_metadata:
        usage = chunk.usage_metadata  # genelde son chunk'ta dolu

if usage:
    log_cost(
        prompt=usage.prompt_token_count,
        output=usage.candidates_token_count,
        cached=usage.cached_content_token_count or 0,
        thoughts=usage.thoughts_token_count or 0,
    )

Hata #5 — LiteLLM Stream'inde

stream_options

Eksik#

from litellm import completion

response = completion(
    model="gpt-5",
    messages=[...],
    stream=True,
    stream_options={"include_usage": True},  # ← litellm de aynı opsiyona muhtaç
)

for chunk in response:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="", flush=True)
    if chunk.usage:
        log_cost(chunk.usage)

Hata #6 — Idle Timeout Sırasında Stream Kopması#

Çözüm#

location /api/llm {
    proxy_read_timeout 600s;
    proxy_send_timeout 600s;
    proxy_buffering off;       # ← SSE/stream için kritik
    proxy_cache off;
}

• Free plan: idle timeout sabit (100s) — work-around için keepalive event göndermen gerek
• Enterprise plan: timeout yapılandırılabilir

Hata #7 — Tool Call Stream'inde Double Counting#

Doğru muhasebe#

# Agent loop:
# 1. LLM çağrı 1 → tool_use_request → usage 1 logla
# 2. Tool execute (telemetry'ye yazma — bu LLM değil)
# 3. LLM çağrı 2 (tool result eklenmiş) → usage 2 logla
# 4. ...

# Agent toplam = usage 1 + usage 2 + ...
# Her iteration ayrı log, "total" iş yükünün toplamı