TensorRT-LLM: NVIDIA Native Engine — INT8 SmoothQuant + FP8 + In-Flight Batching

# === TensorRT-LLM Engine build (Llama 3.1 8B FP8) ===
# 1. TRT-LLM container
docker run --gpus all -it --rm \
    -v $PWD:/workspace \
    nvcr.io/nvidia/tensorrt:24.10-py3 bash
 
# 2. Install
pip install tensorrt-llm
 
# 3. Convert HF → TRT-LLM checkpoint
python convert_checkpoint.py \
    --model_dir meta-llama/Meta-Llama-3.1-8B-Instruct \
    --output_dir trt_ckpt \
    --dtype bfloat16 \
    --use_fp8 \
    --calib_dataset c4 \
    --calib_size 512
 
# 4. Build engine — 1-2 saat RTX 4090
trtllm-build \
    --checkpoint_dir trt_ckpt \
    --output_dir trt_engine \
    --gemm_plugin auto \
    --gpt_attention_plugin auto \
    --max_batch_size 32 \
    --max_input_len 4096 \
    --max_seq_len 8192 \
    --max_num_tokens 16384 \
    --use_paged_kv_cache enable \
    --use_fp8 enable
 
# 5. Inference
python run.py --engine_dir trt_engine \
    --max_output_len 500 \
    --input_text "Türkiye'nin başkenti?"