vLLM 生产级部署完全指南

flowchart TD
    A["客户端请求"] --> B["API Server<br/>(OpenAI兼容)"]
    B --> C["调度器 Scheduler"]
    C --> D["连续批处理引擎"]
    D --> E["PagedAttention<br/>KV Cache管理"]
    E --> F["GPU执行器"]
    F --> G["模型权重"]
    E --> H["CPU Swap<br/>显存卸载"]
    style C fill:#dbeafe,stroke:#2563eb
    style E fill:#fef3c7,stroke:#f59e0b

# 方式一：pip 安装（推荐生产环境）
pip install vllm==0.24.0

# 方式二：conda 安装
conda install -c conda-forge vllm

# 方式三：源码编译（最新特性）
git clone https://github.com/vllm-project/vllm.git
cd vllm
pip install -e .

# 验证安装
python -c "import vllm; print(vllm.__version__)"

# 本地开发快速启动
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --host 127.0.0.1 \
  --port 8000 \
  --dtype bfloat16

# 单机多卡（默认，使用 multiprocessing）
vllm serve meta-llama/Llama-3-70b -tp 4

# 多机多卡（使用 Ray 后端）
# 1. 在所有节点启动 Ray
ray start --head --node-ip-address=192.168.1.100  # 主节点
ray start --address=192.168.1.100:6379             # 工作节点

# 2. 启动 vLLM（在主节点）
vllm serve meta-llama/Llama-3-405b \
  -tp 8 -pp 4 \
  --distributed-executor-backend ray \
  --pipeline-parallel-size 4

flowchart LR
    subgraph 传统方式
        A["请求1<br/>预留2048token"] --> A1["已用512<br/>剩余1536<br/>(浪费)"]
        B["请求2<br/>预留2048token"] --> B1["已用1024<br/>剩余1024<br/>(浪费)"]
    end
    subgraph PagedAttention
        C["请求1<br/>512token"] --> C1["块1<br/>16token"]
        C1 --> C2["块2<br/>16token"]
        C2 --> C3["..."]
        D["请求2<br/>1024token"] --> D1["块N<br/>16token"]
        D1 --> D2["..."]
        style C1 fill:#d1fae5
        style C2 fill:#d1fae5
        style D1 fill:#dbeafe
    end

# 前缀缓存优化配置
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --enable-prefix-caching \
  --block-size 16 \
  --kv-cache-dtype fp8 \
  --gpu-memory-utilization 0.9

# 长文本场景配置（长文档总结、RAG）
vllm serve Qwen/Qwen2.5-72B-Instruct \
  -tp 4 \
  --max-model-len 131072 \
  --enable-chunked-prefill \
  --max-num-batched-tokens 16384 \
  --max-num-seqs 16 \
  --max-num-partial-prefills 2

--max-num-batched-tokens 32768
--max-num-seqs 512
--gpu-memory-utilization 0.95

--max-num-batched-tokens 8192
--max-num-seqs 64
--enable-chunked-prefill
--max-num-partial-prefills 2

vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --enable-lora \
  --max-loras 4 \
  --max-lora-rank 64 \
  --max-cpu-loras 16 \
  --lora-modules \
    medical=/data/lora/medical_lora \
    legal=/data/lora/legal_lora \
    code=/data/lora/code_lora

curl http://localhost:8000/v1/chat/completions \
  -H "Authorization: Bearer sk-xxx" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "medical",
    "messages": [{"role": "user", "content": "头疼怎么办？"}]
  }'

# 推测解码示例
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  -tp 4 \
  --speculative-model meta-llama/Llama-3.2-1B-Instruct \
  --num-speculative-tokens 5 \
  --speculative-max-model-len 4096 \
  --dtype bfloat16

# Qwen2-VL 部署示例
vllm serve Qwen/Qwen2-VL-7B-Instruct \
  --dtype bfloat16 \
  --max-model-len 8192 \
  --limit-mm-per-prompt image=4,video=1 \
  --mm-processor-kwargs '{"num_crops": 4}'

# 支持工具调用的模型无需特殊启动参数，正常启动即可
vllm serve Qwen/Qwen2.5-14B-Instruct \
  --dtype bfloat16 \
  --max-model-len 32768

# O3 极致优化配置
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  -O3 \
  --compilation-config '{"mode": 3, "cudagraph_capture_sizes": [1,2,4,8,16,32,64,128]}'

# 拉取官方镜像（替换对应 CUDA 版本）
docker pull vllm/vllm-openai:v0.24.0

# 支持的镜像标签：
# v0.24.0 (CUDA 12.4)
# latest (跟踪最新 release)

version: '3.8'

services:
  vllm:
    image: vllm/vllm-openai:v0.24.0
    container_name: vllm-server
    runtime: nvidia
    ipc: host
    ports:
      - "8000:8000"
    environment:
      - VLLM_DEVICE_IDS=0,1,2,3
      - NCCL_SOCKET_IFNAME=eth0
      - VLLM_WORKER_MULTIPROC_METHOD=spawn
    volumes:
      - /data/models:/root/.cache/huggingface/hub
      - /data/lora:/app/lora
    command: >
      --model /root/.cache/huggingface/hub/models--meta-llama--Llama-3.1-70b
      -tp 4
      --dtype bfloat16
      --max-model-len 32768
      --gpu-memory-utilization 0.9
      --kv-cache-dtype fp8
      --enable-prefix-caching
      --enable-chunked-prefill
      --api-key sk-your-production-key
      --host 0.0.0.0
      --port 8000
      -O2
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 120s
    logging:
      driver: json-file
      options:
        max-size: "100m"
        max-file: "5"

FROM vllm/vllm-openai:v0.24.0

# 安装自定义依赖
RUN pip install --no-cache-dir \
    prometheus-client \
    langchain \
    sentence-transformers

# 复制自定义配置
COPY config.yaml /app/config.yaml

# 健康检查脚本
COPY healthcheck.sh /app/healthcheck.sh
RUN chmod +x /app/healthcheck.sh

EXPOSE 8000

ENTRYPOINT ["vllm", "serve"]
CMD ["--config", "/app/config.yaml"]

apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-llama3-70b
  namespace: llm
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: vllm-llama3-70b
  template:
    metadata:
      labels:
        app: vllm-llama3-70b
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8000"
        prometheus.io/path: "/metrics"
    spec:
      containers:
      - name: vllm
        image: vllm/vllm-openai:v0.24.0
        command: ["vllm", "serve"]
        args:
        - "meta-llama/Llama-3.1-70B-Instruct"
        - "-tp"
        - "4"
        - "--dtype"
        - "bfloat16"
        - "--max-model-len"
        - "32768"
        - "--gpu-memory-utilization"
        - "0.9"
        - "--api-key"
        - "$(API_KEY)"
        - "--host"
        - "0.0.0.0"
        - "--port"
        - "8000"
        - "-O2"
        ports:
        - containerPort: 8000
        env:
        - name: API_KEY
          valueFrom:
            secretKeyRef:
              name: vllm-secrets
              key: api-key
        - name: NCCL_SOCKET_IFNAME
          value: eth0
        - name: VLLM_WORKER_MULTIPROC_METHOD
          value: spawn
        - name: VLLM_DEVICE_IDS  # v0.24+ 推荐方式（替代 CUDA_VISIBLE_DEVICES）
          value: "0,1,2,3"
        resources:
          limits:
            nvidia.com/gpu: 4
            cpu: "32"
            memory: "128Gi"
          requests:
            nvidia.com/gpu: 4
            cpu: "16"
            memory: "64Gi"
        volumeMounts:
        - name: model-cache
          mountPath: /root/.cache/huggingface
        livenessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 180
          periodSeconds: 30
          timeoutSeconds: 10
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 120
          periodSeconds: 10
      volumes:
      - name: model-cache
        persistentVolumeClaim:
          claimName: model-cache-pvc
      terminationGracePeriodSeconds: 300
---
apiVersion: v1
kind: Service
metadata:
  name: vllm-llama3-70b
  namespace: llm
spec:
  selector:
    app: vllm-llama3-70b
  ports:
  - port: 80
    targetPort: 8000
  type: ClusterIP

server {
    listen 443 ssl http2;
    server_name llm-api.yourcompany.com;

    ssl_certificate /etc/nginx/ssl/fullchain.pem;
    ssl_certificate_key /etc/nginx/ssl/privkey.pem;
    ssl_protocols TLSv1.2 TLSv1.3;

    # 限流配置
    limit_req_zone $binary_remote_addr zone=llm:10m rate=10r/s;
    limit_conn_zone $binary_remote_addr zone=llm_conn:10m;

    location / {
        limit_req zone=llm burst=20 nodelay;
        limit_conn llm_conn 50;

        proxy_pass http://127.0.0.1:8000;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;

        # SSE 和长连接支持
        proxy_http_version 1.1;
        proxy_set_header Connection "";
        proxy_buffering off;
        proxy_cache off;
        proxy_read_timeout 300s;
    }
}

vllm serve ... \
  --otlp-traces-endpoint http://jaeger:4317/v1/traces

# 基础压测：随机请求
vllm bench serve \
  --host localhost \
  --port 8000 \
  --model meta-llama/Llama-3.1-8B-Instruct \
  --num-prompts 1000 \
  --random-input-len 1024 \
  --random-output-len 256 \
  --max-concurrency 64 \
  --api-key sk-xxx

# ShareGPT 数据集压测（更贴近真实对话）
vllm bench serve \
  --backend openai \
  --host localhost \
  --port 8000 \
  --dataset-name sharegpt \
  --dataset-path ./ShareGPT_V3_unfiltered_cleaned_split.json \
  --max-concurrency 128

# config.yaml - 生产环境配置示例 (v0.24.x)
model: meta-llama/Llama-3.1-70B-Instruct
tokenizer: null
revision: main

# 设备选择 (v0.24 新参数，替代 CUDA_VISIBLE_DEVICES)
device_ids: 0,1,2,3

# 模型配置
dtype: bfloat16
max_model_len: 32768
quantization: null
load_format: auto
trust_remote_code: false
task: generate

# 并行配置
tensor_parallel_size: 4
pipeline_parallel_size: 1
data_parallel_size: 1
distributed_executor_backend: mp
enable_expert_parallel: false

# KV Cache
block_size: 16
gpu_memory_utilization: 0.9
kv_cache_dtype: fp8
swap_space: 16
cpu_offload_gb: 0
enable_prefix_caching: true

# 调度器
max_num_batched_tokens: 16384
max_num_seqs: 128
enable_chunked_prefill: true
max_num_partial_prefills: 2
scheduling_policy: fcfs

# 网络
host: 0.0.0.0
port: 8000
api_key: sk-your-production-key-change-me
cors_allow_origins:
  - https://your-app.com
  - https://admin.your-app.com
ssl_keyfile: null
ssl_certfile: null
root_path: null

# LoRA
enable_lora: false
max_loras: 4
max_lora_rank: 64
max_cpu_loras: 16
lora_modules: {}

# 推测解码
speculative_model: null
num_speculative_tokens: 5

# 编译优化
optimization_level: 2
compilation_config:
  mode: 2

# 日志
uvicorn_log_level: warning
disable_log_stats: true
enable_log_requests: false

# 可观测性
otlp_traces_endpoint: http://jaeger.monitoring:4317/v1/traces

vllm serve --config config.yaml

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="sk-xxx"
)

# 普通对话
response = client.chat.completions.create(
    model="meta-llama/Llama-3.1-8B-Instruct",
    messages=[
        {"role": "system", "content": "你是一个有用的助手。"},
        {"role": "user", "content": "你好，请介绍一下自己。"}
    ],
    temperature=0.7,
    max_tokens=512,
    stream=True  # 流式输出
)

for chunk in response:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="", flush=True)

tools = [
    {
        "type": "function",
        "function": {
            "name": "get_weather",
            "description": "获取指定城市的天气",
            "parameters": {
                "type": "object",
                "properties": {
                    "city": {"type": "string", "description": "城市名称"}
                },
                "required": ["city"]
            }
        }
    }
]

response = client.chat.completions.create(
    model="Qwen/Qwen2.5-14B-Instruct",
    messages=[{"role": "user", "content": "北京今天天气怎么样？"}],
    tools=tools,
    tool_choice="auto"
)

# 启动嵌入模型
# vllm serve BAAI/bge-m3 --task embedding

# 调用
response = client.embeddings.create(
    model="BAAI/bge-m3",
    input=["这是一段测试文本", "这是另一段文本"]
)

for emb in response.data:
    print(f"文本 {emb.index} 向量维度: {len(emb.embedding)}")

pip install vllm-ascend
vllm serve Qwen/Qwen2.5-7B \
  --device npu \
  -tp 1 \
  --dtype float16 \
  --gpu-memory-utilization 0.85 \
  --max-model-len 8192

# 检查 GPU 状态
nvidia-smi
nvidia-smi topo -m  # 检查 GPU 拓扑

# 检查 NCCL
python -c "import torch; print(torch.cuda.nccl.version())"

# 检查 vLLM 版本
python -c "import vllm; print(vllm.__version__)"

# 测试模型加载
python -c "
from vllm import LLM
llm = LLM(model='meta-llama/Llama-3.1-8B-Instruct', dtype='bfloat16')
output = llm.generate('Hello, my name is')
print(output[0].outputs[0].text)
"

# 健康检查
curl http://localhost:8000/health
curl http://localhost:8000/v1/models

# 智能客服场景：低延迟 + 多轮对话 + LoRA
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --dtype bfloat16 \
  --quantization awq \
  --max-model-len 8192 \
  --gpu-memory-utilization 0.9 \
  --kv-cache-dtype fp8 \
  --enable-prefix-caching \
  --block-size 16 \
  --enable-chunked-prefill \
  --max-num-batched-tokens 8192 \
  --max-num-seqs 128 \
  --max-num-partial-prefills 2 \
  --enable-lora \
  --max-loras 4 \
  --api-key sk-chatbot-prod \
  -O2 \
  --host 0.0.0.0 --port 8000

# RAG 场景：长上下文 + 分块预填充 + 前缀缓存
vllm serve Qwen/Qwen2.5-72B-Instruct \
  -tp 4 \
  --dtype bfloat16 \
  --max-model-len 131072 \
  --gpu-memory-utilization 0.85 \
  --kv-cache-dtype fp8 \
  --enable-prefix-caching \
  --enable-chunked-prefill \
  --max-num-batched-tokens 32768 \
  --max-num-seqs 8 \
  --max-num-partial-prefills 2 \
  --max-long-partial-prefills 1 \
  -O2 \
  --host 0.0.0.0 --port 8000

# 代码场景：长上下文 + 前缀缓存（代码文件重复度高）
vllm serve deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct \
  -tp 2 \
  --dtype bfloat16 \
  --max-model-len 16384 \
  --gpu-memory-utilization 0.9 \
  --kv-cache-dtype fp8 \
  --enable-prefix-caching \
  --block-size 16 \
  --enable-chunked-prefill \
  --max-num-batched-tokens 16384 \
  --max-num-seqs 64 \
  -O2 \
  --host 0.0.0.0 --port 8000

# 高吞吐场景：最大化批处理，不关注单请求延迟
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  -tp 4 \
  --dtype bfloat16 \
  --max-model-len 32768 \
  --gpu-memory-utilization 0.95 \
  --kv-cache-dtype fp8 \
  --max-num-batched-tokens 65536 \
  --max-num-seqs 512 \
  -O3 \
  --disable-log-stats \
  --host 0.0.0.0 --port 8000

# Embedding 服务：高并发
vllm serve BAAI/bge-m3 \
  --task embedding \
  --dtype float16 \
  --max-model-len 8192 \
  --gpu-memory-utilization 0.9 \
  --max-num-batched-tokens 32768 \
  --max-num-seqs 512 \
  --enforce-eager \
  --host 0.0.0.0 --port 8001

# 多模态：图文理解
vllm serve Qwen/Qwen2-VL-72B-Instruct \
  -tp 4 \
  --dtype bfloat16 \
  --max-model-len 32768 \
  --limit-mm-per-prompt image=8,video=2 \
  --mm-processor-kwargs '{"num_crops": 8}' \
  --gpu-memory-utilization 0.9 \
  --enable-prefix-caching \
  --enable-chunked-prefill \
  --max-num-seqs 32 \
  -O2 \
  --host 0.0.0.0 --port 8000

# RTX 4090 24GB 运行 8B 模型
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --dtype float16 \
  --quantization awq \
  --max-model-len 8192 \
  --gpu-memory-utilization 0.88 \
  --kv-cache-dtype fp8 \
  --enable-prefix-caching \
  --max-num-seqs 64 \
  --host 0.0.0.0 --port 8000

# Mixtral 8x22B（经典 MoE）
vllm serve mistralai/Mixtral-8x22B-Instruct-v0.1 \
  -tp 4 \
  --enable-expert-parallel \
  --dtype bfloat16 \
  --max-model-len 32768 \
  --gpu-memory-utilization 0.9 \
  --kv-cache-dtype fp8 \
  --enable-prefix-caching \
  -O2 \
  --host 0.0.0.0 --port 8000

# DeepSeek-V4（v0.24 新增支持，含 EPLB + DeepEP v2）
vllm serve deepseek-ai/DeepSeek-V4 \
  -tp 8 \
  --enable-expert-parallel \
  --dtype bfloat16 \
  --max-model-len 131072 \
  --gpu-memory-utilization 0.9 \
  --kv-cache-dtype fp8 \
  --enable-prefix-caching \
  --enable-chunked-prefill \
  -O2 \
  --host 0.0.0.0 --port 8000