pip install vllm

vllm serve Qwen/Qwen2.5-7B-Instruct \
  --host 0.0.0.0 \
  --port 8000

curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "Qwen/Qwen2.5-7B-Instruct",
    "messages": [
      {"role": "system", "content": "你是一个助手"},
      {"role": "user", "content": "解释一下什么是 KV Cache？"}
    ],
    "max_tokens": 200
  }'

watch -n 1 nvidia-smi

vllm serve Qwen/Qwen2.5-7B-Instruct \
  --gpu-memory-utilization 0.90 \
  --max-model-len 4096 \
  --max-num-seqs 256 \
  --enforce-eager false \
  --host 0.0.0.0 \
  --port 8000