47 lines
1.1 KiB
YAML
47 lines
1.1 KiB
YAML
|
services:
|
||
|
|
qwen38-vllm:
|
||
|
|
image: ghcr.io/aeon-7/aeon-vllm-ultimate:latest
|
||
|
|
container_name: qwen38-vllm
|
||
|
|
restart: unless-stopped
|
||
|
|
gpus: all
|
||
|
|
ipc: host
|
||
|
|
shm_size: 16g
|
||
|
|
network_mode: host
|
||
|
|
volumes:
|
||
|
|
- /home/daniel/aeon-spark-test/h3/models/Qwen3.8-27B-NVFP4:/model:ro
|
||
|
|
- /home/daniel/aeon-spark-test/h3/vllm-cache:/root/.cache/vllm
|
||
|
|
entrypoint: ["vllm"]
|
||
|
|
command:
|
||
|
|
- serve
|
||
|
|
- /model
|
||
|
|
- --served-model-name
|
||
|
|
- qwen38
|
||
|
|
- --host
|
||
|
|
- 0.0.0.0
|
||
|
|
- --port
|
||
|
|
- "8002"
|
||
|
|
- --tensor-parallel-size
|
||
|
|
- "1"
|
||
|
|
- --gpu-memory-utilization
|
||
|
|
- "0.45"
|
||
|
|
- --max-model-len
|
||
|
|
- "262144"
|
||
|
|
- --max-num-seqs
|
||
|
|
- "4"
|
||
|
|
- --max-num-batched-tokens
|
||
|
|
- "8192"
|
||
|
|
- --enable-chunked-prefill
|
||
|
|
- --enable-prefix-caching
|
||
|
|
- --kv-cache-dtype
|
||
|
|
- fp8_e4m3
|
||
|
|
- --reasoning-parser
|
||
|
|
- qwen3
|
||
|
|
- --tool-call-parser
|
||
|
|
- qwen3_xml
|
||
|
|
- --enable-auto-tool-choice
|
||
|
|
- --distributed-executor-backend
|
||
|
|
- mp
|
||
|
|
- --speculative-config
|
||
|
|
- '{"method":"mtp","num_speculative_tokens":5}'
|
||
|
|
- --trust-remote-code
|